feat: per-task/per-job statistics from logs (MAPCO-11688)

MergerLogic/Extensions/ServiceCollectionExtensions.cs

@@ -152,6 +152,8 @@ public static IServiceCollection RegisterOpenTelemetry(this IServiceCollection c
                         options.AddProcessor(
                             new SimpleLogRecordExportProcessor(new OpenTelemetryFormattedConsoleExporter(new ConsoleExporterOptions()))); //lgtm [cs/local-not-disposed]
                         options.SetResourceBuilder(resourceBuilder);
+                        // required for the exporter to emit BeginScope correlation fields (jobId/taskId)
+                        options.IncludeScopes = true;
                     });
             });
             #endregion Logger

MergerLogic/ImageProcessing/ITileMerger.cs

@@ -6,5 +6,7 @@ namespace MergerLogic.ImageProcessing
     public interface ITileMerger
     {
         Tile? MergeTiles(List<CorrespondingTileBuilder> tiles, Coord targetCoords, TileFormatStrategy strategy, bool uploadOnly = false);
+
+        Tile? MergeTiles(List<CorrespondingTileBuilder> tiles, Coord targetCoords, TileFormatStrategy strategy, out MergeStats stats, bool uploadOnly = false);
     }
 }

MergerLogic/ImageProcessing/MergeStats.cs

@@ -0,0 +1,18 @@
+namespace MergerLogic.ImageProcessing
+{
+    /// <summary>
+    /// Describes which inputs contributed to a merged tile, used to classify the
+    /// write as added / merged / replaced. TargetUsed is false in upload-only mode.
+    /// </summary>
+    public readonly struct MergeStats
+    {
+        public bool TargetUsed { get; }
+        public bool AnySourceUsed { get; }
+
+        public MergeStats(bool targetUsed, bool anySourceUsed)
+        {
+            this.TargetUsed = targetUsed;
+            this.AnySourceUsed = anySourceUsed;
+        }
+    }
+}

MergerLogic/ImageProcessing/TileMerger.cs

@@ -20,6 +20,15 @@ public TileMerger(ITileScaler tileScaler, ILogger<TileMerger> logger)
 
         public Tile? MergeTiles(List<CorrespondingTileBuilder> tiles, Coord targetCoords, TileFormatStrategy strategy, bool uploadOnly = false)
         {
+            return this.MergeTiles(tiles, targetCoords, strategy, out _, uploadOnly);
+        }
+
+        public Tile? MergeTiles(List<CorrespondingTileBuilder> tiles, Coord targetCoords, TileFormatStrategy strategy,
+            out MergeStats stats, bool uploadOnly = false)
+        {
+            bool targetUsed = false;
+            bool anySourceUsed = false;
+
             if(uploadOnly) {
                 this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] Configured to upload only mode");
                 // Ignore target if in upload only mode
@@ -30,18 +39,21 @@ public TileMerger(ITileScaler tileScaler, ILogger<TileMerger> logger)
                     this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] Only one source was found, using raw image");
                     Tile? rawTile = tiles[0]();
                     rawTile?.ConvertToFormat(strategy.ApplyStrategy(rawTile.Format));
+                    stats = new MergeStats(false, rawTile != null);
                     return rawTile;
                 }
             }
 
-            var images = this.GetImageList(tiles, targetCoords, uploadOnly);
+            bool hasTarget = !uploadOnly && tiles.Count > 0;
+            var images = this.GetImageList(tiles, targetCoords, uploadOnly, hasTarget, out targetUsed, out anySourceUsed);
             IMagickImage<byte> image;
 
             switch (images.Count)
             {
                 case 0:
                     // There are no images
                     this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] No images where found return null");
+                    stats = new MergeStats(targetUsed, anySourceUsed);
                     return null;
                 case 1:
                     ImageFormatter.RemoveImageDateAttributes(images[0]);
@@ -73,14 +85,18 @@ public TileMerger(ITileScaler tileScaler, ILogger<TileMerger> logger)
             Tile tile = new Tile(targetCoords, image);
             image.Dispose();
             tile.ConvertToFormat(strategy.ApplyStrategy(tile.Format));
+            stats = new MergeStats(targetUsed, anySourceUsed);
             return tile;
         }
 
-        private List<MagickImage> GetImageList(List<CorrespondingTileBuilder> tiles, Coord targetCoords, bool uploadOnly)
+        private List<MagickImage> GetImageList(List<CorrespondingTileBuilder> tiles, Coord targetCoords, bool uploadOnly,
+            bool hasTarget, out bool targetUsed, out bool anySourceUsed)
         {
             var images = new List<MagickImage>();
             int i = tiles.Count - 1;
             Tile? tile = null;
+            targetUsed = false;
+            anySourceUsed = false;
 
             bool hasAlpha = false;
             try
@@ -100,7 +116,20 @@ private List<MagickImage> GetImageList(List<CorrespondingTileBuilder> tiles, Coo
                         continue;
                     }
 
+                    int before = images.Count;
                     this.AddTileToImageList(targetCoords, tile, images, out hasAlpha);
+                    if (images.Count > before)
+                    {
+                        if (hasTarget && i == 0)
+                        {
+                            targetUsed = true;
+                        }
+                        else
+                        {
+                            anySourceUsed = true;
+                        }
+                    }
+
                     if (!hasAlpha)
                     {
                         return images;

MergerLogic/Monitoring/OpenTelemetryFormattedConsoleExporter.cs

@@ -1,6 +1,7 @@
-using OpenTelemetry;
+using OpenTelemetry;
 using OpenTelemetry.Exporter;
 using OpenTelemetry.Logs;
+using System.Text.Json;
 
 namespace MergerLogic.Monitoring
 {
@@ -29,9 +30,44 @@
             var resource = this.ParseResource();
             var serviceName = this.GetResourceAttribute(resource, SERVICE_NAME_ATTRIBUTE, "unknown_service");
             var serviceVersion = this.GetResourceAttribute(resource, SERVICE_VERSION_ATTRIBUTE, "unknown_version");
-            var exception = record.Exception != null ? $" [{record.Exception}]" : string.Empty;
 
-            return $"[{this.FormatTime(record.Timestamp)}] [{record.LogLevel}] [{serviceName}] [{serviceVersion}] [{record.CategoryName}] [{Environment.CurrentManagedThreadId}] {record.State}{exception}";
+            var entry = new Dictionary<string, object?>
+            {
+                ["time"] = this.FormatTime(record.Timestamp),
+                ["level"] = record.LogLevel.ToString(),
+                ["service"] = serviceName,
+                ["version"] = serviceVersion,
+                ["category"] = record.CategoryName,
+                ["thread"] = Environment.CurrentManagedThreadId,
+                ["message"] = record.State?.ToString(),
+            };
+
+            if (record.Exception != null)
+            {
+                entry["exception"] = record.Exception.ToString();
+            }
+
+            this.AddScopes(record, entry);
+
+            return JsonSerializer.Serialize(entry);
+        }
+
+        // Flatten ILogger.BeginScope key/value pairs to top-level fields (e.g. jobId/taskId) so they
+        // are queryable in Loki. Empty unless IncludeScopes is enabled (see DI setup).
+        private void AddScopes(LogRecord record, Dictionary<string, object?> entry)
+        {
+            record.ForEachScope((scope, state) =>
+            {
+                foreach (var pair in scope)
+                {
+                    if (pair.Key == "{OriginalFormat}")
+                    {
+                        continue;
+                    }
+
+                    state[pair.Key] = pair.Value;
+                }
+            }, entry);
         }
 
         private string FormatTime(DateTime time)
@@ -41,7 +77,7 @@
 
         private Dictionary<string, object> ParseResource()
         {
             var attributes = this.ParentProvider.GetResource()?.Attributes;
             return attributes != null ? new Dictionary<string, object>(attributes) : new Dictionary<string, object>();
         }
 

MergerLogicUnitTests/ImageProcessing/TileMergerTest.cs

@@ -339,6 +339,65 @@ public void MergeTiles(Tile[] tiles, Coord targetCoord, TileFormatStrategy strat
             CollectionAssert.AreEqual(expectedTileBytes, result.GetImageBytes());
         }
 
+        [TestMethod]
+        [TestCategory("MergeTiles")]
+        public void MergeTilesStatsBlendedTargetAndSource()
+        {
+            var targetCoord = new Coord(15, 0, 0);
+            // target (index 0) is transparent, source (last) is transparent -> both enter the stack
+            var tiles = new[]
+            {
+                new Tile(targetCoord, File.ReadAllBytes("2.png")),
+                new Tile(targetCoord, File.ReadAllBytes("1.png"))
+            };
+            var tileBuilders = tiles.Select<Tile, CorrespondingTileBuilder>(tile => () => tile).ToList();
+
+            var result = this._testTileMerger.MergeTiles(tileBuilders, targetCoord, new TileFormatStrategy(TileFormat.Png), out var stats);
+
+            Assert.IsNotNull(result);
+            Assert.IsTrue(stats.TargetUsed);
+            Assert.IsTrue(stats.AnySourceUsed);
+        }
+
+        [TestMethod]
+        [TestCategory("MergeTiles")]
+        public void MergeTilesStatsOpaqueSourceOverTarget()
+        {
+            var targetCoord = new Coord(15, 0, 0);
+            // opaque source (last) short-circuits before the target (index 0) is reached
+            var tiles = new[]
+            {
+                new Tile(targetCoord, File.ReadAllBytes("1.png")),
+                new Tile(targetCoord, File.ReadAllBytes("3.jpeg"))
+            };
+            var tileBuilders = tiles.Select<Tile, CorrespondingTileBuilder>(tile => () => tile).ToList();
+
+            var result = this._testTileMerger.MergeTiles(tileBuilders, targetCoord, new TileFormatStrategy(TileFormat.Jpeg), out var stats);
+
+            Assert.IsNotNull(result);
+            Assert.IsFalse(stats.TargetUsed);
+            Assert.IsTrue(stats.AnySourceUsed);
+        }
+
+        [TestMethod]
+        [TestCategory("MergeTiles")]
+        public void MergeTilesStatsUploadOnly()
+        {
+            var targetCoord = new Coord(15, 0, 0);
+            var tiles = new[]
+            {
+                new Tile(targetCoord, File.ReadAllBytes("2.png")),
+                new Tile(targetCoord, File.ReadAllBytes("1.png"))
+            };
+            var tileBuilders = tiles.Select<Tile, CorrespondingTileBuilder>(tile => () => tile).ToList();
+
+            var result = this._testTileMerger.MergeTiles(tileBuilders, targetCoord, new TileFormatStrategy(TileFormat.Jpeg), out var stats, uploadOnly: true);
+
+            Assert.IsNotNull(result);
+            Assert.IsFalse(stats.TargetUsed);
+            Assert.IsTrue(stats.AnySourceUsed);
+        }
+
         #endregion
     }
 }

MergerLogicUnitTests/Monitoring/OpenTelemetryFormattedConsoleExporterTest.cs

@@ -0,0 +1,92 @@
+using MergerLogic.Monitoring;
+using Microsoft.Extensions.Logging;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using OpenTelemetry;
+using OpenTelemetry.Exporter;
+using OpenTelemetry.Logs;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text.Json;
+
+namespace MergerLogicUnitTests.Monitoring
+{
+    [TestClass]
+    [TestCategory("unit")]
+    [TestCategory("monitoring")]
+    public class OpenTelemetryFormattedConsoleExporterTest
+    {
+        private TextWriter _originalOut = null!;
+
+        [TestInitialize]
+        public void BeforeEach()
+        {
+            this._originalOut = Console.Out;
+        }
+
+        [TestCleanup]
+        public void AfterEach()
+        {
+            Console.SetOut(this._originalOut);
+        }
+
+        private static ILoggerFactory BuildFactory()
+        {
+            return LoggerFactory.Create(builder =>
+            {
+                builder.ClearProviders();
+                builder.AddOpenTelemetry(options =>
+                {
+                    options.IncludeScopes = true;
+                    options.AddProcessor(new SimpleLogRecordExportProcessor(
+                        new OpenTelemetryFormattedConsoleExporter(new ConsoleExporterOptions())));
+                });
+            });
+        }
+
+        private static JsonElement CaptureSingleLine(Action<ILogger> log)
+        {
+            var writer = new StringWriter();
+            Console.SetOut(writer);
+
+            using (var factory = BuildFactory())
+            {
+                log(factory.CreateLogger("TestCategory"));
+            }
+
+            string line = writer.ToString().Trim();
+            return JsonDocument.Parse(line).RootElement;
+        }
+
+        [TestMethod]
+        public void WhenLoggingInsideAScope_ShouldEmitScopeAsTopLevelJsonFields()
+        {
+            JsonElement entry = CaptureSingleLine(logger =>
+            {
+                using (logger.BeginScope(new Dictionary<string, object>
+                {
+                    ["jobId"] = "job-123",
+                    ["taskId"] = "task-456"
+                }))
+                {
+                    logger.LogInformation("processing tiles");
+                }
+            });
+
+            Assert.AreEqual("processing tiles", entry.GetProperty("message").GetString());
+            Assert.AreEqual("job-123", entry.GetProperty("jobId").GetString());
+            Assert.AreEqual("task-456", entry.GetProperty("taskId").GetString());
+            Assert.AreEqual("Information", entry.GetProperty("level").GetString());
+        }
+
+        [TestMethod]
+        public void WhenLoggingWithoutAScope_ShouldNotEmitScopeFields()
+        {
+            JsonElement entry = CaptureSingleLine(logger => logger.LogInformation("no scope here"));
+
+            Assert.AreEqual("no scope here", entry.GetProperty("message").GetString());
+            Assert.IsFalse(entry.TryGetProperty("jobId", out _), "unexpected jobId field");
+            Assert.IsFalse(entry.TryGetProperty("taskId", out _), "unexpected taskId field");
+        }
+    }
+}

MergerService/Models/Jobs/JobParamersAdditiomalParams.cs

@@ -7,13 +7,15 @@ namespace MergerService.Models.Jobs
      public class AdditionalParams
     {
         [JsonInclude] public string? JobTrackerServiceURL { get; }
+        [JsonInclude] public string? ReportOutputPath { get; }
 
         [System.Text.Json.Serialization.JsonIgnore]
         private JsonSerializerSettings _jsonSerializerSettings;
 
-        public AdditionalParams(string jobTrackerServiceURL)
+        public AdditionalParams(string jobTrackerServiceURL, string? reportOutputPath = null)
         {
             this.JobTrackerServiceURL = jobTrackerServiceURL;
+            this.ReportOutputPath = reportOutputPath;
 
             this._jsonSerializerSettings = new JsonSerializerSettings();
             this._jsonSerializerSettings.Converters.Add(new StringEnumConverter());