feat(landing): log parsing guide

apps/landing/src/lib/guide-highlight.ts

@@ -18,7 +18,8 @@ export type GuideLang =
     | "json"
     | "shell"
     | "python-logging"
-    | "loguru-output";
+    | "loguru-output"
+    | "otel-debug";
 
 const escape = (s: string) => s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
 const span = (cls: string, text: string) => `<span class="tok-${cls}">${escape(text)}</span>`;
@@ -205,15 +206,32 @@ const loguruOutput = (code: string) =>
         })
         .join("\n");
 
-// Shell commands: the program name, flags and quoted arguments.
+// Shell commands: the program name, flags and quoted arguments. Single-quoted
+// arguments (awk programs, jq filters) are one string, whatever quotes they hold.
 const shell = (code: string) =>
     tokenize(code, [
         [/#.*/y, (m) => span("comment", m[0])],
         [/(?<=^|\n|\| )(\s*)([\w.-]+)/y, (m) => escape(m[1]) + span("fn", m[2])],
+        [/'[^'\n]*'/y, (m) => span("labels", m[0])],
         [/"[^"\n]*"/y, (m) => span("labels", m[0])],
         [/(?<=\s)--?[\w-]+/y, (m) => span("value", m[0])],
     ]);
 
+// The OpenTelemetry Collector's debug exporter output: field and attribute keys
+// as names, values as values, the type wrappers (Str, Int) as comments. A value
+// that runs over several lines leaves its closing parenthesis to the last one.
+const otelDebug = (code: string) =>
+    code
+        .split("\n")
+        .map((line) => {
+            const attr = /^(\s*-> )([\w.]+)(: )(\w+\()(.*?)(\)?)$/.exec(line);
+            if (attr) return span("comment", attr[1]) + span("name", attr[2]) + escape(attr[3]) + span("comment", attr[4]) + span("value", attr[5]) + span("comment", attr[6]);
+            const field = /^(\w[\w ]*)(:)(.*)$/.exec(line);
+            if (field) return span("name", field[1]) + escape(field[2]) + span("value", field[3]);
+            return escape(line);
+        })
+        .join("\n");
+
 export const highlight = (code: string, lang: GuideLang) =>
     ({
         exposition,
@@ -233,4 +251,5 @@ export const highlight = (code: string, lang: GuideLang) =>
         shell,
         "python-logging": pythonLogging,
         "loguru-output": loguruOutput,
+        "otel-debug": otelDebug,
     })[lang](code);

apps/landing/src/pages/guides/_log-parsing/debug.yaml

@@ -0,0 +1,9 @@
+# debug.yaml: print parsed records instead of exporting them
+exporters:
+    debug:
+        verbosity: detailed
+
+service:
+    pipelines:
+        logs:
+            exporters: [debug]

apps/landing/src/pages/guides/_log-parsing/json.yaml

@@ -0,0 +1,34 @@
+receivers:
+    file_log/app:
+        include: [/var/log/app/app.log]
+        resource:
+            service.name: app
+        operators:
+            - type: json_parser
+              timestamp:
+                  parse_from: attributes.ts
+                  layout_type: gotime
+                  layout: "2006-01-02T15:04:05Z07:00"
+              severity:
+                  parse_from: attributes.level
+            - type: move
+              from: attributes.msg
+              to: body
+              on_error: send_quiet
+
+processors:
+    batch: {}
+
+exporters:
+    otlp_http/maple:
+        endpoint: https://ingest.maple.dev
+        compression: gzip
+        headers:
+            x-maple-ingest-key: ${env:MAPLE_INGEST_KEY}
+
+service:
+    pipelines:
+        logs:
+            receivers: [file_log/app]
+            processors: [batch]
+            exporters: [otlp_http/maple]

apps/landing/src/pages/guides/_log-parsing/logfmt.yaml

@@ -0,0 +1,30 @@
+receivers:
+    file_log/worker:
+        include: [/var/log/worker/worker.log]
+        resource:
+            service.name: worker
+        operators:
+            - type: key_value_parser
+              timestamp:
+                  parse_from: attributes.time
+                  layout_type: gotime
+                  layout: "2006-01-02T15:04:05Z07:00"
+              severity:
+                  parse_from: attributes.level
+
+processors:
+    batch: {}
+
+exporters:
+    otlp_http/maple:
+        endpoint: https://ingest.maple.dev
+        compression: gzip
+        headers:
+            x-maple-ingest-key: ${env:MAPLE_INGEST_KEY}
+
+service:
+    pipelines:
+        logs:
+            receivers: [file_log/worker]
+            processors: [batch]
+            exporters: [otlp_http/maple]

apps/landing/src/pages/guides/_log-parsing/multiline.yaml

@@ -0,0 +1,35 @@
+receivers:
+    file_log/checkout:
+        include: [/var/log/checkout/app.log]
+        resource:
+            service.name: checkout
+        # A new entry starts at a line beginning with a date. Other lines, such
+        # as stack frames, are appended to the entry before them.
+        multiline:
+            line_start_pattern: '^\d{4}-\d{2}-\d{2} '
+        operators:
+            - type: regex_parser
+              regex: '(?s)^(?P<time>\S+ \S+) (?P<level>[A-Z]+)\s+\[(?P<thread>[^\]]+)\] (?P<logger>\S+) - (?P<message>.*)$'
+              timestamp:
+                  parse_from: attributes.time
+                  layout: "%Y-%m-%d %H:%M:%S,%L"
+                  location: UTC
+              severity:
+                  parse_from: attributes.level
+
+processors:
+    batch: {}
+
+exporters:
+    otlp_http/maple:
+        endpoint: https://ingest.maple.dev
+        compression: gzip
+        headers:
+            x-maple-ingest-key: ${env:MAPLE_INGEST_KEY}
+
+service:
+    pipelines:
+        logs:
+            receivers: [file_log/checkout]
+            processors: [batch]
+            exporters: [otlp_http/maple]

apps/landing/src/pages/guides/_log-parsing/nginx.yaml

@@ -0,0 +1,72 @@
+receivers:
+    file_log/nginx:
+        include: [/var/log/nginx/access.log]
+        resource:
+            service.name: nginx
+        operators:
+            - type: regex_parser
+              regex: '^(?P<client_address>\S+) \S+ \S+ \[(?P<time_local>[^\]]+)\] "(?P<method>[A-Z]+) (?P<path>[^ ?"]+)[^ "]* HTTP/[0-9.]+" (?P<status>\d{3}) (?P<body_bytes>\d+|-) "[^"]*" "(?P<user_agent>[^"]*)"$'
+              timestamp:
+                  parse_from: attributes.time_local
+                  layout: "%d/%b/%Y:%H:%M:%S %z"
+              severity:
+                  parse_from: attributes.status
+                  overwrite_text: true
+                  mapping:
+                      info:
+                          - min: 100
+                            max: 399
+                      warn: 4xx
+                      error: 5xx
+            # Operators run top to bottom, so the renames come after the parser
+            # that creates the fields. Dots are not allowed in group names.
+            - type: move
+              from: attributes.client_address
+              to: attributes["client.address"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.method
+              to: attributes["http.request.method"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.path
+              to: attributes["url.path"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.status
+              to: attributes["http.response.status_code"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.body_bytes
+              to: attributes["http.response.body.size"]
+              on_error: send_quiet
+            - type: move
+              from: attributes.user_agent
+              to: attributes["user_agent.original"]
+              on_error: send_quiet
+            - type: remove
+              field: attributes.time_local
+              on_error: send_quiet
+
+processors:
+    transform/nginx:
+        error_mode: ignore
+        log_statements:
+            - delete_key(log.attributes, "http.response.body.size") where log.attributes["http.response.body.size"] == "-"
+            - set(log.attributes["http.response.status_code"], Int(log.attributes["http.response.status_code"])) where log.attributes["http.response.status_code"] != nil
+            - set(log.attributes["http.response.body.size"], Int(log.attributes["http.response.body.size"])) where log.attributes["http.response.body.size"] != nil
+    batch: {}
+
+exporters:
+    otlp_http/maple:
+        endpoint: https://ingest.maple.dev
+        compression: gzip
+        headers:
+            x-maple-ingest-key: ${env:MAPLE_INGEST_KEY}
+
+service:
+    pipelines:
+        logs:
+            receivers: [file_log/nginx]
+            processors: [transform/nginx, batch]
+            exporters: [otlp_http/maple]

apps/landing/src/pages/guides/_log-parsing/syslog.yaml

@@ -0,0 +1,25 @@
+receivers:
+    file_log/syslog:
+        include: [/var/log/remote/hosts.log]
+        resource:
+            service.name: syslog
+        operators:
+            - type: syslog_parser
+              protocol: rfc5424
+
+processors:
+    batch: {}
+
+exporters:
+    otlp_http/maple:
+        endpoint: https://ingest.maple.dev
+        compression: gzip
+        headers:
+            x-maple-ingest-key: ${env:MAPLE_INGEST_KEY}
+
+service:
+    pipelines:
+        logs:
+            receivers: [file_log/syslog]
+            processors: [batch]
+            exporters: [otlp_http/maple]

apps/landing/src/pages/guides/index.astro

@@ -115,6 +115,13 @@ const investigationGuides = [
         description: "An incident response process for small teams: an afternoon of preparation, the response steps for one or two engineers, and lightweight postmortems.",
         read: "12 min",
     },
+    {
+        href: "/guides/log-parsing",
+        label: "Logs",
+        title: "Log parsing: from raw lines to structured fields",
+        description: "How a raw log line becomes named fields, with a tested OpenTelemetry Collector config for nginx, JSON, logfmt, multiline and syslog logs.",
+        read: "9 min",
+    },
 ];
 
 const fieldNotes = (await getCollection("blog") as BlogPost[])