From 1d3a61c171325274923a89a3eac07ade6c2884f8 Mon Sep 17 00:00:00 2001 From: HardMax71 Date: Sat, 24 Oct 2026 23:44:41 +0200 Subject: [PATCH] fix: the error alert lists each kind of error, scanner 404s stop counting as errors, and nginx sees the visitor's address behind Cloudflare --- docs/architecture/rate-limiting.md | 2 +- docs/operations/grafana-integration.md | 18 +++++++- docs/operations/loki.md | 4 +- docs/operations/nginx-configuration.md | 14 +++++- frontend/nginx.conf.template | 38 ++++++++++++++-- grafana/provisioning/alerting/alerting.yml | 52 ++++++++++++++++++++-- otel-collector/otel-collector-config.yaml | 22 ++++++++- 7 files changed, 135 insertions(+), 15 deletions(-) diff --git a/docs/architecture/rate-limiting.md b/docs/architecture/rate-limiting.md index ee3fd28e..7962b76f 100644 --- a/docs/architecture/rate-limiting.md +++ b/docs/architecture/rate-limiting.md @@ -31,7 +31,7 @@ The bucket key is the user id from a valid access token, or the client IP when t --8<-- "backend/app/core/middlewares/rate_limit.py:extract_user_id" ``` -`decode_user_id` checks the signature and expiry and reads the token's `user_id` claim, which login sets. A forged or expired cookie fails that check, so it can't pick someone else's bucket. Tokens issued before the claim existed fall back to the IP until they expire. `get_client_ip` uses the socket peer address. It reads `X-Forwarded-For` (first entry) or `X-Real-IP` only when the peer is a loopback or private address, which is the case behind the frontend nginx. +`decode_user_id` checks the signature and expiry and reads the token's `user_id` claim, which login sets. A forged or expired cookie fails that check, so it can't pick someone else's bucket. Tokens issued before the claim existed fall back to the IP until they expire. `get_client_ip` uses the socket peer address. It reads `X-Forwarded-For` (first entry) or `X-Real-IP` only when the peer is a loopback or private address, which is the case behind the frontend nginx. nginx sets both to the visitor's address and drops any `X-Forwarded-For` the client sent. Behind Cloudflare it takes that address from `CF-Connecting-IP`, and only on connections from Cloudflare's ranges ([Nginx configuration](../operations/nginx-configuration.md#client-addresses-and-logging)). ## Rules diff --git a/docs/operations/grafana-integration.md b/docs/operations/grafana-integration.md index 78e8e132..9390c1bb 100644 --- a/docs/operations/grafana-integration.md +++ b/docs/operations/grafana-integration.md @@ -41,13 +41,13 @@ All dashboards land in one folder, `Integr8sCode`, with the fixed UID `integr8sc --8<-- "grafana/provisioning/alerting/alerting.yml" ``` -Every rule is an instant query evaluated every minute, and each summary names the service or mount and the value. Rules that point at a panel carry its `dashboardUid` and `panelId`, so the mail links to it. +Every rule is an instant query evaluated every minute, and each summary names the service or mount and the value. Rules that point at a panel carry its `dashboardUid` and `panelId`, and the same two as the `__dashboardUid__` and `__panelId__` annotations, since Grafana puts the panel link into the mail only from those. | Rule | Fires when | Severity | |---|---|---| | Host Memory, Host Disk | memory over 85% for 5 minutes or 95% for 2; disk over 80% for 10 minutes or 90% for 5 | warning, critical | | Container restarted, Container stopped | Docker restarted a container in the last 10 minutes; a container seen in the last 10 minutes hasn't reported for a minute, for 1 minute | critical | -| Error lines | a service logged any error line in 5 minutes, one alert per service, from Loki's `detected_level` | warning | +| Error lines | any error line in 5 minutes, from Loki's `detected_level`; one alert per service and line template ([Loki](loki.md#container-output)), so the mail lists each kind of error with its count | warning | | API 5xx, Kafka lag, MongoDB down, DLQ | 5xx over 1% for 5 minutes; a group with consumers 100 messages behind for 10 minutes; health under 1 or no data for 2 minutes; any DLQ message for 30 minutes or over 50 for 5 | critical or warning | The memory and disk rules read the host-metrics collector's scrapers; `system.memory.utilization` is off by default and `otel-host-metrics.yaml` turns it on. The container rules read the collector's docker_stats receiver. @@ -56,6 +56,20 @@ The Heartbeat rule always fires. A child policy sends it to the `heartbeat` cont All other alerts go to the `email-alerts` contact point, which sends one email addressed to everyone in `GRAFANA_ALERT_RECIPIENTS` (`singleEmail: true`). SMTP is configured in grafana.ini from env vars that compose fills in: `GF_SMTP_HOST`, `MAILJET_API_KEY` as the user, `MAILJET_SECRET_KEY` as the password and `MAILJET_FROM_ADDRESS` as the sender. Locally `GF_SMTP_HOST` is empty, so alerts fire but no mail goes out. Production sets it from the `MAILJET_HOST` secret. +The `email` template in the same file writes the subject and the body. The first line is the rule's `description` when it has one, then comes one line per alert from its `summary`, the resolved ones under "Resolved:", and the panel link, or the rule's page for a rule without a panel. An Error lines mail from a local run, with the backend unreachable and the worker failing to create pods, read: + +```text +16 errors in the last 5 minutes: +- 1x frontend: 2026/10/24 21:36:35 [error] 41#41: *6 backend could not be resolved (3: Host not found), client: 172.19.0.1, server: _, request: "GET /api/v1/x1 HTTP/1.1", host: "localhost:15001" +- 10x frontend: 2026/10/24 <*> [error] <*> <*> backend could not be resolved (3: Host not found), client: 172.19.0.1, server: _, request: "GET <*> HTTP/1.1", host: "localhost:15001" +- 1x integr8scode-worker: {"event": "Pod creation failed", "execution_id": "a1b2", "error": "boom", "logger": "integr8scode", "level": "error", "timestamp": "2026-10-24T21:30:00.000Z", } +- 4x integr8scode-worker: {"event": "Pod creation failed", "execution_id": <*> "error": <*> "logger": "integr8scode", "level": "error", "timestamp": <*> } + +https://grafana.integr8scode.cc/d/errors?orgId=1&viewPanel=5 +``` + +The 1x lines are the first of each shape: the collector keeps a line whole until a second one like it arrives. + The rule group names its folder by title, `Integr8sCode`. If the rules load before the dashboard provider creates that folder, Grafana makes a second `Integr8sCode` folder with a random UID and puts the rules there. The compose entrypoint avoids that: it starts Grafana without the alerting files, waits for `/api/folders/integr8scode`, then copies them in and calls `/api/admin/provisioning/alerting/reload`. A Grafana volume from before that fix may still have the stray folder holding the rules. Delete it once in the UI (it's the `Integr8sCode` folder whose URL isn't `/dashboards/f/integr8scode/`), then `docker compose restart grafana` so the entrypoint loads the rules into the right one. diff --git a/docs/operations/loki.md b/docs/operations/loki.md index 978b3326..49642e66 100644 --- a/docs/operations/loki.md +++ b/docs/operations/loki.md @@ -21,7 +21,7 @@ The OTel resource on every record has `service.name` from `SERVICE_NAME` (`integ ## Collector -The logs pipeline receives OTLP and container output (`fluent_forward`), runs `memory_limiter`, `filter/app_json_stdout`, `filter/mongo_info`, `groupbyattrs/container`, `transform/container`, `transform/level`, `batch` and `resource`, and exports with `otlp_http/loki` to `http://loki:3100/otlp`. It also hands every line to `forward/search`, the pipeline that indexes the same lines in [OpenSearch](opensearch.md). The full config is [`otel-collector/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/otel-collector/otel-collector-config.yaml). +The logs pipeline receives OTLP and container output (`fluent_forward`), runs `memory_limiter`, `filter/app_json_stdout`, `filter/mongo_info`, `groupbyattrs/container`, `transform/container`, `transform/level`, `drain`, `transform/template`, `batch` and `resource`, and exports with `otlp_http/loki` to `http://loki:3100/otlp`. It also hands every line to `forward/search`, the pipeline that indexes the same lines in [OpenSearch](opensearch.md). The full config is [`otel-collector/otel-collector-config.yaml`](https://github.com/HardMax71/Integr8sCode/blob/main/otel-collector/otel-collector-config.yaml). The browser's records come in on a pipeline of their own, `logs/browser`. Its receiver, `otlp/browser`, takes OTLP over HTTP on port 4319 with bodies up to 64 KB, and only nginx's `/otlp/v1/logs` reaches it ([Frontend logging](logging.md#frontend)). The browser sends every level, and `filter/browser` is where lines get dropped: below `warning` when the collector's `ENVIRONMENT` is `production`, none anywhere else. Anyone can post to that path, so `transform/browser` drops whatever resource a record arrives with and sets `service.name` to `browser`: a posted record can't pass as the backend or any other service. The Errors dashboard and the error-lines alert group by `service_name`, so `browser` shows up there like any container. @@ -31,6 +31,8 @@ Every compose service logs through Docker's `fluentd` log driver (the `x-logging Docker's lines arrive without a level, and Loki's own guess marks Mongo's and Redis's info lines as errors, so `transform/level` gives each one from its format before Loki sees it: a JSON or logfmt `level`, Mongo's `"s"`, a zap tab-separated level, nginx's `[error]`, then a level word such as log4j's or gunicorn's `ERROR`, a traceback's first line and Redis's marker after the time, and only then a plain `error` or `warn` word. Records that already have a level, like the backend's and worker's over OTLP, keep theirs. The [Errors dashboard](grafana-integration.md#dashboards) and the error-lines alert read the result through `detected_level`. +Then `drain`, the collector's [drain processor](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/drainprocessor), gives each line a template: it groups lines of the same shape and puts `<*>` where they differ, so five failed requests to a dead backend share one `connect() failed (111: Connection refused) while connecting to upstream, client: <*>` line. Two masks turn a traceback into one token, a plain one after the first line break and the `exception` field in structlog's JSON, so a deeper traceback doesn't make a new template. `transform/template` keeps the template, `log.record.template`, on error lines only, since Loki and OpenSearch store every attribute of every line. Loki has it as the structured metadata field `log_record_template`, and the error-lines alert groups by it. A new shape keeps its full text as the template until a second line like it arrives, and the collector learns the shapes again after a restart. The browser's pipeline runs both processors too. + With `fluentd-async` a container starts and buffers its lines while the collector is down, and `docker logs` reads Docker's local copy (3 files of 10 MB per container). The collector's own container keeps Docker's `local` driver, since its debug exporter would otherwise print its own lines back to itself. Loki 3 accepts OTLP natively. Resource attributes such as `service.name` and `service.namespace` become stream labels (`service_name`, `service_namespace`). The record's `trace_id` and `span_id` become structured metadata. For our own events the log line is the JSON that structlog rendered. Third-party library records arrive as plain text, so `| json` can't pull fields out of them. diff --git a/docs/operations/nginx-configuration.md b/docs/operations/nginx-configuration.md index 77c783c9..11bbd35e 100644 --- a/docs/operations/nginx-configuration.md +++ b/docs/operations/nginx-configuration.md @@ -19,7 +19,17 @@ flowchart LR --8<-- "frontend/nginx.conf.template:server_block" ``` -Both server blocks listen on `5001 ssl` with `/etc/nginx/certs/server.crt` and `server.key`. Mount your own pair there and nginx uses it; otherwise a script the image puts in `/docker-entrypoint.d/` makes a self-signed pair before nginx starts and keeps it for the container's lifetime. Compose publishes the port as `5001:5001`. +Both server blocks listen on `5001 ssl` with `/etc/nginx/certs/server.crt` and `server.key`. Mount your own pair there and nginx uses it; otherwise a script the image puts in `/docker-entrypoint.d/` makes a self-signed pair before nginx starts and keeps it for the container's lifetime. Compose publishes the port as `5001:5001`. In production the host's firewall lets only Cloudflare's ranges reach it, so every visitor comes through Cloudflare. + +### Client addresses and logging + +```nginx +--8<-- "frontend/nginx.conf.template:client_address" +``` + +Behind Cloudflare every connection comes from a Cloudflare address. `set_real_ip_from` lists Cloudflare's published ranges, and `real_ip_header CF-Connecting-IP` makes nginx take the visitor's address from that header on connections from those ranges only. A connection from anywhere else keeps its own address whatever headers it sends. Everything after that sees the visitor: the access log, the browser-log rate limit below, and the `X-Real-IP` and `X-Forwarded-For` headers nginx sends upstream. Both carry `$remote_addr` alone, so an `X-Forwarded-For` a client made up never reaches the backend's rate limiter. Cloudflare changes the list rarely; [cloudflare.com/ips](https://www.cloudflare.com/ips/) has the current one. + +`log_not_found off` keeps requests for missing files out of the error log; they're still 404s in the access log. Before it, every scanner probing for `/wp-includes/` or `/firebaseConfig.js` wrote an `[error]` line, and the [error-lines alert](grafana-integration.md#alerting) mailed it. ### Security headers @@ -69,7 +79,7 @@ The SSE location under `/api/v1/events/` speaks HTTP/1.1 with an empty `Connecti --8<-- "frontend/nginx.conf.template:browser_logs" ``` -The app sends its log records here as OTLP JSON, at most one batch a second, and nginx passes them to the collector's `otlp/browser` receiver ([Loki](loki.md#collector) has the rest of the path). Anyone can post to this path, so it takes POST only, bodies up to 64 KB, and 30 requests a minute per address with a burst of 30, from the `limit_req_zone` at the top of the file; past that nginx answers 429. The upstream goes through a variable like the backend's, so nginx starts while the collector is down and this path returns 502 until it's back. +The app sends its log records here as OTLP JSON, at most one batch a second, and nginx passes them to the collector's `otlp/browser` receiver ([Loki](loki.md#collector) has the rest of the path). Anyone can post to this path, so it takes POST only, bodies up to 64 KB, and 30 requests a minute per visitor address with a burst of 30, from the `limit_req_zone` at the top of the file; past that nginx answers 429. The upstream goes through a variable like the backend's, so nginx starts while the collector is down and this path returns 502 until it's back. ### Static files and SPA routing diff --git a/frontend/nginx.conf.template b/frontend/nginx.conf.template index fb8485d9..d5a26c99 100644 --- a/frontend/nginx.conf.template +++ b/frontend/nginx.conf.template @@ -1,4 +1,34 @@ +# --8<-- [start:client_address] +# Cloudflare's ranges from https://www.cloudflare.com/ips/; only they may name the visitor's address +set_real_ip_from 173.245.48.0/20; +set_real_ip_from 103.21.244.0/22; +set_real_ip_from 103.22.200.0/22; +set_real_ip_from 103.31.4.0/22; +set_real_ip_from 141.101.64.0/18; +set_real_ip_from 108.162.192.0/18; +set_real_ip_from 190.93.240.0/20; +set_real_ip_from 188.114.96.0/20; +set_real_ip_from 197.234.240.0/22; +set_real_ip_from 198.41.128.0/17; +set_real_ip_from 162.158.0.0/15; +set_real_ip_from 104.16.0.0/13; +set_real_ip_from 104.24.0.0/14; +set_real_ip_from 172.64.0.0/13; +set_real_ip_from 131.0.72.0/22; +set_real_ip_from 2400:cb00::/32; +set_real_ip_from 2606:4700::/32; +set_real_ip_from 2803:f800::/32; +set_real_ip_from 2405:b500::/32; +set_real_ip_from 2405:8100::/32; +set_real_ip_from 2a06:98c0::/29; +set_real_ip_from 2c0f:f248::/32; +real_ip_header CF-Connecting-IP; + +# A request for a missing file is a 404 in the access log, not an error line +log_not_found off; + limit_req_zone $binary_remote_addr zone=browser_logs:1m rate=30r/m; +# --8<-- [end:client_address] # --8<-- [start:server_block] server { @@ -53,7 +83,7 @@ server { proxy_ssl_trusted_certificate /etc/nginx/backend/server.crt; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; # Forward cookies @@ -67,7 +97,7 @@ server { proxy_pass $backend_upstream; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; # Forward cookies @@ -96,7 +126,7 @@ server { } # --8<-- [end:browser_logs] - # Browsers and crawlers ask for /favicon.ico, the app ships favicon.png, and each miss logged an nginx [error] + # Browsers and crawlers ask for /favicon.ico, and the app ships favicon.png location = /favicon.ico { try_files /favicon.png =404; expires 1y; @@ -139,7 +169,7 @@ server { proxy_pass $grafana_upstream; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto https; } diff --git a/grafana/provisioning/alerting/alerting.yml b/grafana/provisioning/alerting/alerting.yml index 7e0effaa..531c8f70 100644 --- a/grafana/provisioning/alerting/alerting.yml +++ b/grafana/provisioning/alerting/alerting.yml @@ -9,6 +9,8 @@ contactPoints: settings: addresses: ${GF_SMTP_ALERT_RECIPIENTS} singleEmail: true + subject: '{{ template "email.subject" . }}' + message: '{{ template "email.message" . }}' # An outside dead man's switch; when its pings stop, the host or Grafana is down - orgId: 1 @@ -21,6 +23,20 @@ contactPoints: httpMethod: POST disableResolveMessage: true +# One line per alert from its summary, under the rule's description when it has one +templates: + - orgId: 1 + name: email + template: | + {{ define "email.subject" }}[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}{{ with .Alerts.Firing }}{{ with (index . 0).Annotations.description }}: {{ . }}{{ end }}{{ end }}{{ end }} + {{ define "email.message" }}{{ with .Alerts.Firing }}{{ with (index . 0).Annotations.description }}{{ . }}: + {{ end }}{{ range . }}- {{ .Annotations.summary }} + {{ end }}{{ end }}{{ with .Alerts.Resolved }} + Resolved: + {{ range . }}- {{ .Annotations.summary }} + {{ end }}{{ end }} + {{ with index .Alerts 0 }}{{ or .PanelURL .GeneratorURL }}{{ end }}{{ end }} + policies: - orgId: 1 receiver: email-alerts @@ -36,6 +52,7 @@ policies: group_interval: 1m repeat_interval: 5m +# The email links a rule's panel only through the __dashboardUid__ and __panelId__ annotations; dashboardUid and panelId alone don't reach it groups: - orgId: 1 name: infrastructure-alerts @@ -135,6 +152,8 @@ groups: dashboardUid: http-middleware panelId: 108 annotations: + __dashboardUid__: http-middleware + __panelId__: "108" summary: "Disk {{ $labels.mountpoint }} is {{ printf \"%.0f\" $values.A.Value }}% full" labels: severity: warning @@ -166,6 +185,8 @@ groups: dashboardUid: http-middleware panelId: 108 annotations: + __dashboardUid__: http-middleware + __panelId__: "108" summary: "Disk {{ $labels.mountpoint }} is {{ printf \"%.0f\" $values.A.Value }}% full; a full disk stopped prod before" labels: severity: critical @@ -197,6 +218,8 @@ groups: dashboardUid: errors panelId: 3 annotations: + __dashboardUid__: errors + __panelId__: "3" summary: "{{ $labels.container_name }} restarted {{ printf \"%.0f\" $values.A.Value }} times in 10 minutes" labels: severity: critical @@ -229,6 +252,8 @@ groups: dashboardUid: errors panelId: 4 annotations: + __dashboardUid__: errors + __panelId__: "4" summary: "{{ $labels.container_name }} stopped reporting; it was up {{ printf \"%.0f\" $values.A.Value }} s" labels: severity: critical @@ -248,7 +273,15 @@ groups: to: 0 datasourceUid: loki model: - expr: sum by (service_name) (count_over_time({service_namespace="integr8scode"} | detected_level="error" [5m])) + expr: sum by (service_name, log_record_template) (count_over_time({service_namespace="integr8scode"} | detected_level="error" [5m])) + queryType: instant + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: loki + model: + expr: sum(count_over_time({service_namespace="integr8scode"} | detected_level="error" [5m])) queryType: instant - refId: C datasourceUid: __expr__ @@ -262,9 +295,12 @@ groups: for: 0s noDataState: OK dashboardUid: errors - panelId: 1 + panelId: 5 annotations: - summary: "{{ $labels.service_name }} logged {{ printf \"%.0f\" $values.A.Value }} error lines in 5 minutes" + __dashboardUid__: errors + __panelId__: "5" + description: "{{ printf \"%.0f\" $values.B.Value }} {{ if eq $values.B.Value 1.0 }}error{{ else }}errors{{ end }} in the last 5 minutes" + summary: "{{ printf \"%.0f\" $values.A.Value }}x {{ $labels.service_name }}: {{ $labels.log_record_template }}" labels: severity: warning @@ -295,6 +331,8 @@ groups: dashboardUid: http-middleware panelId: 1 annotations: + __dashboardUid__: http-middleware + __panelId__: "1" summary: "{{ printf \"%.1f\" $values.A.Value }}% of API requests answered 5xx over 5 minutes" labels: severity: critical @@ -327,6 +365,8 @@ groups: dashboardUid: kafka-events-monitoring panelId: 21 annotations: + __dashboardUid__: kafka-events-monitoring + __panelId__: "21" summary: "{{ $labels.group }} is {{ printf \"%.0f\" $values.A.Value }} messages behind on {{ $labels.topic }}" labels: severity: warning @@ -359,6 +399,8 @@ groups: dashboardUid: mongodb-monitoring panelId: 1 annotations: + __dashboardUid__: mongodb-monitoring + __panelId__: "1" summary: "MongoDB reports unhealthy or stopped reporting" labels: severity: critical @@ -390,6 +432,8 @@ groups: dashboardUid: dlq-monitoring panelId: 1 annotations: + __dashboardUid__: dlq-monitoring + __panelId__: "1" summary: "{{ printf \"%.0f\" $values.A.Value }} messages have waited in the DLQ for 30 minutes" labels: severity: warning @@ -421,6 +465,8 @@ groups: dashboardUid: dlq-monitoring panelId: 5 annotations: + __dashboardUid__: dlq-monitoring + __panelId__: "5" summary: "{{ printf \"%.0f\" $values.A.Value }} messages are waiting in the DLQ" labels: severity: critical diff --git a/otel-collector/otel-collector-config.yaml b/otel-collector/otel-collector-config.yaml index cecd4a40..285f79b8 100644 --- a/otel-collector/otel-collector-config.yaml +++ b/otel-collector/otel-collector-config.yaml @@ -909,6 +909,24 @@ processors: - 'set(log.severity_text, "WARN") where log.severity_text == "" and log.severity_number == SEVERITY_NUMBER_WARN' - 'set(log.severity_text, "INFO") where log.severity_text == "" and log.severity_number == SEVERITY_NUMBER_INFO' + # Lines of one shape share a template ("open() <*> failed (2: No such file or directory)"); the error-lines alert lists them by it + drain: + max_clusters: 1000 + masking_rules: + # A traceback, after a line break or in structlog's exception field, becomes one token, so its depth doesn't split the template + - name: more + pattern: '\n[\s\S]*' + - name: exception + pattern: '"exception": "(?:[^"\\]|\\.)*"' + + # Only error lines keep the template, since Loki and OpenSearch store every attribute of every line + transform/template: + error_mode: ignore + log_statements: + - context: log + statements: + - delete_key(log.attributes, "log.record.template") where log.severity_number < SEVERITY_NUMBER_ERROR + exporters: # Export to Jaeger for traces via OTLP otlp_grpc/jaeger: @@ -995,12 +1013,12 @@ service: logs: receivers: [otlp, fluent_forward] - processors: [memory_limiter, filter/app_json_stdout, filter/mongo_info, groupbyattrs/container, transform/container, transform/level, batch, resource] + processors: [memory_limiter, filter/app_json_stdout, filter/mongo_info, groupbyattrs/container, transform/container, transform/level, drain, transform/template, batch, resource] exporters: [otlp_http/loki, forward/search, debug] logs/browser: receivers: [otlp/browser] - processors: [memory_limiter, filter/browser, transform/browser, batch, resource] + processors: [memory_limiter, filter/browser, transform/browser, drain, transform/template, batch, resource] exporters: [otlp_http/loki, forward/search] logs/search: