From 19efa3c2a695b8bc0016fd941cf209c6dca1e991 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Thu, 5 Nov 2026 17:00:15 +0100 Subject: [PATCH 1/7] feat: Collect the containerdebug logs --- CHANGELOG.md | 10 + .../opa/pages/usage-guide/logging.adoc | 8 + rust/operator-binary/Cargo.toml | 1 + .../product_logging/vector-test.yaml | 279 ++++++++++++++++++ .../properties/product_logging/vector.yaml | 30 +- .../build/resource/workload/daemonset.rs | 9 +- .../controller/build/resource/workload/mod.rs | 27 +- .../opa-vector-aggregator-values.yaml.j2 | 56 +++- 8 files changed, 389 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f6e9786..77e14e61 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ All notable changes to this project will be documented in this file. - A `PodDisruptionBudget` is now written out for the `servers` role when it runs as a `Deployment`, with `maxUnavailable: 1`. Configurable via `spec.servers.roleConfig.podDisruptionBudget` ([#873]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#894]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator ([#912]). ### Changed @@ -92,6 +94,13 @@ All notable changes to this project will be documented in this file. See [our internal issue](https://github.com/stackabletech/hdfs-operator/issues/626) and [the fix](https://github.com/kube-rs/kube/pull/2042) for details ([#871]). - The reconciler now applies resources and derives the cluster status in discrete apply and update_status steps for the `opa_controller` ([#872]). +- The `containerdebug` process writes its log file again. The operator set + `CONTAINERDEBUG_LOG_DIRECTORY` on the OPA container, which containerdebug stopped reading in its + version 0.2.0 when it moved to `stackable-telemetry`; the directory is now set via + `FILE_LOG_DIRECTORY`, with an hourly rotation that keeps at most two files ([#912]). +- The `tracing-rs` transform of the Vector agent no longer discards log events. A message which is + not a string is kept among the other fields, a `fields` object which is not an object falls back + to the raw line, and a missing timestamp is recorded as an error ([#912]). [#873]: https://github.com/stackabletech/opa-operator/pull/873 [#852]: https://github.com/stackabletech/opa-operator/pull/852 @@ -113,6 +122,7 @@ All notable changes to this project will be documented in this file. [#905]: https://github.com/stackabletech/opa-operator/pull/905 [#906]: https://github.com/stackabletech/opa-operator/pull/906 [#908]: https://github.com/stackabletech/opa-operator/pull/908 +[#912]: https://github.com/stackabletech/opa-operator/pull/912 ## [26.7.0] - 2026-07-21 diff --git a/docs/modules/opa/pages/usage-guide/logging.adoc b/docs/modules/opa/pages/usage-guide/logging.adoc index 23e69883..7662bdb6 100644 --- a/docs/modules/opa/pages/usage-guide/logging.adoc +++ b/docs/modules/opa/pages/usage-guide/logging.adoc @@ -56,5 +56,13 @@ spec: The decision logs are still filtered by the log level set for the console and file appenders. Therefore, the configuration above would result in decision logs being present on file but not on console. +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `opa` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/opa` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the size-capped log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index a06c0b9b..f74d8f16 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -32,4 +32,5 @@ tracing.workspace = true built.workspace = true [dev-dependencies] +stackable-operator = { workspace = true, features = ["test-support"] } yaml_serde.workspace = true diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml index d427a072..8fcd13f3 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -107,6 +107,285 @@ tests: source: | assert!(exists(.errors), "unparsable timestamp should be recorded as an error") + - name: Containerdebug tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk\n\ndisk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Containerdebug tracing-rs log which is unparsable + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{ unparsable' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + .errors[0] + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{ unparsable", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "JSON not parsable: ")) + - name: Containerdebug tracing-rs log which is not a JSON object + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: 'false' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "false", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Containerdebug tracing-rs log with invalid fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + .errors[0], + "Logger/target not found.", + "Level \"INVALID\" unknown, using \"INFO\" instead.", + "Fields are not an object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: ")) + - name: Containerdebug tracing-rs log without a timestamp + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The current time is used and not the timestamp which the file + # source attached to the event. The latter reaches the transform + # as a string, whereas now() yields a timestamp. + assert!(is_timestamp(.timestamp)) + - name: Containerdebug tracing-rs log without a message but with other fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Containerdebug tracing-rs log with a message which is not a string + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay\nmessage=42", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Containerdebug tracing-rs log without any fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + "Message not found." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) - name: Prepare stdout log inputs: - type: log diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index 804d1597..5586b9ef 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -84,8 +84,10 @@ transforms: if err == null { .timestamp = parsed_timestamp } else { - .errors = push(.errors, "Timestamp not parsable, trying current time instead: " + err) + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") } .logger, err = string(event.target) @@ -103,19 +105,29 @@ transforms: } fields, err = object(event.fields) - if err != null { + if err == null { + message, err = string(fields.message) + if err == null { + # The message is prepended to the other fields, so remove it from + # them. + del(fields.message) + } else { + # Keep an unusable message among the other fields instead of + # discarding its content. + message = "" + } + + other_fields = encode_key_value(fields, field_delimiter: "\n") + .message = join!(compact([message, other_fields]), "\n\n") + } else { .errors = push(.errors, "Fields are not an object.") + # Fall back to the raw line, so that its content is not lost. + .message = raw_message } - .message, err = string(fields.message) - if err != null || is_empty(.message) { + if is_empty(.message) { .errors = push(.errors, "Message not found.") } - - del(fields.message) - - other_fields = encode_key_value(fields, field_delimiter: "\n") - .message = join!(compact([.message, other_fields]), "\n\n") } # Transforms OPA's own JSON logs diff --git a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs index 702f35a9..90925b9d 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs @@ -98,7 +98,6 @@ mod tests { let _ = *USER_INFO_FETCHER_KERBEROS_VOLUME_NAME; let _ = *RESOURCE_INFO_FETCHER_CREDENTIALS_VOLUME_NAME; let _ = *TLS_VOLUME_NAME; - let _ = *CONTAINERDEBUG_LOG_DIRECTORY; let _ = *WATCH_NAMESPACE; let _ = *CONSOLE_LOG_LEVEL; let _ = *FILE_LOG_LEVEL; @@ -812,12 +811,10 @@ mod tests { } /// The user-supplied `envOverrides` must be merged in after all operator-set environment - /// variables, so that they can override any of them. `CONTAINERDEBUG_LOG_DIRECTORY` is used + /// variables, so that they can override any of them. `FILE_LOG_DIRECTORY` is used /// as the example here because it is set unconditionally by the operator. #[test] fn env_overrides_override_operator_set_env_vars() { - use std::str::FromStr; - use stackable_operator::v2::builder::pod::container::EnvVarName; let cluster = validated_cluster_from_spec(json!({ @@ -830,7 +827,7 @@ mod tests { .expect("the default role group should exist"); let mut role_group = role_group.clone(); role_group.env_overrides = EnvVarSet::new().with_value( - &EnvVarName::from_str("CONTAINERDEBUG_LOG_DIRECTORY").expect("valid env var name"), + &EnvVarName::from_str_unsafe("FILE_LOG_DIRECTORY"), "/custom/log/dir", ); @@ -860,7 +857,7 @@ mod tests { let containerdebug: Vec<_> = env .iter() - .filter(|env_var| env_var.name == "CONTAINERDEBUG_LOG_DIRECTORY") + .filter(|env_var| env_var.name == "FILE_LOG_DIRECTORY") .collect(); assert_eq!( containerdebug.len(), diff --git a/rust/operator-binary/src/controller/build/resource/workload/mod.rs b/rust/operator-binary/src/controller/build/resource/workload/mod.rs index dae0ce3e..1ea3f2c3 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/mod.rs @@ -6,6 +6,7 @@ use std::{collections::BTreeMap, str::FromStr}; +use const_format::concatcp; use indoc::formatdoc; use snafu::{ResultExt, Snafu}; use stackable_opa_operator::crd::{Container, DEFAULT_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, OpaRole}; @@ -87,7 +88,6 @@ const RESOURCE_INFO_FETCHER_CREDENTIALS_DIR: &str = "/stackable/credentials"; constant!(TLS_VOLUME_NAME: VolumeName = "tls"); const TLS_STORE_DIR: &str = "/stackable/tls"; -constant!(CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); constant!(WATCH_NAMESPACE: EnvVarName = "WATCH_NAMESPACE"); // HTTP probe configuration shared by the bundle-builder and OPA containers. They differ in the @@ -335,10 +335,23 @@ pub fn build_server_rolegroup_pod_template( // All operator-set environment variables of the OPA container, collected into an // `EnvVarSet` so that every name occurs only once. - let opa_env_vars = EnvVarSet::new().with_value( - &CONTAINERDEBUG_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/containerdebug"), - ); + // + // These are the generic `stackable-telemetry` variables, here for the `containerdebug` + // process which runs in the background of the OPA container. `FILE_LOG_LEVEL` is + // deliberately not set: unlike our own Rust binaries, containerdebug's level is not derived + // from the log configuration. The rotation differs from the one in + // [`stackable_rust_cli_env_vars`] as well, because containerdebug writes only a handful of + // lines per run and a minutely rotation with five files would keep just a few minutes of it. + let opa_env_vars = EnvVarSet::new() + // The directory must be exactly one level below the log directory, because the Vector + // agent derives the container name from that path segment. + .with_value( + &FILE_LOG_DIRECTORY, + concatcp!(STACKABLE_LOG_DIR, "/containerdebug"), + ) + // Rotation is opt-in, so without these the file would grow unbounded. + .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly") + .with_value(&FILE_LOG_MAX_FILES, "2"); // Environment variable overrides (highest precedence), merged from role and role group. // They are merged in last so that they override any operator-set environment variable. let opa_env_vars = opa_env_vars.merge(rolegroup_config.env_overrides.clone()); @@ -518,6 +531,9 @@ pub fn build_server_rolegroup_pod_template( /// The size limit of the shared `log` [`EmptyDirVolumeSource`], which has to accommodate the log /// files of every container that mounts it. The always-present containers are budgeted for /// unconditionally; the optional info-fetcher sidecars only when the cluster configures them. +/// +/// The `containerdebug` files in this volume are rotated by time, not by size, so they have no +/// budget; raising `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. fn log_volume_size_limit(cluster: &ValidatedCluster) -> Quantity { let mut max_log_file_sizes = vec![ MAX_OPA_BUNDLE_BUILDER_LOG_FILE_SIZE, @@ -656,6 +672,7 @@ fn build_opa_start_command( {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers + mkdir --parents \"$FILE_LOG_DIRECTORY\" containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & opa run -s -a 0.0.0.0:{bind_port} -c {CONFIG_DIR}/{config_file} -l {opa_log_level} --shutdown-grace-period {shutdown_grace_period_s} --disable-telemetry {tls_flags} {extra_cli_args} {logging_redirects} & wait_for_termination $! diff --git a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 index f084db8e..5d065e3f 100644 --- a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 @@ -22,55 +22,71 @@ customConfig: transforms: validEvents: type: filter - inputs: [vector] + inputs: + - vector condition: is_null(.errors) filteredAutomaticLogConfigServerOpaDecision: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "opa" && .logger == "decision" + filteredAutomaticLogConfigServerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + starts_with(string!(.pod), "test-opa-server-automatic-log-config") && + .container == "containerdebug" filteredAutomaticLogConfigServerOpa: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "opa" && .logger != "decision" filteredAutomaticLogConfigServerBundleBuilder: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "bundle-builder" filteredAutomaticLogConfigServerUserInfoFetcher: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "user-info-fetcher" filteredAutomaticLogConfigServerResourceInfoFetcher: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "resource-info-fetcher" filteredAutomaticLogConfigServerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "vector" filteredAutomaticLogConfigServerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "prepare" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -78,11 +94,29 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. + stdout: + inputs: + - vector + type: console + encoding: + codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector routing: endpoints: From cd5acb01f62cfcc7e29892ffdfeee252e257c749 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 6 Nov 2026 09:15:50 +0100 Subject: [PATCH 2/7] chore: Upgrade serde-saphyr --- Cargo.lock | 6 +++--- Cargo.nix | 13 +++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bdb77a15..43c8506d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3339,12 +3339,12 @@ dependencies = [ [[package]] name = "serde-saphyr" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f4dc2714fc5632417a6e979602d7044bad8f2880c63fc7217087180acec31b6" +checksum = "d43fb82d550b07e6f929a58d7d0935182602f48ddead875f5ee313c41c8f07fe" dependencies = [ "annotate-snippets", - "base64 0.23.1", + "base64 0.22.1", "encoding_rs_io", "granit-parser", "nohash-hasher", diff --git a/Cargo.nix b/Cargo.nix index 361690a4..122a7f50 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -11470,10 +11470,10 @@ rec { }; "serde-saphyr" = rec { crateName = "serde-saphyr"; - version = "2.1.0"; + version = "2.1.1"; edition = "2024"; crateBin = []; - sha256 = "1diixjn80w882xrgqqqci3rdifj4f0nn0yg9lqbj8qy59xqw4k9z"; + sha256 = "1zh7iwfc84z3brgqgbfyips049hq6l4pv3d557wyc1qbalnvhgyl"; libName = "serde_saphyr"; license = "MIT OR Apache-2.0"; dependencies = [ @@ -11484,7 +11484,7 @@ rec { } { name = "base64"; - packageId = "base64 0.23.1"; + packageId = "base64 0.22.1"; optional = true; } { @@ -12638,6 +12638,11 @@ rec { } ]; devDependencies = [ + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "webhook" "test-support" ]; + } { name = "yaml_serde"; packageId = "yaml_serde"; @@ -13093,7 +13098,7 @@ rec { "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; }; - resolvedDefaultFeatures = [ "crds" "default" "webhook" ]; + resolvedDefaultFeatures = [ "crds" "default" "test-support" "webhook" ]; }; "stackable-operator-derive" = rec { crateName = "stackable-operator-derive"; From 8d1a82ab716d9b121c86853e55683551c99add06 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 9 Nov 2026 17:23:12 +0100 Subject: [PATCH 3/7] fix: Do not hard-code the environment variable name "FILE_LOG_DIRECTORY" --- .../src/controller/build/resource/workload/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/controller/build/resource/workload/mod.rs b/rust/operator-binary/src/controller/build/resource/workload/mod.rs index 1ea3f2c3..abc8968b 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/mod.rs @@ -672,7 +672,7 @@ fn build_opa_start_command( {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers - mkdir --parents \"$FILE_LOG_DIRECTORY\" + mkdir --parents \"${file_log_directory}\" || true containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & opa run -s -a 0.0.0.0:{bind_port} -c {CONFIG_DIR}/{config_file} -l {opa_log_level} --shutdown-grace-period {shutdown_grace_period_s} --disable-telemetry {tls_flags} {extra_cli_args} {logging_redirects} & wait_for_termination $! @@ -682,6 +682,7 @@ fn build_opa_start_command( remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + file_log_directory = FILE_LOG_DIRECTORY.as_ref(), shutdown_grace_period_s = merged_config.graceful_shutdown_timeout.unwrap_or(DEFAULT_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT).as_secs(), opa_log_level = [console_log_level, file_log_level].iter().min().unwrap_or(&LogLevel::INFO).to_opa_literal(), extra_cli_args = extra_cli_args From 28c60fd419f4ffc3fc183d0f5974e7f4e4bedc5e Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 9 Nov 2026 20:27:02 +0100 Subject: [PATCH 4/7] fix: Create the log directories of the info-fetcher sidecars --- CHANGELOG.md | 3 ++ .../build/resource/workload/daemonset.rs | 45 +++++++++++++++-- .../controller/build/resource/workload/mod.rs | 48 +++++++++++++------ 3 files changed, 79 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 355c3692..b8cac36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,6 +110,9 @@ All notable changes to this project will be documented in this file. - The `tracing-rs` transform of the Vector agent no longer discards log events. A message which is not a string is kept among the other fields, a `fields` object which is not an object falls back to the raw line, and a missing timestamp is recorded as an error ([#912]). +- The user-info-fetcher and resource-info-fetcher containers no longer log an error about a missing + log directory on startup. Their log directory is now created by the `prepare` container, because + those containers run their binary without a shell which could create it ([#912]). [#873]: https://github.com/stackabletech/opa-operator/pull/873 [#852]: https://github.com/stackabletech/opa-operator/pull/852 diff --git a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs index 90925b9d..9c857061 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs @@ -389,7 +389,7 @@ mod tests { // The redirect is appended directly after the bundle-builder invocation. (`/dev/null` also // appears in the shared bash trap helpers, so match the specific redirect.) assert!( - build_bundle_builder_start_command(&silenced, "bundle-builder") + build_bundle_builder_start_command(&silenced) .contains("stackable-opa-bundle-builder > /dev/null") ); @@ -404,8 +404,7 @@ mod tests { }, })); assert!( - build_bundle_builder_start_command(&logging, "bundle-builder") - .contains("stackable-opa-bundle-builder &") + build_bundle_builder_start_command(&logging).contains("stackable-opa-bundle-builder &") ); } @@ -690,6 +689,46 @@ mod tests { } } + /// The info-fetcher sidecars run their binary without a shell, so they cannot create their own + /// log directory. The prepare container does it for them, and the path it creates has to be + /// exactly the one the sidecar logs into: otherwise the telemetry library prunes against a + /// missing directory and complains on every start. + #[test] + fn prepare_creates_the_info_fetcher_log_directories() { + let ds = build(&cluster_with_both_info_fetchers()); + let prepare_args = ds + .spec + .as_ref() + .expect("the DaemonSet has a spec") + .template + .spec + .as_ref() + .expect("the pod template has a spec") + .init_containers + .as_ref() + .expect("the pod has init containers") + .iter() + .find(|c| c.name == "prepare") + .expect("the prepare container exists") + .args + .clone() + .expect("the prepare container has args") + .join(" && "); + // The commands are one `&&` chain, and the match below has to be exact: a substring + // check would also accept a longer path that merely starts with the expected one. + let commands: Vec<&str> = prepare_args.split(" && ").map(str::trim).collect(); + + for container_name in ["user-info-fetcher", "resource-info-fetcher"] { + let container = container_by_name(&ds, container_name); + let directory = env_var(&container, "FILE_LOG_DIRECTORY"); + let expected = format!("mkdir --parents {directory}"); + assert!( + commands.contains(&expected.as_str()), + "prepare must run {expected:?} for {container_name}, got: {commands:?}" + ); + } + } + /// The sidecars only read their config and credentials, so those mounts must not be writable. /// The log volume must stay writable, because that is where they write their file logs. #[test] diff --git a/rust/operator-binary/src/controller/build/resource/workload/mod.rs b/rust/operator-binary/src/controller/build/resource/workload/mod.rs index abc8968b..544b8a75 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/mod.rs @@ -289,7 +289,7 @@ pub fn build_server_rolegroup_pod_template( .image_from_product_image(resolved_product_image) .command(bash_entrypoint_command()) .args(vec![ - build_prepare_start_command(merged_config, Container::Prepare.name().as_ref()) + build_prepare_start_command(cluster, merged_config, Container::Prepare.name().as_ref()) .join(" && "), ]) .add_volume_mount(BUNDLES_VOLUME_NAME.as_ref(), BUNDLES_DIR) @@ -312,10 +312,7 @@ pub fn build_server_rolegroup_pod_template( .image_from_product_image(resolved_product_image) // inherit the pull policy and pull secrets, and then... .image(opa_bundle_builder_image) // ...override the image .command(bash_entrypoint_command()) - .args(vec![build_bundle_builder_start_command( - merged_config, - Container::BundleBuilder.name().as_ref(), - )]) + .args(vec![build_bundle_builder_start_command(merged_config)]) .add_env_vars(bundle_builder_env_vars) .add_volume_mount(BUNDLES_VOLUME_NAME.as_ref(), BUNDLES_DIR) .expect("The mount paths are statically defined and there should be no duplicates.") @@ -551,6 +548,13 @@ fn log_volume_size_limit(cluster: &ValidatedCluster) -> Quantity { product_logging::framework::calculate_log_volume_size_limit(&max_log_file_sizes) } +/// The directory the given container's Rust binary writes its file logs into, which is the value +/// of its `FILE_LOG_DIRECTORY`. It is exactly one level below [`STACKABLE_LOG_DIR`], because the +/// Vector agent derives the container name from that path segment. +fn file_log_directory(container: &Container) -> String { + format!("{STACKABLE_LOG_DIR}/{container}") +} + /// Env variables that are need to run stackable Rust binaries, such as /// * opa-bundle-builder /// * user-info-fetcher @@ -567,10 +571,7 @@ fn stackable_rust_cli_env_vars( EnvVarSet::new() .with_value(&CONSOLE_LOG_LEVEL, log_level.clone()) .with_value(&FILE_LOG_LEVEL, log_level) - .with_value( - &FILE_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/{container}"), - ) + .with_value(&FILE_LOG_DIRECTORY, file_log_directory(container)) .with_value(&FILE_LOG_ROTATION_PERIOD, DEFAULT_FILE_LOG_ROTATION_PERIOD) .with_value(&FILE_LOG_MAX_FILES, DEFAULT_FILE_LOG_MAX_FILES.to_string()) // `FieldPathEnvVar` has no variant for `spec.nodeName`, so the `EnvVar` is built by hand. @@ -689,10 +690,7 @@ fn build_opa_start_command( } } -fn build_bundle_builder_start_command( - merged_config: &ValidatedOpaConfig, - container_name: &str, -) -> String { +fn build_bundle_builder_start_command(merged_config: &ValidatedOpaConfig) -> String { let mut console_logging_off = false; // We need to check if the console logging is deactivated (NONE) @@ -711,10 +709,11 @@ fn build_bundle_builder_start_command( formatdoc! {" {COMMON_BASH_TRAP_FUNCTIONS} prepare_signal_handlers - mkdir -p {STACKABLE_LOG_DIR}/{container_name} + mkdir --parents \"${file_log_directory}\" || true stackable-opa-bundle-builder{logging_redirects} & wait_for_termination $! ", + file_log_directory = FILE_LOG_DIRECTORY.as_ref(), logging_redirects = if console_logging_off { " > /dev/null" } else { @@ -769,6 +768,7 @@ fn sidecar_container_log_level( } fn build_prepare_start_command( + cluster: &ValidatedCluster, merged_config: &ValidatedOpaConfig, container_name: &str, ) -> Vec { @@ -788,5 +788,25 @@ fn build_prepare_start_command( prepare_container_args.push(format!("mkdir -p {dir}")); } + // The log directories of the info-fetcher sidecars are created here, because those containers + // run their binary directly without a shell that could do it. Setting `FILE_LOG_MAX_FILES` + // makes the telemetry library prune old log files before it creates the first one, which + // complains about the directory not existing yet. The sidecars with a shell create their own + // directory from `$FILE_LOG_DIRECTORY` instead, so that an `envOverrides` entry for it takes + // effect; here the path has to be rebuilt, which an override would therefore not move. + let mut sidecars = vec![]; + if cluster.cluster_config.user_info.is_some() { + sidecars.push(Container::UserInfoFetcher); + } + if cluster.cluster_config.resource_info.is_some() { + sidecars.push(Container::ResourceInfoFetcher); + } + for sidecar in sidecars { + prepare_container_args.push(format!( + "mkdir --parents {dir}", + dir = file_log_directory(&sidecar) + )); + } + prepare_container_args } From 0feff607006a91bfe020f91bacef1868677cd4d5 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 9 Nov 2026 20:27:43 +0100 Subject: [PATCH 5/7] chore: Update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b8cac36d..03f7dbc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,13 +114,13 @@ All notable changes to this project will be documented in this file. log directory on startup. Their log directory is now created by the `prepare` container, because those containers run their binary without a shell which could create it ([#912]). -[#873]: https://github.com/stackabletech/opa-operator/pull/873 [#852]: https://github.com/stackabletech/opa-operator/pull/852 [#861]: https://github.com/stackabletech/opa-operator/pull/861 [#863]: https://github.com/stackabletech/opa-operator/pull/863 [#867]: https://github.com/stackabletech/opa-operator/pull/867 [#871]: https://github.com/stackabletech/opa-operator/pull/871 [#872]: https://github.com/stackabletech/opa-operator/pull/872 +[#873]: https://github.com/stackabletech/opa-operator/pull/873 [#880]: https://github.com/stackabletech/opa-operator/pull/880 [#882]: https://github.com/stackabletech/opa-operator/pull/882 [#886]: https://github.com/stackabletech/opa-operator/pull/886 From 452b01a147f8e11bffa3109dc6dc218f880e4ee5 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Tue, 10 Nov 2026 08:40:48 +0100 Subject: [PATCH 6/7] fix: Do not create the log directories of the info-fetcher sidecars --- CHANGELOG.md | 3 -- .../build/resource/workload/daemonset.rs | 40 ------------------- .../controller/build/resource/workload/mod.rs | 23 +---------- 3 files changed, 1 insertion(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03f7dbc5..0f85942d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -110,9 +110,6 @@ All notable changes to this project will be documented in this file. - The `tracing-rs` transform of the Vector agent no longer discards log events. A message which is not a string is kept among the other fields, a `fields` object which is not an object falls back to the raw line, and a missing timestamp is recorded as an error ([#912]). -- The user-info-fetcher and resource-info-fetcher containers no longer log an error about a missing - log directory on startup. Their log directory is now created by the `prepare` container, because - those containers run their binary without a shell which could create it ([#912]). [#852]: https://github.com/stackabletech/opa-operator/pull/852 [#861]: https://github.com/stackabletech/opa-operator/pull/861 diff --git a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs index 9c857061..4a647805 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs @@ -689,46 +689,6 @@ mod tests { } } - /// The info-fetcher sidecars run their binary without a shell, so they cannot create their own - /// log directory. The prepare container does it for them, and the path it creates has to be - /// exactly the one the sidecar logs into: otherwise the telemetry library prunes against a - /// missing directory and complains on every start. - #[test] - fn prepare_creates_the_info_fetcher_log_directories() { - let ds = build(&cluster_with_both_info_fetchers()); - let prepare_args = ds - .spec - .as_ref() - .expect("the DaemonSet has a spec") - .template - .spec - .as_ref() - .expect("the pod template has a spec") - .init_containers - .as_ref() - .expect("the pod has init containers") - .iter() - .find(|c| c.name == "prepare") - .expect("the prepare container exists") - .args - .clone() - .expect("the prepare container has args") - .join(" && "); - // The commands are one `&&` chain, and the match below has to be exact: a substring - // check would also accept a longer path that merely starts with the expected one. - let commands: Vec<&str> = prepare_args.split(" && ").map(str::trim).collect(); - - for container_name in ["user-info-fetcher", "resource-info-fetcher"] { - let container = container_by_name(&ds, container_name); - let directory = env_var(&container, "FILE_LOG_DIRECTORY"); - let expected = format!("mkdir --parents {directory}"); - assert!( - commands.contains(&expected.as_str()), - "prepare must run {expected:?} for {container_name}, got: {commands:?}" - ); - } - } - /// The sidecars only read their config and credentials, so those mounts must not be writable. /// The log volume must stay writable, because that is where they write their file logs. #[test] diff --git a/rust/operator-binary/src/controller/build/resource/workload/mod.rs b/rust/operator-binary/src/controller/build/resource/workload/mod.rs index 544b8a75..605aabfb 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/mod.rs @@ -289,7 +289,7 @@ pub fn build_server_rolegroup_pod_template( .image_from_product_image(resolved_product_image) .command(bash_entrypoint_command()) .args(vec![ - build_prepare_start_command(cluster, merged_config, Container::Prepare.name().as_ref()) + build_prepare_start_command(merged_config, Container::Prepare.name().as_ref()) .join(" && "), ]) .add_volume_mount(BUNDLES_VOLUME_NAME.as_ref(), BUNDLES_DIR) @@ -768,7 +768,6 @@ fn sidecar_container_log_level( } fn build_prepare_start_command( - cluster: &ValidatedCluster, merged_config: &ValidatedOpaConfig, container_name: &str, ) -> Vec { @@ -788,25 +787,5 @@ fn build_prepare_start_command( prepare_container_args.push(format!("mkdir -p {dir}")); } - // The log directories of the info-fetcher sidecars are created here, because those containers - // run their binary directly without a shell that could do it. Setting `FILE_LOG_MAX_FILES` - // makes the telemetry library prune old log files before it creates the first one, which - // complains about the directory not existing yet. The sidecars with a shell create their own - // directory from `$FILE_LOG_DIRECTORY` instead, so that an `envOverrides` entry for it takes - // effect; here the path has to be rebuilt, which an override would therefore not move. - let mut sidecars = vec![]; - if cluster.cluster_config.user_info.is_some() { - sidecars.push(Container::UserInfoFetcher); - } - if cluster.cluster_config.resource_info.is_some() { - sidecars.push(Container::ResourceInfoFetcher); - } - for sidecar in sidecars { - prepare_container_args.push(format!( - "mkdir --parents {dir}", - dir = file_log_directory(&sidecar) - )); - } - prepare_container_args } From 8da19616f1f3635c5b24af6f85f605478af19f4c Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Tue, 10 Nov 2026 09:37:09 +0100 Subject: [PATCH 7/7] chore: Sort the FILE_LOG_* constants like in the other operators --- .../src/controller/build/resource/workload/daemonset.rs | 2 +- .../src/controller/build/resource/workload/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs index 4a647805..4140d518 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs @@ -102,8 +102,8 @@ mod tests { let _ = *CONSOLE_LOG_LEVEL; let _ = *FILE_LOG_LEVEL; let _ = *FILE_LOG_DIRECTORY; - let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *FILE_LOG_MAX_FILES; + let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *KUBERNETES_NODE_NAME; let _ = *KUBERNETES_CLUSTER_DOMAIN; } diff --git a/rust/operator-binary/src/controller/build/resource/workload/mod.rs b/rust/operator-binary/src/controller/build/resource/workload/mod.rs index 605aabfb..e3e7d87f 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/mod.rs @@ -103,8 +103,8 @@ const LIVENESS_PROBE_INITIAL_DELAY_SECONDS: i32 = 30; constant!(CONSOLE_LOG_LEVEL: EnvVarName = "CONSOLE_LOG_LEVEL"); constant!(FILE_LOG_LEVEL: EnvVarName = "FILE_LOG_LEVEL"); constant!(FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); -constant!(FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); constant!(FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +constant!(FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); constant!(KUBERNETES_NODE_NAME: EnvVarName = "KUBERNETES_NODE_NAME"); constant!(KUBERNETES_CLUSTER_DOMAIN: EnvVarName = "KUBERNETES_CLUSTER_DOMAIN");