diff --git a/CHANGELOG.md b/CHANGELOG.md index b4e557dd..0f85942d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,8 @@ All notable changes to this project will be documented in this file. - A `PodDisruptionBudget` is now written out for the `servers` role when it runs as a `Deployment`, with `maxUnavailable: 1`. Configurable via `spec.servers.roleConfig.podDisruptionBudget` ([#873]). - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#894]). +- The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to + the Vector aggregator ([#912]). ### Changed @@ -101,14 +103,21 @@ All notable changes to this project will be documented in this file. containing a line break, is rejected with `400` ([#909]). - The user-info-fetcher Keycloak backend no longer drops a configured `rootPath` from the URLs it requests, so a Keycloak served below a sub-path, like `/auth`, can now be reached ([#909]). +- The `containerdebug` process writes its log file again. The operator set + `CONTAINERDEBUG_LOG_DIRECTORY` on the OPA container, which containerdebug stopped reading in its + version 0.2.0 when it moved to `stackable-telemetry`; the directory is now set via + `FILE_LOG_DIRECTORY`, with an hourly rotation that keeps at most two files ([#912]). +- The `tracing-rs` transform of the Vector agent no longer discards log events. A message which is + not a string is kept among the other fields, a `fields` object which is not an object falls back + to the raw line, and a missing timestamp is recorded as an error ([#912]). -[#873]: https://github.com/stackabletech/opa-operator/pull/873 [#852]: https://github.com/stackabletech/opa-operator/pull/852 [#861]: https://github.com/stackabletech/opa-operator/pull/861 [#863]: https://github.com/stackabletech/opa-operator/pull/863 [#867]: https://github.com/stackabletech/opa-operator/pull/867 [#871]: https://github.com/stackabletech/opa-operator/pull/871 [#872]: https://github.com/stackabletech/opa-operator/pull/872 +[#873]: https://github.com/stackabletech/opa-operator/pull/873 [#880]: https://github.com/stackabletech/opa-operator/pull/880 [#882]: https://github.com/stackabletech/opa-operator/pull/882 [#886]: https://github.com/stackabletech/opa-operator/pull/886 @@ -123,6 +132,7 @@ All notable changes to this project will be documented in this file. [#906]: https://github.com/stackabletech/opa-operator/pull/906 [#908]: https://github.com/stackabletech/opa-operator/pull/908 [#909]: https://github.com/stackabletech/opa-operator/pull/909 +[#912]: https://github.com/stackabletech/opa-operator/pull/912 ## [26.7.0] - 2026-07-21 diff --git a/Cargo.nix b/Cargo.nix index 0f24cd2b..940adbe5 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -12638,6 +12638,11 @@ rec { } ]; devDependencies = [ + { + name = "stackable-operator"; + packageId = "stackable-operator"; + features = [ "webhook" "test-support" ]; + } { name = "yaml_serde"; packageId = "yaml_serde"; @@ -13093,7 +13098,7 @@ rec { "time" = [ "stackable-shared/time" ]; "webhook" = [ "dep:stackable-webhook" ]; }; - resolvedDefaultFeatures = [ "crds" "default" "webhook" ]; + resolvedDefaultFeatures = [ "crds" "default" "test-support" "webhook" ]; }; "stackable-operator-derive" = rec { crateName = "stackable-operator-derive"; diff --git a/docs/modules/opa/pages/usage-guide/logging.adoc b/docs/modules/opa/pages/usage-guide/logging.adoc index 23e69883..7662bdb6 100644 --- a/docs/modules/opa/pages/usage-guide/logging.adoc +++ b/docs/modules/opa/pages/usage-guide/logging.adoc @@ -56,5 +56,13 @@ spec: The decision logs are still filtered by the log level set for the console and file appenders. Therefore, the configuration above would result in decision logs being present on file but not on console. +NOTE: The logs of the xref:concepts:observability/containerdebug.adoc[`containerdebug` process] are collected as well. +They are labelled `container: containerdebug`, although no such Kubernetes container exists. +The process runs in the background of the `opa` container. +The label comes from `/stackable/log/containerdebug`, the directory which the operator sets with the `FILE_LOG_DIRECTORY` environment variable and from which the Vector agent derives the container name. +Overriding `FILE_LOG_DIRECTORY` with `envOverrides` therefore changes how these logs are aggregated. +The agent only collects files one level below `/stackable/log`, so a sibling directory such as `/stackable/log/opa` relabels the events instead of dropping them, while any deeper or shallower path stops the aggregation. +A path outside `/stackable/log` also leaves the size-capped log volume. + Further information on how to configure logging, can be found in xref:concepts:logging.adoc[]. diff --git a/rust/operator-binary/Cargo.toml b/rust/operator-binary/Cargo.toml index a06c0b9b..f74d8f16 100644 --- a/rust/operator-binary/Cargo.toml +++ b/rust/operator-binary/Cargo.toml @@ -32,4 +32,5 @@ tracing.workspace = true built.workspace = true [dev-dependencies] +stackable-operator = { workspace = true, features = ["test-support"] } yaml_serde.workspace = true diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml index d427a072..8fcd13f3 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector-test.yaml @@ -107,6 +107,285 @@ tests: source: | assert!(exists(.errors), "unparsable timestamp should be recorded as an error") + - name: Containerdebug tracing-rs log + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":"found disk","disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk\n\ndisk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Containerdebug tracing-rs log which is unparsable + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{ unparsable' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + .errors[0] + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{ unparsable", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "JSON not parsable: ")) + - name: Containerdebug tracing-rs log which is not a JSON object + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: 'false' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + "Parsed event is not a JSON object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "false", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Containerdebug tracing-rs log with invalid fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"unparsable","level":"INVALID","fields":"none","target":null}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + .errors[0], + "Logger/target not found.", + "Level \"INVALID\" unknown, using \"INFO\" instead.", + "Fields are not an object." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "", + "message": "{\"timestamp\":\"unparsable\",\"level\":\"INVALID\",\"fields\":\"none\",\"target\":null}", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The remainder of the message contains character offsets into + # the VRL source, so only the stable prefix is asserted. + assert!(starts_with!(.errors[0], "Timestamp not parsable, using current time instead: ")) + - name: Containerdebug tracing-rs log without a timestamp + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"level":"INFO","fields":{"message":"found disk"},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + "Timestamp not found, using current time instead." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "found disk", + "timestamp": .timestamp, + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + + # The current time is used and not the timestamp which the file + # source attached to the event. The latter reaches the transform + # as a string, whereas now() yields a timestamp. + assert!(is_timestamp(.timestamp)) + - name: Containerdebug tracing-rs log without a message but with other fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Containerdebug tracing-rs log with a message which is not a string + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{"message":42,"disk.name":"overlay"},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "disk.name=overlay\nmessage=42", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) + - name: Containerdebug tracing-rs log without any fields + inputs: + - type: log + insert_at: processed_files_tracing_rs + log_fields: + file: /stackable/log/containerdebug/containerdebug.2026-11-02-14.tracing-rs.json + message: '{"timestamp":"2026-11-02T14:49:28.028922Z","level":"INFO","fields":{},"target":"containerdebug::system_information::disk"}' + pod: test-opa-server-default-0 + source_type: file + timestamp: 2026-11-02T14:49:29.473487331Z + outputs: + - extract_from: extended_logs + expected_event_count: 1 + conditions: + - type: vrl + source: | + expected_log_event = { + "cluster": "opa", + "container": "containerdebug", + "errors": [ + "Message not found." + ], + "file": "containerdebug.2026-11-02-14.tracing-rs.json", + "level": "INFO", + "logger": "containerdebug::system_information::disk", + "message": "", + "timestamp": t'2026-11-02T14:49:28.028922Z', + "namespace": "default", + "pod": "test-opa-server-default-0", + "role": "server", + "roleGroup": "default" + } + + assert_eq!(expected_log_event, .) - name: Prepare stdout log inputs: - type: log diff --git a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml index 804d1597..5586b9ef 100644 --- a/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml +++ b/rust/operator-binary/src/controller/build/properties/product_logging/vector.yaml @@ -84,8 +84,10 @@ transforms: if err == null { .timestamp = parsed_timestamp } else { - .errors = push(.errors, "Timestamp not parsable, trying current time instead: " + err) + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") } .logger, err = string(event.target) @@ -103,19 +105,29 @@ transforms: } fields, err = object(event.fields) - if err != null { + if err == null { + message, err = string(fields.message) + if err == null { + # The message is prepended to the other fields, so remove it from + # them. + del(fields.message) + } else { + # Keep an unusable message among the other fields instead of + # discarding its content. + message = "" + } + + other_fields = encode_key_value(fields, field_delimiter: "\n") + .message = join!(compact([message, other_fields]), "\n\n") + } else { .errors = push(.errors, "Fields are not an object.") + # Fall back to the raw line, so that its content is not lost. + .message = raw_message } - .message, err = string(fields.message) - if err != null || is_empty(.message) { + if is_empty(.message) { .errors = push(.errors, "Message not found.") } - - del(fields.message) - - other_fields = encode_key_value(fields, field_delimiter: "\n") - .message = join!(compact([.message, other_fields]), "\n\n") } # Transforms OPA's own JSON logs diff --git a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs index 702f35a9..4140d518 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/daemonset.rs @@ -98,13 +98,12 @@ mod tests { let _ = *USER_INFO_FETCHER_KERBEROS_VOLUME_NAME; let _ = *RESOURCE_INFO_FETCHER_CREDENTIALS_VOLUME_NAME; let _ = *TLS_VOLUME_NAME; - let _ = *CONTAINERDEBUG_LOG_DIRECTORY; let _ = *WATCH_NAMESPACE; let _ = *CONSOLE_LOG_LEVEL; let _ = *FILE_LOG_LEVEL; let _ = *FILE_LOG_DIRECTORY; - let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *FILE_LOG_MAX_FILES; + let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *KUBERNETES_NODE_NAME; let _ = *KUBERNETES_CLUSTER_DOMAIN; } @@ -390,7 +389,7 @@ mod tests { // The redirect is appended directly after the bundle-builder invocation. (`/dev/null` also // appears in the shared bash trap helpers, so match the specific redirect.) assert!( - build_bundle_builder_start_command(&silenced, "bundle-builder") + build_bundle_builder_start_command(&silenced) .contains("stackable-opa-bundle-builder > /dev/null") ); @@ -405,8 +404,7 @@ mod tests { }, })); assert!( - build_bundle_builder_start_command(&logging, "bundle-builder") - .contains("stackable-opa-bundle-builder &") + build_bundle_builder_start_command(&logging).contains("stackable-opa-bundle-builder &") ); } @@ -812,12 +810,10 @@ mod tests { } /// The user-supplied `envOverrides` must be merged in after all operator-set environment - /// variables, so that they can override any of them. `CONTAINERDEBUG_LOG_DIRECTORY` is used + /// variables, so that they can override any of them. `FILE_LOG_DIRECTORY` is used /// as the example here because it is set unconditionally by the operator. #[test] fn env_overrides_override_operator_set_env_vars() { - use std::str::FromStr; - use stackable_operator::v2::builder::pod::container::EnvVarName; let cluster = validated_cluster_from_spec(json!({ @@ -830,7 +826,7 @@ mod tests { .expect("the default role group should exist"); let mut role_group = role_group.clone(); role_group.env_overrides = EnvVarSet::new().with_value( - &EnvVarName::from_str("CONTAINERDEBUG_LOG_DIRECTORY").expect("valid env var name"), + &EnvVarName::from_str_unsafe("FILE_LOG_DIRECTORY"), "/custom/log/dir", ); @@ -860,7 +856,7 @@ mod tests { let containerdebug: Vec<_> = env .iter() - .filter(|env_var| env_var.name == "CONTAINERDEBUG_LOG_DIRECTORY") + .filter(|env_var| env_var.name == "FILE_LOG_DIRECTORY") .collect(); assert_eq!( containerdebug.len(), diff --git a/rust/operator-binary/src/controller/build/resource/workload/mod.rs b/rust/operator-binary/src/controller/build/resource/workload/mod.rs index dae0ce3e..e3e7d87f 100644 --- a/rust/operator-binary/src/controller/build/resource/workload/mod.rs +++ b/rust/operator-binary/src/controller/build/resource/workload/mod.rs @@ -6,6 +6,7 @@ use std::{collections::BTreeMap, str::FromStr}; +use const_format::concatcp; use indoc::formatdoc; use snafu::{ResultExt, Snafu}; use stackable_opa_operator::crd::{Container, DEFAULT_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, OpaRole}; @@ -87,7 +88,6 @@ const RESOURCE_INFO_FETCHER_CREDENTIALS_DIR: &str = "/stackable/credentials"; constant!(TLS_VOLUME_NAME: VolumeName = "tls"); const TLS_STORE_DIR: &str = "/stackable/tls"; -constant!(CONTAINERDEBUG_LOG_DIRECTORY: EnvVarName = "CONTAINERDEBUG_LOG_DIRECTORY"); constant!(WATCH_NAMESPACE: EnvVarName = "WATCH_NAMESPACE"); // HTTP probe configuration shared by the bundle-builder and OPA containers. They differ in the @@ -103,8 +103,8 @@ const LIVENESS_PROBE_INITIAL_DELAY_SECONDS: i32 = 30; constant!(CONSOLE_LOG_LEVEL: EnvVarName = "CONSOLE_LOG_LEVEL"); constant!(FILE_LOG_LEVEL: EnvVarName = "FILE_LOG_LEVEL"); constant!(FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); -constant!(FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); constant!(FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +constant!(FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); constant!(KUBERNETES_NODE_NAME: EnvVarName = "KUBERNETES_NODE_NAME"); constant!(KUBERNETES_CLUSTER_DOMAIN: EnvVarName = "KUBERNETES_CLUSTER_DOMAIN"); @@ -312,10 +312,7 @@ pub fn build_server_rolegroup_pod_template( .image_from_product_image(resolved_product_image) // inherit the pull policy and pull secrets, and then... .image(opa_bundle_builder_image) // ...override the image .command(bash_entrypoint_command()) - .args(vec![build_bundle_builder_start_command( - merged_config, - Container::BundleBuilder.name().as_ref(), - )]) + .args(vec![build_bundle_builder_start_command(merged_config)]) .add_env_vars(bundle_builder_env_vars) .add_volume_mount(BUNDLES_VOLUME_NAME.as_ref(), BUNDLES_DIR) .expect("The mount paths are statically defined and there should be no duplicates.") @@ -335,10 +332,23 @@ pub fn build_server_rolegroup_pod_template( // All operator-set environment variables of the OPA container, collected into an // `EnvVarSet` so that every name occurs only once. - let opa_env_vars = EnvVarSet::new().with_value( - &CONTAINERDEBUG_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/containerdebug"), - ); + // + // These are the generic `stackable-telemetry` variables, here for the `containerdebug` + // process which runs in the background of the OPA container. `FILE_LOG_LEVEL` is + // deliberately not set: unlike our own Rust binaries, containerdebug's level is not derived + // from the log configuration. The rotation differs from the one in + // [`stackable_rust_cli_env_vars`] as well, because containerdebug writes only a handful of + // lines per run and a minutely rotation with five files would keep just a few minutes of it. + let opa_env_vars = EnvVarSet::new() + // The directory must be exactly one level below the log directory, because the Vector + // agent derives the container name from that path segment. + .with_value( + &FILE_LOG_DIRECTORY, + concatcp!(STACKABLE_LOG_DIR, "/containerdebug"), + ) + // Rotation is opt-in, so without these the file would grow unbounded. + .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly") + .with_value(&FILE_LOG_MAX_FILES, "2"); // Environment variable overrides (highest precedence), merged from role and role group. // They are merged in last so that they override any operator-set environment variable. let opa_env_vars = opa_env_vars.merge(rolegroup_config.env_overrides.clone()); @@ -518,6 +528,9 @@ pub fn build_server_rolegroup_pod_template( /// The size limit of the shared `log` [`EmptyDirVolumeSource`], which has to accommodate the log /// files of every container that mounts it. The always-present containers are budgeted for /// unconditionally; the optional info-fetcher sidecars only when the cluster configures them. +/// +/// The `containerdebug` files in this volume are rotated by time, not by size, so they have no +/// budget; raising `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. fn log_volume_size_limit(cluster: &ValidatedCluster) -> Quantity { let mut max_log_file_sizes = vec![ MAX_OPA_BUNDLE_BUILDER_LOG_FILE_SIZE, @@ -535,6 +548,13 @@ fn log_volume_size_limit(cluster: &ValidatedCluster) -> Quantity { product_logging::framework::calculate_log_volume_size_limit(&max_log_file_sizes) } +/// The directory the given container's Rust binary writes its file logs into, which is the value +/// of its `FILE_LOG_DIRECTORY`. It is exactly one level below [`STACKABLE_LOG_DIR`], because the +/// Vector agent derives the container name from that path segment. +fn file_log_directory(container: &Container) -> String { + format!("{STACKABLE_LOG_DIR}/{container}") +} + /// Env variables that are need to run stackable Rust binaries, such as /// * opa-bundle-builder /// * user-info-fetcher @@ -551,10 +571,7 @@ fn stackable_rust_cli_env_vars( EnvVarSet::new() .with_value(&CONSOLE_LOG_LEVEL, log_level.clone()) .with_value(&FILE_LOG_LEVEL, log_level) - .with_value( - &FILE_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/{container}"), - ) + .with_value(&FILE_LOG_DIRECTORY, file_log_directory(container)) .with_value(&FILE_LOG_ROTATION_PERIOD, DEFAULT_FILE_LOG_ROTATION_PERIOD) .with_value(&FILE_LOG_MAX_FILES, DEFAULT_FILE_LOG_MAX_FILES.to_string()) // `FieldPathEnvVar` has no variant for `spec.nodeName`, so the `EnvVar` is built by hand. @@ -656,6 +673,7 @@ fn build_opa_start_command( {COMMON_BASH_TRAP_FUNCTIONS} {remove_vector_shutdown_file_command} prepare_signal_handlers + mkdir --parents \"${file_log_directory}\" || true containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop & opa run -s -a 0.0.0.0:{bind_port} -c {CONFIG_DIR}/{config_file} -l {opa_log_level} --shutdown-grace-period {shutdown_grace_period_s} --disable-telemetry {tls_flags} {extra_cli_args} {logging_redirects} & wait_for_termination $! @@ -665,16 +683,14 @@ fn build_opa_start_command( remove_vector_shutdown_file_command(STACKABLE_LOG_DIR), create_vector_shutdown_file_command = create_vector_shutdown_file_command(STACKABLE_LOG_DIR), + file_log_directory = FILE_LOG_DIRECTORY.as_ref(), shutdown_grace_period_s = merged_config.graceful_shutdown_timeout.unwrap_or(DEFAULT_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT).as_secs(), opa_log_level = [console_log_level, file_log_level].iter().min().unwrap_or(&LogLevel::INFO).to_opa_literal(), extra_cli_args = extra_cli_args } } -fn build_bundle_builder_start_command( - merged_config: &ValidatedOpaConfig, - container_name: &str, -) -> String { +fn build_bundle_builder_start_command(merged_config: &ValidatedOpaConfig) -> String { let mut console_logging_off = false; // We need to check if the console logging is deactivated (NONE) @@ -693,10 +709,11 @@ fn build_bundle_builder_start_command( formatdoc! {" {COMMON_BASH_TRAP_FUNCTIONS} prepare_signal_handlers - mkdir -p {STACKABLE_LOG_DIR}/{container_name} + mkdir --parents \"${file_log_directory}\" || true stackable-opa-bundle-builder{logging_redirects} & wait_for_termination $! ", + file_log_directory = FILE_LOG_DIRECTORY.as_ref(), logging_redirects = if console_logging_off { " > /dev/null" } else { diff --git a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 index f084db8e..5d065e3f 100644 --- a/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 +++ b/tests/templates/kuttl/logging/opa-vector-aggregator-values.yaml.j2 @@ -22,55 +22,71 @@ customConfig: transforms: validEvents: type: filter - inputs: [vector] + inputs: + - vector condition: is_null(.errors) filteredAutomaticLogConfigServerOpaDecision: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "opa" && .logger == "decision" + filteredAutomaticLogConfigServerContainerdebug: + type: filter + inputs: + - validEvents + condition: >- + starts_with(string!(.pod), "test-opa-server-automatic-log-config") && + .container == "containerdebug" filteredAutomaticLogConfigServerOpa: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "opa" && .logger != "decision" filteredAutomaticLogConfigServerBundleBuilder: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "bundle-builder" filteredAutomaticLogConfigServerUserInfoFetcher: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "user-info-fetcher" filteredAutomaticLogConfigServerResourceInfoFetcher: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "resource-info-fetcher" filteredAutomaticLogConfigServerVector: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "vector" filteredAutomaticLogConfigServerPrepare: type: filter - inputs: [validEvents] + inputs: + - validEvents condition: >- starts_with(string!(.pod), "test-opa-server-automatic-log-config") && .container == "prepare" filteredInvalidEvents: type: filter - inputs: [vector] + inputs: + - vector condition: |- .timestamp == from_unix_timestamp!(0) || is_null(.level) || @@ -78,11 +94,29 @@ customConfig: is_null(.message) sinks: test: - inputs: [filtered*] + inputs: + - filtered* type: blackhole + # Shows all events received from the agents. This sink is intentional and + # not a debugging leftover: if a "filtered*" transform gathers no event, + # then the assertion only reports a count, whereas this log shows which + # pod and container names actually arrived. A failure in CI cannot be + # reproduced by enabling this afterwards, so it is kept enabled at the + # price of the log volume. + stdout: + inputs: + - vector + type: console + encoding: + codec: json + buffer: + # Avoid back pressure from a stalling stdout. The test should not + # fail because of a sink which only aids debugging. + when_full: drop_newest {% if lookup('env', 'VECTOR_AGGREGATOR') %} aggregator: - inputs: [vector] + inputs: + - vector type: vector routing: endpoints: