From 26807752dd1f0ec940616814f480c9138d6f845a Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 17:13:07 +0100 Subject: [PATCH 1/3] chore: Follow-ups to the containerdebug log collection --- CHANGELOG.md | 3 ++- .../src/controller/build/properties/env_vars.rs | 8 +++----- .../src/controller/build/volumes.rs | 8 +++----- rust/operator-binary/src/crd/mod.rs | 14 +++++++++++++- 4 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c6cd879..33b7d1fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#868]). - Webservers now have a default affinity to the OPA Pods when OPA authorization is configured ([#872]). - The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to - the Vector aggregator ([#888]). + the Vector aggregator ([#888], [#892]). ### Changed @@ -112,6 +112,7 @@ [#886]: https://github.com/stackabletech/airflow-operator/pull/886 [#887]: https://github.com/stackabletech/airflow-operator/pull/887 [#888]: https://github.com/stackabletech/airflow-operator/pull/888 +[#892]: https://github.com/stackabletech/airflow-operator/pull/892 ## [26.7.0] - 2026-07-21 diff --git a/rust/operator-binary/src/controller/build/properties/env_vars.rs b/rust/operator-binary/src/controller/build/properties/env_vars.rs index a74d6dd3..91674d8c 100644 --- a/rust/operator-binary/src/controller/build/properties/env_vars.rs +++ b/rust/operator-binary/src/controller/build/properties/env_vars.rs @@ -15,7 +15,8 @@ use stackable_operator::{ use crate::{ controller::{ValidatedCluster, ValidatedLogging}, crd::{ - AirflowExecutor, AirflowRole, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, + AirflowExecutor, AirflowRole, CONTAINERDEBUG_LOG_DIR, HTTP_PORT, LOG_CONFIG_DIR, + TEMPLATE_LOCATION, TEMPLATE_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -188,10 +189,7 @@ pub fn build_airflow_statefulset_envs( // other telemetry-based process in it would log into the same directory and the Vector // agent would label its events with `container: containerdebug`. env_vars = env_vars - .with_value( - &FILE_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/containerdebug"), - ) + .with_value(&FILE_LOG_DIRECTORY, CONTAINERDEBUG_LOG_DIR) .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly") .with_value(&FILE_LOG_MAX_FILES, "2") // apply overrides last of all; `EnvVarSet` is keyed by name, so iteration is already diff --git a/rust/operator-binary/src/controller/build/volumes.rs b/rust/operator-binary/src/controller/build/volumes.rs index 30f9821b..06e6d787 100644 --- a/rust/operator-binary/src/controller/build/volumes.rs +++ b/rust/operator-binary/src/controller/build/volumes.rs @@ -32,11 +32,9 @@ pub fn create_volumes( name: LOG_VOLUME_NAME.to_string(), empty_dir: Some(EmptyDirVolumeSource { medium: None, - // Only the Airflow log files are budgeted here. The `containerdebug` files in this - // volume are rotated by time, which bounds their number but not their size, so they - // are an unbudgeted writer. They stay negligible at the default INFO level and a 30 - // minute loop, but raising `FILE_LOG_LEVEL` via `envOverrides` can exceed the limit, - // which makes the kubelet evict the Pod. + // Only the Airflow log files are budgeted. The `containerdebug` files in this + // volume are rotated by time, not by size, so they have no budget; raising + // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. size_limit: Some(product_logging::framework::calculate_log_volume_size_limit( &[MAX_LOG_FILES_SIZE], )), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 89b8bce3..50c3742b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeSet, ops::Deref, str::FromStr}; +use const_format::concatcp; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -84,6 +85,11 @@ pub const CONFIG_PATH: &str = "/stackable/app/config"; pub const LOG_CONFIG_DIR: &str = "/stackable/app/log_config"; pub const AIRFLOW_HOME: &str = "/stackable/airflow"; +/// Directory which the `containerdebug` process writes its log files to. It must be exactly one +/// level below the log directory, because the Vector agent derives the container name from that +/// path segment. +pub const CONTAINERDEBUG_LOG_DIR: &str = concatcp!(STACKABLE_LOG_DIR, "/containerdebug"); + constant!(pub TEMPLATE_VOLUME_NAME: VolumeName = "airflow-executor-pod-template"); pub const TEMPLATE_LOCATION: &str = "/templates"; pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; @@ -776,7 +782,13 @@ impl Deref for AirflowRole { } fn container_debug_command() -> String { - format!("containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &") + // The log directory is created here, because `FILE_LOG_MAX_FILES` makes the telemetry library + // prune old log files before it creates the first one, which complains about the directory not + // existing yet. + format!( + "mkdir --parents {CONTAINERDEBUG_LOG_DIR}\n\ + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &" + ) } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] From 905db68f6965989b42c49516011fe6d5da85a7b5 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 4 Nov 2026 17:50:46 +0100 Subject: [PATCH 2/3] chore: Create the containerdebug log directory via the environment variable --- .../src/controller/build/properties/env_vars.rs | 11 ++++++++--- rust/operator-binary/src/crd/mod.rs | 8 +------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/env_vars.rs b/rust/operator-binary/src/controller/build/properties/env_vars.rs index 91674d8c..6bb930af 100644 --- a/rust/operator-binary/src/controller/build/properties/env_vars.rs +++ b/rust/operator-binary/src/controller/build/properties/env_vars.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeSet, str::FromStr}; +use const_format::concatcp; use stackable_operator::{ constant, crd::{authentication::oidc, git_sync}, @@ -15,8 +16,7 @@ use stackable_operator::{ use crate::{ controller::{ValidatedCluster, ValidatedLogging}, crd::{ - AirflowExecutor, AirflowRole, CONTAINERDEBUG_LOG_DIR, HTTP_PORT, LOG_CONFIG_DIR, - TEMPLATE_LOCATION, TEMPLATE_NAME, + AirflowExecutor, AirflowRole, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -189,7 +189,12 @@ pub fn build_airflow_statefulset_envs( // other telemetry-based process in it would log into the same directory and the Vector // agent would label its events with `container: containerdebug`. env_vars = env_vars - .with_value(&FILE_LOG_DIRECTORY, CONTAINERDEBUG_LOG_DIR) + // The directory must be exactly one level below the log directory, because the Vector + // agent derives the container name from that path segment. + .with_value( + &FILE_LOG_DIRECTORY, + concatcp!(STACKABLE_LOG_DIR, "/containerdebug"), + ) .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly") .with_value(&FILE_LOG_MAX_FILES, "2") // apply overrides last of all; `EnvVarSet` is keyed by name, so iteration is already diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 50c3742b..b7cf46a2 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,6 +1,5 @@ use std::{collections::BTreeSet, ops::Deref, str::FromStr}; -use const_format::concatcp; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -85,11 +84,6 @@ pub const CONFIG_PATH: &str = "/stackable/app/config"; pub const LOG_CONFIG_DIR: &str = "/stackable/app/log_config"; pub const AIRFLOW_HOME: &str = "/stackable/airflow"; -/// Directory which the `containerdebug` process writes its log files to. It must be exactly one -/// level below the log directory, because the Vector agent derives the container name from that -/// path segment. -pub const CONTAINERDEBUG_LOG_DIR: &str = concatcp!(STACKABLE_LOG_DIR, "/containerdebug"); - constant!(pub TEMPLATE_VOLUME_NAME: VolumeName = "airflow-executor-pod-template"); pub const TEMPLATE_LOCATION: &str = "/templates"; pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; @@ -786,7 +780,7 @@ fn container_debug_command() -> String { // prune old log files before it creates the first one, which complains about the directory not // existing yet. format!( - "mkdir --parents {CONTAINERDEBUG_LOG_DIR}\n\ + "mkdir --parents \"$FILE_LOG_DIRECTORY\"\n\ containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &" ) } From b95573a5f275a14aeeeeabc84d594a33cd53c8c6 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 9 Nov 2026 15:03:03 +0100 Subject: [PATCH 3/3] fix: Do not hard-code the environment variable name "FILE_LOG_DIRECTORY" --- .../controller/build/properties/env_vars.rs | 9 ++------ rust/operator-binary/src/crd/mod.rs | 21 ++++++++++++++++--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/rust/operator-binary/src/controller/build/properties/env_vars.rs b/rust/operator-binary/src/controller/build/properties/env_vars.rs index 6bb930af..7bd0549c 100644 --- a/rust/operator-binary/src/controller/build/properties/env_vars.rs +++ b/rust/operator-binary/src/controller/build/properties/env_vars.rs @@ -16,7 +16,8 @@ use stackable_operator::{ use crate::{ controller::{ValidatedCluster, ValidatedLogging}, crd::{ - AirflowExecutor, AirflowRole, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, + AirflowExecutor, AirflowRole, FILE_LOG_DIRECTORY, FILE_LOG_MAX_FILES, + FILE_LOG_ROTATION_PERIOD, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -78,9 +79,6 @@ constant!(ADMIN_PASSWORD_SECRET_KEY: SecretKey = "adminUser.password"); constant!(PYTHONPATH: EnvVarName = "PYTHONPATH"); -constant!(FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); -constant!(FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); -constant!(FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); constant!(STACKABLE_POST_HOOK: EnvVarName = "_STACKABLE_POST_HOOK"); /// Return environment variables to be applied to the statefulsets for the scheduler, webserver (and worker, @@ -539,9 +537,6 @@ mod tests { let _ = *AIRFLOW_WEBSERVER_ENABLE_PROXY_FIX; let _ = *AIRFLOW_WEBSERVER_EXPOSE_CONFIG; let _ = *AIRFLOW_WEBSERVER_PROXY_FIX_X_FOR; - let _ = *FILE_LOG_DIRECTORY; - let _ = *FILE_LOG_MAX_FILES; - let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *FORWARDED_ALLOW_IPS; let _ = *PYTHONPATH; let _ = *STACKABLE_POST_HOOK; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index b7cf46a2..f4eedf94 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeSet, ops::Deref, str::FromStr}; +use indoc::formatdoc; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -38,6 +39,7 @@ use stackable_operator::{ status::condition::{ClusterCondition, HasStatusCondition}, utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema}, v2::{ + builder::pod::container::EnvVarName, config_overrides::KeyValueConfigOverrides, flask_config_writer::{FlaskAppConfigOptions, PythonType}, product_logging::framework::STACKABLE_LOG_DIR, @@ -84,6 +86,14 @@ pub const CONFIG_PATH: &str = "/stackable/app/config"; pub const LOG_CONFIG_DIR: &str = "/stackable/app/log_config"; pub const AIRFLOW_HOME: &str = "/stackable/airflow"; +// The `stackable-telemetry` variables which configure the file logs of the `containerdebug` +// process. They are set on the Airflow container in `env_vars`, and the start command below +// references the directory as a shell variable, so that an `envOverrides` entry for it takes +// effect in both places. +constant!(pub FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); +constant!(pub FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +constant!(pub FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); + constant!(pub TEMPLATE_VOLUME_NAME: VolumeName = "airflow-executor-pod-template"); pub const TEMPLATE_LOCATION: &str = "/templates"; pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; @@ -779,9 +789,11 @@ fn container_debug_command() -> String { // The log directory is created here, because `FILE_LOG_MAX_FILES` makes the telemetry library // prune old log files before it creates the first one, which complains about the directory not // existing yet. - format!( - "mkdir --parents \"$FILE_LOG_DIRECTORY\"\n\ - containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &" + formatdoc!( + r#" + mkdir --parents "${file_log_directory}" || true + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &"#, + file_log_directory = FILE_LOG_DIRECTORY.as_ref(), ) } @@ -1035,6 +1047,9 @@ mod tests { let _ = *WORKER_ROLE_NAME; let _ = *DAG_PROCESSOR_ROLE_NAME; let _ = *TRIGGERER_ROLE_NAME; + let _ = *FILE_LOG_DIRECTORY; + let _ = *FILE_LOG_MAX_FILES; + let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *TEMPLATE_VOLUME_NAME; let _ = *LISTENER_PVC_NAME; let _ = *METRICS_CONTAINER_NAME;