diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c6cd879..33b7d1fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ - Add `/ready` endpoint to the operator Deployment, which reports the CRD installation status ([#868]). - Webservers now have a default affinity to the OPA Pods when OPA authorization is configured ([#872]). - The logs of the `containerdebug` process are now collected by the Vector agent and forwarded to - the Vector aggregator ([#888]). + the Vector aggregator ([#888], [#892]). ### Changed @@ -112,6 +112,7 @@ [#886]: https://github.com/stackabletech/airflow-operator/pull/886 [#887]: https://github.com/stackabletech/airflow-operator/pull/887 [#888]: https://github.com/stackabletech/airflow-operator/pull/888 +[#892]: https://github.com/stackabletech/airflow-operator/pull/892 ## [26.7.0] - 2026-07-21 diff --git a/rust/operator-binary/src/controller/build/properties/env_vars.rs b/rust/operator-binary/src/controller/build/properties/env_vars.rs index a74d6dd3..7bd0549c 100644 --- a/rust/operator-binary/src/controller/build/properties/env_vars.rs +++ b/rust/operator-binary/src/controller/build/properties/env_vars.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeSet, str::FromStr}; +use const_format::concatcp; use stackable_operator::{ constant, crd::{authentication::oidc, git_sync}, @@ -15,7 +16,8 @@ use stackable_operator::{ use crate::{ controller::{ValidatedCluster, ValidatedLogging}, crd::{ - AirflowExecutor, AirflowRole, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, + AirflowExecutor, AirflowRole, FILE_LOG_DIRECTORY, FILE_LOG_MAX_FILES, + FILE_LOG_ROTATION_PERIOD, HTTP_PORT, LOG_CONFIG_DIR, TEMPLATE_LOCATION, TEMPLATE_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -77,9 +79,6 @@ constant!(ADMIN_PASSWORD_SECRET_KEY: SecretKey = "adminUser.password"); constant!(PYTHONPATH: EnvVarName = "PYTHONPATH"); -constant!(FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); -constant!(FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); -constant!(FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); constant!(STACKABLE_POST_HOOK: EnvVarName = "_STACKABLE_POST_HOOK"); /// Return environment variables to be applied to the statefulsets for the scheduler, webserver (and worker, @@ -188,9 +187,11 @@ pub fn build_airflow_statefulset_envs( // other telemetry-based process in it would log into the same directory and the Vector // agent would label its events with `container: containerdebug`. env_vars = env_vars + // The directory must be exactly one level below the log directory, because the Vector + // agent derives the container name from that path segment. .with_value( &FILE_LOG_DIRECTORY, - format!("{STACKABLE_LOG_DIR}/containerdebug"), + concatcp!(STACKABLE_LOG_DIR, "/containerdebug"), ) .with_value(&FILE_LOG_ROTATION_PERIOD, "hourly") .with_value(&FILE_LOG_MAX_FILES, "2") @@ -536,9 +537,6 @@ mod tests { let _ = *AIRFLOW_WEBSERVER_ENABLE_PROXY_FIX; let _ = *AIRFLOW_WEBSERVER_EXPOSE_CONFIG; let _ = *AIRFLOW_WEBSERVER_PROXY_FIX_X_FOR; - let _ = *FILE_LOG_DIRECTORY; - let _ = *FILE_LOG_MAX_FILES; - let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *FORWARDED_ALLOW_IPS; let _ = *PYTHONPATH; let _ = *STACKABLE_POST_HOOK; diff --git a/rust/operator-binary/src/controller/build/volumes.rs b/rust/operator-binary/src/controller/build/volumes.rs index 30f9821b..06e6d787 100644 --- a/rust/operator-binary/src/controller/build/volumes.rs +++ b/rust/operator-binary/src/controller/build/volumes.rs @@ -32,11 +32,9 @@ pub fn create_volumes( name: LOG_VOLUME_NAME.to_string(), empty_dir: Some(EmptyDirVolumeSource { medium: None, - // Only the Airflow log files are budgeted here. The `containerdebug` files in this - // volume are rotated by time, which bounds their number but not their size, so they - // are an unbudgeted writer. They stay negligible at the default INFO level and a 30 - // minute loop, but raising `FILE_LOG_LEVEL` via `envOverrides` can exceed the limit, - // which makes the kubelet evict the Pod. + // Only the Airflow log files are budgeted. The `containerdebug` files in this + // volume are rotated by time, not by size, so they have no budget; raising + // `FILE_LOG_LEVEL` via `envOverrides` can therefore evict the Pod. size_limit: Some(product_logging::framework::calculate_log_volume_size_limit( &[MAX_LOG_FILES_SIZE], )), diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 89b8bce3..f4eedf94 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,5 +1,6 @@ use std::{collections::BTreeSet, ops::Deref, str::FromStr}; +use indoc::formatdoc; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; use stackable_operator::{ @@ -38,6 +39,7 @@ use stackable_operator::{ status::condition::{ClusterCondition, HasStatusCondition}, utils::{COMMON_BASH_TRAP_FUNCTIONS, crds::raw_object_list_schema}, v2::{ + builder::pod::container::EnvVarName, config_overrides::KeyValueConfigOverrides, flask_config_writer::{FlaskAppConfigOptions, PythonType}, product_logging::framework::STACKABLE_LOG_DIR, @@ -84,6 +86,14 @@ pub const CONFIG_PATH: &str = "/stackable/app/config"; pub const LOG_CONFIG_DIR: &str = "/stackable/app/log_config"; pub const AIRFLOW_HOME: &str = "/stackable/airflow"; +// The `stackable-telemetry` variables which configure the file logs of the `containerdebug` +// process. They are set on the Airflow container in `env_vars`, and the start command below +// references the directory as a shell variable, so that an `envOverrides` entry for it takes +// effect in both places. +constant!(pub FILE_LOG_DIRECTORY: EnvVarName = "FILE_LOG_DIRECTORY"); +constant!(pub FILE_LOG_MAX_FILES: EnvVarName = "FILE_LOG_MAX_FILES"); +constant!(pub FILE_LOG_ROTATION_PERIOD: EnvVarName = "FILE_LOG_ROTATION_PERIOD"); + constant!(pub TEMPLATE_VOLUME_NAME: VolumeName = "airflow-executor-pod-template"); pub const TEMPLATE_LOCATION: &str = "/templates"; pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; @@ -776,7 +786,15 @@ impl Deref for AirflowRole { } fn container_debug_command() -> String { - format!("containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &") + // The log directory is created here, because `FILE_LOG_MAX_FILES` makes the telemetry library + // prune old log files before it creates the first one, which complains about the directory not + // existing yet. + formatdoc!( + r#" + mkdir --parents "${file_log_directory}" || true + containerdebug --output={STACKABLE_LOG_DIR}/containerdebug-state.json --loop &"#, + file_log_directory = FILE_LOG_DIRECTORY.as_ref(), + ) } #[derive(Clone, Debug, Deserialize, JsonSchema, PartialEq, Serialize)] @@ -1029,6 +1047,9 @@ mod tests { let _ = *WORKER_ROLE_NAME; let _ = *DAG_PROCESSOR_ROLE_NAME; let _ = *TRIGGERER_ROLE_NAME; + let _ = *FILE_LOG_DIRECTORY; + let _ = *FILE_LOG_MAX_FILES; + let _ = *FILE_LOG_ROTATION_PERIOD; let _ = *TEMPLATE_VOLUME_NAME; let _ = *LISTENER_PVC_NAME; let _ = *METRICS_CONTAINER_NAME;