fix(airflow): Allow overriding the logging configuration in Airflow 3.1 (#1445)

* fix(airflow): Allow overriding the logging configuration

* fix(airflow): Use the log levels defined in stdlib_config for structlog

Author: siegfriedweber

CHANGELOG.md

@@ -80,6 +80,7 @@ All notable changes to this project will be documented in this file.
 - superset: Pin setup-tools to ensure pkg_resources are installed (needed for `4.1.4` builds) ([#1428]).
 - ubi10-rust-builder: Add gzip dependency for the ONBUILD step ([#1436]).
 - airflow: Pin virtualenv to prevent hatch pulling in a version with a breaking change ([#1437]).
+- airflow: Allow overriding the logging configuration in Airflow 3.1.x ([#1445]).
 
 [#1336]: https://github.com/stackabletech/docker-images/pull/1336
 [#1337]: https://github.com/stackabletech/docker-images/pull/1337
@@ -127,6 +128,7 @@ All notable changes to this project will be documented in this file.
 [#1436]: https://github.com/stackabletech/docker-images/pull/1436
 [#1437]: https://github.com/stackabletech/docker-images/pull/1437
 [#1442]: https://github.com/stackabletech/docker-images/pull/1442
+[#1445]: https://github.com/stackabletech/docker-images/pull/1445
 [#1447]: https://github.com/stackabletech/docker-images/pull/1447
 
 ## [25.11.0] - 2025-11-07

airflow/stackable/patches/3.1.6/0001-Allow-overriding-the-logging-configuration.patch

@@ -0,0 +1,71 @@
+From 583a680fc1c6aba63d44c53db487de35a9bf6ed5 Mon Sep 17 00:00:00 2001
+From: Siegfried Weber <mail@siegfriedweber.net>
+Date: Mon, 9 Mar 2026 10:11:28 +0100
+Subject: Allow overriding the logging configuration
+
+---
+ .../src/airflow_shared/logging/structlog.py   | 30 ++++++++++++++++++-
+ 1 file changed, 29 insertions(+), 1 deletion(-)
+
+diff --git a/shared/logging/src/airflow_shared/logging/structlog.py b/shared/logging/src/airflow_shared/logging/structlog.py
+index 2d901b8303..949db2732c 100644
+--- a/shared/logging/src/airflow_shared/logging/structlog.py
++++ b/shared/logging/src/airflow_shared/logging/structlog.py
+@@ -376,6 +376,18 @@ def structlog_processors(
+     return shared_processors, console, console
+ 
+ 
++def update_config(
++    config: dict,
++    updates: Mapping,
++):
++    for key, value in updates.items():
++        if isinstance(value, Mapping):
++            config[key] = update_config(config.get(key, {}), value)
++        else:
++            config[key] = value
++    return config
++
++
+ def configure_logging(
+     *,
+     json_output: bool = False,
+@@ -452,6 +464,19 @@ def configure_logging(
+             else:
+                 PER_LOGGER_LEVELS[log] = loglevel
+ 
++    # Use the log levels defined in stdlib_config for structlog
++    logger_configs = stdlib_config.get("loggers", {})
++    logger_configs[""] = stdlib_config.get("root", {})
++    for logger, logger_config in logger_configs.items():
++        if "level" in logger_config:
++            loglevel = logger_config["level"]
++            if isinstance(loglevel, str):
++                try:
++                    loglevel = NAME_TO_LEVEL[loglevel.lower()]
++                except KeyError:
++                    raise ValueError(f"Invalid log level for logger {logger!r}: {loglevel!r}") from None
++            PER_LOGGER_LEVELS[logger] = loglevel
++
+     shared_pre_chain, for_stdlib, for_structlog = structlog_processors(
+         json_output,
+         log_format=log_format,
+@@ -494,7 +519,7 @@ def configure_logging(
+ 
+     import logging.config
+ 
+-    config = {**stdlib_config}
++    config = {}
+     config.setdefault("version", 1)
+     config.setdefault("disable_existing_loggers", False)
+     config["formatters"] = {**config.get("formatters", {})}
+@@ -547,6 +572,9 @@ def configure_logging(
+         "propagate": True,
+     }
+ 
++    # Merge stdlib_config into config and override existing values
++    update_config(config, stdlib_config)
++
+     logging.config.dictConfig(config)
+ 
+