From deb16de16255fc70965b47de709226e3eb5b0313 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Fri, 18 Sep 2026 13:05:27 +0200 Subject: [PATCH 1/2] TEZ-4756: DAGUtils: configuration handling improvements Route each key/value pair through Hadoop's ConfigRedactor when building the ATS configuration map, so values matching the cluster's configured sensitive-key pattern are masked in the map published to Timeline. Adds a unit test. Co-Authored-By: Claude Code --- .../tez/dag/history/utils/DAGUtils.java | 14 ++++++++-- .../tez/dag/history/utils/TestDAGUtils.java | 28 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java index 6cadd2e96d..37f8bdd06c 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java @@ -30,6 +30,7 @@ import java.util.TreeMap; import java.util.zip.Inflater; +import org.apache.hadoop.conf.ConfigRedactor; import org.apache.hadoop.conf.Configuration; import org.apache.tez.common.ATSConstants; import org.apache.tez.common.Preconditions; @@ -505,11 +506,20 @@ public static Map convertEdgeManagerPluginDescriptor( public static Map convertConfigurationToATSMap(Configuration conf) { // Copy configuration to avoid CME since iterator is not thread safe until HADOOP-13500 - Iterator> iter = new Configuration(conf).iterator(); + Configuration snapshot = new Configuration(conf); + // The AM configuration typically holds the union of every hadoop site + // file loaded at startup (core-site, hdfs-site, yarn-site, ssl-*, + // credential providers, keytab paths, …). Publishing it verbatim to + // Timeline exposes those secrets to anyone with timeline read access, so + // route each value through Hadoop's ConfigRedactor, which masks keys + // matched by hadoop.security.sensitive-config-keys (the default pattern + // covers *password*, *secret*, *keystore*, aws credential keys, etc.). + ConfigRedactor redactor = new ConfigRedactor(snapshot); + Iterator> iter = snapshot.iterator(); Map atsConf = new TreeMap(); while (iter.hasNext()) { Entry entry = iter.next(); - atsConf.put(entry.getKey(), entry.getValue()); + atsConf.put(entry.getKey(), redactor.redact(entry.getKey(), entry.getValue())); } return atsConf; } diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java index 2bf3746bbb..71da4eccd3 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java @@ -222,4 +222,32 @@ public void testConvertDAGPlanToATSMap() throws IOException, JSONException { } } + @Test + public void testConvertConfigurationToATSMapRedactsSecrets() { + Configuration conf = new Configuration(false); + // Non-sensitive properties must pass through unchanged. + conf.set("tez.am.dag.scheduler.class", + "org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder"); + conf.set("mapreduce.job.name", "normal-job"); + // These keys are covered by Hadoop's default + // hadoop.security.sensitive-config-keys pattern. + conf.set("fs.s3a.secret.key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRET"); + conf.set("fs.s3a.access.key", "AKIAIOSFODNN7EXAMPLE"); + conf.set("ssl.server.keystore.password", "SuperSecretKeystorePass!"); + conf.set("hadoop.security.credential.provider.password", "credpass"); + + Map ats = DAGUtils.convertConfigurationToATSMap(conf); + + assertEquals("org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder", + ats.get("tez.am.dag.scheduler.class")); + assertEquals("normal-job", ats.get("mapreduce.job.name")); + assertFalse(ats.get("fs.s3a.secret.key").contains("SECRET"), + "s3a secret key must be redacted, got: " + ats.get("fs.s3a.secret.key")); + assertFalse(ats.get("ssl.server.keystore.password").contains("SuperSecret"), + "keystore password must be redacted, got: " + ats.get("ssl.server.keystore.password")); + assertFalse(ats.get("hadoop.security.credential.provider.password").contains("credpass"), + "credential provider password must be redacted, got: " + + ats.get("hadoop.security.credential.provider.password")); + } + } From 40752a1e9b0e9fce4e7ea248b82e4504085a87f9 Mon Sep 17 00:00:00 2001 From: Laszlo Bodor Date: Mon, 21 Sep 2026 14:11:20 +0200 Subject: [PATCH 2/2] PR comments --- .../tez/dag/history/utils/DAGUtils.java | 10 ++-- .../tez/dag/history/utils/TestDAGUtils.java | 47 +++++++++++++------ 2 files changed, 35 insertions(+), 22 deletions(-) diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java index 37f8bdd06c..9973931922 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java @@ -507,13 +507,9 @@ public static Map convertEdgeManagerPluginDescriptor( public static Map convertConfigurationToATSMap(Configuration conf) { // Copy configuration to avoid CME since iterator is not thread safe until HADOOP-13500 Configuration snapshot = new Configuration(conf); - // The AM configuration typically holds the union of every hadoop site - // file loaded at startup (core-site, hdfs-site, yarn-site, ssl-*, - // credential providers, keytab paths, …). Publishing it verbatim to - // Timeline exposes those secrets to anyone with timeline read access, so - // route each value through Hadoop's ConfigRedactor, which masks keys - // matched by hadoop.security.sensitive-config-keys (the default pattern - // covers *password*, *secret*, *keystore*, aws credential keys, etc.). + // The AM config holds every hadoop site file loaded at startup, so + // publishing it as is would expose credentials to anyone with timeline + // read access. Mask via hadoop.security.sensitive-config-keys. ConfigRedactor redactor = new ConfigRedactor(snapshot); Iterator> iter = snapshot.iterator(); Map atsConf = new TreeMap(); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java index 71da4eccd3..488f993f38 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java @@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.tez.client.CallerContext; @@ -224,30 +225,46 @@ public void testConvertDAGPlanToATSMap() throws IOException, JSONException { @Test public void testConvertConfigurationToATSMapRedactsSecrets() { - Configuration conf = new Configuration(false); - // Non-sensitive properties must pass through unchanged. + // Defaults must load: ConfigRedactor's fallback pattern is narrower than + // core-default.xml and misses fs.s3a.session.token. + Configuration conf = new Configuration(); conf.set("tez.am.dag.scheduler.class", "org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder"); conf.set("mapreduce.job.name", "normal-job"); - // These keys are covered by Hadoop's default - // hadoop.security.sensitive-config-keys pattern. - conf.set("fs.s3a.secret.key", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRET"); - conf.set("fs.s3a.access.key", "AKIAIOSFODNN7EXAMPLE"); - conf.set("ssl.server.keystore.password", "SuperSecretKeystorePass!"); - conf.set("hadoop.security.credential.provider.password", "credpass"); + conf.set("fs.s3a.secret.key", "fake-s3a-secret-key"); + conf.set("fs.s3a.session.token", "fake-s3a-session-token"); + conf.set("ssl.server.keystore.password", "fake-keystore-password"); + conf.set("hadoop.security.credential.provider.password", "fake-credential-password"); + // An access key id is not a secret; the default pattern omits it. + conf.set("fs.s3a.access.key", "fake-s3a-access-key-id"); Map ats = DAGUtils.convertConfigurationToATSMap(conf); assertEquals("org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder", ats.get("tez.am.dag.scheduler.class")); assertEquals("normal-job", ats.get("mapreduce.job.name")); - assertFalse(ats.get("fs.s3a.secret.key").contains("SECRET"), - "s3a secret key must be redacted, got: " + ats.get("fs.s3a.secret.key")); - assertFalse(ats.get("ssl.server.keystore.password").contains("SuperSecret"), - "keystore password must be redacted, got: " + ats.get("ssl.server.keystore.password")); - assertFalse(ats.get("hadoop.security.credential.provider.password").contains("credpass"), - "credential provider password must be redacted, got: " - + ats.get("hadoop.security.credential.provider.password")); + assertEquals("fake-s3a-access-key-id", ats.get("fs.s3a.access.key")); + for (String key : new String[] {"fs.s3a.secret.key", "fs.s3a.session.token", + "ssl.server.keystore.password", "hadoop.security.credential.provider.password"}) { + assertFalse(ats.get(key).contains("fake"), + key + " must be redacted, got: " + ats.get(key)); + } + } + + @Test + public void testConvertConfigurationToATSMapHonoursConfiguredPattern() { + // A configured pattern replaces the defaults and must be honoured. + Configuration conf = new Configuration(); + conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS, + "my\\.site\\.internal\\.id"); + conf.set("my.site.internal.id", "fake-internal-id"); + conf.set("mapreduce.job.name", "normal-job"); + + Map ats = DAGUtils.convertConfigurationToATSMap(conf); + + assertFalse(ats.get("my.site.internal.id").contains("fake"), + "configured sensitive key must be redacted, got: " + ats.get("my.site.internal.id")); + assertEquals("normal-job", ats.get("mapreduce.job.name")); } }