TEZ-4756: DAGUtils: configuration handling improvements

tez-dag/src/main/java/org/apache/tez/dag/history/utils/DAGUtils.java

@@ -30,6 +30,7 @@
 import java.util.TreeMap;
 import java.util.zip.Inflater;
 
+import org.apache.hadoop.conf.ConfigRedactor;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.tez.common.ATSConstants;
 import org.apache.tez.common.Preconditions;
@@ -505,11 +506,16 @@ public static Map<String,Object> convertEdgeManagerPluginDescriptor(
 
   public static Map<String, String> convertConfigurationToATSMap(Configuration conf) {
     // Copy configuration to avoid CME since iterator is not thread safe until HADOOP-13500
-    Iterator<Entry<String, String>> iter = new Configuration(conf).iterator();
+    Configuration snapshot = new Configuration(conf);
+    // The AM config holds every hadoop site file loaded at startup, so
+    // publishing it as is would expose credentials to anyone with timeline
+    // read access. Mask via hadoop.security.sensitive-config-keys.
+    ConfigRedactor redactor = new ConfigRedactor(snapshot);
+    Iterator<Entry<String, String>> iter = snapshot.iterator();
     Map<String, String> atsConf = new TreeMap<String, String>();
     while (iter.hasNext()) {
       Entry<String, String> entry = iter.next();
-      atsConf.put(entry.getKey(), entry.getValue());
+      atsConf.put(entry.getKey(), redactor.redact(entry.getKey(), entry.getValue()));
     }
     return atsConf;
   }

tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java

@@ -31,6 +31,7 @@
 import java.util.concurrent.TimeUnit;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Resource;
 import org.apache.tez.client.CallerContext;
@@ -222,4 +223,48 @@ public void testConvertDAGPlanToATSMap() throws IOException, JSONException {
     }
   }
 
+  @Test
+  public void testConvertConfigurationToATSMapRedactsSecrets() {
+    // Defaults must load: ConfigRedactor's fallback pattern is narrower than
+    // core-default.xml and misses fs.s3a.session.token.
+    Configuration conf = new Configuration();
+    conf.set("tez.am.dag.scheduler.class",
+        "org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder");
+    conf.set("mapreduce.job.name", "normal-job");
+    conf.set("fs.s3a.secret.key", "fake-s3a-secret-key");
+    conf.set("fs.s3a.session.token", "fake-s3a-session-token");
+    conf.set("ssl.server.keystore.password", "fake-keystore-password");
+    conf.set("hadoop.security.credential.provider.password", "fake-credential-password");
+    // An access key id is not a secret; the default pattern omits it.
+    conf.set("fs.s3a.access.key", "fake-s3a-access-key-id");
+
+    Map<String, String> ats = DAGUtils.convertConfigurationToATSMap(conf);
+
+    assertEquals("org.apache.tez.dag.app.dag.impl.DAGSchedulerNaturalOrder",
+        ats.get("tez.am.dag.scheduler.class"));
+    assertEquals("normal-job", ats.get("mapreduce.job.name"));
+    assertEquals("fake-s3a-access-key-id", ats.get("fs.s3a.access.key"));
+    for (String key : new String[] {"fs.s3a.secret.key", "fs.s3a.session.token",
+        "ssl.server.keystore.password", "hadoop.security.credential.provider.password"}) {
+      assertFalse(ats.get(key).contains("fake"),
+          key + " must be redacted, got: " + ats.get(key));
+    }
+  }
+
+  @Test
+  public void testConvertConfigurationToATSMapHonoursConfiguredPattern() {
+    // A configured pattern replaces the defaults and must be honoured.
+    Configuration conf = new Configuration();
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_SENSITIVE_CONFIG_KEYS,
+        "my\\.site\\.internal\\.id");
+    conf.set("my.site.internal.id", "fake-internal-id");
+    conf.set("mapreduce.job.name", "normal-job");
+
+    Map<String, String> ats = DAGUtils.convertConfigurationToATSMap(conf);
+
+    assertFalse(ats.get("my.site.internal.id").contains("fake"),
+        "configured sensitive key must be redacted, got: " + ats.get("my.site.internal.id"));
+    assertEquals("normal-job", ats.get("mapreduce.job.name"));
+  }
+
 }