sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2618: Sqoop2: Mask S3 sensitive properties in HDFS configuration
Date Thu, 15 Oct 2015 23:53:26 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 1ea3db992 -> 77be2a8f9


SQOOP-2618: Sqoop2: Mask S3 sensitive properties in HDFS configuration

(Abraham Fine via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/77be2a8f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/77be2a8f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/77be2a8f

Branch: refs/heads/sqoop2
Commit: 77be2a8f9c8be384ae453b3f26296d30d4a40893
Parents: 1ea3db9
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Thu Oct 15 16:53:07 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Thu Oct 15 16:53:07 2015 -0700

----------------------------------------------------------------------
 .../hdfs/configuration/LinkConfig.java          |  3 +-
 .../sqoop/connector/hdfs/TestLinkConfig.java    | 42 ++++++++++++++++++++
 2 files changed, 44 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/77be2a8f/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
index 39f3752..5852e53 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
@@ -36,7 +36,8 @@ public class LinkConfig {
   @Input(size = 255, validators = { @Validator(DirectoryExistsValidator.class)})
   public String confDir;
 
-  @Input public Map<String, String> configOverrides;
+  @Input(sensitiveKeyPattern = "(?i)(.*(password|key|secret).*)")
+  public Map<String, String> configOverrides;
 
   public LinkConfig() {
     configOverrides = new HashMap<>();

http://git-wip-us.apache.org/repos/asf/sqoop/blob/77be2a8f/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java
b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java
index 05178ac..900e3db 100644
--- a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestLinkConfig.java
@@ -18,8 +18,15 @@
 package org.apache.sqoop.connector.hdfs;
 
 import org.apache.sqoop.connector.hdfs.configuration.LinkConfig;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MMapInput;
 import org.testng.annotations.Test;
 
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertTrue;
 
@@ -68,4 +75,39 @@ public class TestLinkConfig {
       assertFalse(validator.getStatus().canProceed(), uri);
     }
   }
+
+  @Test
+  public void testSensitiveConfigOverridesKeys() {
+    String nonSensitiveKey = "public";
+    String valueString = "value";
+    String filler = "blah";
+
+    String[] sensitiveWords = new String[] {"password", "key", "secret"};
+
+    LinkConfiguration linkConfiguration = new LinkConfiguration();
+    LinkConfig config = new LinkConfig();
+    linkConfiguration.linkConfig = config;
+    config.configOverrides.put(nonSensitiveKey, valueString);
+    for (String sensitiveWord : sensitiveWords) {
+      for (String sensitiveWordWithCase : new String[] {sensitiveWord, sensitiveWord.toUpperCase()})
{
+        config.configOverrides.put(filler + sensitiveWordWithCase + filler, valueString);
+        config.configOverrides.put(sensitiveWordWithCase + filler, valueString);
+        config.configOverrides.put(filler + sensitiveWordWithCase, valueString);
+      }
+    }
+
+    MConfig mLinkConfig = ConfigUtils.toConfigs(linkConfiguration).get(0);
+    MMapInput mConfigOverrides = mLinkConfig.getMapInput("linkConfig.configOverrides");
+
+    Map<String, String> redactedMap = mConfigOverrides.getNonsenstiveValue();
+    assertEquals(redactedMap.get(nonSensitiveKey), valueString);
+
+    for (String sensitiveWord : sensitiveWords) {
+      for (String sensitiveWordWithCase : new String[] {sensitiveWord, sensitiveWord.toUpperCase()})
{
+        assertEquals(redactedMap.get(filler + sensitiveWordWithCase + filler), MMapInput.SENSITIVE_VALUE_PLACEHOLDER);
+        assertEquals(redactedMap.get(sensitiveWordWithCase + filler), MMapInput.SENSITIVE_VALUE_PLACEHOLDER);
+        assertEquals(redactedMap.get(filler + sensitiveWordWithCase), MMapInput.SENSITIVE_VALUE_PLACEHOLDER);
+      }
+    }
+  }
 }


Mime
View raw message