sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject sqoop git commit: SQOOP-3410: Test S3 import with fs.s3a.security.credential.provider.path
Date Fri, 23 Nov 2018 14:16:41 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 2bf6f3ccd -> 08eb5bdc4


SQOOP-3410: Test S3 import with fs.s3a.security.credential.provider.path

(Boglarka Egyed via Szabolcs Vasas)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/08eb5bdc
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/08eb5bdc
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/08eb5bdc

Branch: refs/heads/trunk
Commit: 08eb5bdc405c22a25cf680390bbd438f6513199c
Parents: 2bf6f3c
Author: Szabolcs Vasas <vasas@apache.org>
Authored: Fri Nov 23 15:15:02 2018 +0100
Committer: Szabolcs Vasas <vasas@apache.org>
Committed: Fri Nov 23 15:15:02 2018 +0100

----------------------------------------------------------------------
 src/docs/user/s3.txt                            |  6 ++++--
 .../util/password/CredentialProviderHelper.java |  6 ++++--
 .../credentials/TestPassingSecurePassword.java  |  4 ++--
 .../s3/TestS3ImportWithHadoopCredProvider.java  | 21 +++++++++++++++++++-
 4 files changed, 30 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/08eb5bdc/src/docs/user/s3.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/s3.txt b/src/docs/user/s3.txt
index 6ff828c..e3d9771 100644
--- a/src/docs/user/s3.txt
+++ b/src/docs/user/s3.txt
@@ -173,8 +173,10 @@ https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Pr
 For a guide to the Hadoop Credential Provider API please see the Hadoop documentation at
 https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/CredentialProviderAPI.html.
 
-After creating a credential file with the credential entries the URL to the provider can
be set via the
-+hadoop.security.credential.provider.path+ property.
+After creating a credential file with the credential entries the URL to the provider can
be set via either the
++hadoop.security.credential.provider.path+ or the +fs.s3a.security.credential.provider.path+
property. For learning
+more about the precedence of these please see the Hadoop AWS documentation at
+https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html#Configure_the_hadoop.security.credential.provider.path_property.
 
 Hadoop Credential Provider is often protected by password supporting three options:
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/08eb5bdc/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
index 4e79f0a..e659def 100644
--- a/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
+++ b/src/java/org/apache/sqoop/util/password/CredentialProviderHelper.java
@@ -83,8 +83,10 @@ public class CredentialProviderHelper {
   // Should track what is specified in JavaKeyStoreProvider class.
   public static final String SCHEME_NAME = "jceks";
   // Should track what is in CredentialProvider class.
-  public static final String CREDENTIAL_PROVIDER_PATH =
+  public static final String HADOOP_CREDENTIAL_PROVIDER_PATH =
     "hadoop.security.credential.provider.path";
+  public static final String S3A_CREDENTIAL_PROVIDER_PATH =
+          "fs.s3a.security.credential.provider.path";
   public static final String CREDENTIAL_PROVIDER_PASSWORD_FILE =
           "hadoop.security.credstore.java-keystore-provider.password-file";
 
@@ -103,7 +105,7 @@ public class CredentialProviderHelper {
   public static String resolveAlias(Configuration conf, String alias)
     throws IOException {
     LOG.debug("Resolving alias with credential provider path set to "
-      + conf.get(CREDENTIAL_PROVIDER_PATH));
+      + conf.get(HADOOP_CREDENTIAL_PROVIDER_PATH));
     try {
       char[] cred = (char[])
         methGetPassword.invoke(conf, new Object[] { alias });

http://git-wip-us.apache.org/repos/asf/sqoop/blob/08eb5bdc/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
index dca3195..4a8e7dc 100644
--- a/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
+++ b/src/test/org/apache/sqoop/credentials/TestPassingSecurePassword.java
@@ -397,7 +397,7 @@ public class TestPassingSecurePassword extends BaseSqoopTestCase {
         "://file/" + credDir.getAbsolutePath() + "/" + jksFile;
       File file = new File(credDir, jksFile);
       file.delete();
-      conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
+      conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
         ourUrl);
       CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
 
@@ -439,7 +439,7 @@ public class TestPassingSecurePassword extends BaseSqoopTestCase {
         "://file/" + credDir.getAbsolutePath() + "/" + jksFile;
       File file = new File(credDir, jksFile);
       file.delete();
-      conf.set(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH,
+      conf.set(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
         ourUrl);
       CredentialProviderHelper.createCredentialEntry(conf, alias, pw);
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/08eb5bdc/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
index e1d7cbd..534d5a9 100644
--- a/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
+++ b/src/test/org/apache/sqoop/s3/TestS3ImportWithHadoopCredProvider.java
@@ -32,6 +32,7 @@ import org.apache.sqoop.testutil.ImportJobTestCase;
 import org.apache.sqoop.testutil.S3CredentialGenerator;
 import org.apache.sqoop.testutil.S3TestUtils;
 import org.apache.sqoop.testutil.TextFileTestUtils;
+import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
 import org.apache.sqoop.util.password.CredentialProviderHelper;
 import org.junit.After;
 import org.junit.AfterClass;
@@ -43,18 +44,36 @@ import org.junit.Test;
 import org.junit.contrib.java.lang.system.EnvironmentVariables;
 import org.junit.experimental.categories.Category;
 import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.Arrays;
 
 import static junit.framework.TestCase.fail;
 
 @Category(S3Test.class)
+@RunWith(Parameterized.class)
+@Parameterized.UseParametersRunnerFactory(BlockJUnit4ClassRunnerWithParametersFactory.class)
 public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase {
+
+    @Parameterized.Parameters(name = "credentialProviderPathProperty = {0}")
+    public static Iterable<? extends Object> parameters() {
+        return Arrays.asList(CredentialProviderHelper.HADOOP_CREDENTIAL_PROVIDER_PATH,
+            CredentialProviderHelper.S3A_CREDENTIAL_PROVIDER_PATH);
+    }
+
     public static final Log LOG = LogFactory.getLog(
             TestS3ImportWithHadoopCredProvider.class.getName());
 
+    private String credentialProviderPathProperty;
+
+    public TestS3ImportWithHadoopCredProvider(String credentialProviderPathProperty) {
+        this.credentialProviderPathProperty = credentialProviderPathProperty;
+    }
+
     private static S3CredentialGenerator s3CredentialGenerator;
 
     private static String providerPathDefault;
@@ -156,7 +175,7 @@ public class TestS3ImportWithHadoopCredProvider extends ImportJobTestCase
{
     private String[] getArgs(String providerPath, boolean withPwdFile, String pwdFile) {
         ArgumentArrayBuilder builder = S3TestUtils.getArgumentArrayBuilderForHadoopCredProviderS3UnitTests(this);
 
-        builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PATH, providerPath);
+        builder.withProperty(credentialProviderPathProperty, providerPath);
         if (withPwdFile) {
             builder.withProperty(CredentialProviderHelper.CREDENTIAL_PROVIDER_PASSWORD_FILE,
pwdFile);
         }


Mime
View raw message