sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2845: Sqoop2: Derive keystore password from a script passed to configuration
Date Fri, 04 Mar 2016 17:23:32 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 506104b0c -> 7bf9eacfc


SQOOP-2845: Sqoop2: Derive keystore password from a script passed to configuration

(Abraham Fine via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/7bf9eacf
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/7bf9eacf
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/7bf9eacf

Branch: refs/heads/sqoop2
Commit: 7bf9eacfcbfda56dd02d979e7eed1b5e79a48081
Parents: 506104b
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Fri Mar 4 09:23:02 2016 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Fri Mar 4 09:23:02 2016 -0800

----------------------------------------------------------------------
 .../sqoop/security/SecurityConstants.java       |  18 ++-
 .../apache/sqoop/server/SqoopJettyServer.java   |  60 +++++++-
 .../apache/sqoop/server/common/ServerError.java |   3 +
 .../test/infrastructure/SqoopTestCase.java      |   2 +-
 .../integration/serverproperties/SslTest.java   | 138 ++++++++++++++-----
 5 files changed, 179 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/7bf9eacf/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
index 91a1b8b..0241c86 100644
--- a/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
+++ b/core/src/main/java/org/apache/sqoop/security/SecurityConstants.java
@@ -187,12 +187,20 @@ public final class SecurityConstants {
 
   /**
    * The config specifies the password of the JKS formatted keystore
-   * <tt>org.apache.sqoop.security.tls.keystorepassword</tt>.
+   * <tt>org.apache.sqoop.security.tls.keystore_password</tt>.
    */
   public static final String KEYSTORE_PASSWORD =
           PREFIX_TLS_CONFIG + "keystore_password";
 
   /**
+   * The config specifies a script that outputs the password of the JKS formatted keystore
+   * to standard out
+   * <tt>org.apache.sqoop.security.tls.keystore_password_generator</tt>.
+   */
+  public static final String KEYSTORE_PASSWORD_GENERATOR =
+    PREFIX_TLS_CONFIG + "keystore_password_generator";
+
+  /**
    * The config specifies the password for the private key in the JKS formatted
    * keystore
    * <tt>org.apache.sqoop.security.tls.keymanagerpassword</tt>.
@@ -201,6 +209,14 @@ public final class SecurityConstants {
     PREFIX_TLS_CONFIG + "keymanager_password";
 
   /**
+   * The config specifies a script that outputs the password for the
+   * private key in the JKS formatted keystore to standard out
+   * <tt>org.apache.sqoop.security.tls.keymanager_password_generator</tt>.
+   */
+  public static final String KEYMANAGER_PASSWORD_GENERATOR =
+    PREFIX_TLS_CONFIG + "keymanager_password_generator";
+
+  /**
    * The config specifies the token kind in delegation token.
    */
   public static final String TOKEN_KIND = "sqoop_token_kind";

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7bf9eacf/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
index 60368af..4696a87 100644
--- a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
+++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
@@ -18,6 +18,7 @@
 
 package org.apache.sqoop.server;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.common.MapContext;
 import org.apache.sqoop.common.SqoopException;
@@ -26,18 +27,28 @@ import org.apache.sqoop.core.SqoopServer;
 import org.apache.sqoop.filter.SqoopAuthenticationFilter;
 import org.apache.sqoop.security.SecurityConstants;
 import org.apache.sqoop.server.common.ServerError;
-import org.apache.sqoop.server.v1.*;
+import org.apache.sqoop.server.v1.AuthorizationServlet;
+import org.apache.sqoop.server.v1.ConfigurableServlet;
+import org.apache.sqoop.server.v1.ConnectorServlet;
+import org.apache.sqoop.server.v1.DriverServlet;
+import org.apache.sqoop.server.v1.JobServlet;
+import org.apache.sqoop.server.v1.LinkServlet;
+import org.apache.sqoop.server.v1.SubmissionsServlet;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.SecureRequestCustomizer;
 import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.server.SslConnectionFactory;
 import org.eclipse.jetty.servlet.ServletContextHandler;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.ExecutorThreadPool;
-import org.eclipse.jetty.server.ServerConnector;
 
 import javax.servlet.DispatcherType;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.util.EnumSet;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
@@ -86,13 +97,37 @@ public class SqoopJettyServer {
       }
 
       String keyStorePassword = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD);
-      if (keyStorePassword != null && keyStorePassword.length() > 0) {
+      String keyStorePasswordGenerator = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD_GENERATOR);
+      if (StringUtils.isNotBlank(keyStorePassword)) {
+        if (StringUtils.isNotBlank(keyStorePasswordGenerator)) {
+          LOG.warn(SecurityConstants.KEYSTORE_PASSWORD + " and " + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR
+            + "are both set, using " + SecurityConstants.KEYSTORE_PASSWORD);
+        }
         sslContextFactory.setKeyStorePassword(keyStorePassword);
+      } else if (StringUtils.isNotBlank(keyStorePasswordGenerator)) {
+        try {
+          String passwordFromGenerator = readPasswordFromGenerator(keyStorePasswordGenerator);
+          sslContextFactory.setKeyStorePassword(passwordFromGenerator);
+        } catch (IOException exception) {
+          throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator:
" + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR, exception);
+        }
       }
 
       String keyManagerPassword = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD);
-      if (keyManagerPassword != null && keyManagerPassword.length() > 0) {
+      String keyManagerPasswordGenerator = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR);
+      if (StringUtils.isNotBlank(keyManagerPassword)) {
         sslContextFactory.setKeyManagerPassword(keyManagerPassword);
+        if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) {
+          LOG.warn(SecurityConstants.KEYMANAGER_PASSWORD + " and " + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR
+            + "are both set, using " + SecurityConstants.KEYMANAGER_PASSWORD);
+        }
+      } else if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) {
+        try {
+          String passwordFromGenerator = readPasswordFromGenerator(keyManagerPasswordGenerator);
+          sslContextFactory.setKeyManagerPassword(passwordFromGenerator);
+        } catch (IOException exception) {
+          throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator:
" + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR, exception);
+        }
       }
 
       HttpConfiguration https = new HttpConfiguration();
@@ -111,6 +146,21 @@ public class SqoopJettyServer {
     webServer.setHandler(createServletContextHandler());
   }
 
+  private String readPasswordFromGenerator(String generatorCommand) throws IOException {
+    ProcessBuilder processBuilder = new ProcessBuilder("/bin/sh", "-c", generatorCommand);
+    Process process = processBuilder.start();
+    String output;
+    try (
+      InputStreamReader inputStreamReader = new InputStreamReader(process.getInputStream(),
Charset.forName("UTF-8"));
+      BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
+    ) {
+      output =  bufferedReader.readLine();
+    } catch(IOException exception) {
+      throw exception;
+    }
+    return output;
+  }
+
   public synchronized void startServer() {
     try {
       webServer.start();
@@ -162,7 +212,7 @@ public class SqoopJettyServer {
     return context;
   }
 
-  public static void main(String[] args) {
+  public static void main(String[] args){
     SqoopJettyServer sqoopJettyServer = new SqoopJettyServer();
     sqoopJettyServer.startServer();
     sqoopJettyServer.joinServerThread();

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7bf9eacf/server/src/main/java/org/apache/sqoop/server/common/ServerError.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/common/ServerError.java b/server/src/main/java/org/apache/sqoop/server/common/ServerError.java
index a644e9f..e820951 100644
--- a/server/src/main/java/org/apache/sqoop/server/common/ServerError.java
+++ b/server/src/main/java/org/apache/sqoop/server/common/ServerError.java
@@ -41,6 +41,9 @@ public enum ServerError implements ErrorCode {
 
   /** TLS enabled but keystore location not set*/
   SERVER_0007("TLS enabled but keystore location not set"),
+
+  /** Execution of password generator failed*/
+  SERVER_0008("Execution of password generator failed"),
   ;
 
   private final String message;

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7bf9eacf/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java b/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java
index 9e1c683..5602023 100644
--- a/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java
@@ -115,7 +115,7 @@ public class SqoopTestCase implements ITest {
 
   private SqoopClient client;
 
-  private DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
+  protected DelegationTokenAuthenticatedURL.Token authToken = new DelegationTokenAuthenticatedURL.Token();
 
   protected FileSystem hdfsClient;
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7bf9eacf/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java
b/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java
index 17503f3..c981db2 100644
--- a/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/serverproperties/SslTest.java
@@ -31,10 +31,11 @@ import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
 import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
 import org.apache.sqoop.test.utils.HdfsUtils;
 import org.apache.sqoop.test.utils.SecurityUtils;
-import org.apache.sqoop.test.utils.SqoopUtils;
 import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.testng.annotations.AfterMethod;
+import org.testng.annotations.AfterSuite;
 import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeSuite;
 import org.testng.annotations.Test;
 
 import javax.net.ssl.HostnameVerifier;
@@ -53,25 +54,36 @@ import java.util.Map;
 
 import static org.testng.Assert.assertEquals;
 
-@Infrastructure(dependencies = {KdcInfrastructureProvider.class, HadoopInfrastructureProvider.class,
DatabaseInfrastructureProvider.class})
+@Infrastructure(dependencies = {HadoopInfrastructureProvider.class, DatabaseInfrastructureProvider.class,
KdcInfrastructureProvider.class})
 @Test(groups = {"no-real-cluster"})
 public class SslTest extends SqoopTestCase {
 
   private SqoopMiniCluster sqoopMiniCluster;
   private SSLContext defaultSslContext;
   private HostnameVerifier defaultHostNameVerifier;
+  private X509Certificate serverCertificate;
+  private String sslKeystoreDir;
+
+  private final String KEYSTORE_DIR = "ssltestkeystores/";
 
   public static class SslSqoopMiniCluster extends JettySqoopMiniCluster {
 
     private String keyStoreFilePath;
     private String keyStorePassword;
+    private String keyStorePasswordGenerator;
     private String keyManagerPassword;
+    private String keyManagerPasswordGenerator;
 
-    public SslSqoopMiniCluster(String temporaryPath, Configuration configuration, String
keyStoreFilePath, String keyStorePassword, String keyManagerPassword) throws Exception {
+    public SslSqoopMiniCluster(String temporaryPath, Configuration configuration,
+                               String keyStoreFilePath, String keyStorePassword,
+                               String keyStorePasswordGenerator, String keyManagerPassword,
+                               String keyManagerPasswordGenerator) throws Exception {
       super(temporaryPath, configuration);
       this.keyStoreFilePath = keyStoreFilePath;
       this.keyStorePassword = keyStorePassword;
+      this.keyStorePasswordGenerator = keyStorePasswordGenerator;
       this.keyManagerPassword = keyManagerPassword;
+      this.keyManagerPasswordGenerator = keyManagerPasswordGenerator;
     }
 
     @Override
@@ -82,74 +94,127 @@ public class SslTest extends SqoopTestCase {
       properties.put(SecurityConstants.TLS_PROTOCOL, "TLSv1.2");
       properties.put(SecurityConstants.KEYSTORE_LOCATION, keyStoreFilePath);
       properties.put(SecurityConstants.KEYSTORE_PASSWORD, keyStorePassword);
+      properties.put(SecurityConstants.KEYSTORE_PASSWORD_GENERATOR, keyStorePasswordGenerator);
       properties.put(SecurityConstants.KEYMANAGER_PASSWORD, keyManagerPassword);
+      properties.put(SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR, keyManagerPasswordGenerator);
 
       return properties;
     }
   }
 
+  @BeforeSuite
+  public void createCertificates() throws Exception {
+    sslKeystoreDir = getTemporaryPath() + KEYSTORE_DIR;
+    serverCertificate = setupKeystore(sslKeystoreDir);
+  }
+
   @BeforeMethod
-  public void backupSslContext() throws Exception {
+  public void backupState() throws Exception {
+    authToken = new DelegationTokenAuthenticatedURL.Token();
     defaultSslContext = SSLContext.getDefault();
     defaultHostNameVerifier = HttpsURLConnection.getDefaultHostnameVerifier();
   }
 
+  @BeforeMethod(dependsOnMethods = { "backupState" })
+  public void bypassHostnameVerification() throws Exception {
+    HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
+      public boolean verify(String hostname, SSLSession session) {
+        try {
+          if (hostname.equals((new URL(sqoopMiniCluster.getServerUrl())).getHost())) {
+            return true;
+          }
+        } catch (MalformedURLException e) {
+          return false;
+        }
+        return false;
+      }
+    });
+  }
+
+  private void prepareKDC() {
+    KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class);
+    if (kdcProvider != null) {
+      sqoopMiniCluster.setKdc(kdcProvider.getInstance());
+    }
+  }
+
+  private X509Certificate setupKeystore(String sslKeystoreDir) throws Exception {
+    String sslConfDir = sslKeystoreDir;
+    FileUtils.deleteDirectory(new File(sslKeystoreDir));
+    FileUtils.forceMkdir(new File(sslKeystoreDir));
+    return SecurityUtils.setupSSLConfig(
+      sslKeystoreDir, sslConfDir, new Configuration(), false, true);
+  }
+
+  @AfterSuite
+  public void deleteCertificates() throws Exception {
+    FileUtils.deleteDirectory(new File(sslKeystoreDir));
+  }
+
   @AfterMethod
-  public void restoreSslContext() {
+  public void restoreState() {
     SSLContext.setDefault(defaultSslContext);
     HttpsURLConnection.setDefaultHostnameVerifier(defaultHostNameVerifier);
   }
 
   @AfterMethod
   public void stopCluster() throws Exception {
-    sqoopMiniCluster.stop();
+    try {
+      sqoopMiniCluster.stop();
+    } catch (Exception e) {
+      throw e;
+    }
+
   }
 
   @Test
-  public void testSslInUse() throws Exception {
-    String sslKeystoresDir = getTemporaryPath() + "ssl-keystore/";
-    String sslConfDir = SqoopUtils.getClasspathDir(SslTest.class);
-    FileUtils.deleteDirectory(new File(sslKeystoresDir));
-    FileUtils.forceMkdir(new File(sslKeystoresDir));
-    X509Certificate serverCertificate = SecurityUtils.setupSSLConfig(
-      sslKeystoresDir, sslConfDir, new Configuration(), false, true);
-
+  public void testSslInUseWithPassword() throws Exception {
     sqoopMiniCluster =
-      new SslSqoopMiniCluster(HdfsUtils.joinPathFragments(getTemporaryPath(), getTestName()),
getHadoopConf(), sslKeystoresDir + SecurityUtils.SERVER_KEYSTORE, SecurityUtils.SERVER_KEY_STORE_PASSWORD,
SecurityUtils.SERVER_KEY_PASSWORD);
+      new SslSqoopMiniCluster(HdfsUtils.joinPathFragments(getTemporaryPath(), getTestName()),
+        getHadoopConf(), sslKeystoreDir + SecurityUtils.SERVER_KEYSTORE,
+        SecurityUtils.SERVER_KEY_STORE_PASSWORD, "",
+        SecurityUtils.SERVER_KEY_PASSWORD, "");
 
-    KdcInfrastructureProvider kdcProvider = getInfrastructureProvider(KdcInfrastructureProvider.class);
-    if (kdcProvider != null) {
-      sqoopMiniCluster.setKdc(kdcProvider.getInstance());
-    }
+    prepareKDC();
 
     sqoopMiniCluster.start();
 
-    // Bypass hostname verification
-    HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {
-      public boolean verify(String hostname, SSLSession session) {
-        try {
-          if (hostname.equals((new URL(sqoopMiniCluster.getServerUrl())).getHost())) {
-            return true;
-          }
-        } catch (MalformedURLException e) {
-          return false;
-        }
-        return false;
-      }
-    });
-
     SslContextFactory sslContextFactory = new SslContextFactory();
-    sslContextFactory.setKeyStorePath(sslKeystoresDir + SecurityUtils.TRUSTSTORE);
+    sslContextFactory.setKeyStorePath(sslKeystoreDir + SecurityUtils.TRUSTSTORE);
+    sslContextFactory.start();
+    SSLContext.setDefault(sslContextFactory.getSslContext());
+
+    initSqoopClient(sqoopMiniCluster.getServerUrl());
+
+    verifySsl(serverCertificate);
+  }
+
+  @Test
+  public void testSslInUseWithPasswordGenerator() throws Exception {
+    sqoopMiniCluster =
+      new SslSqoopMiniCluster(HdfsUtils.joinPathFragments(getTemporaryPath(), getTestName()),
+        getHadoopConf(), sslKeystoreDir + SecurityUtils.SERVER_KEYSTORE,
+        "", "echo " + SecurityUtils.SERVER_KEY_STORE_PASSWORD,
+        "", "echo " + SecurityUtils.SERVER_KEY_PASSWORD );
 
+    prepareKDC();
+
+    sqoopMiniCluster.start();
+
+    SslContextFactory sslContextFactory = new SslContextFactory();
+    sslContextFactory.setKeyStorePath(sslKeystoreDir + SecurityUtils.TRUSTSTORE);
     sslContextFactory.start();
 
     SSLContext.setDefault(sslContextFactory.getSslContext());
 
     initSqoopClient(sqoopMiniCluster.getServerUrl());
+    verifySsl(serverCertificate);
+  }
 
-    // Make a request and check the cert
+  private void verifySsl(X509Certificate serverCertificate) throws Exception {
     URL url = new URL(sqoopMiniCluster.getServerUrl() + "version?" +
       PseudoAuthenticator.USER_NAME + "=" + System.getProperty("user.name"));
+
     HttpURLConnection conn = new DelegationTokenAuthenticatedURL().openConnection(url, getAuthToken());
     conn.setRequestMethod(HttpMethod.GET);
     conn.setRequestProperty("Accept", MediaType.APPLICATION_JSON);
@@ -157,7 +222,10 @@ public class SslTest extends SqoopTestCase {
     assertEquals(conn.getResponseCode(), 200);
 
     HttpsURLConnection secured = (HttpsURLConnection) conn;
+
     Certificate actualCertificate = secured.getServerCertificates()[0];
+
+    secured.disconnect();
     assertEquals(actualCertificate, serverCertificate);
   }
 


Mime
View raw message