sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject [1/2] sqoop git commit: SQOOP-2890: Provide tooling to encrypt non-encrypted repository and rotate keys
Date Mon, 11 Apr 2016 20:21:24 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 5b897a46f -> c6fc7f95a


http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryEncryptionToolTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryEncryptionToolTest.java
b/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryEncryptionToolTest.java
new file mode 100644
index 0000000..5f588f5
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/tools/RepositoryEncryptionToolTest.java
@@ -0,0 +1,444 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.integration.tools;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.common.MapContext;
+import org.apache.sqoop.core.PropertiesConfigurationProvider;
+import org.apache.sqoop.core.SqoopConfiguration;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MStringInput;
+import org.apache.sqoop.repository.MasterKeyManager;
+import org.apache.sqoop.repository.RepositoryManager;
+import org.apache.sqoop.repository.common.CommonRepoUtils;
+import org.apache.sqoop.repository.common
+  .CommonRepositoryInsertUpdateDeleteSelectQuery;
+import org.apache.sqoop.security.SecurityConstants;
+import org.apache.sqoop.test.infrastructure.Infrastructure;
+import org.apache.sqoop.test.infrastructure.SqoopTestCase;
+import org.apache.sqoop.test.infrastructure.providers.DatabaseInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
+import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.apache.sqoop.tools.tool.RepositoryEncryptionTool;
+import org.testng.Assert;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Map;
+
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_ENCRYPTED;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_HMAC;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_INPUT;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_IV;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_VALUE;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.SCHEMA_SQOOP;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_LINK_INPUT_NAME;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@Test(groups = "no-real-cluster")
+@Infrastructure(dependencies = {DatabaseInfrastructureProvider.class, HadoopInfrastructureProvider.class})
+public class RepositoryEncryptionToolTest extends SqoopTestCase {
+
+  private SqoopMiniCluster sqoopMiniCluster;
+  private String temporaryPath;
+
+  public static final String JDBC_URL = "jdbc:derby:memory:myDB";
+  public static final String INPUT_VALUE_QUERY =
+    "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + ", "
+    + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", "
+    + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", "
+    + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", "
+    + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC)
+    + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+    + " WHERE " +  CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + " = ?";
+
+  private String passwordGenerator;
+  private String hmacAlgorithm;
+  private String cipherAlgorithm;
+  private int cipherKeySize;
+  private String cipherSpec;
+  private String pbkdf2Algorithm;
+  private int pbkdf2Rounds;
+  private int ivLength;
+
+  public static class SqoopMiniCluster extends JettySqoopMiniCluster {
+
+    private boolean repositoryEncryptionEnabled;
+
+    private String passwordGenerator;
+    private String hmacAlgorithm;
+    private String cipherAlgorithm;
+    private int cipherKeySize;
+    private String cipherSpec;
+    private String pbkdf2Algorithm;
+    private int pbkdf2Rounds;
+    private int ivLength;
+
+    public SqoopMiniCluster(String temporaryPath, Configuration configuration) throws Exception
{
+      super(temporaryPath, configuration);
+      this.repositoryEncryptionEnabled = false;
+    }
+
+    public SqoopMiniCluster(String temporaryPath, Configuration configuration, String passwordGenerator,
String hmacAlgorithm, String cipherAlgorithm, int cipherKeySize, String cipherSpec, String
pbkdf2Algorithm, int pbkdf2Rounds, int ivLength) throws Exception {
+      super(temporaryPath, configuration);
+      this.repositoryEncryptionEnabled = true;
+      this.passwordGenerator = passwordGenerator;
+      this.hmacAlgorithm = hmacAlgorithm;
+      this.cipherAlgorithm = cipherAlgorithm;
+      this.cipherKeySize = cipherKeySize;
+      this.cipherSpec = cipherSpec;
+      this.pbkdf2Algorithm = pbkdf2Algorithm;
+      this.pbkdf2Rounds = pbkdf2Rounds;
+      this.ivLength = ivLength;
+    }
+
+    @Override
+    protected Map<String, String> getSecurityConfiguration() {
+      Map<String, String> properties = super.getSecurityConfiguration();
+
+      // Remove all default repository encryption values
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_ENABLED);
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR);
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM);
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM);
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE);
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE);
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC);
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM);
+      properties.remove(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS);
+
+      properties.put(SecurityConstants.REPO_ENCRYPTION_ENABLED, String.valueOf(repositoryEncryptionEnabled));
+      if (repositoryEncryptionEnabled) {
+        properties.put(SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR, passwordGenerator);
+        properties.put(SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM, hmacAlgorithm);
+        properties.put(SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM, cipherAlgorithm);
+        properties.put(SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE, String.valueOf(cipherKeySize));
+        properties.put(SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE, String.valueOf(ivLength));
+        properties.put(SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC, cipherSpec);
+        properties.put(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM, pbkdf2Algorithm);
+        properties.put(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS, String.valueOf(pbkdf2Rounds));
+      }
+
+      return properties;
+    }
+  }
+
+  @BeforeMethod
+  public void before() throws Exception {
+    dropRepository();
+
+    passwordGenerator = "echo test";
+    hmacAlgorithm = "HmacSHA256";
+    cipherAlgorithm = "AES";
+    cipherKeySize = 16;
+    cipherSpec = "AES/CBC/PKCS5Padding";
+    pbkdf2Algorithm = "PBKDF2WithHmacSHA1";
+    pbkdf2Rounds = 4000;
+    ivLength = 16;
+
+    temporaryPath = HdfsUtils.joinPathFragments(super.getTemporaryPath(), getTestName());
+  }
+
+  @Test
+  public void testNotEncryptedToEncrypted() throws Exception {
+    // Start nonencrypted sqoop instance
+    sqoopMiniCluster = new SqoopMiniCluster(temporaryPath, getHadoopConf());
+    sqoopMiniCluster.start();
+
+    verifyMasterKeyDoesNotExist();
+
+    // Create a link and a job with a secure input
+    SqoopClient client = new SqoopClient(sqoopMiniCluster.getServerUrl());
+    MLink link = client.createLink("generic-jdbc-connector");
+    link.setName("zelda");
+    fillRdbmsLinkConfig(link);
+    client.saveLink(link);
+
+    MStringInput sensitiveInput = link.getConnectorLinkConfig().getStringInput("linkConfig.password");
+    verifyPlaintextInput(sensitiveInput.getPersistenceId(), sensitiveInput.getValue());
+
+    // Stop sqoop instance
+    sqoopMiniCluster.stop();
+
+    // Run tool
+    RepositoryEncryptionTool repositoryEncryptionTool = new RepositoryEncryptionTool();
+    repositoryEncryptionTool.runToolWithConfiguration(new String[] {
+      "-T" + SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR + "=" + passwordGenerator,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM + "=" + hmacAlgorithm,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM + "=" + cipherAlgorithm,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE + "=" + cipherKeySize,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC + "=" + cipherSpec,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM + "=" + pbkdf2Algorithm,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS + "=" + pbkdf2Rounds,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE + "=" + ivLength,
+    });
+
+    cleanUpAfterTool();
+
+    // Verify that the data is encrypted
+    StringBuffer cipherText = new StringBuffer();
+    StringBuffer iv = new StringBuffer();
+    StringBuffer hmac = new StringBuffer();
+    readEncryptedInput(sensitiveInput.getPersistenceId(), cipherText, iv, hmac);
+
+    // Read the encrypted data by using the MasterKeyManager the server initializes
+    sqoopMiniCluster = new SqoopMiniCluster(temporaryPath, getHadoopConf(), passwordGenerator,
+      hmacAlgorithm, cipherAlgorithm, cipherKeySize, cipherSpec, pbkdf2Algorithm, pbkdf2Rounds,
ivLength);
+    sqoopMiniCluster.start();
+
+    String decrypted = MasterKeyManager.getInstance().decryptWithMasterKey(cipherText.toString(),
iv.toString(), hmac.toString());
+
+    Assert.assertEquals(sensitiveInput.getValue(), decrypted);
+  }
+
+  @Test
+  public void testEncryptedToNotEncrypted() throws Exception {
+    sqoopMiniCluster = new SqoopMiniCluster(temporaryPath, getHadoopConf(), passwordGenerator,
+      hmacAlgorithm, cipherAlgorithm, cipherKeySize, cipherSpec, pbkdf2Algorithm, pbkdf2Rounds,
ivLength);
+    sqoopMiniCluster.start();
+
+    SqoopClient client = new SqoopClient(sqoopMiniCluster.getServerUrl());
+    MLink link = client.createLink("generic-jdbc-connector");
+    link.setName("zelda");
+    fillRdbmsLinkConfig(link);
+    client.saveLink(link);
+    MStringInput sensitiveInput = link.getConnectorLinkConfig().getStringInput("linkConfig.password");
+
+    StringBuffer cipherText = new StringBuffer();
+    StringBuffer iv = new StringBuffer();
+    StringBuffer hmac = new StringBuffer();
+    readEncryptedInput(sensitiveInput.getPersistenceId(), cipherText, iv, hmac);
+
+    String decrypted = MasterKeyManager.getInstance().decryptWithMasterKey(cipherText.toString(),
iv.toString(), hmac.toString());
+
+    // Stop sqoop instance
+    sqoopMiniCluster.stop();
+
+    // Run tool
+    RepositoryEncryptionTool repositoryEncryptionTool = new RepositoryEncryptionTool();
+    repositoryEncryptionTool.runToolWithConfiguration(new String[] {
+      "-F" + SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR + "=" + passwordGenerator,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM + "=" + hmacAlgorithm,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM + "=" + cipherAlgorithm,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE + "=" + cipherKeySize,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC + "=" + cipherSpec,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM + "=" + pbkdf2Algorithm,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS + "=" + pbkdf2Rounds,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE + "=" + ivLength,
+    });
+
+    cleanUpAfterTool();
+
+    sqoopMiniCluster = new SqoopMiniCluster(temporaryPath, getHadoopConf());
+    sqoopMiniCluster.start();
+
+    verifyPlaintextInput(sensitiveInput.getPersistenceId(), decrypted);
+
+    verifyMasterKeyDoesNotExist();
+  }
+
+  @Test
+  public void testEncryptedToEncrypted() throws Exception {
+    sqoopMiniCluster = new SqoopMiniCluster(temporaryPath, getHadoopConf(), passwordGenerator,
+      hmacAlgorithm, cipherAlgorithm, cipherKeySize, cipherSpec, pbkdf2Algorithm, pbkdf2Rounds,
ivLength);
+    sqoopMiniCluster.start();
+
+    SqoopClient client = new SqoopClient(sqoopMiniCluster.getServerUrl());
+    MLink link = client.createLink("generic-jdbc-connector");
+    link.setName("zelda");
+    fillRdbmsLinkConfig(link);
+    client.saveLink(link);
+    MStringInput sensitiveInput = link.getConnectorLinkConfig().getStringInput("linkConfig.password");
+
+    StringBuffer cipherTextFrom = new StringBuffer();
+    StringBuffer ivFrom = new StringBuffer();
+    StringBuffer hmacFrom = new StringBuffer();
+    readEncryptedInput(sensitiveInput.getPersistenceId(), cipherTextFrom, ivFrom, hmacFrom);
+
+    String decryptedFirst = MasterKeyManager.getInstance().decryptWithMasterKey(cipherTextFrom.toString(),
ivFrom.toString(), hmacFrom.toString());
+
+    // Stop sqoop instance
+    sqoopMiniCluster.stop();
+
+    // Run tool
+    RepositoryEncryptionTool repositoryEncryptionTool = new RepositoryEncryptionTool();
+    repositoryEncryptionTool.runToolWithConfiguration(new String[] {
+      "-F" + SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR + "=" + passwordGenerator,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM + "=" + hmacAlgorithm,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM + "=" + cipherAlgorithm,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE + "=" + cipherKeySize,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC + "=" + cipherSpec,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM + "=" + pbkdf2Algorithm,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS + "=" + pbkdf2Rounds,
+      "-F" + SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE + "=" + ivLength,
+
+      "-T" + SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR + "=" + passwordGenerator,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM + "=" + hmacAlgorithm,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM + "=" + cipherAlgorithm,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE + "=" + cipherKeySize,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC + "=" + cipherSpec,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM + "=" + pbkdf2Algorithm,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS + "=" + pbkdf2Rounds,
+      "-T" + SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE + "=" + ivLength,
+    });
+
+    cleanUpAfterTool();
+
+    StringBuffer cipherTextTo = new StringBuffer();
+    StringBuffer ivTo = new StringBuffer();
+    StringBuffer hmacTo = new StringBuffer();
+
+    Assert.assertNotEquals(cipherTextFrom, cipherTextTo);
+
+    readEncryptedInput(sensitiveInput.getPersistenceId(), cipherTextTo, ivTo, hmacTo);
+
+    // Read the encrypted data by using the MasterKeyManager the server initializes
+    sqoopMiniCluster = new SqoopMiniCluster(temporaryPath, getHadoopConf(), passwordGenerator,
+      hmacAlgorithm, cipherAlgorithm, cipherKeySize, cipherSpec, pbkdf2Algorithm, pbkdf2Rounds,
ivLength);
+    sqoopMiniCluster.start();
+
+    String decryptedSecond = MasterKeyManager.getInstance().decryptWithMasterKey(cipherTextTo.toString(),
ivTo.toString(), hmacTo.toString());
+
+    Assert.assertEquals(decryptedFirst, decryptedSecond);
+  }
+
+  @Test
+  public void testEncryptedToEncryptedUsingConfiguration() throws Exception {
+    sqoopMiniCluster = new SqoopMiniCluster(temporaryPath, getHadoopConf(), passwordGenerator,
+      hmacAlgorithm, cipherAlgorithm, cipherKeySize, cipherSpec, pbkdf2Algorithm, pbkdf2Rounds,
ivLength);
+    sqoopMiniCluster.start();
+
+    SqoopClient client = new SqoopClient(sqoopMiniCluster.getServerUrl());
+    MLink link = client.createLink("generic-jdbc-connector");
+    link.setName("zelda");
+    fillRdbmsLinkConfig(link);
+    client.saveLink(link);
+    MStringInput sensitiveInput = link.getConnectorLinkConfig().getStringInput("linkConfig.password");
+
+    StringBuffer cipherTextFrom = new StringBuffer();
+    StringBuffer ivFrom = new StringBuffer();
+    StringBuffer hmacFrom = new StringBuffer();
+    readEncryptedInput(sensitiveInput.getPersistenceId(), cipherTextFrom, ivFrom, hmacFrom);
+
+    String decryptedFirst = MasterKeyManager.getInstance().decryptWithMasterKey(cipherTextFrom.toString(),
ivFrom.toString(), hmacFrom.toString());
+
+    // Read the configuration context that we will need for the tool
+    MapContext configurationMapContext = SqoopConfiguration.getInstance().getContext();
+
+    // Stop sqoop instance
+    sqoopMiniCluster.stop();
+
+    // Set the configuration
+    SqoopConfiguration oldSqoopConfiguration = SqoopConfiguration.getInstance();
+    SqoopConfiguration configurationMock = mock(SqoopConfiguration.class);
+    when(configurationMock.getContext()).thenReturn(configurationMapContext);
+    when(configurationMock.getProvider()).thenReturn(new PropertiesConfigurationProvider());
+    SqoopConfiguration.setInstance(configurationMock);
+
+    // Run tool
+    RepositoryEncryptionTool repositoryEncryptionTool = new RepositoryEncryptionTool();
+    repositoryEncryptionTool.runToolWithConfiguration(new String[] {
+      "-FuseConf",
+      "-TuseConf",
+    });
+
+    cleanUpAfterTool();
+
+    StringBuffer cipherTextTo = new StringBuffer();
+    StringBuffer ivTo = new StringBuffer();
+    StringBuffer hmacTo = new StringBuffer();
+
+    Assert.assertNotEquals(cipherTextFrom, cipherTextTo);
+
+    readEncryptedInput(sensitiveInput.getPersistenceId(), cipherTextTo, ivTo, hmacTo);
+
+    // Read the encrypted data by using the MasterKeyManager the server initializes
+    sqoopMiniCluster = new SqoopMiniCluster(temporaryPath, getHadoopConf(), passwordGenerator,
+      hmacAlgorithm, cipherAlgorithm, cipherKeySize, cipherSpec, pbkdf2Algorithm, pbkdf2Rounds,
ivLength);
+    sqoopMiniCluster.start();
+
+    String decryptedSecond = MasterKeyManager.getInstance().decryptWithMasterKey(cipherTextTo.toString(),
ivTo.toString(), hmacTo.toString());
+
+    Assert.assertEquals(decryptedFirst, decryptedSecond);
+
+    SqoopConfiguration.setInstance(oldSqoopConfiguration);
+  }
+
+  private void cleanUpAfterTool() {
+    RepositoryManager.getInstance().destroy();
+    MasterKeyManager.getInstance().destroy();
+    SqoopConfiguration.getInstance().destroy();
+  }
+
+  private void verifyMasterKeyDoesNotExist() throws Exception {
+    try (PreparedStatement inputSelection = DriverManager.getConnection(JDBC_URL).prepareStatement((new
CommonRepositoryInsertUpdateDeleteSelectQuery()).getStmtSelectSqMasterKey())) {
+      try (ResultSet resultSet = inputSelection.executeQuery()) {
+        Assert.assertFalse(resultSet.next());
+      }
+    }
+  }
+
+  private void verifyPlaintextInput(long persistenceId, String expectedValue) throws Exception
{
+    try (PreparedStatement inputSelection = DriverManager.getConnection(JDBC_URL).prepareStatement(INPUT_VALUE_QUERY))
{
+      inputSelection.setLong(1, persistenceId);
+      try (ResultSet resultSet = inputSelection.executeQuery()) {
+        while (resultSet.next()) {
+          Assert.assertEquals(expectedValue, resultSet.getString(2));
+          Assert.assertFalse(resultSet.getBoolean(3));
+          Assert.assertNull(resultSet.getString(4));
+          Assert.assertNull(resultSet.getString(5));
+        }
+      }
+    }
+  }
+
+  private void readEncryptedInput(long inputId, StringBuffer cipherText, StringBuffer iv,
StringBuffer hmac) throws Exception {
+    try (PreparedStatement inputSelection = DriverManager.getConnection(JDBC_URL).prepareStatement(INPUT_VALUE_QUERY))
{
+      inputSelection.setLong(1, inputId);
+      try (ResultSet resultSet = inputSelection.executeQuery()) {
+        while (resultSet.next()) {
+          Assert.assertTrue(resultSet.getBoolean(3));
+          cipherText.append(resultSet.getString(2));
+          iv.append(resultSet.getString(4));
+          hmac.append(resultSet.getString(5));
+        }
+      }
+    }
+  }
+
+  @AfterMethod
+  public void stopCluster() throws Exception {
+    sqoopMiniCluster.stop();
+    dropRepository();
+  }
+
+  private void dropRepository() {
+    try {
+      DriverManager.getConnection(JDBC_URL + ";drop=true");
+    } catch (Exception exception) {
+      // Dropping the database always throws an exception
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/tools/src/main/java/org/apache/sqoop/tools/tool/BuiltinTools.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/BuiltinTools.java b/tools/src/main/java/org/apache/sqoop/tools/tool/BuiltinTools.java
index 13a2c5f..dc52c5d 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/BuiltinTools.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/BuiltinTools.java
@@ -38,6 +38,7 @@ public class BuiltinTools {
     tools.put("verify", VerifyTool.class);
     tools.put("repositorydump", RepositoryDumpTool.class);
     tools.put("repositoryload", RepositoryLoadTool.class);
+    tools.put("repositoryencryption", RepositoryEncryptionTool.class);
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/sqoop/blob/c6fc7f95/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryEncryptionTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryEncryptionTool.java
b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryEncryptionTool.java
new file mode 100644
index 0000000..d0eab40
--- /dev/null
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryEncryptionTool.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.tools.tool;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.cli.SqoopGnuParser;
+import org.apache.sqoop.common.MapContext;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.core.SqoopConfiguration;
+import org.apache.sqoop.repository.MasterKeyManager;
+import org.apache.sqoop.repository.Repository;
+import org.apache.sqoop.repository.RepositoryManager;
+import org.apache.sqoop.repository.RepositoryTransaction;
+import org.apache.sqoop.security.SecurityConstants;
+import org.apache.sqoop.security.SecurityError;
+import org.apache.sqoop.tools.ConfiguredTool;
+import org.apache.sqoop.utils.PasswordUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+public class RepositoryEncryptionTool extends ConfiguredTool {
+  public static final Logger LOG = Logger.getLogger(RepositoryDumpTool.class);
+
+  private static String FROM_OPTION = "F";
+  private static String TO_OPTION = "T";
+
+  private static String USE_CONFIGURATION = "useConf";
+
+  @Override
+  public boolean runToolWithConfiguration(String[] arguments) {
+    Options options = new Options();
+    options.addOption(OptionBuilder.hasArgs().withValueSeparator().create(FROM_OPTION));
+    options.addOption(OptionBuilder.hasArgs().withValueSeparator().create(TO_OPTION));
+
+    CommandLineParser parser = new SqoopGnuParser();
+    SqoopConfiguration.getInstance().initialize();
+    RepositoryManager.getInstance().initialize();
+    Repository repository = RepositoryManager.getInstance().getRepository();
+
+    CommandLine line;
+    try {
+      line = parser.parse(options, arguments);
+    } catch (ParseException e) {
+      LOG.error("Error parsing command line arguments:", e);
+      System.out.println("Error parsing command line arguments. Please check Server logs
for details.");
+      return false;
+    }
+
+    Properties fromProperties = line.getOptionProperties(FROM_OPTION);
+    Properties toProperties = line.getOptionProperties(TO_OPTION);
+
+    RepositoryTransaction repositoryTransaction = null;
+    boolean successful = true;
+    try {
+      repositoryTransaction = repository.getTransaction();
+      repositoryTransaction.begin();
+
+      MasterKeyManager fromMasterKeyManager = null;
+      MasterKeyManager toMasterKeyManager = null;
+
+      if (!fromProperties.isEmpty()) {
+        fromMasterKeyManager = initializeMasterKeyManagerFromProperties(fromProperties, false,
repositoryTransaction);
+      } else {
+        // Check to make sure there is no master key to prevent corruption
+        if (repository.getMasterKey(repositoryTransaction) != null) {
+          System.out.println("Repository is encrypted, need configuration to decrypt");
+          throw new SqoopException(SecurityError.ENCRYPTION_0013);
+        }
+      }
+
+      if (!toProperties.isEmpty()) {
+        toMasterKeyManager = initializeMasterKeyManagerFromProperties(toProperties, true,
repositoryTransaction);
+      }
+
+      repository.changeMasterKeyManager(fromMasterKeyManager, toMasterKeyManager, repositoryTransaction);
+      if (fromMasterKeyManager != null) {
+        fromMasterKeyManager.deleteMasterKeyFromRepository();
+        fromMasterKeyManager.destroy();
+      }
+
+      repositoryTransaction.commit();
+      System.out.println("Changes committed");
+    } catch (Exception ex) {
+      if (repositoryTransaction != null) {
+        repositoryTransaction.rollback();
+      }
+      System.out.println("Error running tool. Please check Server logs for details.");
+      LOG.error(new SqoopException(SecurityError.ENCRYPTION_0012, ex));
+      successful = false;
+    } finally {
+      if (repositoryTransaction != null) {
+        repositoryTransaction.close();
+      }
+    }
+    return successful;
+  }
+
+  private MasterKeyManager initializeMasterKeyManagerFromProperties(Properties properties,
boolean readFromRepository, RepositoryTransaction transaction) {
+    MasterKeyManager masterKeyManager = new MasterKeyManager();
+    if (properties.getProperty(USE_CONFIGURATION) != null) {
+      masterKeyManager.initialize(true, readFromRepository, transaction);
+    } else {
+      String hmacAlgorithm = properties.getProperty(SecurityConstants.REPO_ENCRYPTION_HMAC_ALGORITHM);
+      String cipherAlgorithm = properties.getProperty(SecurityConstants.REPO_ENCRYPTION_CIPHER_ALGORITHM);
+      String cipherSpec = properties.getProperty(SecurityConstants.REPO_ENCRYPTION_CIPHER_SPEC);
+      int cipherKeySize = Integer.parseInt(properties.getProperty(SecurityConstants.REPO_ENCRYPTION_CIPHER_KEY_SIZE));
+      int ivLength = Integer.parseInt(properties.getProperty(SecurityConstants.REPO_ENCRYPTION_INITIALIZATION_VECTOR_SIZE));
+      String pbkdf2Algorithm = properties.getProperty(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ALGORITHM);
+      int pbkdf2Rounds = Integer.parseInt(properties.getProperty(SecurityConstants.REPO_ENCRYPTION_PBKDF2_ROUNDS));
+
+      // We need to create a MapContext to make reading the password simpler here
+      Map<String, String> passwordProperties = new HashMap<>();
+      passwordProperties.put(SecurityConstants.REPO_ENCRYPTION_PASSWORD, properties.getProperty(SecurityConstants.REPO_ENCRYPTION_PASSWORD));
+      passwordProperties.put(SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR, properties.getProperty(SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR));
+      String password = PasswordUtils.readPassword(new MapContext(passwordProperties), SecurityConstants.REPO_ENCRYPTION_PASSWORD,
SecurityConstants.REPO_ENCRYPTION_PASSWORD_GENERATOR);
+
+
+      masterKeyManager.initialize(true, hmacAlgorithm, cipherAlgorithm, cipherSpec, cipherKeySize,
ivLength, pbkdf2Algorithm, pbkdf2Rounds, password, readFromRepository, transaction);
+    }
+
+    return masterKeyManager;
+  }
+}


Mime
View raw message