sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject [1/2] sqoop git commit: SQOOP-2889: Read and write from encrypted repository
Date Tue, 29 Mar 2016 21:14:32 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 bfcfedfa7 -> 5b897a46f


http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
index ae16b85..d527290 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
@@ -161,6 +161,9 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", "
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
+          + "cast(null as varchar(100)),"
+          + "false,"
+          + "cast(null as varchar(100)),"
           + "cast(null as varchar(100))"
           + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
           + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?"
@@ -223,7 +226,10 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ","
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
-          + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE)
+          + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", "
+          + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", "
+          + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", "
+          + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC)
           + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
           + " LEFT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
           + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID)
@@ -246,7 +252,10 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", "
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
           + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
-          + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE)
+          + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + ", "
+          + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + ", "
+          + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + ", "
+          + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC)
           + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
           + " LEFT OUTER JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
           + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID)
@@ -271,11 +280,14 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
 
   // DML: Insert new link inputs
   private static final String STMT_INSERT_LINK_INPUT =
-      "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ " ("
-          + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_LINK) + ", "
-          + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + ", "
-          + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE)
-          + ") VALUES (?, ?, ?)";
+    "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+ " ("
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_LINK) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_INPUT) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_VALUE) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC)
+      + ") VALUES (?, ?, ?, ?, ?, ?)";
 
   // DML: Update link
   private static final String STMT_UPDATE_LINK =
@@ -385,13 +397,17 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
           + CommonRepoUtils.escapeColumnName(COLUMN_SQB_UPDATE_DATE)
           + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?)";
 
+
   // DML: Insert new job inputs
   private static final String STMT_INSERT_JOB_INPUT =
-      "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ " ("
-          + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_JOB) + ", "
-          + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + ", "
-          + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE)
-          + ") VALUES (?, ?, ?)";
+    "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+ " ("
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_JOB) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_INPUT) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_VALUE) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC)
+      + ") VALUES (?, ?, ?, ?, ?, ?)";
 
   private static final String STMT_UPDATE_JOB =
       "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME) + " SET "
@@ -741,6 +757,22 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
           + " IN (SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_ID) + " FROM
" + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIG_NAME)
           + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_CONFIGURABLE) + "
= ?)";
 
+  private static final String STMT_SELECT_SQ_MASTER_KEY =
+    "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV)
+      + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME);
+
+  private static final String STMT_INSERT_SQ_MASTER_KEY =
+    "INSERT INTO " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME)
+ " ("
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + ", "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV)
+      + ") VALUES (?, ?, ?, ?)";
+
+
   public String getStmtSelectSqdIdBySqdName() {
     return STMT_SELECT_SQD_ID_BY_SQD_NAME;
   }
@@ -1042,4 +1074,12 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
   public String getStmtInsertIntoInputRelation() {
     return STMT_INSERT_INTO_INPUT_RELATION;
   }
+
+  public String getStmtSelectSqMasterKey() {
+     return STMT_SELECT_SQ_MASTER_KEY;
+  }
+
+  public String getStmtInsertSqMasterKey() {
+    return STMT_INSERT_SQ_MASTER_KEY;
+  }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
index d1940e8..6fd9d6c 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
@@ -175,10 +175,16 @@ public final class CommonRepositorySchemaConstants {
 
   public static final String COLUMN_SQ_LNKI_LINK = "SQ_LNKI_LINK";
 
+  public static final String COLUMN_SQ_LNKI_ENCRYPTED = "SQ_LNKI_ENCRYPTED";
+
   public static final String COLUMN_SQ_LNKI_INPUT = "SQ_LNKI_INPUT";
 
   public static final String COLUMN_SQ_LNKI_VALUE = "SQ_LNKI_VALUE";
 
+  public static final String COLUMN_SQ_LNKI_IV = "SQ_LNKI_IV";
+
+  public static final String COLUMN_SQ_LNKI_HMAC = "SQ_LNKI_HMAC";
+
   // SQ_JOB_INPUT
 
   public static final String TABLE_SQ_JOB_INPUT_NAME = "SQ_JOB_INPUT";
@@ -187,8 +193,14 @@ public final class CommonRepositorySchemaConstants {
 
   public static final String COLUMN_SQBI_INPUT = "SQBI_INPUT";
 
+  public static final String COLUMN_SQBI_ENCRYPTED = "SQBI_ENCRYPTED";
+
   public static final String COLUMN_SQBI_VALUE = "SQBI_VALUE";
 
+  public static final String COLUMN_SQBI_IV = "SQBI_IV";
+
+  public static final String COLUMN_SQBI_HMAC = "SQBI_HMAC";
+
   // SQ_SUBMISSION
 
   public static final String TABLE_SQ_SUBMISSION_NAME = "SQ_SUBMISSION";
@@ -273,6 +285,20 @@ public final class CommonRepositorySchemaConstants {
 
   public static final String COLUMN_SQCO_VALUE = "SQCO_VALUE";
 
+  // SQ_MASTER_KEY
+
+  public static final String TABLE_SQ_MASTER_KEY_NAME = "SQ_MASTER_KEY";
+
+  public static final String COLUMN_SQMK_ID = "SQMK_ID";
+
+  public static final String COLUMN_SQMK_SECRET = "SQMK_SECRET";
+
+  public static final String COLUMN_SQMK_HMAC = "SQMK_HMAC";
+
+  public static final String COLUMN_SQMK_SALT = "SQMK_SALT";
+
+  public static final String COLUMN_SQMK_IV = "SQMK_IV";
+
   // Constraints
 
   public static final String CONSTRAINT_SQCD_SQC_NAME

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index ee5e8d1..75636d4 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -319,6 +319,14 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler {
       runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_COLUMN_SQB_NAME, conn);
       runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ALTER_COLUMN_SQB_NAME_NOT_NULL, conn);
       runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ALTER_COLUMN_SQB_NAME_NOT_NULL, conn);
+
+      runQuery(QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED, conn);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV, conn);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC, conn);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED, conn);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV, conn);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC, conn);
     }
 
     // last step upgrade the repository version to the latest value in the code

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
index 1770036..b4f66bb 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaCreateQuery.java
@@ -181,6 +181,10 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
  * <p>
  * <strong>SQ_LINK_INPUT</strong>: N:M relationship link and input
  *
+ * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum
length
+ * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where
+ * 1024 is the maximum possible key size in bits for the IV and the digest.
+ *
  * <pre>
  *    +----------------------------+
  *    | SQ_LINK_INPUT              |
@@ -188,6 +192,9 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
  *    | SQ_LNKI_LINK: BIGINT PK    | FK SQ_LINK(SQ_LNK_ID)
  *    | SQ_LNKI_INPUT: BIGINT PK   | FK SQ_INPUT(SQI_ID)
  *    | SQ_LNKI_VALUE: LONG VARCHAR|
+ *    | SQ_LNKI_ENCRYPTED: BOOLEAN |
+ *    | SQ_LNKI_IV: VARCHAR(171)    |
+ *    | SQ_LNKI_HMAC: VARCHAR(171) |
  *    +----------------------------+
  * </pre>
  *
@@ -195,6 +202,10 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
  * <p>
  * <strong>SQ_JOB_INPUT</strong>: N:M relationship job and input
  *
+ * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum
length
+ * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where
+ * 1024 is the maximum possible key size in bits for the IV and the digest.
+ *
  * <pre>
  *    +----------------------------+
  *    | SQ_JOB_INPUT               |
@@ -202,6 +213,9 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
  *    | SQBI_JOB: BIGINT PK        | FK SQ_JOB(SQB_ID)
  *    | SQBI_INPUT: BIGINT PK      | FK SQ_INPUT(SQI_ID)
  *    | SQBI_VALUE: LONG VARCHAR   |
+ *    | SQBI_ENCRYPTED: BOOLEAN    |
+ *    | SQBI_IV: VARCHAR(24)       |
+ *    | SQBI_HMAC: VARCHAR(171)    |
  *    +----------------------------+
  * </pre>
  *
@@ -310,6 +324,34 @@ import static org.apache.sqoop.repository.derby.DerbySchemaConstants.*;
  *    +----------------------------+
  * </pre>
  * </p>
+ * <p>
+ * <strong>SQ_MASTER_KEY</strong>: Master Key for sensitive value encryption
+ *
+ * SQMK_SECRET length determined by a generous assumption of the maximum length
+ * of the base64 encoded secret. Calculated by ceil((4/3)(1024/8+1024/8)). Where
+ * 1024 is the maximum possible key size in bits for encryption and HMAC.
+ *
+ * SQMK_HMAC length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8))
+ * where 1024 is the maximum possible size of the digest in bits.
+ *
+ * SQMK_SALT length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8))
+ * where 1024 is the maximum possible size of the salt in bits.
+ *
+ * SQMK_IV length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(128))
+ * where 128 is the maximum possible size of the salt in bytes.
+ *
+ * <pre>
+ *    +---------------------------+
+ *    | SQ_MASTER_KEY             |
+ *    +---------------------------+
+ *    | SQMK_ID: BIGINT PK        |
+ *    | SQMK_SECRET: VARCHAR(342) |
+ *    | SQMK_HMAC: VARCHAR(171)   |
+ *    | SQMK_SALT: VARCHAR(171)   |
+ *    | SQMK_IV: VARCHAR(171)     |
+ *    +---------------------------+
+ * </pre>
+ * </p>
  */
 
 // NOTE: If you have signed yourself to modify the schema for the repository

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
index 5081b82..a427421 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
@@ -662,6 +662,41 @@ public final class DerbySchemaUpgradeQuery {
         + " ALTER COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME)
         + " NOT NULL";
 
+  // DDL: Create table SQ_MASTER_KEY
+  public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY =
+    "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME)
+ " ("
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + " BIGINT GENERATED ALWAYS AS IDENTITY
(START WITH 1, INCREMENT BY 1), "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + " VARCHAR(342), "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + " VARCHAR(171), "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + " VARCHAR(171), "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + " VARCHAR(171)"
+      + ")";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED =
+      "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+        + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + " BOOLEAN
DEFAULT FALSE";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + " VARCHAR(171)";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " VARCHAR(171)";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + " BOOLEAN
DEFAULT FALSE";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + " VARCHAR(171)";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " VARCHAR(171)";
+
+
   private DerbySchemaUpgradeQuery() {
     // Disable explicit object creation
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index e4cca07..9bcfe01 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -247,6 +247,14 @@ abstract public class DerbyTestCase {
       runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT_TYPE);
       runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT_PROPERTY);
       runQuery(QUERY_CREATE_TABLE_SQ_CONTEXT);
+
+      runQuery(QUERY_CREATE_TABLE_SQ_MASTER_KEY);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV);
+      runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC);
     }
 
     // deprecated repository version

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
index 2c74c32..87bd6d8 100644
--- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
+++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlRepositoryHandler.java
@@ -146,6 +146,7 @@ public class MySqlRepositoryHandler extends CommonRepositoryHandler {
       runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT_TYPE, conn);
       runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT_PROPERTY, conn);
       runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_CONTEXT, conn);
+      runQuery(MySqlSchemaCreateQuery.QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn);
 
       // Insert FROM and TO directions.
       insertDirections(conn);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
index 47f12fe..29355d5 100644
--- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
+++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MySqlSchemaCreateQuery.java
@@ -24,7 +24,7 @@ import org.apache.sqoop.repository.common.CommonRepositorySchemaConstants;
 
 public class MySqlSchemaCreateQuery {
 
-  public static final String QUERY_CREATE_DATABASE_SQOOP = "CREATE DATABASE " + CommonRepoUtils.escapeDatabaseName(SCHEMA_SQOOP);
+  public static final String QUERY_CREATE_DATABASE_SQOOP = "CREATE DATABASE IF NOT EXISTS"
+ CommonRepoUtils.escapeDatabaseName(SCHEMA_SQOOP);
 
   public static final String QUERY_CREATE_TABLE_SQ_SYSTEM =
       "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, CommonRepositorySchemaConstants.TABLE_SQ_SYSTEM_NAME)
+ " ("
@@ -177,6 +177,9 @@ public class MySqlSchemaCreateQuery {
           + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_LINK)
+ " BIGINT, "
           + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_INPUT)
+ " BIGINT, "
           + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_VALUE)
+ " VARCHAR(1000), "
+          + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_ENCRYPTED)
+ " BOOLEAN DEFAULT FALSE, "
+          + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_IV)
+ " VARCHAR(171), "
+          + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_HMAC)
+ " VARCHAR(171),"
           + "PRIMARY KEY (" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_LINK)
+ ", "
             + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_INPUT)
+ "), "
           + "CONSTRAINT " + CommonRepoUtils.escapeConstraintName(CommonRepositorySchemaConstants.CONSTRAINT_SQ_LNKI_SQ_LNK_NAME)
@@ -194,6 +197,9 @@ public class MySqlSchemaCreateQuery {
           + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_JOB)
+ " BIGINT, "
           + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_INPUT)
+ " BIGINT, "
           + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_VALUE)
+ " VARCHAR(1000), "
+          + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_ENCRYPTED)
+ " BOOLEAN DEFAULT FALSE, "
+          + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_IV)
+ " VARCHAR(171), "
+          + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_HMAC)
+ " VARCHAR(171),"
           + "PRIMARY KEY (" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_JOB)
+ ", "
             + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQBI_INPUT)
+ "), "
           + "CONSTRAINT " + CommonRepoUtils.escapeConstraintName(CommonRepositorySchemaConstants.CONSTRAINT_SQBI_SQB_NAME)
@@ -291,6 +297,16 @@ public class MySqlSchemaCreateQuery {
            + "(" + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQCP_ID)
+ ") "
        + ")";
 
+  // DDL: Create table SQ_MASTER_KEY
+  public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY =
+    "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, CommonRepositorySchemaConstants.TABLE_SQ_MASTER_KEY_NAME)
+ " ("
+      + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_ID)
+ " BIGINT NOT NULL AUTO_INCREMENT PRIMARY KEY, "
+      + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_SECRET)
+ " VARCHAR(342), "
+      + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_HMAC)
+ " VARCHAR(171), "
+      + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_SALT)
+ " VARCHAR(171), "
+      + CommonRepoUtils.escapeColumnName(CommonRepositorySchemaConstants.COLUMN_SQMK_IV)
+ " VARCHAR(171)"
+      + ")";
+
   private MySqlSchemaCreateQuery() {
     // Disable explicit object creation
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
index 4c295c0..08d3374 100644
--- a/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
+++ b/repository/repository-mysql/src/main/java/org/apache/sqoop/repository/mysql/MysqlRepositoryInsertUpdateDeleteSelectQuery.java
@@ -36,6 +36,7 @@ public class MysqlRepositoryInsertUpdateDeleteSelectQuery extends
     CommonRepositoryInsertUpdateDeleteSelectQuery {
 
   // DML: Get inputs for a given config
+  // MySQL requires that we cast to null to char instead of varchar
   private static final String STMT_SELECT_INPUT = "SELECT "
       + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ID) + ", "
       + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + ", "
@@ -46,6 +47,9 @@ public class MysqlRepositoryInsertUpdateDeleteSelectQuery extends
       + CommonRepoUtils.escapeColumnName(COLUMN_SQI_STRLENGTH) + ", "
       + CommonRepoUtils.escapeColumnName(COLUMN_SQI_EDITABLE) + ", "
       + CommonRepoUtils.escapeColumnName(COLUMN_SQI_ENUMVALS) + ", "
+      + "cast(null as char(100)),"
+      + "false,"
+      + "cast(null as char(100)),"
       + "cast(null as char(100))" + " FROM "
       + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
       + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG)

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
index 400d706..b18a7d5 100644
--- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
+++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlRepositoryHandler.java
@@ -145,6 +145,13 @@ public class PostgresqlRepositoryHandler extends CommonRepositoryHandler
{
       runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_UPDATE_COLUMN_SQB_NAME,
conn);
       runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_ALTER_COLUMN_SQB_NAME_NOT_NULL,
conn);
       runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ALTER_COLUMN_SQB_NAME_NOT_NULL,
conn);
+      runQuery(PostgresqlSchemaUpgradeQuery.QUERY_CREATE_TABLE_SQ_MASTER_KEY, conn);
+      runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED,
conn);
+      runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV, conn);
+      runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC, conn);
+      runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED,
conn);
+      runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV, conn);
+      runQuery(PostgresqlSchemaUpgradeQuery.QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC,
conn);
     }
 
     try (PreparedStatement stmtDel = conn.prepareStatement(PostgresqlSchemaQuery.STMT_DELETE_SYSTEM);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
index 8358df0..e31886e 100644
--- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
+++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaCreateQuery.java
@@ -169,6 +169,11 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants
  * </p>
  * <p>
  * <strong>SQ_LINK_INPUT</strong>: N:M relationship link and input
+ *
+ * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum
length
+ * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where
+ * 1024 is the maximum possible key size in bits for the IV and the digest.
+ *
  * <pre>
  *    +----------------------------+
  *    | SQ_LINK_INPUT              |
@@ -176,11 +181,19 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants
  *    | SQ_LNK_LINK: BIGSERIAL     | FK SQ_LINK(SQ_LNK_ID)
  *    | SQ_LNK_INPUT: BIGINT       | FK SQ_INPUT(SQI_ID)
  *    | SQ_LNK_VALUE: VARCHAR      |
+ *    | SQ_LNK_ENCRYPTED: BOOLEAN  |
+ *    | SQ_LNKI_IV: VARCHAR(171)   |
+ *    | SQ_LNKI_HMAC: VARCHAR(171) |
  *    +----------------------------+
  * </pre>
  * </p>
  * <p>
  * <strong>SQ_JOB_INPUT</strong>: N:M relationship job and input
+ *
+ * SQ_LNKI_IV and SQ_LNKI_HMAC length determined by a generous assumption of the maximum
length
+ * of the base64 encoded values. Calculated by ceil((4/3)(1024/8)). Where
+ * 1024 is the maximum possible key size in bits for the IV and the digest.
+ *
  * <pre>
  *    +----------------------------+
  *    | SQ_JOB_INPUT               |
@@ -188,6 +201,9 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants
  *    | SQBI_JOB: BIGINT           | FK SQ_JOB(SQB_ID)
  *    | SQBI_INPUT: BIGINT         | FK SQ_INPUT(SQI_ID)
  *    | SQBI_VALUE: VARCHAR(1000)  |
+ *    | SQBI_ENCRYPTED: BOOLEAN    |
+ *    | SQBI_IV: VARCHAR(171)      |
+ *    | SQBI_HMAC: VARCHAR(171)    |
  *    +----------------------------+
  * </pre>
  * </p>
@@ -287,6 +303,34 @@ import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants
  *    +----------------------------+
  * </pre>
  * </p>
+ * <p>
+ * <strong>SQ_MASTER_KEY</strong>: Master Key for sensitive value encryption
+ *
+ * SQMK_SECRET length determined by a generous assumption of the maximum length
+ * of the base64 encoded secret. Calculated by ceil((4/3)(1024/8+1024/8)). Where
+ * 1024 is the maximum possible key size in bits for encryption and HMAC.
+ *
+ * SQMK_HMAC length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8))
+ * where 1024 is the maximum possible size of the digest in bits.
+ *
+ * SQMK_SALT length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(1024/8))
+ * where 1024 is the maximum possible size of the salt in bits.
+ *
+ * SQMK_IV length is determined in a similar way to SQ_MASTER_KEY: ceil((4/3)(128))
+ * where 128 is the maximum possible size of the salt in bytes.
+ *
+ * <pre>
+ *    +---------------------------+
+ *    | SQ_MASTER_KEY             |
+ *    +---------------------------+
+ *    | SQMK_ID: BIGINT PK        |
+ *    | SQMK_SECRET: VARCHAR(342) |
+ *    | SQMK_HMAC: VARCHAR(171)   |
+ *    | SQMK_SALT: VARCHAR(171)   |
+ *    | SQMK_IV: VARCHAR(171)     |
+ *    +---------------------------+
+ * </pre>
+ * </p>
  */
 public class PostgresqlSchemaCreateQuery {
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
index 52954e6..3021039 100644
--- a/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
+++ b/repository/repository-postgresql/src/main/java/org/apache/sqoop/repository/postgresql/PostgresqlSchemaUpgradeQuery.java
@@ -17,15 +17,30 @@
  */
 package org.apache.sqoop.repository.postgresql;
 
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_ENCRYPTED;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_HMAC;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQBI_IV;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQB_ID;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQC_NAME;
+
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_HMAC;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_ID;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_IV;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_SALT;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQMK_SECRET;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_ENCRYPTED;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_HMAC;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNKI_IV;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNK_ID;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQ_LNK_NAME;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQB_NAME;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.SCHEMA_SQOOP;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_CONFIGURABLE_NAME;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_JOB_INPUT_NAME;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_JOB_NAME;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_LINK_INPUT_NAME;
 import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_LINK_NAME;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_MASTER_KEY_NAME;
 
 import org.apache.sqoop.repository.common.CommonRepoUtils;
 
@@ -58,4 +73,37 @@ public class PostgresqlSchemaUpgradeQuery {
       "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIGURABLE_NAME)
       + " ALTER COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQC_NAME)
       + " SET NOT NULL";
+
+  public static final String QUERY_CREATE_TABLE_SQ_MASTER_KEY =
+    "CREATE TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_MASTER_KEY_NAME)
+ " ("
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_ID) + " BIGSERIAL PRIMARY KEY NOT NULL,
"
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SECRET) + " VARCHAR(342), "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_HMAC) + " VARCHAR(171), "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_SALT) + " VARCHAR(171), "
+      + CommonRepoUtils.escapeColumnName(COLUMN_SQMK_IV) + " VARCHAR(171)"
+      + ")";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_ENCRYPTED =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_ENCRYPTED) + " BOOLEAN
DEFAULT FALSE";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_IV =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_IV) + " VARCHAR(171)";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_JOB_INPUT_WITH_HMAC =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQBI_HMAC) + " VARCHAR(171)";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_ENCRYPTED =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_ENCRYPTED) + " BOOLEAN
DEFAULT FALSE";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_IV =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_IV) + " VARCHAR(171)";
+
+  public static final String QUERY_UPGRADE_TABLE_SQ_LINK_INPUT_WITH_HMAC =
+    "ALTER TABLE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_LINK_INPUT_NAME)
+      + " ADD COLUMN " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_LNKI_HMAC) + " VARCHAR(171)";
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
----------------------------------------------------------------------
diff --git a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
index 00aa214..c2f5b84 100644
--- a/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
+++ b/server/src/main/java/org/apache/sqoop/server/SqoopJettyServer.java
@@ -34,7 +34,7 @@ import org.apache.sqoop.server.v1.DriverServlet;
 import org.apache.sqoop.server.v1.JobServlet;
 import org.apache.sqoop.server.v1.LinkServlet;
 import org.apache.sqoop.server.v1.SubmissionsServlet;
-import org.apache.sqoop.utils.ProcessUtils;
+import org.apache.sqoop.utils.PasswordUtils;
 import org.eclipse.jetty.server.HttpConfiguration;
 import org.eclipse.jetty.server.HttpConnectionFactory;
 import org.eclipse.jetty.server.SecureRequestCustomizer;
@@ -46,7 +46,6 @@ import org.eclipse.jetty.util.ssl.SslContextFactory;
 import org.eclipse.jetty.util.thread.ExecutorThreadPool;
 
 import javax.servlet.DispatcherType;
-import java.io.IOException;
 import java.util.EnumSet;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.SynchronousQueue;
@@ -94,40 +93,19 @@ public class SqoopJettyServer {
         sslContextFactory.setProtocol(protocol.trim());
       }
 
-      String keyStorePassword = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD);
-      String keyStorePasswordGenerator = configurationContext.getString(SecurityConstants.KEYSTORE_PASSWORD_GENERATOR);
-      if (StringUtils.isNotBlank(keyStorePassword)) {
-        if (StringUtils.isNotBlank(keyStorePasswordGenerator)) {
-          LOG.warn(SecurityConstants.KEYSTORE_PASSWORD + " and " + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR
-            + "are both set, using " + SecurityConstants.KEYSTORE_PASSWORD);
-        }
+      String keyStorePassword = PasswordUtils.readPassword(configurationContext, SecurityConstants.KEYSTORE_PASSWORD,
+        SecurityConstants.KEYSTORE_PASSWORD_GENERATOR);
+      if (StringUtils.isNotEmpty(keyStorePassword)) {
         sslContextFactory.setKeyStorePassword(keyStorePassword);
-      } else if (StringUtils.isNotBlank(keyStorePasswordGenerator)) {
-        try {
-          String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyStorePasswordGenerator);
-          sslContextFactory.setKeyStorePassword(passwordFromGenerator);
-        } catch (IOException exception) {
-          throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator:
" + SecurityConstants.KEYSTORE_PASSWORD_GENERATOR, exception);
-        }
       }
 
-      String keyManagerPassword = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD);
-      String keyManagerPasswordGenerator = configurationContext.getString(SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR);
-      if (StringUtils.isNotBlank(keyManagerPassword)) {
+      String keyManagerPassword = PasswordUtils.readPassword(configurationContext, SecurityConstants.KEYMANAGER_PASSWORD,
+        SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR);
+      if (StringUtils.isNotEmpty(keyManagerPassword)) {
         sslContextFactory.setKeyManagerPassword(keyManagerPassword);
-        if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) {
-          LOG.warn(SecurityConstants.KEYMANAGER_PASSWORD + " and " + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR
-            + "are both set, using " + SecurityConstants.KEYMANAGER_PASSWORD);
-        }
-      } else if (StringUtils.isNotBlank(keyManagerPasswordGenerator)) {
-        try {
-          String passwordFromGenerator = ProcessUtils.readOutputFromGenerator(keyManagerPasswordGenerator);
-          sslContextFactory.setKeyManagerPassword(passwordFromGenerator);
-        } catch (IOException exception) {
-          throw new SqoopException(ServerError.SERVER_0008, "failed to execute generator:
" + SecurityConstants.KEYMANAGER_PASSWORD_GENERATOR, exception);
-        }
       }
 
+
       HttpConfiguration https = new HttpConfiguration();
       https.addCustomizer(new SecureRequestCustomizer());
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java
----------------------------------------------------------------------
diff --git a/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java b/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java
index 0b603a1..acd055d 100644
--- a/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java
+++ b/shell/src/main/java/org/apache/sqoop/shell/SetTruststoreFunction.java
@@ -20,7 +20,7 @@ package org.apache.sqoop.shell;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.sqoop.shell.core.Constants;
-import org.apache.sqoop.utils.ProcessUtils;
+import org.apache.sqoop.utils.PasswordUtils;
 import org.apache.sqoop.validation.Status;
 
 import javax.net.ssl.HttpsURLConnection;
@@ -70,7 +70,7 @@ public class SetTruststoreFunction extends SqoopFunction {
           truststorePassword = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD).toCharArray();
         } else if (line.hasOption(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR)) {
           String generator = line.getOptionValue(Constants.OPT_TRUSTSTORE_PASSWORD_GENERATOR);
-          truststorePassword = ProcessUtils.readOutputFromGenerator(generator).toCharArray();
+          truststorePassword = PasswordUtils.readOutputFromGenerator(generator).toCharArray();
         }
 
         KeyStore keyStore = KeyStore.getInstance("JKS");

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5b897a46/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index c7a4db8..9fa9467 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -22,6 +22,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.sqoop.core.ConfigurationConstants;
 import org.apache.sqoop.common.test.repository.RepositoryProviderFactory;
+import org.apache.sqoop.security.SecurityConstants;
 import org.apache.sqoop.test.kdc.KdcRunner;
 
 import java.io.File;
@@ -231,6 +232,16 @@ public abstract class SqoopMiniCluster {
     properties.put("org.apache.sqoop.authentication.proxyuser." + "hadoop" + ".groups", "*");
     properties.put("org.apache.sqoop.authentication.proxyuser." + "hadoop" + ".hosts", "*");
 
+    properties.put("org.apache.sqoop.security.repo_encryption.enabled", String.valueOf(true));
+    properties.put("org.apache.sqoop.security.repo_encryption.password_generator", "echo
youwillnevergetthis");
+    properties.put("org.apache.sqoop.security.repo_encryption.hmac_algorithm", "HmacSHA256");
+    properties.put("org.apache.sqoop.security.repo_encryption.cipher_algorithm", "AES");
+    properties.put("org.apache.sqoop.security.repo_encryption.cipher_key_size", "16");
+    properties.put("org.apache.sqoop.security.repo_encryption.initialization_vector_size",
"16");
+    properties.put("org.apache.sqoop.security.repo_encryption.cipher_spec", "AES/CBC/PKCS5Padding");
+    properties.put("org.apache.sqoop.security.repo_encryption.pbkdf2_algorithm", "PBKDF2WithHmacSHA1");
+    properties.put("org.apache.sqoop.security.repo_encryption.pbkdf2_rounds", "4000");
+
     return properties;
   }
 


Mime
View raw message