sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-1790: Sqoop2: Upgrade configs with name conflicts
Date Thu, 26 Feb 2015 15:10:15 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 37839d9ec -> cb037c16a


SQOOP-1790: Sqoop2: Upgrade configs with name conflicts

(Abraham Elmahrek via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/cb037c16
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/cb037c16
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/cb037c16

Branch: refs/heads/sqoop2
Commit: cb037c16a00d401ad5eba1d4e7b779929de927cc
Parents: 37839d9
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Thu Feb 26 07:09:39 2015 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Thu Feb 26 07:09:39 2015 -0800

----------------------------------------------------------------------
 .../jdbc/GenericJdbcConnectorUpgrader.java      |  65 ------
 .../jdbc/TestGenericJdbcConnectorUpgrader.java  |  71 -------
 .../common/CommonRepositoryHandler.java         |   2 +-
 .../common/CommonRepositorySchemaConstants.java |   1 +
 .../derby/DerbyRepositoryHandler.java           |   7 +
 .../derby/DerbySchemaUpgradeQuery.java          |  31 +++
 ...GenericJdbcConnectorConfigAndInputNames.java | 196 +++++++++++++++++++
 .../derby/upgrade/Derby1_99_3UpgradeTest.java   |  83 ++++++++
 .../derby/derby-repository-1.99.3.tar.gz        | Bin 0 -> 119907 bytes
 9 files changed, 319 insertions(+), 137 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
index 6e85c03..87af723 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorUpgrader.java
@@ -32,83 +32,18 @@ import java.util.Map;
 // NOTE: All config types have the similar upgrade path at this point
 public class GenericJdbcConnectorUpgrader extends ConnectorConfigurableUpgrader {
 
-  private static final String JOB_CONFIGURATION_FORM_NAME = "table";
-  private static final String CONNECTION_CONFIGURATION_FORM_NAME = "connection";
-  private static final Map<String, String> CONNECTION_TO_LINK_CONFIG_INPUT_MAP;
-  private static final Map<String, String> IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP;
-  private static final Map<String, String> EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP;
-
-  static {
-    CONNECTION_TO_LINK_CONFIG_INPUT_MAP = new HashMap<String, String>();
-    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".jdbcDriver",
"linkConfig.jdbcDriver");
-    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".connectionString",
"linkConfig.connectionString");
-    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".username",
"linkConfig.username");
-    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".password",
"linkConfig.password");
-    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".jdbcProperties",
"linkConfig.jdbcProperties");
-
-    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP = new HashMap<String, String>();
-    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".schemaName",
"fromJobConfig.schemaName");
-    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".tableName",
"fromJobConfig.tableName");
-    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".sql", "fromJobConfig.sql");
-    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".columns",
"fromJobConfig.columns");
-    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".partitionColumn",
"fromJobConfig.partitionColumn");
-    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".partitionColumnNull",
"fromJobConfig.allowNullValueInPartitionColumn");
-    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".boundaryQuery",
"fromJobConfig.boundaryQuery");
-
-    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP = new HashMap<String, String>();
-    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".schemaName",
"toJobConfig.schemaName");
-    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".tableName",
"toJobConfig.tableName");
-    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".sql", "toJobConfig.sql");
-    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".columns",
"toJobConfig.columns");
-    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".stageTableName",
"toJobConfig.stageTableName");
-    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".clearStageTable",
"toJobConfig.shouldClearStageTable");
-  }
-
   @Override
   public void upgradeLinkConfig(MLinkConfig original, MLinkConfig upgradeTarget) {
-    // Upgrade from 1.99.3 to 1.99.4
-    for (MConfig config : original.getConfigs()) {
-      if (config.getName().equals(CONNECTION_CONFIGURATION_FORM_NAME)) {
-        for (MInput originalInput : config.getInputs()) {
-          String inputName = CONNECTION_TO_LINK_CONFIG_INPUT_MAP.get(originalInput.getName());
-          MInput input = upgradeTarget.getInput(inputName);
-          input.setValue(originalInput.getValue());
-        }
-      }
-    }
-
     ConfigurableUpgradeUtil.doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
   }
 
   @Override
   public void upgradeFromJobConfig(MFromConfig original, MFromConfig upgradeTarget) {
-    // Upgrade from 1.99.3 to 1.99.4
-    for (MConfig config : original.getConfigs()) {
-      if (config.getName().equals(JOB_CONFIGURATION_FORM_NAME)) {
-        for (MInput originalInput : config.getInputs()) {
-          String inputName = IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.get(originalInput.getName());
-          MInput input = upgradeTarget.getInput(inputName);
-          input.setValue(originalInput.getValue());
-        }
-      }
-    }
-
     ConfigurableUpgradeUtil.doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
   }
 
   @Override
   public void upgradeToJobConfig(MToConfig original, MToConfig upgradeTarget) {
-    // Upgrade from 1.99.3 to 1.99.4
-    for (MConfig config : original.getConfigs()) {
-      if (config.getName().equals(JOB_CONFIGURATION_FORM_NAME)) {
-        for (MInput originalInput : config.getInputs()) {
-          String inputName = EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.get(originalInput.getName());
-          MInput input = upgradeTarget.getInput(inputName);
-          input.setValue(originalInput.getValue());
-        }
-      }
-    }
-
     ConfigurableUpgradeUtil.doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
index 220e42e..2a3ab9b 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
@@ -68,31 +68,6 @@ public class TestGenericJdbcConnectorUpgrader {
     assertEquals("test-columns", newConfigs.getInput("fromJobConfig.columns").getValue());
     assertEquals("test-partitionColumn", newConfigs.getInput("fromJobConfig.partitionColumn").getValue());
     assertEquals("test-allowNullValueInPartitionColumn", newConfigs.getInput("fromJobConfig.allowNullValueInPartitionColumn").getValue());
-
-    // 1.99.3 upgrade to 1.99.4
-    originalConfigs = new MFromConfig(new LinkedList<MConfig>());
-    newConfigs = new MFromConfig(ConfigUtils.toConfigs(FromJobConfiguration.class));
-    originalConfigs.getConfigs().add(new MConfig("table", new LinkedList<MInput<?>>()));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.schemaName",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.tableName",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.sql", false,
InputEditable.ANY, StringUtils.EMPTY, (short)2000));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.columns",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.partitionColumn",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MBooleanInput("table.partitionColumnNull",
false, InputEditable.ANY, StringUtils.EMPTY));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.boundaryQuery",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getInput("table.schemaName").setValue("test-schema");
-    originalConfigs.getInput("table.tableName").setValue("test-tableName");
-    originalConfigs.getInput("table.sql").setValue("test-sql");
-    originalConfigs.getInput("table.columns").setValue("test-columns");
-    originalConfigs.getInput("table.partitionColumn").setValue("test-partitionColumn");
-    originalConfigs.getInput("table.partitionColumnNull").setValue("test-allowNullValueInPartitionColumn");
-    upgrader.upgradeFromJobConfig(originalConfigs, newConfigs);
-    assertEquals("test-schema", newConfigs.getInput("fromJobConfig.schemaName").getValue());
-    assertEquals("test-tableName", newConfigs.getInput("fromJobConfig.tableName").getValue());
-    assertEquals("test-sql", newConfigs.getInput("fromJobConfig.sql").getValue());
-    assertEquals("test-columns", newConfigs.getInput("fromJobConfig.columns").getValue());
-    assertEquals("test-partitionColumn", newConfigs.getInput("fromJobConfig.partitionColumn").getValue());
-    assertEquals("test-allowNullValueInPartitionColumn", newConfigs.getInput("fromJobConfig.allowNullValueInPartitionColumn").getValue());
   }
 
   @Test
@@ -114,31 +89,6 @@ public class TestGenericJdbcConnectorUpgrader {
     assertEquals("test-columns", newConfigs.getInput("toJobConfig.columns").getValue());
     assertEquals("test-stageTableName", newConfigs.getInput("toJobConfig.stageTableName").getValue());
     assertEquals("test-shouldClearStageTable", newConfigs.getInput("toJobConfig.shouldClearStageTable").getValue());
-
-    // 1.99.3 upgrade to 1.99.4
-    originalConfigs = new MToConfig(new LinkedList<MConfig>());
-    newConfigs = new MToConfig(ConfigUtils.toConfigs(ToJobConfiguration.class));
-    originalConfigs.getConfigs().add(new MConfig("table", new LinkedList<MInput<?>>()));
-
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.schemaName",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.tableName",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.sql", false,
InputEditable.ANY, StringUtils.EMPTY,(short)2000));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.columns",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("table.stageTableName",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MBooleanInput("table.clearStageTable",
false, InputEditable.ANY, StringUtils.EMPTY));
-    originalConfigs.getInput("table.schemaName").setValue("test-schema");
-    originalConfigs.getInput("table.tableName").setValue("test-tableName");
-    originalConfigs.getInput("table.sql").setValue("test-sql");
-    originalConfigs.getInput("table.columns").setValue("test-columns");
-    originalConfigs.getInput("table.stageTableName").setValue("test-stageTableName");
-    originalConfigs.getInput("table.clearStageTable").setValue("test-shouldClearStageTable");
-    upgrader.upgradeToJobConfig(originalConfigs, newConfigs);
-    assertEquals("test-schema", newConfigs.getInput("toJobConfig.schemaName").getValue());
-    assertEquals("test-tableName", newConfigs.getInput("toJobConfig.tableName").getValue());
-    assertEquals("test-sql", newConfigs.getInput("toJobConfig.sql").getValue());
-    assertEquals("test-columns", newConfigs.getInput("toJobConfig.columns").getValue());
-    assertEquals("test-stageTableName", newConfigs.getInput("toJobConfig.stageTableName").getValue());
-    assertEquals("test-shouldClearStageTable", newConfigs.getInput("toJobConfig.shouldClearStageTable").getValue());
   }
 
   @Test
@@ -158,26 +108,5 @@ public class TestGenericJdbcConnectorUpgrader {
     assertEquals("test-username", newConfigs.getInput("linkConfig.username").getValue());
     assertEquals("test-password", newConfigs.getInput("linkConfig.password").getValue());
     assertEquals("test-jdbcProperties", newConfigs.getInput("linkConfig.jdbcProperties").getValue());
-
-    // 1.99.3 upgrade to 1.99.4
-    originalConfigs = new MLinkConfig(new LinkedList<MConfig>());
-    newConfigs = new MLinkConfig(ConfigUtils.toConfigs(LinkConfiguration.class));
-    originalConfigs.getConfigs().add(new MConfig("connection", new LinkedList<MInput<?>>()));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.jdbcDriver",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.connectionString",
false,InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.username",
false, InputEditable.ANY, StringUtils.EMPTY, (short)2000));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.password",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getConfigs().get(0).getInputs().add(new MStringInput("connection.jdbcProperties",
false, InputEditable.ANY, StringUtils.EMPTY, (short)50));
-    originalConfigs.getInput("connection.jdbcDriver").setValue("test-jdbcDriver");
-    originalConfigs.getInput("connection.connectionString").setValue("test-connectionString");
-    originalConfigs.getInput("connection.username").setValue("test-username");
-    originalConfigs.getInput("connection.password").setValue("test-password");
-    originalConfigs.getInput("connection.jdbcProperties").setValue("test-jdbcProperties");
-    upgrader.upgradeLinkConfig(originalConfigs, newConfigs);
-    assertEquals("test-jdbcDriver", newConfigs.getInput("linkConfig.jdbcDriver").getValue());
-    assertEquals("test-connectionString", newConfigs.getInput("linkConfig.connectionString").getValue());
-    assertEquals("test-username", newConfigs.getInput("linkConfig.username").getValue());
-    assertEquals("test-password", newConfigs.getInput("linkConfig.password").getValue());
-    assertEquals("test-jdbcProperties", newConfigs.getInput("linkConfig.jdbcProperties").getValue());
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
index 87d2d9c..039524a 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
@@ -2523,7 +2523,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler
{
    *
    * @param stmts Statements to close
    */
-  protected void closeStatements(Statement... stmts) {
+  public void closeStatements(Statement... stmts) {
     if(stmts == null) {
       return;
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
index 6e183de..545b990 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
@@ -77,6 +77,7 @@ public final class CommonRepositorySchemaConstants {
 
   public static final String COLUMN_SQ_CFG_CONFIGURABLE = "SQ_CFG_CONFIGURABLE";
 
+  @Deprecated
   public static final String COLUMN_SQ_CFG_DIRECTION = "SQ_CFG_DIRECTION";
 
   public static final String COLUMN_SQ_CFG_NAME = "SQ_CFG_NAME";

http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index 1b1270e..7af912d 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -291,6 +291,13 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler {
         updateDriverConfigInput(conn, driverId);
         LOG.info("Finished Updating config and inputs for the driver.");
       }
+
+      // Update generic jdbc connector
+      if (repositoryVersion > 0) {
+        DerbyUpgradeGenericJdbcConnectorConfigAndInputNames derbyUpgradeGenericJdbcConnectorConfigAndInputNames
+            = new DerbyUpgradeGenericJdbcConnectorConfigAndInputNames(this, conn);
+        derbyUpgradeGenericJdbcConnectorConfigAndInputNames.execute();
+      }
     }
 
     if (repositoryVersion < 5) {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
index 5a5880c..662da9f 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
@@ -598,6 +598,37 @@ public final class DerbySchemaUpgradeQuery {
     return queryBuilder.toString();
   }
 
+  // Update Generic Jdbc Connector configs
+
+  public static final String QUERY_UPDATE_TABLE_SQ_CONFIG_NAME =
+      "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIG_NAME)
+          + " SET " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_NAME) + " = ?"
+          + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_ID) + " = ?";
+
+  public static final String QUERY_UPDATE_TABLE_SQ_INPUT_SQI_NAME =
+      "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_INPUT_NAME)
+          + " SET " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + " = ?"
+          + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_NAME) + " = ?"
+          + " AND " + CommonRepoUtils.escapeColumnName(COLUMN_SQI_CONFIG) + " = ?";
+
+  public static final String QUERY_SELECT_CONFIG_ID_BY_NAME =
+      "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_ID)
+          + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIG_NAME)
+          + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_NAME) + " = ?";
+
+  public static final String QUERY_SELECT_DIRECTION_CONFIG_BY_DIRECTION_NAME =
+      "SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_DIR_CONFIG)
+          + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_CONFIG_DIRECTIONS_NAME)
+          + " LEFT JOIN " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_DIRECTION_NAME)
+            + " ON " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_DIR_DIRECTION)
+            + " = " + CommonRepoUtils.escapeColumnName(COLUMN_SQD_ID)
+          + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQD_NAME) + " = ?";
+
+  public static final String QUERY_SELECT_CONFIG_ID_BY_NAME_AND_DIRECTION = QUERY_SELECT_CONFIG_ID_BY_NAME
+      + " AND " + CommonRepoUtils.escapeColumnName(COLUMN_SQ_CFG_ID) + " IN ("
+        + QUERY_SELECT_DIRECTION_CONFIG_BY_DIRECTION_NAME
+      + ")";
+
   private DerbySchemaUpgradeQuery() {
     // Disable explicit object creation
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyUpgradeGenericJdbcConnectorConfigAndInputNames.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyUpgradeGenericJdbcConnectorConfigAndInputNames.java
b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyUpgradeGenericJdbcConnectorConfigAndInputNames.java
new file mode 100644
index 0000000..eb8a905
--- /dev/null
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyUpgradeGenericJdbcConnectorConfigAndInputNames.java
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.repository.derby;
+
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.error.code.DerbyRepoError;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Sqoop 1.99.4 release included the following changes:
+ * 1. FROM/TO refactoring
+ * 2. Nomenclature improvements
+ *
+ * With the above improvements, the Generic JDBC Connector
+ * and it's configurations need to be updated.
+ *
+ * This class is intended to handle the updates to the Generic JDBC Connector.
+ */
+public class DerbyUpgradeGenericJdbcConnectorConfigAndInputNames {
+  private static final Logger LOG =
+      Logger.getLogger(DerbyUpgradeGenericJdbcConnectorConfigAndInputNames.class);
+
+  private static final String JOB_CONFIGURATION_FORM_NAME = "table";
+  private static final String CONNECTION_CONFIGURATION_FORM_NAME = "connection";
+  private static final String LINK_CONFIG_NAME = "linkConfig";
+  private static final String FROM_JOB_CONFIG_NAME = "fromJobConfig";
+  private static final String TO_JOB_CONFIG_NAME = "toJobConfig";
+  private static final Map<String, String> CONNECTION_TO_LINK_CONFIG_INPUT_MAP;
+  private static final Map<String, String> IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP;
+  private static final Map<String, String> EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP;
+
+  static {
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP = new HashMap<String, String>();
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".jdbcDriver",
LINK_CONFIG_NAME + ".jdbcDriver");
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".connectionString",
LINK_CONFIG_NAME + ".connectionString");
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".username",
LINK_CONFIG_NAME + ".username");
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".password",
LINK_CONFIG_NAME + ".password");
+    CONNECTION_TO_LINK_CONFIG_INPUT_MAP.put(CONNECTION_CONFIGURATION_FORM_NAME + ".jdbcProperties",
LINK_CONFIG_NAME + ".jdbcProperties");
+
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP = new HashMap<String, String>();
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".schemaName",
FROM_JOB_CONFIG_NAME + ".schemaName");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".tableName",
FROM_JOB_CONFIG_NAME + ".tableName");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".sql", FROM_JOB_CONFIG_NAME
+ ".sql");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".columns",
FROM_JOB_CONFIG_NAME + ".columns");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".partitionColumn",
FROM_JOB_CONFIG_NAME + ".partitionColumn");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".partitionColumnNull",
FROM_JOB_CONFIG_NAME + ".allowNullValueInPartitionColumn");
+    IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".boundaryQuery",
FROM_JOB_CONFIG_NAME + ".boundaryQuery");
+
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP = new HashMap<String, String>();
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".schemaName",
TO_JOB_CONFIG_NAME + ".schemaName");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".tableName",
TO_JOB_CONFIG_NAME + ".tableName");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".sql", TO_JOB_CONFIG_NAME
+ ".sql");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".columns",
TO_JOB_CONFIG_NAME + ".columns");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".stageTableName",
TO_JOB_CONFIG_NAME + ".stageTableName");
+    EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP.put(JOB_CONFIGURATION_FORM_NAME + ".clearStageTable",
TO_JOB_CONFIG_NAME + ".shouldClearStageTable");
+  }
+
+  private Connection connection;
+  private DerbyRepositoryHandler handler;
+
+  public DerbyUpgradeGenericJdbcConnectorConfigAndInputNames(DerbyRepositoryHandler handler,
Connection connection) {
+    this.handler = handler;
+    this.connection = connection;
+  }
+
+  public void execute() {
+    LOG.info("Renaming Generic JDBC Connector configs and inputs.");
+
+    Long linkConfigId = getConfigId(false, CONNECTION_CONFIGURATION_FORM_NAME);
+    Long fromJobConfigId = getConfigId(true, JOB_CONFIGURATION_FORM_NAME, Direction.FROM.toString());
+    Long toJobConfigId = getConfigId(true, JOB_CONFIGURATION_FORM_NAME, Direction.TO.toString());
+
+    if (linkConfigId != null) {
+      LOG.info("Renaming LINK config (" + linkConfigId + ") and inputs.");
+      renameConfig(linkConfigId, LINK_CONFIG_NAME);
+      renameConfigInputs(linkConfigId, CONNECTION_TO_LINK_CONFIG_INPUT_MAP);
+    } else {
+      LOG.info("Renaming LINK config and inputs skipped.");
+    }
+
+    if (fromJobConfigId != null) {
+      LOG.info("Renaming FROM config (" + fromJobConfigId + ") and inputs.");
+      renameConfig(fromJobConfigId, FROM_JOB_CONFIG_NAME);
+      renameConfigInputs(fromJobConfigId, IMPORT_JOB_TABLE_TO_FROM_CONFIG_INPUT_MAP);
+    } else {
+      LOG.info("Renaming FROM config and inputs skipped.");
+    }
+
+    if (toJobConfigId != null) {
+      LOG.info("Renaming TO config (" + fromJobConfigId + ") and inputs.");
+      renameConfig(toJobConfigId, TO_JOB_CONFIG_NAME);
+      renameConfigInputs(toJobConfigId, EXPORT_JOB_TABLE_TO_TO_CONFIG_INPUT_MAP);
+    } else {
+      LOG.info("Renaming TO config and inputs skipped.");
+    }
+
+    LOG.info("Done Generic JDBC Connector configs and inputs.");
+  }
+
+  private Long getConfigId(boolean direction, String ... args) {
+    PreparedStatement statement = null;
+    String configIdQuery = (direction) ?
+        DerbySchemaUpgradeQuery.QUERY_SELECT_CONFIG_ID_BY_NAME_AND_DIRECTION : DerbySchemaUpgradeQuery.QUERY_SELECT_CONFIG_ID_BY_NAME;
+
+    try {
+      statement = connection.prepareStatement(configIdQuery);
+
+      for (int i = 0; i < args.length; ++i) {
+        statement.setString(i + 1, args[i]);
+      }
+
+      ResultSet configIdResultSet = statement.executeQuery();
+
+      LOG.debug("QUERY(" + configIdQuery + ") with args [" + StringUtils.join(args, ",")
+ "] fetch size: " + configIdResultSet.getFetchSize());
+
+      if (!configIdResultSet.next() || configIdResultSet.getFetchSize() != 1) {
+        return null;
+      }
+
+      return configIdResultSet.getLong(1);
+    } catch (SQLException e) {
+      throw new SqoopException(DerbyRepoError.DERBYREPO_0002, e);
+    } finally {
+      handler.closeStatements(statement);
+    }
+  }
+
+  private void renameConfig(long configId, String configName) {
+    PreparedStatement statement = null;
+    String query = DerbySchemaUpgradeQuery.QUERY_UPDATE_TABLE_SQ_CONFIG_NAME;
+
+    try {
+      statement = connection.prepareStatement(query);
+      statement.setString(1, configName);
+      statement.setLong(2, configId);
+
+      int updateCount = statement.executeUpdate();
+      LOG.debug("QUERY(" + query + ") with args [" + configName + ", " + configId + "] update
count: " + updateCount);
+    } catch (SQLException e) {
+      throw new SqoopException(DerbyRepoError.DERBYREPO_0002, e);
+    } finally {
+      handler.closeStatements(statement);
+    }
+  }
+
+  private void renameConfigInputs(long configId, Map<String, String> inputNameMap)
{
+    PreparedStatement statement = null;
+
+    try {
+      statement = connection.prepareStatement(DerbySchemaUpgradeQuery.QUERY_UPDATE_TABLE_SQ_INPUT_SQI_NAME);
+
+      for (String inputName : inputNameMap.keySet()) {
+        statement.setString(1, inputNameMap.get(inputName));
+        statement.setString(2, inputName);
+        statement.setLong(3, configId);
+        statement.addBatch();
+
+        LOG.debug("QUERY(" + DerbySchemaUpgradeQuery.QUERY_UPDATE_TABLE_SQ_INPUT_SQI_NAME
+ ") args ["
+            + inputNameMap.get(inputName) + "," + inputName + "," + configId + "]");
+      }
+
+      int[] updateCounts = statement.executeBatch();
+      LOG.debug("QUERY(" + DerbySchemaUpgradeQuery.QUERY_UPDATE_TABLE_SQ_INPUT_SQI_NAME +
") update count: "
+          + StringUtils.join(ArrayUtils.toObject(updateCounts), ","));
+    } catch (SQLException e) {
+      throw new SqoopException(DerbyRepoError.DERBYREPO_0002, e);
+    } finally {
+      handler.closeStatements(statement);
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
new file mode 100644
index 0000000..849ab7b
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.integration.repository.derby.upgrade;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * This version contains the following structures:
+ * Generic JDBC Connector link with name "Link1" and id 1
+ * Generic JDBC Connector link with name "Link2" and id 2
+ * Generic JDBC Connector link with name "Link3" and id 3
+ * Generic JDBC Connector link with name "Link4" and id 4
+ * Job IMPORT with name "Job1" and id 1
+ * Job IMPORT with name "Job2" and id 2
+ * Job IMPORT with name "Job3" and id 3
+ * Job EXPORT with name "Job4" and id 4
+ * Link with id 4 has been disabled
+ * Job with id 3 has been disabled
+ * Job with id 1 has been run 5 times
+ */
+public class Derby1_99_3UpgradeTest extends DerbyRepositoryUpgradeTest {
+
+  @Override
+  public String getPathToRepositoryTarball() {
+    return "/repository/derby/derby-repository-1.99.3.tar.gz";
+  }
+
+  @Override
+  public int getNumberOfLinks() {
+    return 5;
+  }
+
+  @Override
+  public int getNumberOfJobs() {
+    return 4;
+  }
+
+  @Override
+  public Map<Integer, Integer> getNumberOfSubmissions() {
+    HashMap<Integer, Integer> ret = new HashMap<Integer, Integer>();
+    ret.put(1, 5);
+    ret.put(2, 0);
+    ret.put(3, 0);
+    ret.put(4, 0);
+    return ret;
+  }
+
+  @Override
+  public Integer[] getDisabledLinkIds() {
+    return new Integer[] {4};
+  }
+
+  @Override
+  public Integer[] getDisabledJobIds() {
+    return new Integer[] {3};
+  }
+
+  @Override
+  public Integer[] getDeleteLinkIds() {
+    return new Integer[] {1, 2, 3, 4, 5};
+  }
+
+  @Override
+  public Integer[] getDeleteJobIds() {
+    return new Integer[] {1, 2, 3, 4};
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/cb037c16/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz
----------------------------------------------------------------------
diff --git a/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz b/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz
new file mode 100644
index 0000000..bf9bf85
Binary files /dev/null and b/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz
differ


Mime
View raw message