sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject sqoop git commit: SQOOP-2441: Sqoop2: Generic JDBC: Drop support for specifying custom query when exporting data
Date Mon, 24 Aug 2015 20:59:23 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 548dcca42 -> 09a21649d


SQOOP-2441: Sqoop2: Generic JDBC: Drop support for specifying custom query when exporting
data

(Jarek Jarcec Cecho via Abraham Elmahrek)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/09a21649
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/09a21649
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/09a21649

Branch: refs/heads/sqoop2
Commit: 09a21649d4f8df8beef16c6a511b588db39403e4
Parents: 548dcca
Author: Abraham Elmahrek <abe@apache.org>
Authored: Mon Aug 24 13:59:03 2015 -0700
Committer: Abraham Elmahrek <abe@apache.org>
Committed: Mon Aug 24 13:59:03 2015 -0700

----------------------------------------------------------------------
 .../error/code/GenericJdbcConnectorError.java   |  10 --
 .../jdbc/GenericJdbcToInitializer.java          | 114 +++++++------------
 .../jdbc/configuration/ToJobConfig.java         |  16 +--
 .../jdbc/TestGenericJdbcConnectorUpgrader.java  |  12 --
 .../sqoop/connector/jdbc/TestToInitializer.java |  47 +-------
 5 files changed, 43 insertions(+), 156 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/09a21649/common/src/main/java/org/apache/sqoop/error/code/GenericJdbcConnectorError.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/error/code/GenericJdbcConnectorError.java
b/common/src/main/java/org/apache/sqoop/error/code/GenericJdbcConnectorError.java
index 9a9bb66..c0744bd 100644
--- a/common/src/main/java/org/apache/sqoop/error/code/GenericJdbcConnectorError.java
+++ b/common/src/main/java/org/apache/sqoop/error/code/GenericJdbcConnectorError.java
@@ -42,14 +42,6 @@ public enum GenericJdbcConnectorError implements ErrorCode {
   /** No boundaries are found for partition column. */
   GENERIC_JDBC_CONNECTOR_0006("No boundaries are found for partition column"),
 
-  /** The table name and the table sql cannot be specify together. */
-  GENERIC_JDBC_CONNECTOR_0007("The table name and the table sql "
-      + "cannot be specified together"),
-
-  /** Neither the table name nor the table sql are specified. */
-  GENERIC_JDBC_CONNECTOR_0008("Neither the table name nor the table sql "
-      + "are specified"),
-
   /** No substitute token in the specified sql. */
   GENERIC_JDBC_CONNECTOR_0010("No substitute token in the specified sql"),
 
@@ -79,8 +71,6 @@ public enum GenericJdbcConnectorError implements ErrorCode {
   GENERIC_JDBC_CONNECTOR_0018("Error occurred while transferring data from " +
     "stage table to destination table."),
 
-  GENERIC_JDBC_CONNECTOR_0019("Table name extraction not supported."),
-
   GENERIC_JDBC_CONNECTOR_0020("Unknown direction."),
 
   GENERIC_JDBC_CONNECTOR_0021("Schema column size do not match the result set column size"),

http://git-wip-us.apache.org/repos/asf/sqoop/blob/09a21649/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
index ad375fd..80fbece 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcToInitializer.java
@@ -65,10 +65,7 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration,
ToJ
     executor = new GenericJdbcExecutor(linkConfig);
 
     String schemaName = executor.encloseIdentifiers(toJobConfig.toJobConfig.schemaName, toJobConfig.toJobConfig.tableName);
-    if (schemaName == null) {
-      throw new SqoopException(GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0019,
-          "Table name extraction not supported yet.");
-    }
+    assert schemaName != null;
 
     Schema schema = new Schema(schemaName);
     try (Statement statement = executor.getConnection().createStatement(
@@ -106,86 +103,55 @@ public class GenericJdbcToInitializer extends Initializer<LinkConfiguration,
ToJ
       false : toJobConfig.toJobConfig.shouldClearStageTable;
     final boolean stageEnabled =
       stageTableName != null && stageTableName.length() > 0;
-    String tableSql = toJobConfig.toJobConfig.sql;
     String tableColumns = toJobConfig.toJobConfig.columns;
 
-    if (tableName != null && tableSql != null) {
-      // when both fromTable name and fromTable sql are specified:
-      throw new SqoopException(
-          GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0007);
-
-    } else if (tableName != null) {
-      // when fromTable name is specified:
-      if(stageEnabled) {
-        LOG.info("Stage has been enabled.");
-        LOG.info("Use stageTable: " + stageTableName +
-          " with clearStageTable: " + clearStageTable);
-
-        if(clearStageTable) {
-          executor.deleteTableData(stageTableName);
-        } else {
-          long stageRowCount = executor.getTableRowCount(stageTableName);
-          if(stageRowCount > 0) {
-            throw new SqoopException(
-              GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0017);
-          }
-        }
-      }
-
-      // For databases that support schemas (IE: postgresql).
-      final String tableInUse = stageEnabled ? stageTableName : tableName;
-      String fullTableName = executor.encloseIdentifiers(schemaName, tableInUse);
-
-      if (tableColumns == null) {
-        String[] columns = executor.getQueryColumns("SELECT * FROM "
-            + fullTableName + " WHERE 1 = 0");
-        StringBuilder builder = new StringBuilder();
-        builder.append("INSERT INTO ");
-        builder.append(fullTableName);
-        builder.append(" VALUES (?");
-        for (int i = 1; i < columns.length; i++) {
-          builder.append(",?");
-        }
-        builder.append(")");
-        dataSql = builder.toString();
+    // when fromTable name is specified:
+    if(stageEnabled) {
+      LOG.info("Stage has been enabled.");
+      LOG.info("Use stageTable: " + stageTableName + " with clearStageTable: " + clearStageTable);
 
+      if(clearStageTable) {
+        executor.deleteTableData(stageTableName);
       } else {
-        String[] columns = StringUtils.split(tableColumns, ',');
-        StringBuilder builder = new StringBuilder();
-        builder.append("INSERT INTO ");
-        builder.append(fullTableName);
-        builder.append(" (");
-        builder.append(tableColumns);
-        builder.append(") VALUES (?");
-        for (int i = 1; i < columns.length; i++) {
-          builder.append(",?");
+        long stageRowCount = executor.getTableRowCount(stageTableName);
+        if(stageRowCount > 0) {
+          throw new SqoopException(GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0017,
"Found rows: " + stageRowCount);
         }
-        builder.append(")");
-        dataSql = builder.toString();
-      }
-    } else if (tableSql != null) {
-      // when fromTable sql is specified:
-
-      if (tableSql.indexOf(
-          GenericJdbcConnectorConstants.SQL_PARAMETER_MARKER) == -1) {
-        // make sure parameter marker is in the specified sql
-        throw new SqoopException(
-            GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0013);
       }
+    }
 
-      if (tableColumns == null) {
-        dataSql = tableSql;
-      } else {
-        throw new SqoopException(
-            GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0014);
+    // For databases that support schemas (IE: postgresql).
+    final String tableInUse = stageEnabled ? stageTableName : tableName;
+    String fullTableName = executor.encloseIdentifiers(schemaName, tableInUse);
+
+    if (tableColumns == null) {
+      String[] columns = executor.getQueryColumns("SELECT * FROM " + fullTableName + " WHERE
1 = 0");
+      StringBuilder builder = new StringBuilder();
+      builder.append("INSERT INTO ");
+      builder.append(fullTableName);
+      builder.append(" VALUES (?");
+      for (int i = 1; i < columns.length; i++) {
+        builder.append(",?");
       }
+      builder.append(")");
+      dataSql = builder.toString();
+
     } else {
-      // when neither are specified:
-      throw new SqoopException(
-          GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0008);
+      String[] columns = StringUtils.split(tableColumns, ',');
+      StringBuilder builder = new StringBuilder();
+      builder.append("INSERT INTO ");
+      builder.append(fullTableName);
+      builder.append(" (");
+      builder.append(tableColumns);
+      builder.append(") VALUES (?");
+      for (int i = 1; i < columns.length; i++) {
+        builder.append(",?");
+      }
+      builder.append(")");
+      dataSql = builder.toString();
     }
 
-    context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_TO_DATA_SQL,
-        dataSql);
+    LOG.info("Using query to insert data: " + dataSql);
+    context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_TO_DATA_SQL, dataSql);
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/09a21649/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
index c9651d5..2384112 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/configuration/ToJobConfig.java
@@ -22,6 +22,7 @@ import org.apache.sqoop.model.Input;
 import org.apache.sqoop.model.Validator;
 import org.apache.sqoop.validation.Status;
 import org.apache.sqoop.validation.validators.AbstractValidator;
+import org.apache.sqoop.validation.validators.NotEmpty;
 
 /**
  *
@@ -31,13 +32,10 @@ public class ToJobConfig {
   @Input(size = 50)
   public String schemaName;
 
-  @Input(size = 2000)
+  @Input(size = 2000, validators = { @Validator(NotEmpty.class)})
   public String tableName;
 
   @Input(size = 50)
-  public String sql;
-
-  @Input(size = 50)
   public String columns;
 
   @Input(size = 2000)
@@ -49,16 +47,6 @@ public class ToJobConfig {
   public static class ConfigValidator extends AbstractValidator<ToJobConfig> {
     @Override
     public void validate(ToJobConfig config) {
-      if (config.tableName == null && config.sql == null) {
-        addMessage(Status.ERROR, "Either table name or SQL must be specified");
-      }
-      if (config.tableName != null && config.sql != null) {
-        addMessage(Status.ERROR, "Both table name and SQL cannot be specified");
-      }
-      if (config.tableName == null && config.stageTableName != null) {
-        addMessage(Status.ERROR,
-            "Stage table name cannot be specified without specifying table name");
-      }
       if (config.stageTableName == null && config.shouldClearStageTable != null)
{
         addMessage(Status.ERROR,
             "Should Clear stage table cannot be specified without specifying the name of
the stage table.");

http://git-wip-us.apache.org/repos/asf/sqoop/blob/09a21649/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
index c39aabc..e05b93d 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestGenericJdbcConnectorUpgrader.java
@@ -17,24 +17,16 @@
  */
 package org.apache.sqoop.connector.jdbc;
 
-import org.apache.commons.lang.StringUtils;
 import org.apache.sqoop.connector.jdbc.configuration.FromJobConfiguration;
 import org.apache.sqoop.connector.jdbc.configuration.LinkConfiguration;
 import org.apache.sqoop.connector.jdbc.configuration.ToJobConfiguration;
 import org.apache.sqoop.model.ConfigUtils;
-import org.apache.sqoop.model.InputEditable;
-import org.apache.sqoop.model.MBooleanInput;
-import org.apache.sqoop.model.MConfig;
 import org.apache.sqoop.model.MFromConfig;
-import org.apache.sqoop.model.MInput;
 import org.apache.sqoop.model.MLinkConfig;
-import org.apache.sqoop.model.MStringInput;
 import org.apache.sqoop.model.MToConfig;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-import java.util.LinkedList;
-
 import static org.testng.Assert.assertEquals;
 
 /**
@@ -56,7 +48,6 @@ public class TestGenericJdbcConnectorUpgrader {
     MFromConfig newConfigs = new MFromConfig(ConfigUtils.toConfigs(FromJobConfiguration.class));
     originalConfigs.getInput("fromJobConfig.schemaName").setValue("test-schema");
     originalConfigs.getInput("fromJobConfig.tableName").setValue("test-tableName");
-    originalConfigs.getInput("fromJobConfig.sql").setValue("test-sql");
     originalConfigs.getInput("fromJobConfig.columns").setValue("test-columns");
     originalConfigs.getInput("fromJobConfig.partitionColumn").setValue("test-partitionColumn");
     originalConfigs.getInput("fromJobConfig.allowNullValueInPartitionColumn").setValue("test-allowNullValueInPartitionColumn");
@@ -64,7 +55,6 @@ public class TestGenericJdbcConnectorUpgrader {
     assertEquals(originalConfigs, newConfigs);
     assertEquals("test-schema", newConfigs.getInput("fromJobConfig.schemaName").getValue());
     assertEquals("test-tableName", newConfigs.getInput("fromJobConfig.tableName").getValue());
-    assertEquals("test-sql", newConfigs.getInput("fromJobConfig.sql").getValue());
     assertEquals("test-columns", newConfigs.getInput("fromJobConfig.columns").getValue());
     assertEquals("test-partitionColumn", newConfigs.getInput("fromJobConfig.partitionColumn").getValue());
     assertEquals("test-allowNullValueInPartitionColumn", newConfigs.getInput("fromJobConfig.allowNullValueInPartitionColumn").getValue());
@@ -77,7 +67,6 @@ public class TestGenericJdbcConnectorUpgrader {
     MToConfig newConfigs = new MToConfig(ConfigUtils.toConfigs(ToJobConfiguration.class));
     originalConfigs.getInput("toJobConfig.schemaName").setValue("test-schema");
     originalConfigs.getInput("toJobConfig.tableName").setValue("test-tableName");
-    originalConfigs.getInput("toJobConfig.sql").setValue("test-sql");
     originalConfigs.getInput("toJobConfig.columns").setValue("test-columns");
     originalConfigs.getInput("toJobConfig.stageTableName").setValue("test-stageTableName");
     originalConfigs.getInput("toJobConfig.shouldClearStageTable").setValue("test-shouldClearStageTable");
@@ -85,7 +74,6 @@ public class TestGenericJdbcConnectorUpgrader {
     assertEquals(originalConfigs, newConfigs);
     assertEquals("test-schema", newConfigs.getInput("toJobConfig.schemaName").getValue());
     assertEquals("test-tableName", newConfigs.getInput("toJobConfig.tableName").getValue());
-    assertEquals("test-sql", newConfigs.getInput("toJobConfig.sql").getValue());
     assertEquals("test-columns", newConfigs.getInput("toJobConfig.columns").getValue());
     assertEquals("test-stageTableName", newConfigs.getInput("toJobConfig.stageTableName").getValue());
     assertEquals("test-shouldClearStageTable", newConfigs.getInput("toJobConfig.shouldClearStageTable").getValue());

http://git-wip-us.apache.org/repos/asf/sqoop/blob/09a21649/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
index 870ce98..df405c8 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestToInitializer.java
@@ -40,8 +40,6 @@ public class TestToInitializer {
   private final String tableName;
   private final String schemalessTableName;
   private final String stageTableName;
-  private final String tableSql;
-  private final String schemalessTableSql;
   private final String tableColumns;
 
   private GenericJdbcExecutor executor;
@@ -50,10 +48,7 @@ public class TestToInitializer {
     schemaName = getClass().getSimpleName().toUpperCase() + "SCHEMA";
     tableName = getClass().getSimpleName().toUpperCase() + "TABLEWITHSCHEMA";
     schemalessTableName = getClass().getSimpleName().toUpperCase() + "TABLE";
-    stageTableName = getClass().getSimpleName().toUpperCase() +
-      "_STAGE_TABLE";
-    tableSql = "INSERT INTO " + tableName + " VALUES (?,?,?)";
-    schemalessTableSql = "INSERT INTO " + schemalessTableName + " VALUES (?,?,?)";
+    stageTableName = getClass().getSimpleName().toUpperCase() + "_STAGE_TABLE";
     tableColumns = "ICOL,VCOL";
   }
 
@@ -125,26 +120,6 @@ public class TestToInitializer {
 
   @Test
   @SuppressWarnings("unchecked")
-  public void testTableSql() throws Exception {
-    LinkConfiguration linkConfig = new LinkConfiguration();
-    ToJobConfiguration jobConfig = new ToJobConfiguration();
-
-    linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
-    linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
-    jobConfig.toJobConfig.sql = schemalessTableSql;
-
-    MutableContext context = new MutableMapContext();
-    InitializerContext initializerContext = new InitializerContext(context);
-
-    @SuppressWarnings("rawtypes")
-    Initializer initializer = new GenericJdbcToInitializer();
-    initializer.initialize(initializerContext, linkConfig, jobConfig);
-
-    verifyResult(context, schemalessTableSql);
-  }
-
-  @Test
-  @SuppressWarnings("unchecked")
   public void testTableNameWithSchema() throws Exception {
     LinkConfiguration linkConfig = new LinkConfiguration();
     ToJobConfiguration jobConfig = new ToJobConfiguration();
@@ -287,26 +262,6 @@ public class TestToInitializer {
   }
 
   @Test
-  public void testStageTableWithoutTable() throws Exception {
-    LinkConfiguration linkConfig = new LinkConfiguration();
-    ToJobConfiguration jobConfig = new ToJobConfiguration();
-
-    linkConfig.linkConfig.jdbcDriver = GenericJdbcTestConstants.DRIVER;
-    linkConfig.linkConfig.connectionString = GenericJdbcTestConstants.URL;
-    //specifying stage table without specifying table name
-    jobConfig.toJobConfig.stageTableName = stageTableName;
-    jobConfig.toJobConfig.sql = "";
-
-    ConfigValidationRunner validationRunner = new ConfigValidationRunner();
-    ConfigValidationResult result = validationRunner.validate(jobConfig);
-    assertEquals(Status.ERROR, result.getStatus(),
-            "Stage table name cannot be specified without specifying " +
-            "table name");
-    assertTrue(result.getMessages().containsKey(
-      "toJobConfig"));
-  }
-
-  @Test
   @SuppressWarnings("unchecked")
   public void testClearStageTable() throws Exception {
     LinkConfiguration linkConfig = new LinkConfiguration();


Mime
View raw message