sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject [1/2] git commit: SQOOP-814: Start using configuration objects in Generic JDBC Connector
Date Wed, 23 Jan 2013 00:29:49 GMT
SQOOP-814: Start using configuration objects in Generic JDBC Connector

(Jarek Jarcec Cecho via Cheolsoo Park)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f89c8e59
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f89c8e59
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f89c8e59

Branch: refs/heads/sqoop2
Commit: f89c8e59f2826f899b7a85815be7940d705d5499
Parents: acfc0ae
Author: Cheolsoo Park <cheolsoo@apache.org>
Authored: Tue Jan 22 15:41:26 2013 -0800
Committer: Cheolsoo Park <cheolsoo@apache.org>
Committed: Tue Jan 22 15:41:26 2013 -0800

----------------------------------------------------------------------
 .../jdbc/GenericJdbcConnectorConstants.java        |    9 ---
 .../jdbc/GenericJdbcExportInitializer.java         |   19 ------
 .../connector/jdbc/GenericJdbcExportLoader.java    |   18 ++----
 .../connector/jdbc/GenericJdbcImportExtractor.java |   27 +++-----
 .../jdbc/GenericJdbcImportInitializer.java         |   18 -----
 .../jdbc/GenericJdbcImportPartitioner.java         |   12 +---
 .../sqoop/connector/jdbc/TestExportLoader.java     |   20 +++---
 .../sqoop/connector/jdbc/TestImportExtractor.java  |   49 +++++++--------
 8 files changed, 54 insertions(+), 118 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
index 09056d3..23f19b5 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnectorConstants.java
@@ -33,15 +33,6 @@ public final class GenericJdbcConnectorConstants {
   public static final String PREFIX_CONNECTOR_JDBC_CONFIG =
       "org.apache.sqoop.connector.jdbc.";
 
-  public static final String CONNECTOR_JDBC_DRIVER =
-      PREFIX_CONNECTOR_JDBC_CONFIG + "driver";
-  public static final String CONNECTOR_JDBC_URL =
-      PREFIX_CONNECTOR_JDBC_CONFIG + "url";
-  public static final String CONNECTOR_JDBC_USERNAME =
-      PREFIX_CONNECTOR_JDBC_CONFIG + "username";
-  public static final String CONNECTOR_JDBC_PASSWORD =
-      PREFIX_CONNECTOR_JDBC_CONFIG + "password";
-
   public static final String CONNECTOR_JDBC_PARTITION_COLUMNNAME =
       PREFIX_CONNECTOR_JDBC_CONFIG + "partition.columnname";
   public static final String CONNECTOR_JDBC_PARTITION_COLUMNTYPE =

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java
index 2b0ed00..531dd2e 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportInitializer.java
@@ -26,7 +26,6 @@ import org.apache.sqoop.common.MutableContext;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
 import org.apache.sqoop.connector.jdbc.configuration.ExportJobConfiguration;
-import org.apache.sqoop.job.Constants;
 import org.apache.sqoop.job.etl.Initializer;
 import org.apache.sqoop.utils.ClassUtils;
 
@@ -64,30 +63,12 @@ public class GenericJdbcExportInitializer extends Initializer<ConnectionConfigur
           GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0012,
           "JDBC Driver");
     }
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER,
-        driver);
 
     if (url == null) {
       throw new SqoopException(
           GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0012,
           "Connection string");
     }
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL,
-        url);
-
-    if (username != null) {
-      context.setString(
-          GenericJdbcConnectorConstants.CONNECTOR_JDBC_USERNAME,
-          username);
-    }
-
-    if (password != null) {
-      context.setString(
-          GenericJdbcConnectorConstants.CONNECTOR_JDBC_PASSWORD,
-          password);
-    }
 
     executor = new GenericJdbcExecutor(driver, url, username, password);
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java
index ec047b4..31dbcea 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcExportLoader.java
@@ -32,19 +32,13 @@ public class GenericJdbcExportLoader extends Loader<ConnectionConfiguration,
Exp
 
   @Override
   public void load(ImmutableContext context, ConnectionConfiguration connection, ExportJobConfiguration
job, DataReader reader) throws Exception{
-    String driver = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER);
-    String url = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL);
-    String username = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_USERNAME);
-    String password = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_PASSWORD);
-    GenericJdbcExecutor executor = new GenericJdbcExecutor(
-        driver, url, username, password);
+    String driver = connection.connection.jdbcDriver;
+    String url = connection.connection.connectionString;
+    String username = connection.connection.username;
+    String password = connection.connection.password;
+    GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password);
 
-    String sql = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL);
+    String sql = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL);
     executor.beginBatch(sql);
     try {
       int numberOfRows = 0;

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java
index c3ecdda..f4389a3 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportExtractor.java
@@ -37,24 +37,17 @@ public class GenericJdbcImportExtractor extends Extractor<ConnectionConfiguratio
  private long rowsRead = 0;
   @Override
   public void run(ImmutableContext context, ConnectionConfiguration connection, ImportJobConfiguration
job, Partition partition, DataWriter writer) {
-    String driver = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER);
-    String url = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL);
-    String username = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_USERNAME);
-    String password = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_PASSWORD);
-    GenericJdbcExecutor executor = new GenericJdbcExecutor(
-        driver, url, username, password);
+    String driver = connection.connection.jdbcDriver;
+    String url = connection.connection.connectionString;
+    String username = connection.connection.username;
+    String password = connection.connection.password;
+    GenericJdbcExecutor executor = new GenericJdbcExecutor(driver, url, username, password);
+
+    String query = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL);
+    String conditions = ((GenericJdbcImportPartition)partition).getConditions();
+    query = query.replace(GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN, conditions);
+    LOG.info("Using query: " + query);
 
-    String query = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL);
-    String conditions =
-        ((GenericJdbcImportPartition)partition).getConditions();
-    query = query.replace(
-        GenericJdbcConnectorConstants.SQL_CONDITIONS_TOKEN, conditions);
-    LOG.debug("Using query: " + query);
     rowsRead = 0;
     ResultSet resultSet = executor.executeQuery(query);
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java
index 781c7f2..c6336fa 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportInitializer.java
@@ -73,30 +73,12 @@ public class GenericJdbcImportInitializer extends Initializer<ConnectionConfigur
           GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0012,
           "JDBC Driver");
     }
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER,
-        driver);
 
     if (url == null) {
       throw new SqoopException(
           GenericJdbcConnectorError.GENERIC_JDBC_CONNECTOR_0012,
           "Connection string");
     }
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL,
-        url);
-
-    if (username != null) {
-      context.setString(
-          GenericJdbcConnectorConstants.CONNECTOR_JDBC_USERNAME,
-          username);
-    }
-
-    if (password != null) {
-      context.setString(
-          GenericJdbcConnectorConstants.CONNECTOR_JDBC_PASSWORD,
-          password);
-    }
 
     executor = new GenericJdbcExecutor(driver, url, username, password);
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
index 2313e65..b5dbc61 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
@@ -39,14 +39,10 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
   @Override
   public List<Partition> getPartitions(ImmutableContext context, long maxPartitions,
ConnectionConfiguration connection, ImportJobConfiguration job) {
     numberPartitions = maxPartitions;
-    partitionColumnName = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME);
-    partitionColumnType = Integer.parseInt(context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNTYPE));
-    partitionMinValue = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE);
-    partitionMaxValue = context.getString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE);
+    partitionColumnName = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNNAME);
+    partitionColumnType = context.getInt(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_COLUMNTYPE,
-1);
+    partitionMinValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MINVALUE);
+    partitionMaxValue = context.getString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_PARTITION_MAXVALUE);
 
     switch (partitionColumnType) {
     case Types.TINYINT:

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java
b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java
index bf3c5e2..5892cbb 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestExportLoader.java
@@ -23,6 +23,8 @@ import junit.framework.TestCase;
 
 import org.apache.sqoop.common.MutableContext;
 import org.apache.sqoop.common.MutableMapContext;
+import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
+import org.apache.sqoop.connector.jdbc.configuration.ExportJobConfiguration;
 import org.apache.sqoop.job.etl.Loader;
 import org.apache.sqoop.job.io.DataReader;
 
@@ -59,20 +61,20 @@ public class TestExportLoader extends TestCase {
   public void testInsert() throws Exception {
     MutableContext context = new MutableMapContext();
 
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER,
-        GenericJdbcTestConstants.DRIVER);
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL,
-        GenericJdbcTestConstants.URL);
+    ConnectionConfiguration connectionConfig = new ConnectionConfiguration();
+
+    connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+    connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL;
+
+    ExportJobConfiguration jobConfig = new ExportJobConfiguration();
+
     context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL,
-        "INSERT INTO " + executor.delimitIdentifier(tableName)
-            + " VALUES (?,?,?)");
+        "INSERT INTO " + executor.delimitIdentifier(tableName) + " VALUES (?,?,?)");
 
     Loader loader = new GenericJdbcExportLoader();
     DummyReader reader = new DummyReader();
 
-    loader.load(context, null, null, reader);
+    loader.load(context, connectionConfig, jobConfig, reader);
 
     int index = START;
     ResultSet rs = executor.executeQuery("SELECT * FROM "

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f89c8e59/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java
b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java
index 168e572..0be713e 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportExtractor.java
@@ -65,18 +65,16 @@ public class TestImportExtractor extends TestCase {
 
   public void testQuery() throws Exception {
     MutableContext context = new MutableMapContext();
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER,
-        GenericJdbcTestConstants.DRIVER);
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL,
-        GenericJdbcTestConstants.URL);
-    context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL,
-        "SELECT * FROM " + executor.delimitIdentifier(tableName)
-            + " WHERE ${CONDITIONS}");
 
-    ConnectionConfiguration connConf = new ConnectionConfiguration();
-    ImportJobConfiguration jobConf = new ImportJobConfiguration();
+    ConnectionConfiguration connectionConfig = new ConnectionConfiguration();
+
+    connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+    connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL;
+
+    ImportJobConfiguration jobConfig = new ImportJobConfiguration();
+
+    context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL,
+        "SELECT * FROM " + executor.delimitIdentifier(tableName) + " WHERE ${CONDITIONS}");
 
     GenericJdbcImportPartition partition;
 
@@ -85,33 +83,32 @@ public class TestImportExtractor extends TestCase {
 
     partition = new GenericJdbcImportPartition();
     partition.setConditions("-50.0 <= DCOL AND DCOL < -16.6666666666666665");
-    extractor.run(context, connConf, jobConf, partition, writer);
+    extractor.run(context, connectionConfig, jobConfig, partition, writer);
 
     partition = new GenericJdbcImportPartition();
     partition.setConditions("-16.6666666666666665 <= DCOL AND DCOL < 16.666666666666667");
-    extractor.run(context, connConf, jobConf, partition, writer);
+    extractor.run(context, connectionConfig, jobConfig, partition, writer);
 
     partition = new GenericJdbcImportPartition();
     partition.setConditions("16.666666666666667 <= DCOL AND DCOL <= 50.0");
-    extractor.run(context, connConf, jobConf, partition, writer);
+    extractor.run(context, connectionConfig, jobConfig, partition, writer);
   }
 
   public void testSubquery() throws Exception {
     MutableContext context = new MutableMapContext();
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_DRIVER,
-        GenericJdbcTestConstants.DRIVER);
-    context.setString(
-        GenericJdbcConnectorConstants.CONNECTOR_JDBC_URL,
-        GenericJdbcTestConstants.URL);
+
+    ConnectionConfiguration connectionConfig = new ConnectionConfiguration();
+
+    connectionConfig.connection.jdbcDriver = GenericJdbcTestConstants.DRIVER;
+    connectionConfig.connection.connectionString = GenericJdbcTestConstants.URL;
+
+    ImportJobConfiguration jobConfig = new ImportJobConfiguration();
+
     context.setString(GenericJdbcConnectorConstants.CONNECTOR_JDBC_DATA_SQL,
         "SELECT SQOOP_SUBQUERY_ALIAS.ICOL,SQOOP_SUBQUERY_ALIAS.VCOL FROM "
             + "(SELECT * FROM " + executor.delimitIdentifier(tableName)
             + " WHERE ${CONDITIONS}) SQOOP_SUBQUERY_ALIAS");
 
-    ConnectionConfiguration connConf = new ConnectionConfiguration();
-    ImportJobConfiguration jobConf = new ImportJobConfiguration();
-
     GenericJdbcImportPartition partition;
 
     Extractor extractor = new GenericJdbcImportExtractor();
@@ -119,15 +116,15 @@ public class TestImportExtractor extends TestCase {
 
     partition = new GenericJdbcImportPartition();
     partition.setConditions("-50 <= ICOL AND ICOL < -16");
-    extractor.run(context, connConf, jobConf, partition, writer);
+    extractor.run(context, connectionConfig, jobConfig, partition, writer);
 
     partition = new GenericJdbcImportPartition();
     partition.setConditions("-16 <= ICOL AND ICOL < 17");
-    extractor.run(context, connConf, jobConf, partition, writer);
+    extractor.run(context, connectionConfig, jobConfig, partition, writer);
 
     partition = new GenericJdbcImportPartition();
     partition.setConditions("17 <= ICOL AND ICOL < 50");
-    extractor.run(context, connConf, jobConf, partition, writer);
+    extractor.run(context, connectionConfig, jobConfig, partition, writer);
   }
 
   public class DummyWriter extends DataWriter {


Mime
View raw message