sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject git commit: SQOOP-908: Create MySQL and PostgreSQL database providers
Date Sat, 02 Mar 2013 00:07:21 GMT
Updated Branches:
  refs/heads/sqoop2 2642b66cf -> f64dba2e0


SQOOP-908: Create MySQL and PostgreSQL database providers

(Jarcec Cecho via Cheolsoo Park)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f64dba2e
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f64dba2e
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f64dba2e

Branch: refs/heads/sqoop2
Commit: f64dba2e01e61e92ae6fd3bdcb6e794f198f41ae
Parents: 2642b66
Author: Cheolsoo Park <cheolsoo@apache.org>
Authored: Fri Mar 1 16:06:20 2013 -0800
Committer: Cheolsoo Park <cheolsoo@apache.org>
Committed: Fri Mar 1 16:06:20 2013 -0800

----------------------------------------------------------------------
 pom.xml                                            |   12 ++
 test/pom.xml                                       |   10 ++
 .../org/apache/sqoop/test/db/DatabaseProvider.java |    8 ++
 .../sqoop/test/db/DatabaseProviderFactory.java     |   39 +++++++
 .../org/apache/sqoop/test/db/DerbyProvider.java    |    7 +-
 .../org/apache/sqoop/test/db/MySQLProvider.java    |   83 +++++++++++++++
 .../apache/sqoop/test/db/PostgreSQLProvider.java   |   83 +++++++++++++++
 .../test/minicluster/TomcatSqoopMiniCluster.java   |    2 +
 .../apache/sqoop/integration/TomcatTestCase.java   |    8 +-
 .../integration/connector/ConnectorTestCase.java   |   10 ++-
 .../connector/jdbc/generic/TableImportTest.java    |   11 ++-
 11 files changed, 262 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b569473..0620d08 100644
--- a/pom.xml
+++ b/pom.xml
@@ -106,6 +106,8 @@ limitations under the License.
     <tomcat.major.version>6</tomcat.major.version>
     <tomcat.minor.version>0.36</tomcat.minor.version>
     <tomcat.version>${tomcat.major.version}.${tomcat.minor.version}</tomcat.version>
+    <jdbc.mysql.version>5.1.23</jdbc.mysql.version>
+    <jdbc.postgresql.version>9.1-901.jdbc4</jdbc.postgresql.version>
   </properties>
 
   <dependencies>
@@ -350,6 +352,16 @@ limitations under the License.
         <artifactId>cargo-core-container-tomcat</artifactId>
         <version>${cargo.version}</version>
       </dependency>
+      <dependency>
+        <groupId>mysql</groupId>
+        <artifactId>mysql-connector-java</artifactId>
+        <version>${jdbc.mysql.version}</version>
+      </dependency>
+      <dependency>
+        <groupId>postgresql</groupId>
+        <artifactId>postgresql</artifactId>
+        <version>${jdbc.postgresql.version}</version>
+      </dependency>
    </dependencies>
   </dependencyManagement>
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index fe47ff7..26aa556 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -83,6 +83,16 @@ limitations under the License.
       <artifactId>derbyclient</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>mysql</groupId>
+      <artifactId>mysql-connector-java</artifactId>
+    </dependency>
+
+    <dependency>
+      <groupId>postgresql</groupId>
+      <artifactId>postgresql</artifactId>
+    </dependency>
+
   </dependencies>
 
   <!-- Add classifier name to the JAR name -->

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java b/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java
index 364ff61..dce1043 100644
--- a/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java
+++ b/test/src/main/java/org/apache/sqoop/test/db/DatabaseProvider.java
@@ -104,10 +104,18 @@ abstract public class DatabaseProvider {
     return "NULL";
   }
 
+  public String getJdbcDriver() {
+    return null;
+  }
+
   /**
    * Start the handler.
    */
   public void start() {
+    if(getJdbcDriver() != null) {
+      loadClass(getJdbcDriver());
+    }
+
     // Create connection to the database server
     try {
       setConnection(DriverManager.getConnection(getConnectionUrl(), getConnectionUsername(),
getConnectionPassword()));

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java b/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java
new file mode 100644
index 0000000..bc430f1
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/db/DatabaseProviderFactory.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.db;
+
+import java.util.Properties;
+
+/**
+ * Create database provider.
+ */
+public class DatabaseProviderFactory {
+
+  public static final String PROVIDER_PROPERTY = "sqoop.provider.class";
+
+  public static DatabaseProvider getProvider(Properties properties) throws ClassNotFoundException,
IllegalAccessException, InstantiationException {
+    String className = properties.getProperty(PROVIDER_PROPERTY);
+    if(className == null) {
+      return new DerbyProvider();
+    }
+
+    Class klass = Class.forName(className);
+    return (DatabaseProvider)klass.newInstance();
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/main/java/org/apache/sqoop/test/db/DerbyProvider.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/db/DerbyProvider.java b/test/src/main/java/org/apache/sqoop/test/db/DerbyProvider.java
index 402fab9..aa6b897 100644
--- a/test/src/main/java/org/apache/sqoop/test/db/DerbyProvider.java
+++ b/test/src/main/java/org/apache/sqoop/test/db/DerbyProvider.java
@@ -46,8 +46,6 @@ public class DerbyProvider extends DatabaseProvider {
       throw new RuntimeException("Can't derby server", e);
     }
 
-    // Load JDBC driver and create connection
-    loadClass(DRIVER);
     super.start();
   }
 
@@ -83,6 +81,11 @@ public class DerbyProvider extends DatabaseProvider {
   }
 
   @Override
+  public String getJdbcDriver() {
+    return DRIVER;
+  }
+
+  @Override
   public String getConnectionUrl() {
     return "jdbc:derby://localhost:1527/memory:sqoop;create=true";
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/main/java/org/apache/sqoop/test/db/MySQLProvider.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/db/MySQLProvider.java b/test/src/main/java/org/apache/sqoop/test/db/MySQLProvider.java
new file mode 100644
index 0000000..4f21935
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/db/MySQLProvider.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.db;
+
+/**
+ * MySQL Provider that will connect to remote MySQL server.
+ *
+ * JDBC can be configured via system properties. Default value is server running
+ * on the same box (localhost) that is access via sqoop/sqoop credentials.
+ */
+public class MySQLProvider extends DatabaseProvider {
+
+  public static final String DRIVER = "com.mysql.jdbc.Driver";
+
+  private static final String CONNECTION = System.getProperties().getProperty(
+    "sqoop.provider.mysql.jdbc",
+    "jdbc:mysql://localhost/test"
+  );
+
+  private static final String USERNAME = System.getProperties().getProperty(
+    "sqoop.provider.mysql.username",
+    "sqoop"
+  );
+
+  private static final String PASSWORD = System.getProperties().getProperty(
+    "sqoop.provider.mysql.password",
+    "sqoop"
+  );
+
+  @Override
+  public String getConnectionUrl() {
+    return CONNECTION;
+  }
+
+  @Override
+  public String getConnectionUsername() {
+    return USERNAME;
+  }
+
+  @Override
+  public String getConnectionPassword() {
+    return PASSWORD;
+  }
+
+  @Override
+  public String escapeColumnName(String columnName) {
+    return escape(columnName);
+  }
+
+  @Override
+  public String escapeTableName(String tableName) {
+    return escape(tableName);
+  }
+
+  @Override
+  public String escapeValueString(String value) {
+    return "\"" + value + "\"";
+  }
+
+  @Override
+  public String getJdbcDriver() {
+    return DRIVER;
+  }
+
+  public String escape(String entity) {
+    return "`" + entity + "`";
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/main/java/org/apache/sqoop/test/db/PostgreSQLProvider.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/db/PostgreSQLProvider.java b/test/src/main/java/org/apache/sqoop/test/db/PostgreSQLProvider.java
new file mode 100644
index 0000000..3259768
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/db/PostgreSQLProvider.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.db;
+
+/**
+ * PostgreSQL Provider that will connect to remote PostgreSQL server.
+ *
+ * JDBC can be configured via system properties. Default value is server running
+ * on the same box (localhost) that is access via sqoop/sqoop credentials.
+ */
+public class PostgreSQLProvider extends DatabaseProvider {
+
+  public static final String DRIVER = "org.postgresql.Driver";
+
+  private static final String CONNECTION = System.getProperties().getProperty(
+    "sqoop.provider.postgresql.jdbc",
+    "jdbc:postgresql://localhost/test"
+  );
+
+  private static final String USERNAME = System.getProperties().getProperty(
+    "sqoop.provider.postgresql.username",
+    "sqoop"
+  );
+
+  private static final String PASSWORD = System.getProperties().getProperty(
+    "sqoop.provider.postgresql.password",
+    "sqoop"
+  );
+
+  @Override
+  public String getConnectionUrl() {
+    return CONNECTION;
+  }
+
+  @Override
+  public String getConnectionUsername() {
+    return USERNAME;
+  }
+
+  @Override
+  public String getConnectionPassword() {
+    return PASSWORD;
+  }
+
+  @Override
+  public String escapeColumnName(String columnName) {
+    return escape(columnName);
+  }
+
+  @Override
+  public String escapeTableName(String tableName) {
+    return escape(tableName);
+  }
+
+  @Override
+  public String escapeValueString(String value) {
+    return "'" + value + "'";
+  }
+
+  @Override
+  public String getJdbcDriver() {
+    return DRIVER;
+  }
+
+  public String escape(String entity) {
+    return "\"" + entity + "\"";
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
index ed6c596..567ddbc 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
@@ -93,6 +93,8 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster {
          jar.contains("jackson-") || // Jackson
          jar.contains("derby")    || // Derby drivers
          jar.contains("avro-")    || // Avro
+         jar.contains("mysql")    || // MySQL JDBC driver
+         jar.contains("postgre")  || // PostgreSQL JDBC driver
          jar.contains("google")      // Google libraries (guava, ...)
        ) {
         extraClassPath.add(jar);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
index 7c8a978..8b46179 100644
--- a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
+++ b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
@@ -63,7 +63,7 @@ abstract public class TomcatTestCase {
 
   @Before
   public void startServer() throws Exception {
-    cluster = new TomcatSqoopMiniCluster(TMP_PATH);
+    cluster = new TomcatSqoopMiniCluster(getTemporaryPath());
     cluster.start();
   }
 
@@ -72,6 +72,10 @@ abstract public class TomcatTestCase {
     cluster.stop();
   }
 
+  public String getTemporaryPath() {
+    return TMP_PATH;
+  }
+
   /**
    * Return testing server URL
    *
@@ -87,7 +91,7 @@ abstract public class TomcatTestCase {
    * @return
    */
   public String getMapreduceDirectory() {
-    return cluster.getTemporaryPath() + "/mapreduce-job-io";
+    return getTemporaryPath() + "/mapreduce-job-io";
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
index 595810f..14f7dc4 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
@@ -17,9 +17,10 @@
  */
 package org.apache.sqoop.integration.connector;
 
+import org.apache.log4j.Logger;
 import org.apache.sqoop.integration.TomcatTestCase;
 import org.apache.sqoop.test.db.DatabaseProvider;
-import org.apache.sqoop.test.db.DerbyProvider;
+import org.apache.sqoop.test.db.DatabaseProviderFactory;
 import org.junit.After;
 import org.junit.Before;
 
@@ -30,11 +31,14 @@ import org.junit.Before;
  */
 abstract public class ConnectorTestCase extends TomcatTestCase {
 
+  private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class);
+
   protected DatabaseProvider provider;
 
   @Before
-  public void startProvider() {
-    provider = new DerbyProvider();
+  public void startProvider() throws Exception {
+    provider = DatabaseProviderFactory.getProvider(System.getProperties());
+    LOG.info("Starting database provider: " + provider.getClass().getName());
     provider.start();
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f64dba2e/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
index 3a8b1b5..20588d3 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
@@ -36,12 +36,13 @@ import org.apache.sqoop.model.MFramework;
 import org.apache.sqoop.model.MJob;
 import org.apache.sqoop.model.MStringInput;
 import org.apache.sqoop.model.MSubmission;
-import org.apache.sqoop.test.db.DerbyProvider;
 import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.Validation;
 import org.junit.Test;
 
+import java.util.Map;
+
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
 
@@ -82,8 +83,10 @@ public class TableImportTest extends ConnectorTestCase {
                                              framework.getConnectionForms());
 
     // Connector values
-    ((MStringInput) (connection.getConnectorPart().getForms().get(0).getInputs().get(0))).setValue(DerbyProvider.DRIVER);
+    ((MStringInput) (connection.getConnectorPart().getForms().get(0).getInputs().get(0))).setValue(provider.getJdbcDriver());
     ((MStringInput) (connection.getConnectorPart().getForms().get(0).getInputs().get(1))).setValue(provider.getConnectionUrl());
+    ((MStringInput) (connection.getConnectorPart().getForms().get(0).getInputs().get(2))).setValue(provider.getConnectionUsername());
+    ((MStringInput) (connection.getConnectorPart().getForms().get(0).getInputs().get(3))).setValue(provider.getConnectionPassword());
     // Framework values
     // No need to set anything
 
@@ -124,7 +127,7 @@ public class TableImportTest extends ConnectorTestCase {
 
     // Wait until the job finish
     do {
-      Thread.sleep(1000);
+      Thread.sleep(5000);
       submission = submissionRequest.read(getServerUrl(), "" + job.getPersistenceId()).getSubmission();
     } while(submission.getStatus().isRunning());
 


Mime
View raw message