sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject git commit: SQOOP-1124: Direct export to PostgreSQL with PGBulkloadManager fails without explicit port number settings
Date Thu, 11 Jul 2013 15:35:12 GMT
Updated Branches:
  refs/heads/trunk 73cc549d9 -> ab4bfb9a8


SQOOP-1124: Direct export to PostgreSQL with PGBulkloadManager fails without explicit port
number settings

(Masatake Iwasaki via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ab4bfb9a
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ab4bfb9a
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ab4bfb9a

Branch: refs/heads/trunk
Commit: ab4bfb9a8281d714b6549ba9d394d39e5c9c4d4a
Parents: 73cc549
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Thu Jul 11 08:34:34 2013 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Thu Jul 11 08:34:34 2013 -0700

----------------------------------------------------------------------
 build.xml                                       | 24 +++++++++++++++++++
 .../postgresql/PGBulkloadExportJob.java         |  1 -
 .../postgresql/PGBulkloadExportMapper.java      |  6 +++--
 .../manager/PGBulkloadManagerManualTest.java    | 25 +++++++++++++++-----
 4 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index ef657e3..e3683f8 100644
--- a/build.xml
+++ b/build.xml
@@ -254,6 +254,18 @@
   <property name="sqoop.test.postgresql.connectstring.host_url"
             value="jdbc:postgresql://localhost/"/>
 
+  <property name="sqoop.test.postgresql.database"
+            value="sqooptest" />
+
+  <property name="sqoop.test.postgresql.tablespace"
+            value="sqooptest" />
+
+  <property name="sqoop.test.postgresql.username"
+            value="sqooptest" />
+
+  <property name="sqoop.test.postgresql.pg_bulkload"
+            value="pg_bulkload" />
+
   <property name="sqoop.test.sqlserver.connectstring.host_url"
             value="jdbc:sqlserver://sqlserverhost:1433"/>
 
@@ -818,6 +830,18 @@
       <sysproperty key="sqoop.test.postgresql.connectstring.host_url"
                    value="${sqoop.test.postgresql.connectstring.host_url}"/>
 
+      <sysproperty key="sqoop.test.postgresql.database"
+                   value="${sqoop.test.postgresql.database}" />
+
+      <sysproperty key="sqoop.test.postgresql.tablespace"
+                   value="${sqoop.test.postgresql.tablespace}" />
+
+      <sysproperty key="sqoop.test.postgresql.username"
+                   value="${sqoop.test.postgresql.username}" />
+
+      <sysproperty key="sqoop.test.postgresql.pg_bulkload"
+                   value="${sqoop.test.postgresql.pg_bulkload}" />
+
       <sysproperty key="sqoop.test.sqlserver.connectstring.host_url"
                    value="${sqoop.test.sqlserver.connectstring.host_url}"/>
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java
index 79fb7da..1e2ad9f 100644
--- a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportJob.java
@@ -148,7 +148,6 @@ public class PGBulkloadExportJob extends ExportJobBase {
 
   @Override
   public void runExport() throws ExportException, IOException {
-    ConnManager cmgr = context.getConnManager();
     SqoopOptions options = context.getOptions();
     Configuration conf = options.getConf();
     DBConfiguration dbConf = null;

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java
index 333546f..16af84c 100644
--- a/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java
+++ b/src/java/org/apache/sqoop/mapreduce/postgresql/PGBulkloadExportMapper.java
@@ -125,8 +125,10 @@ public class PGBulkloadExportMapper
           + JdbcUrl.getDatabaseName(conf.get(DBConfiguration.URL_PROPERTY)));
       args.add("--host="
           + JdbcUrl.getHostName(conf.get(DBConfiguration.URL_PROPERTY)));
-      args.add("--port="
-          + JdbcUrl.getPort(conf.get(DBConfiguration.URL_PROPERTY)));
+      int port = JdbcUrl.getPort(conf.get(DBConfiguration.URL_PROPERTY));
+      if (port != -1) {
+        args.add("--port=" + port);
+      }
       args.add("--input=stdin");
       args.add("--output=" + tmpTableName);
       args.add("-o");

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ab4bfb9a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java b/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
index 0403614..4d03a8b 100644
--- a/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PGBulkloadManagerManualTest.java
@@ -26,7 +26,7 @@ import java.util.Arrays;
 import java.util.ArrayList;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
 import com.cloudera.sqoop.TestExport;
 import com.cloudera.sqoop.mapreduce.db.DBConfiguration;
 
@@ -64,10 +64,21 @@ public class PGBulkloadManagerManualTest extends TestExport {
   public static final Log LOG =
       LogFactory.getLog(PGBulkloadManagerManualTest.class.getName());
   private DBConfiguration dbConf;
-
+  static final String HOST_URL =
+    System.getProperty("sqoop.test.postgresql.connectstring.host_url",
+                       "jdbc:postgresql://localhost/");
+  static final String DATABASE =
+    System.getProperty("sqoop.test.postgresql.database", "sqooptest");
+  static final String TABLESPACE =
+    System.getProperty("sqoop.test.postgresql.tablespace", "sqooptest");
+  static final String USERNAME =
+    System.getProperty("sqoop.test.postgresql.username", "sqooptest");
+  static final String PG_BULKLOAD =
+    System.getProperty("sqoop.test.postgresql.pg_bulkload", "pg_bulkload");
+  static final String CONNECT_STRING = HOST_URL + DATABASE;
 
   public PGBulkloadManagerManualTest() {
-    Configuration conf = getConf();
+    JobConf conf = new JobConf(getConf());
     DBConfiguration.configureDB(conf,
                                 "org.postgresql.Driver",
                                 getConnectString(),
@@ -85,12 +96,12 @@ public class PGBulkloadManagerManualTest extends TestExport {
 
   @Override
   protected String getConnectString() {
-    return "jdbc:postgresql://localhost:5432/sqooptest";
+    return CONNECT_STRING;
   }
 
 
   protected String getUserName() {
-    return "sqooptest";
+    return USERNAME;
   }
 
 
@@ -144,6 +155,8 @@ public class PGBulkloadManagerManualTest extends TestExport {
                              String... additionalArgv) {
     ArrayList<String> args =
         new ArrayList<String>(Arrays.asList(additionalArgv));
+    args.add("-D");
+    args.add("pgbulkload.bin=" + PG_BULKLOAD);
     args.add("--username");
     args.add(getUserName());
     args.add("--connection-manager");
@@ -181,7 +194,7 @@ public class PGBulkloadManagerManualTest extends TestExport {
 
   public void testExportWithTablespace() throws IOException, SQLException {
     String[] genericargs =
-      newStrArray(null, "-Dpgbulkload.staging.tablespace=sqooptest");
+      newStrArray(null, "-Dpgbulkload.staging.tablespace=" + TABLESPACE);
     multiFileTestWithGenericArgs(1, 10, 1, genericargs);
   }
 


Mime
View raw message