sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject git commit: SQOOP-901: Allow user to override hardcoded boolean strings in direct PostgreSQL connector
Date Sun, 24 Feb 2013 03:43:45 GMT
Updated Branches:
  refs/heads/trunk 0affc4ba9 -> 8c3c78b11


SQOOP-901: Allow user to override hardcoded boolean strings in direct PostgreSQL connector

(Jarcec Cecho via Cheolsoo Park)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/8c3c78b1
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/8c3c78b1
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/8c3c78b1

Branch: refs/heads/trunk
Commit: 8c3c78b11cb5c2330e31cbf867d32974a011f268
Parents: 0affc4b
Author: Cheolsoo Park <cheolsoo@apache.org>
Authored: Sat Feb 23 19:31:59 2013 -0800
Committer: Cheolsoo Park <cheolsoo@apache.org>
Committed: Sat Feb 23 19:31:59 2013 -0800

----------------------------------------------------------------------
 src/docs/user/connectors.txt                       |   16 ++++
 .../sqoop/manager/DirectPostgresqlManager.java     |   69 ++++++++++++++-
 .../apache/sqoop/manager/PostgresqlManager.java    |    9 ++-
 .../sqoop/manager/PostgresqlImportTest.java        |   57 ++++++++----
 4 files changed, 125 insertions(+), 26 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/docs/user/connectors.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/connectors.txt b/src/docs/user/connectors.txt
index ff424f7..7dd2a2e 100644
--- a/src/docs/user/connectors.txt
+++ b/src/docs/user/connectors.txt
@@ -98,6 +98,22 @@ Argument                                 Description
                                          Default is "public".
 ---------------------------------------------------------------------------------
 
+The direct connector (used when specified +\--direct+ parameter), offers also
+additional extra arguments:
+
+.Additional supported PostgreSQL extra arguments in direct mode:
+[grid="all"]
+`----------------------------------------`---------------------------------------
+Argument                                 Description
+---------------------------------------------------------------------------------
++\--boolean-true-string <str>+           String that will be used to encode \
+                                         +true+ value of +boolean+ columns.
+                                         Default is "TRUE".
++\--boolean-false-string <str>+          String that will be used to encode \
+                                         +false+ value of +boolean+ columns.
+                                         Default is "FALSE".
+---------------------------------------------------------------------------------
+
 Schema support
 ^^^^^^^^^^^^^^
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
index a05bf60..c085218 100644
--- a/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
+++ b/src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
@@ -30,9 +30,12 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.sqoop.cli.RelatedOptions;
 import org.apache.sqoop.util.PostgreSQLUtils;
 
 import com.cloudera.sqoop.SqoopOptions;
@@ -54,15 +57,33 @@ import org.apache.sqoop.util.SubstitutionUtils;
  */
 public class DirectPostgresqlManager
     extends com.cloudera.sqoop.manager.PostgresqlManager {
+
   public static final Log LOG = LogFactory.getLog(
       DirectPostgresqlManager.class.getName());
 
+  public static final String BOOLEAN_TRUE_STRING = "boolean-true-string";
+  public static final String DEFAULT_BOOLEAN_TRUE_STRING = "TRUE";
+
+  public static final String BOOLEAN_FALSE_STRING = "boolean-false-string";
+  public static final String DEFAULT_BOOLEAN_FALSE_STRING = "FALSE";
+
   public DirectPostgresqlManager(final SqoopOptions opts) {
     super(opts);
+
+    if (this.booleanFalseString == null) {
+      this.booleanFalseString = DEFAULT_BOOLEAN_FALSE_STRING;
+    }
+    if (booleanTrueString == null) {
+      this.booleanTrueString = DEFAULT_BOOLEAN_TRUE_STRING;
+    }
   }
 
   private static final String PSQL_CMD = "psql";
 
+  private String booleanTrueString;
+
+  private String booleanFalseString;
+
   /** Copies data directly into HDFS, adding the user's chosen line terminator
       char to each record.
     */
@@ -196,12 +217,18 @@ public class DirectPostgresqlManager
         sb.append(col);
       } else {
         if ("bool".equalsIgnoreCase(columnTypes.get(col))) {
-          sb.append(String.format("case when %s=true then 'TRUE' "
-          + "when %s=false then 'FALSE' end as %s",
+          sb.append(String.format("case when %s=true then "
+          + "'" + booleanTrueString + "' "
+          + "when %s=false then "
+          +  "'" + booleanFalseString + "'"
+          +  " end as %s",
           colEscaped, colEscaped, colEscaped));
         } else if ("bit".equalsIgnoreCase(columnTypes.get(col))) {
-          sb.append(String.format("case when %s=B'1' then 'TRUE' "
-          + "when %s=B'0' then 'FALSE' end as %s",
+          sb.append(String.format("case when %s=B'1' then "
+          + "'" + booleanTrueString + "' "
+          + "when %s=B'0' then "
+          +  "'" + booleanFalseString + "'"
+          +  " end as %s",
           colEscaped, colEscaped, colEscaped));
         } else {
           sb.append(colEscaped);
@@ -508,4 +535,38 @@ public class DirectPostgresqlManager
     return false;
   }
   // CHECKSTYLE:ON
+
+  /** {@inheritDoc}. */
+  @Override
+  protected void applyExtraArguments(CommandLine cmdLine) {
+    super.applyExtraArguments(cmdLine);
+
+    if (cmdLine.hasOption(BOOLEAN_TRUE_STRING)) {
+      String arg = cmdLine.getOptionValue(BOOLEAN_TRUE_STRING);
+      LOG.info("Loaded TRUE encoding string " + arg);
+      this.booleanTrueString = arg;
+    }
+    if (cmdLine.hasOption(BOOLEAN_FALSE_STRING)) {
+      String arg = cmdLine.getOptionValue(BOOLEAN_FALSE_STRING);
+      LOG.info("Loaded FALSE encoding string " + arg);
+      this.booleanFalseString = arg;
+    }
+  }
+
+  /** {@inheritDoc}. */
+  @Override
+  @SuppressWarnings("static-access")
+  protected RelatedOptions getExtraOptions() {
+    RelatedOptions extraOptions = super.getExtraOptions();
+
+    extraOptions.addOption(OptionBuilder.withArgName("string").hasArg()
+      .withDescription("String to encode TRUE value")
+      .withLongOpt(BOOLEAN_TRUE_STRING).create());
+
+    extraOptions.addOption(OptionBuilder.withArgName("string").hasArg()
+      .withDescription("String to encode FALSE value")
+      .withLongOpt(BOOLEAN_FALSE_STRING).create());
+
+    return extraOptions;
+  }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/java/org/apache/sqoop/manager/PostgresqlManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/PostgresqlManager.java b/src/java/org/apache/sqoop/manager/PostgresqlManager.java
index facdc74..609a457 100644
--- a/src/java/org/apache/sqoop/manager/PostgresqlManager.java
+++ b/src/java/org/apache/sqoop/manager/PostgresqlManager.java
@@ -196,7 +196,7 @@ public class PostgresqlManager
    * @param args Extra arguments array
    * @throws ParseException
    */
-  void parseExtraArgs(String[] args) throws ParseException {
+  private void parseExtraArgs(String[] args) throws ParseException {
     // No-op when no extra arguments are present
     if (args == null || args.length == 0) {
       return;
@@ -207,6 +207,11 @@ public class PostgresqlManager
     CommandLineParser parser = new GnuParser();
     CommandLine cmdLine = parser.parse(getExtraOptions(), args, true);
 
+    // Apply parsed arguments
+    applyExtraArguments(cmdLine);
+  }
+
+  protected void applyExtraArguments(CommandLine cmdLine) {
     // Apply extra options
     if (cmdLine.hasOption(SCHEMA)) {
       String schemaName = cmdLine.getOptionValue(SCHEMA);
@@ -222,7 +227,7 @@ public class PostgresqlManager
    * @return
    */
   @SuppressWarnings("static-access")
-  private RelatedOptions getExtraOptions() {
+  protected RelatedOptions getExtraOptions() {
     // Connection args (common)
     RelatedOptions extraOptions =
       new RelatedOptions("PostgreSQL extra options:");

http://git-wip-us.apache.org/repos/asf/sqoop/blob/8c3c78b1/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
index ee00c41..3fadff7 100644
--- a/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
@@ -139,12 +139,13 @@ public class PostgresqlImportTest extends ImportJobTestCase {
         connection.commit();
       } catch (SQLException e) {
          LOG.info("Couldn't create schema " + schema + " (is o.k. as long as"
-           + "the schema already exists.", e);
+           + "the schema already exists.");
         connection.rollback();
       }
 
       String fullTableName = manager.escapeTableName(schema)
         + "." + manager.escapeTableName(tableName);
+      LOG.info("Creating table: " + fullTableName);
 
       try {
         // Try to remove the table first. DROP TABLE IF EXISTS didn't
@@ -152,8 +153,7 @@ public class PostgresqlImportTest extends ImportJobTestCase {
         // any exception here if one occurs.
         st.executeUpdate("DROP TABLE " + fullTableName);
       } catch (SQLException e) {
-        LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)",
-          e);
+        LOG.info("Couldn't drop table " + schema + "." + tableName + " (ok)");
         // Now we need to reset the transaction.
         connection.rollback();
       }
@@ -163,17 +163,18 @@ public class PostgresqlImportTest extends ImportJobTestCase {
           + manager.escapeColName("name") + " VARCHAR(24) NOT NULL, "
           + manager.escapeColName("start_date") + " DATE, "
           + manager.escapeColName("Salary") + " FLOAT, "
+          + manager.escapeColName("Fired") + " BOOL, "
           + manager.escapeColName("dept") + " VARCHAR(32))");
 
       st.executeUpdate("INSERT INTO " + fullTableName
-          + " VALUES(1,'Aaron','2009-05-14',1000000.00,'engineering')");
+          + " VALUES(1,'Aaron','2009-05-14',1000000.00,TRUE,'engineering')");
       st.executeUpdate("INSERT INTO " + fullTableName
-          + " VALUES(2,'Bob','2009-04-20',400.00,'sales')");
+          + " VALUES(2,'Bob','2009-04-20',400.00,TRUE,'sales')");
       st.executeUpdate("INSERT INTO " + fullTableName
-          + " VALUES(3,'Fred','2009-01-23',15.00,'marketing')");
+          + " VALUES(3,'Fred','2009-01-23',15.00,FALSE,'marketing')");
       if (nullEntry) {
         st.executeUpdate("INSERT INTO " + fullTableName
-          + " VALUES(4,'Mike',NULL,NULL,NULL)");
+          + " VALUES(4,'Mike',NULL,NULL,NULL,NULL)");
 
       }
       connection.commit();
@@ -276,8 +277,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
   @Test
   public void testJdbcBasedImport() throws IOException {
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400.0,sales",
-      "3,Fred,2009-01-23,15.0,marketing",
+      "2,Bob,2009-04-20,400.0,true,sales",
+      "3,Fred,2009-01-23,15.0,false,marketing",
     };
 
     doImportAndVerify(false, expectedResults, TABLE_NAME);
@@ -286,8 +287,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
   @Test
   public void testDirectImport() throws IOException {
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
+      "2,Bob,2009-04-20,400,TRUE,sales",
+      "3,Fred,2009-01-23,15,FALSE,marketing",
     };
 
     doImportAndVerify(true, expectedResults, TABLE_NAME);
@@ -309,8 +310,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
   @Test
   public void testTableNameWithSpecialCharacter() throws IOException {
     String [] expectedResults = {
-        "2,Bob,2009-04-20,400.0,sales",
-        "3,Fred,2009-01-23,15.0,marketing",
+        "2,Bob,2009-04-20,400.0,true,sales",
+        "3,Fred,2009-01-23,15.0,false,marketing",
     };
 
     doImportAndVerify(false, expectedResults, SPECIAL_TABLE_NAME);
@@ -330,8 +331,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
  @Test
   public void testDifferentSchemaImport() throws IOException {
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400.0,sales",
-      "3,Fred,2009-01-23,15.0,marketing",
+      "2,Bob,2009-04-20,400.0,true,sales",
+      "3,Fred,2009-01-23,15.0,false,marketing",
     };
 
     String [] extraArgs = { "--",
@@ -344,8 +345,8 @@ public class PostgresqlImportTest extends ImportJobTestCase {
   @Test
   public void testDifferentSchemaImportDirect() throws IOException {
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
+      "2,Bob,2009-04-20,400,TRUE,sales",
+      "3,Fred,2009-01-23,15,FALSE,marketing",
     };
 
     String [] extraArgs = { "--",
@@ -358,9 +359,9 @@ public class PostgresqlImportTest extends ImportJobTestCase {
   @Test
   public void testNullEscapeCharacters() throws Exception {
      String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
-      "4,Mike,\\N,\\N,\\N",
+      "2,Bob,2009-04-20,400,TRUE,sales",
+      "3,Fred,2009-01-23,15,FALSE,marketing",
+      "4,Mike,\\N,\\N,\\N,\\N",
     };
 
     String [] extraArgs = {
@@ -370,4 +371,20 @@ public class PostgresqlImportTest extends ImportJobTestCase {
 
     doImportAndVerify(true, expectedResults, NULL_TABLE_NAME, extraArgs);
   }
+
+  @Test
+  public void testDifferentBooleanValues() throws Exception {
+    String [] expectedResults = {
+      "2,Bob,2009-04-20,400,REAL_TRUE,sales",
+      "3,Fred,2009-01-23,15,REAL_FALSE,marketing",
+    };
+
+    String [] extraArgs = {
+      "--",
+      "--boolean-true-string", "REAL_TRUE",
+      "--boolean-false-string", "REAL_FALSE",
+    };
+
+    doImportAndVerify(true, expectedResults, TABLE_NAME, extraArgs);
+  }
 }


Mime
View raw message