sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject svn commit: r1180279 - in /incubator/sqoop/trunk/src: java/com/cloudera/sqoop/manager/SqlManager.java java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java test/com/cloudera/sqoop/TestBoundaryQuery.java
Date Fri, 07 Oct 2011 23:42:01 GMT
Author: blee
Date: Fri Oct  7 23:42:00 2011
New Revision: 1180279

URL: http://svn.apache.org/viewvc?rev=1180279&view=rev
Log:
SQOOP-359 Import fails with Unknown SQL datatype exception

Modified:
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java
    incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java?rev=1180279&r1=1180278&r2=1180279&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java Fri Oct  7 23:42:00
2011
@@ -440,11 +440,19 @@ public abstract class SqlManager extends
     }
 
     String splitCol = getSplitColumn(opts, null);
-    if (null == splitCol && opts.getNumMappers() > 1) {
-      // Can't infer a primary key.
-      throw new ImportException("A split-by column must be specified for "
-          + "parallel free-form query imports. Please specify one with "
-          + "--split-by or perform a sequential import with '-m 1'.");
+    if (splitCol == null) {
+      String boundaryQuery = opts.getBoundaryQuery();
+      if (opts.getNumMappers() > 1) {
+        // Can't infer a primary key.
+        throw new ImportException("A split-by column must be specified for "
+            + "parallel free-form query imports. Please specify one with "
+            + "--split-by or perform a sequential import with '-m 1'.");
+      } else if (boundaryQuery != null && !boundaryQuery.isEmpty()) {
+        // Query import with boundary query and no split column specified
+        throw new ImportException("Using a boundary query for a query based "
+            + "import requires specifying the split by column as well. Please "
+            + "specify a column name using --split-by and try again.");
+      }
     }
 
     importer.runImport(null, jarFile, splitCol, opts.getConf());

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java?rev=1180279&r1=1180278&r2=1180279&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/DataDrivenImportJob.java Fri
Oct  7 23:42:00 2011
@@ -171,11 +171,16 @@ public class DataDrivenImportJob extends
         String inputBoundingQuery = options.getBoundaryQuery();
 
         if(inputBoundingQuery == null) {
+          inputBoundingQuery =
             mgr.getInputBoundsQuery(splitByCol, sanitizedQuery);
-        }
-        if (inputBoundingQuery == null) {
-            inputBoundingQuery = "SELECT MIN(" + splitByCol + "), MAX("
-                    + splitByCol + ") FROM (" + sanitizedQuery + ") AS t1";
+          if (inputBoundingQuery == null) {
+            if (splitByCol != null) {
+              inputBoundingQuery = "SELECT MIN(" + splitByCol + "), MAX("
+                      + splitByCol + ") FROM (" + sanitizedQuery + ") AS t1";
+            } else {
+              inputBoundingQuery = "";
+            }
+          }
         }
         DataDrivenDBInputFormat.setInput(job, DBWritable.class,
             inputQuery, inputBoundingQuery);

Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java?rev=1180279&r1=1180278&r2=1180279&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestBoundaryQuery.java Fri Oct  7 23:42:00
2011
@@ -46,8 +46,9 @@ public class TestBoundaryQuery extends I
    * Create the argv to pass to Sqoop.
    * @return the argv as an array of strings.
    */
-  protected String [] getArgv(boolean includeHadoopFlags, String boundaryQuery,
-      String targetDir) {
+  protected String [] getArgv(boolean includeHadoopFlags, boolean tableImport,
+      String boundaryQuery,
+      String targetDir, String... extraArgs) {
 
     ArrayList<String> args = new ArrayList<String>();
 
@@ -55,20 +56,29 @@ public class TestBoundaryQuery extends I
       CommonArgs.addHadoopFlags(args);
     }
 
-    args.add("--table");
-    args.add(HsqldbTestServer.getTableName());
-    args.add("--split-by");
-    args.add("INTFIELD1");
+    if (tableImport) {
+      args.add("--table");
+      args.add(HsqldbTestServer.getTableName());
+    } else {
+      args.add("--query");
+      args.add("SELECT INTFIELD1, INTFIELD2 FROM "
+            + HsqldbTestServer.getTableName() + " WHERE $CONDITIONS");
+    }
     args.add("--connect");
     args.add(HsqldbTestServer.getUrl());
-    args.add("--boundary-query");
-    args.add(boundaryQuery);
+    if (boundaryQuery != null) {
+      args.add("--boundary-query");
+      args.add(boundaryQuery);
+    }
     args.add("--as-sequencefile");
     args.add("--target-dir");
     args.add(targetDir);
     args.add("--class-name");
     args.add(getTableName());
     args.add("--verbose");
+    for (String extraArg : extraArgs) {
+      args.add(extraArg);
+    }
 
     return args.toArray(new String[0]);
   }
@@ -89,18 +99,18 @@ public class TestBoundaryQuery extends I
     return Integer.parseInt(parts[0]);
   }
 
-  public void runQueryTest(String query, int numExpectedResults,
-    int expectedSum, String targetDir)
-      throws IOException {
+  public void runQueryTest(String query, boolean tableImport,
+      int numExpectedResults, int expectedSum, String targetDir,
+      String... extraArgs) throws IOException {
 
     ClassLoader prevClassLoader = null;
     SequenceFile.Reader reader = null;
 
-    String [] argv = getArgv(true, query, targetDir);
+    String [] argv = getArgv(true, tableImport, query, targetDir, extraArgs);
     runImport(argv);
     try {
       SqoopOptions opts = new ImportTool().parseArguments(
-          getArgv(false, query, targetDir),
+          getArgv(false, tableImport, query, targetDir, extraArgs),
           null, null, true);
 
       CompilationManager compileMgr = new CompilationManager(opts);
@@ -161,6 +171,13 @@ public class TestBoundaryQuery extends I
     String query = "select min(intfield1), max(intfield1) from "
       + getTableName() +" where intfield1 in (3, 5)";
 
-    runQueryTest(query, 2, 8, getTablePath().toString());
+    runQueryTest(query, true, 2, 8, getTablePath().toString(),
+        "--split-by", "INTFIELD1");
+  }
+
+  public void testNoBoundaryQuerySingleMapper() throws IOException {
+
+      runQueryTest(null, false, 4, 16, getTablePath().toString(),
+          "--m", "1");
   }
 }



Mime
View raw message