sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From abhij...@apache.org
Subject git commit: SQOOP-824: Sqoop code generation in 'update' export mode incompatible with '--columns' option (Jarek Jarcec Cecho via Abhijeet Gaikwad)
Date Sun, 20 Jan 2013 15:31:36 GMT
Updated Branches:
  refs/heads/trunk 2b0456d6c -> 9a3fd1484


SQOOP-824: Sqoop code generation in 'update' export mode incompatible with '--columns' option
(Jarek Jarcec Cecho via Abhijeet Gaikwad)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9a3fd148
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9a3fd148
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9a3fd148

Branch: refs/heads/trunk
Commit: 9a3fd1484f7ff713a323f5687cb9681fc0c524da
Parents: 2b0456d
Author: Abhijeet Gaikwad <abhijeet@apache.org>
Authored: Sun Jan 20 20:56:44 2013 +0530
Committer: Abhijeet Gaikwad <abhijeet@apache.org>
Committed: Sun Jan 20 20:56:44 2013 +0530

----------------------------------------------------------------------
 src/java/org/apache/sqoop/manager/ConnManager.java |   26 ++++++++-
 src/test/com/cloudera/sqoop/TestExportUpdate.java  |   46 ++++++++++++++-
 2 files changed, 70 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/9a3fd148/src/java/org/apache/sqoop/manager/ConnManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/ConnManager.java b/src/java/org/apache/sqoop/manager/ConnManager.java
index 115186f..358981e 100644
--- a/src/java/org/apache/sqoop/manager/ConnManager.java
+++ b/src/java/org/apache/sqoop/manager/ConnManager.java
@@ -506,6 +506,17 @@ public abstract class ConnManager {
     // last, because the UPDATE-based OutputFormat will generate the SET
     // clause followed by the WHERE clause, and the SqoopRecord needs to
     // serialize to this layout.
+
+    // Check if user specified --columns parameter
+    Set<String> columns = null;
+    if (options.getColumns() != null && options.getColumns().length > 0) {
+      // If so, put all column in uppercase form into our help set
+      columns = new HashSet<String>();
+      for(String c : options.getColumns()) {
+        columns.add(c.toUpperCase());
+      }
+    }
+
     Set<String> updateKeys = new LinkedHashSet<String>();
     Set<String> updateKeysUppercase = new HashSet<String>();
     String updateKeyValue = options.getUpdateKeyCol();
@@ -513,8 +524,16 @@ public abstract class ConnManager {
     while (stok.hasMoreTokens()) {
       String nextUpdateColumn = stok.nextToken().trim();
       if (nextUpdateColumn.length() > 0) {
+        String upperCase = nextUpdateColumn.toUpperCase();
+
+        // We must make sure that --columns is super set of --update-key
+        if (columns != null && !columns.contains(upperCase)) {
+          throw new RuntimeException("You must specify all columns from "
+            + "--update-key parameter in --columns parameter.");
+        }
+
         updateKeys.add(nextUpdateColumn);
-        updateKeysUppercase.add(nextUpdateColumn.toUpperCase());
+        updateKeysUppercase.add(upperCase);
       } else {
         throw new RuntimeException("Invalid update key column value specified"
                     + ": '" + updateKeyValue + "'");
@@ -524,6 +543,11 @@ public abstract class ConnManager {
     List<String> dbOutCols = new ArrayList<String>();
     for (String col : allColNames) {
       if (!updateKeysUppercase.contains(col.toUpperCase())) {
+        // Skip columns that were not explicitly stated on command line
+        if (columns != null && !columns.contains(col.toUpperCase())) {
+          continue;
+        }
+
         dbOutCols.add(col); // add non-key columns to the output order list.
       }
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/9a3fd148/src/test/com/cloudera/sqoop/TestExportUpdate.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestExportUpdate.java b/src/test/com/cloudera/sqoop/TestExportUpdate.java
index f5c30f3..95d7b6a 100644
--- a/src/test/com/cloudera/sqoop/TestExportUpdate.java
+++ b/src/test/com/cloudera/sqoop/TestExportUpdate.java
@@ -98,7 +98,7 @@ public class TestExportUpdate extends ExportJobTestCase {
    *    1   |   1   | 1foo1
    *    1   |   2   | 1foo2
    * </pre></p>
-   * @param firstKeyRange the number of
+   * @param aMax the number of
    * @throws SQLException
    */
   private void createMultiKeyTable(int aMax) throws SQLException {
@@ -642,4 +642,48 @@ public class TestExportUpdate extends ExportJobTestCase {
     verifyRow("A", "9", "9", "foo18", "18");
   }
 
+  /**
+   * Test updating only subset of the columns.
+   *
+   * @throws Exception
+   */
+  public void testUpdateColumnSubset() throws Exception {
+    populateDatabase(4);
+    createUpdateFiles(1, 3, 0);
+
+    runExport(getArgv(true, 2, 2, "-m", "1",
+      "--update-key", "A", "--columns", "A,B"));
+
+    verifyRowCount(4);
+
+    // First column should not have any changes (even though it was updated)
+    verifyRow("A", "0", "0", "foo0", "0");
+
+    // Second column have updated column B, but C should be left untouched
+    verifyRow("A", "1", "1", "foo2", "1");
+
+    // Third column have updated column B, but C should be left untouched
+    verifyRow("A", "2", "2", "foo4", "2");
+
+    // Last columns should be completely untouched
+    verifyRow("A", "3", "3", "foo3", "3");
+  }
+
+  /**
+   * Parameter --columns must be superset of --update-key in order for
+   * CompilationManager and other parts of the framework work correctly.
+   *
+   * @throws Exception
+   */
+  public void testUpdateColumnNotInColumns() throws Exception {
+    populateDatabase(1);
+    try {
+      runExport(getArgv(true, 2, 2, "-m", "1",
+        "--update-key", "A", "--columns", "B"));
+      fail("Expected IOException");
+    } catch (IOException e) {
+      assertTrue(true);
+    }
+  }
+
 }


Mime
View raw message