sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hshreedha...@apache.org
Subject sqoop git commit: SQOOP-2847. Sqoop --incremental + missing parent --target-dir reports success with no data
Date Tue, 01 Mar 2016 18:42:53 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 5c4d06d72 -> c2351511e


SQOOP-2847. Sqoop --incremental + missing parent --target-dir reports success with no data

(Jarcec via Hari)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/c2351511
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/c2351511
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/c2351511

Branch: refs/heads/trunk
Commit: c2351511e618d1abb3cce78eab85545247a233c0
Parents: 5c4d06d
Author: Hari Shreedharan <hshreedharan@apache.org>
Authored: Tue Mar 1 10:42:21 2016 -0800
Committer: Hari Shreedharan <hshreedharan@apache.org>
Committed: Tue Mar 1 10:42:21 2016 -0800

----------------------------------------------------------------------
 src/java/org/apache/sqoop/tool/ImportTool.java  | 12 +++++++++-
 .../cloudera/sqoop/TestIncrementalImport.java   | 23 +++++++++++++++++++-
 2 files changed, 33 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/c2351511/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index 39af42c..ad1d48b 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -430,6 +430,7 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
   protected void lastModifiedMerge(SqoopOptions options, ImportJobContext context) throws
IOException {
     FileSystem fs = FileSystem.get(options.getConf());
     if (context.getDestination() != null && fs.exists(context.getDestination()))
{
+      LOG.info("Final destination exists, will run merge job.");
       Path userDestDir = getOutputPath(options, context.getTableName(), false);
       if (fs.exists(userDestDir)) {
         String tableClassName = null;
@@ -461,7 +462,16 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
 
         unloadJars();
       } else {
-        fs.rename(context.getDestination(), userDestDir);
+        // Create parent directory(ies), otherwise fs.rename would fail
+        if(!fs.exists(userDestDir.getParent())) {
+          fs.mkdirs(userDestDir.getParent());
+        }
+
+        // And finally move the data
+        LOG.info("Moving data from temporary directory " + context.getDestination() + " to
final destination " + userDestDir);
+        if(!fs.rename(context.getDestination(), userDestDir)) {
+          throw new RuntimeException("Couldn't move data from temporary directory " + context.getDestination()
+ " to final destination " + userDestDir);
+        }
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/c2351511/src/test/com/cloudera/sqoop/TestIncrementalImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestIncrementalImport.java b/src/test/com/cloudera/sqoop/TestIncrementalImport.java
index d28a187..d62532c 100644
--- a/src/test/com/cloudera/sqoop/TestIncrementalImport.java
+++ b/src/test/com/cloudera/sqoop/TestIncrementalImport.java
@@ -49,6 +49,8 @@ import com.cloudera.sqoop.testutil.CommonArgs;
 import com.cloudera.sqoop.tool.ImportTool;
 import com.cloudera.sqoop.tool.JobTool;
 
+import javax.management.Query;
+
 /**
  * Test the incremental import functionality.
  *
@@ -543,12 +545,16 @@ public class TestIncrementalImport extends TestCase {
     if (commonArgs) {
       CommonArgs.addHadoopFlags(args);
     }
+
+    String [] directoryNames = directoryName.split("/");
+    String className = directoryNames[directoryNames.length -1];
+
     args.add("--connect");
     args.add(SOURCE_DB_URL);
     args.add("--query");
     args.add(query);
     args.add("--class-name");
-    args.add(directoryName);
+    args.add(className);
     args.add("--target-dir");
     args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR
       + System.getProperty("file.separator") + directoryName);
@@ -775,6 +781,21 @@ public class TestIncrementalImport extends TestCase {
     assertDirOfNumbers(TABLE_NAME, 0);
   }
 
+  public void testEmptyLastModifiedWithNonExistingParentDirectory() throws Exception {
+    final String TABLE_NAME = "emptyLastModifiedNoParent";
+    final String QUERY = "SELECT id, last_modified FROM \"" + TABLE_NAME + "\" WHERE $CONDITIONS";
+    final String DIRECTORY = "non-existing/parents/" + TABLE_NAME;
+    createTimestampTable(TABLE_NAME, 0, null);
+    List<String> args = getArgListForQuery(QUERY, DIRECTORY, true, false, false);
+
+    Configuration conf = newConf();
+    SqoopOptions options = new SqoopOptions();
+    options.setConf(conf);
+    runImport(options, args);
+
+    assertDirOfNumbers(DIRECTORY, 0);
+  }
+
   public void testFullLastModifiedImport() throws Exception {
     // Given a table of rows imported in the past,
     // see that they are imported.


Mime
View raw message