sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-1281: Support of glob paths during export
Date Tue, 06 Oct 2015 15:52:53 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 967f0eb0f -> 02e36db2b


SQOOP-1281: Support of glob paths during export

(Clément MAHTIEU via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/02e36db2
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/02e36db2
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/02e36db2

Branch: refs/heads/trunk
Commit: 02e36db2b8deee01ae08a493369097b6812a164e
Parents: 967f0eb
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Tue Oct 6 08:51:20 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Tue Oct 6 08:51:20 2015 -0700

----------------------------------------------------------------------
 .../apache/sqoop/mapreduce/ExportJobBase.java   | 16 ++++++++++++----
 src/test/com/cloudera/sqoop/TestAvroExport.java | 20 ++++++++++++++++++++
 2 files changed, 32 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/02e36db2/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
index f9fa7f3..068b8f7 100644
--- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
@@ -127,18 +127,26 @@ public class ExportJobBase extends JobBase {
     FileSystem fs = p.getFileSystem(conf);
 
     try {
-      FileStatus stat = fs.getFileStatus(p);
+      FileStatus[] fileStatuses = fs.globStatus(p);
 
-      if (null == stat) {
+      if (null == fileStatuses) {
         // Couldn't get the item.
         LOG.warn("Input path " + p + " does not exist");
         return FileType.UNKNOWN;
       }
 
+      if (fileStatuses.length == 0) {
+        LOG.warn("Input path " + p + " does not match any file");
+        return FileType.UNKNOWN;
+      }
+
+      FileStatus stat = fileStatuses[0];
+
       if (stat.isDir()) {
-        FileStatus [] subitems = fs.listStatus(p);
+        Path dir = stat.getPath();
+        FileStatus [] subitems = fs.listStatus(dir);
         if (subitems == null || subitems.length == 0) {
-          LOG.warn("Input path " + p + " contains no files");
+          LOG.warn("Input path " + dir + " contains no files");
           return FileType.UNKNOWN; // empty dir.
         }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/02e36db2/src/test/com/cloudera/sqoop/TestAvroExport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAvroExport.java b/src/test/com/cloudera/sqoop/TestAvroExport.java
index 1d8f5df..5303048 100644
--- a/src/test/com/cloudera/sqoop/TestAvroExport.java
+++ b/src/test/com/cloudera/sqoop/TestAvroExport.java
@@ -333,6 +333,26 @@ public class TestAvroExport extends ExportJobTestCase {
     }
   }
 
+  public void testPathPatternInExportDir() throws IOException, SQLException {
+    final int TOTAL_RECORDS = 10;
+
+    ColumnGenerator[] gens = new ColumnGenerator[] {
+      colGenerator(true, Schema.create(Schema.Type.BOOLEAN), true, "BIT"),
+    };
+
+    createAvroFile(0, TOTAL_RECORDS, gens);
+    createTable(gens);
+
+    // Converts path to an unary set while preserving the leading '/'
+    String pathPattern = new StringBuilder(getTablePath().toString())
+            .insert(1, "{")
+            .append("}")
+            .toString();
+
+    runExport(getArgv(true, 10, 10, "--export-dir", pathPattern));
+    verifyExport(TOTAL_RECORDS);
+  }
+
   public void testNullableField() throws IOException, SQLException {
     String[] argv = {};
     final int TOTAL_RECORDS = 1 * 10;


Mime
View raw message