sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject sqoop git commit: SQOOP-3330: Sqoop --append does not work with -Dmapreduce.output.basename
Date Fri, 13 Jul 2018 07:35:33 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 0cfbf5671 -> a6bedca4b


SQOOP-3330: Sqoop --append does not work with -Dmapreduce.output.basename

(Eric Lin via Szabolcs Vasas)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/a6bedca4
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/a6bedca4
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/a6bedca4

Branch: refs/heads/trunk
Commit: a6bedca4b272173292ab3aeadedea7283aacd16e
Parents: 0cfbf56
Author: Szabolcs Vasas <vasas@apache.org>
Authored: Fri Jul 13 09:34:38 2018 +0200
Committer: Szabolcs Vasas <vasas@apache.org>
Committed: Fri Jul 13 09:34:38 2018 +0200

----------------------------------------------------------------------
 src/java/org/apache/sqoop/util/AppendUtils.java | 23 ++++++++++--
 src/test/org/apache/sqoop/TestAppendUtils.java  | 38 +++++++++++++++++---
 2 files changed, 53 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/a6bedca4/src/java/org/apache/sqoop/util/AppendUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java
index fa85280..20c0d13 100644
--- a/src/java/org/apache/sqoop/util/AppendUtils.java
+++ b/src/java/org/apache/sqoop/util/AppendUtils.java
@@ -43,7 +43,7 @@ public class AppendUtils {
   private static final String FILEPART_SEPARATOR = "-";
   private static final String FILEEXT_SEPARATOR = ".";
 
-  private static final Pattern DATA_PART_PATTERN = Pattern.compile("part.*-([0-9]{" + PARTITION_DIGITS
+ "}+).*");
+  public static final String DATA_PART_PATTERN_PREFIX = "part";
 
   private ImportJobContext context = null;
 
@@ -115,7 +115,7 @@ public class AppendUtils {
       for (FileStatus fileStat : existingFiles) {
         if (!fileStat.isDir()) {
           String filename = fileStat.getPath().getName();
-          Matcher mat = DATA_PART_PATTERN.matcher(filename);
+          Matcher mat = getDataFileNamePattern().matcher(filename);
           if (mat.matches()) {
             int thisPart = Integer.parseInt(mat.group(1));
             if (thisPart >= nextPartition) {
@@ -204,7 +204,7 @@ public class AppendUtils {
 
           LOG.debug("Directory: " + sourceFilename + " renamed to: " + destPath.getName());
         }
-      } else if (DATA_PART_PATTERN.matcher(sourceFilename).matches()) {    // move only matching
top-level files
+      } else if (getDataFileNamePattern().matcher(sourceFilename).matches()) {    // move
only matching top-level files
         do {
           // clear the builder in case this isn't the first iteration
           destFilename.setLength(0);
@@ -276,4 +276,21 @@ public class AppendUtils {
     return new Path(tempDir);
   }
 
+  /**
+   * Return the Pattern of the file name that will end up in the target directory
+   *
+   * Take into account that user might pass mapreduce.output.basename, which should
+   * override the default filename prefix of "part"
+   *
+   * @return Pattern
+   */
+  private Pattern getDataFileNamePattern() {
+    String prefix = context.getOptions().getConf().get("mapreduce.output.basename");
+
+    if(null == prefix || prefix.length() == 0) {
+      prefix = DATA_PART_PATTERN_PREFIX;
+    }
+
+    return Pattern.compile(prefix + ".*-([0-9]{" + PARTITION_DIGITS + "}+).*");
+  }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a6bedca4/src/test/org/apache/sqoop/TestAppendUtils.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/TestAppendUtils.java b/src/test/org/apache/sqoop/TestAppendUtils.java
index f14fc6a..3d66bec 100644
--- a/src/test/org/apache/sqoop/TestAppendUtils.java
+++ b/src/test/org/apache/sqoop/TestAppendUtils.java
@@ -131,10 +131,10 @@ public class TestAppendUtils extends ImportJobTestCase {
   }
 
   /** @return FileStatus for data files only. */
-  private FileStatus[] listFiles(FileSystem fs, Path path) throws IOException {
+  private FileStatus[] listFiles(FileSystem fs, Path path, String prefix) throws IOException
{
     FileStatus[] fileStatuses = fs.listStatus(path);
     ArrayList files = new ArrayList();
-    Pattern patt = Pattern.compile("part.*-([0-9][0-9][0-9][0-9][0-9]).*");
+    Pattern patt = Pattern.compile(prefix + ".*-([0-9][0-9][0-9][0-9][0-9]).*");
     for (FileStatus fstat : fileStatuses) {
       String fname = fstat.getPath().getName();
       if (!fstat.isDir()) {
@@ -147,6 +147,10 @@ public class TestAppendUtils extends ImportJobTestCase {
     return (FileStatus[]) files.toArray(new FileStatus[files.size()]);
   }
 
+  private FileStatus[] listFiles(FileSystem fs, Path path) throws IOException {
+    return listFiles(fs, path, AppendUtils.DATA_PART_PATTERN_PREFIX);
+  }
+
   private class StatusPathComparator implements Comparator<FileStatus> {
 
     @Override
@@ -183,13 +187,17 @@ public class TestAppendUtils extends ImportJobTestCase {
     }
   }
 
+  public void runAppendTest(ArrayList args, Path outputPath) throws IOException {
+    runAppendTest(args, outputPath, AppendUtils.DATA_PART_PATTERN_PREFIX);
+  }
+
   /**
    * Test for ouput path file-count increase, current files untouched and new
    * correct partition number.
    *
    * @throws IOException
    */
-  public void runAppendTest(ArrayList args, Path outputPath)
+  public void runAppendTest(ArrayList args, Path outputPath, String prefix)
       throws IOException {
 
     try {
@@ -205,7 +213,7 @@ public class TestAppendUtils extends ImportJobTestCase {
       runUncleanImport(argv);
 
       // get current file count
-      FileStatus[] fileStatuses = listFiles(fs, outputPath);
+      FileStatus[] fileStatuses = listFiles(fs, outputPath, prefix);
       Arrays.sort(fileStatuses, new StatusPathComparator());
       int previousFileCount = fileStatuses.length;
 
@@ -223,7 +231,7 @@ public class TestAppendUtils extends ImportJobTestCase {
       runUncleanImport(argv);
 
       // check directory file increase
-      fileStatuses = listFiles(fs, outputPath);
+      fileStatuses = listFiles(fs, outputPath, prefix);
       Arrays.sort(fileStatuses, new StatusPathComparator());
       int currentFileCount = fileStatuses.length;
       assertTrue("Output directory didn't got increased in file count ",
@@ -311,5 +319,25 @@ public class TestAppendUtils extends ImportJobTestCase {
     utils.append();
   }
 
+  /**
+   * Test that when we pass in -Dmapreduce.output.basename=prefix, file should also got appended
+   *
+   * @throws IOException
+   */
+  @Test
+  public void testAppendWithMapreduceOutputBasename() throws IOException {
+    String prefix = "prefix-test";
+
+    ArrayList<String> args = new ArrayList<>();
+    args.add("-D");
+    args.add("mapreduce.output.basename=" + prefix);
+    args.addAll(getOutputlessArgv(false, true, HsqldbTestServer.getFieldNames(), getConf()));
+    String targetDir = getWarehouseDir() + "/tempTargetDirOutputBaseNameTest";
+    args.add("--target-dir");
+    args.add(targetDir);
+
+    Path output = new Path(targetDir);
+    runAppendTest(args, output, prefix);
+  }
 }
 


Mime
View raw message