sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject sqoop git commit: SQOOP-2156: Sqoop2: HdfsUtils.getOutputMapreduceFiles should ignore hidden files
Date Tue, 03 Mar 2015 02:27:54 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 c408ac5a4 -> 934a00976


SQOOP-2156: Sqoop2: HdfsUtils.getOutputMapreduceFiles should ignore hidden files

(Jarek Jarcec Cecho via Abraham Elmahrek)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/934a0097
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/934a0097
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/934a0097

Branch: refs/heads/sqoop2
Commit: 934a009762719a2459aa8f85e0c300645c5b2d20
Parents: c408ac5
Author: Abraham Elmahrek <abe@apache.org>
Authored: Mon Mar 2 18:27:28 2015 -0800
Committer: Abraham Elmahrek <abe@apache.org>
Committed: Mon Mar 2 18:27:28 2015 -0800

----------------------------------------------------------------------
 .../java/org/apache/sqoop/test/utils/HdfsUtils.java    | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/934a0097/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
index 5a55b8b..79700a6 100644
--- a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
+++ b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
@@ -20,6 +20,7 @@ package org.apache.sqoop.test.utils;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
 import org.apache.log4j.Logger;
 
 import java.io.BufferedWriter;
@@ -37,6 +38,14 @@ public class HdfsUtils {
 
   private static final char PATH_SEPARATOR = '/';
 
+  public static PathFilter filterHiddenFiles = new PathFilter() {
+    @Override
+    public boolean accept(Path path) {
+      String fileName = path.getName();
+      return !fileName.startsWith("_") && !fileName.startsWith(".");
+    }
+  };
+
   /**
    * Get list of mapreduce output files from given directory.
    *
@@ -45,9 +54,9 @@ public class HdfsUtils {
    * @throws IOException
    * @throws FileNotFoundException
    */
-  public static Path [] getOutputMapreduceFiles(FileSystem fs, String directory) throws FileNotFoundException,
IOException {
+  public static Path [] getOutputMapreduceFiles(FileSystem fs, String directory) throws IOException
{
     LinkedList<Path> files = new LinkedList<Path>();
-    for (FileStatus fileStatus : fs.listStatus(new Path(directory))) {
+    for (FileStatus fileStatus : fs.listStatus(new Path(directory), filterHiddenFiles)) {
       LOG.debug("Found mapreduce output file: " + fileStatus.getPath() + " with size " +
fileStatus.getLen());
       files.add(fileStatus.getPath());
     }


Mime
View raw message