hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From szets...@apache.org
Subject svn commit: r1499634 - in /hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project: ./ CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
Date Thu, 04 Jul 2013 03:41:36 GMT
Author: szetszwo
Date: Thu Jul  4 03:41:36 2013
New Revision: 1499634

URL: http://svn.apache.org/r1499634
Log:
svn merge -c 1499149 from branch-2 for MAPREDUCE-5355. MiniMRYarnCluster with localFs does
not work on Windows.

Modified:
    hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/   (props changed)
    hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt   (contents,
props changed)
    hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java

Propchange: hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-2/hadoop-mapreduce-project:r1499149

Modified: hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt?rev=1499634&r1=1499633&r2=1499634&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt Thu Jul
 4 03:41:36 2013
@@ -436,6 +436,9 @@ Release 2.1.0-beta - 2013-07-02
     MAPREDUCE-5349. TestClusterMapReduceTestCase and TestJobName fail on Windows
     in branch-2. (Chuan Liu via cnauroth)
 
+    MAPREDUCE-5355. MiniMRYarnCluster with localFs does not work on Windows.
+    (Chuan Liu via cnauroth)
+
     MAPREDUCE-5291. Change MR App to use updated property names in
     container-log4j.properties. (Zhijie Shen via sseth)
 

Propchange: hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
  Merged /hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt:r1499149

Modified: hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1499634&r1=1499633&r2=1499634&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
(original)
+++ hadoop/common/branches/branch-2.1.0-beta/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
Thu Jul  4 03:41:36 2013
@@ -26,6 +26,7 @@ import org.apache.commons.logging.LogFac
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.CommonConfigurationKeys;
 import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.LocalFileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.LocalContainerLauncher;
 import org.apache.hadoop.mapred.ShuffleHandler;
@@ -87,6 +88,21 @@ public class MiniMRYarnCluster extends M
     try {
       Path stagingPath = FileContext.getFileContext(conf).makeQualified(
           new Path(conf.get(MRJobConfig.MR_AM_STAGING_DIR)));
+      /*
+       * Re-configure the staging path on Windows if the file system is localFs.
+       * We need to use a absolute path that contains the drive letter. The unit
+       * test could run on a different drive than the AM. We can run into the
+       * issue that job files are localized to the drive where the test runs on,
+       * while the AM starts on a different drive and fails to find the job
+       * metafiles. Using absolute path can avoid this ambiguity.
+       */
+      if (Path.WINDOWS) {
+        if (LocalFileSystem.class.isInstance(stagingPath.getFileSystem(conf))) {
+          conf.set(MRJobConfig.MR_AM_STAGING_DIR,
+              new File(conf.get(MRJobConfig.MR_AM_STAGING_DIR))
+                  .getAbsolutePath());
+        }
+      }
       FileContext fc=FileContext.getFileContext(stagingPath.toUri(), conf);
       if (fc.util().exists(stagingPath)) {
         LOG.info(stagingPath + " exists! deleting...");



Mime
View raw message