hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jl...@apache.org
Subject svn commit: r1482100 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
Date Mon, 13 May 2013 20:54:59 GMT
Author: jlowe
Date: Mon May 13 20:54:58 2013
New Revision: 1482100

URL: http://svn.apache.org/r1482100
Log:
MAPREDUCE-5211. Reducer intermediate files can collide during merge. Contributed by Jason
Lowe

Modified:
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
    hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1482100&r1=1482099&r2=1482100&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon May 13 20:54:58
2013
@@ -24,6 +24,9 @@ Release 0.23.8 - UNRELEASED
     MAPREDUCE-5147. Maven build should create 
     hadoop-mapreduce-client-app-VERSION.jar directly (Robert Parker via tgraves)
 
+    MAPREDUCE-5211. Reducer intermediate files can collide during merge
+    (jlowe)
+
 Release 0.23.7 - 2013-04-18
 
   INCOMPATIBLE CHANGES

Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java?rev=1482100&r1=1482099&r2=1482100&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManager.java
Mon May 13 20:54:58 2013
@@ -505,7 +505,8 @@ public class MergeManager<K, V> {
   }
   
   private class OnDiskMerger extends MergeThread<MapOutput<K,V>,K,V> {
-    
+    private int mergeCount = 0;
+
     public OnDiskMerger(MergeManager<K, V> manager) {
       super(manager, ioSortFactor, exceptionReporter);
       setName("OnDiskMerger - Thread to merge on-disk map-outputs");
@@ -537,8 +538,10 @@ public class MergeManager<K, V> {
         ChecksumFileSystem.getChecksumLength(approxOutputSize, bytesPerSum);
 
       // 2. Start the on-disk merge process
+      ++mergeCount;
       Path outputPath = 
-        localDirAllocator.getLocalPathForWrite(inputs.get(0).toString(), 
+        localDirAllocator.getLocalPathForWrite(
+            new Path(reduceId.toString(), "diskmerge" + mergeCount).toString(),
             approxOutputSize, jobConf).suffix(Task.MERGED_OUTPUT_PREFIX);
       Writer<K,V> writer = 
         new Writer<K,V>(jobConf, rfs, outputPath, 



Mime
View raw message