hadoop-mapreduce-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t...@apache.org
Subject svn commit: r1333291 [4/4] - in /hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project: ./ conf/ dev-support/ hadoop-mapreduce-client/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/ hado...
Date Thu, 03 May 2012 02:14:30 GMT
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
Thu May  3 02:14:01 2012
@@ -288,23 +288,23 @@ class LoadJob extends GridmixJob {
       final long[] reduceBytes = split.getOutputBytes();
       final long[] reduceRecords = split.getOutputRecords();
 
-      // enable gridmix map output record for compression
-      final boolean emulateMapOutputCompression = 
-        CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
-        && conf.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
-      float compressionRatio = 1.0f;
-      if (emulateMapOutputCompression) {
-        compressionRatio = 
-          CompressionEmulationUtil.getMapOutputCompressionEmulationRatio(conf);
-        LOG.info("GridMix is configured to use a compression ratio of " 
-                 + compressionRatio + " for the map output data.");
-        key.setCompressibility(true, compressionRatio);
-        val.setCompressibility(true, compressionRatio);
-      }
-      
       long totalRecords = 0L;
       final int nReduces = ctxt.getNumReduceTasks();
       if (nReduces > 0) {
+        // enable gridmix map output record for compression
+        boolean emulateMapOutputCompression = 
+          CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
+          && conf.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
+        float compressionRatio = 1.0f;
+        if (emulateMapOutputCompression) {
+          compressionRatio = 
+            CompressionEmulationUtil.getMapOutputCompressionEmulationRatio(conf);
+          LOG.info("GridMix is configured to use a compression ratio of " 
+                   + compressionRatio + " for the map output data.");
+          key.setCompressibility(true, compressionRatio);
+          val.setCompressibility(true, compressionRatio);
+        }
+        
         int idx = 0;
         int id = split.getId();
         for (int i = 0; i < nReduces; ++i) {
@@ -332,7 +332,21 @@ class LoadJob extends GridmixJob {
         }
       } else {
         long mapOutputBytes = reduceBytes[0];
-        if (emulateMapOutputCompression) {
+        
+        // enable gridmix job output compression
+        boolean emulateJobOutputCompression = 
+          CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
+          && conf.getBoolean(FileOutputFormat.COMPRESS, false);
+
+        if (emulateJobOutputCompression) {
+          float compressionRatio = 
+            CompressionEmulationUtil.getJobOutputCompressionEmulationRatio(conf);
+          LOG.info("GridMix is configured to use a compression ratio of " 
+                   + compressionRatio + " for the job output data.");
+          key.setCompressibility(true, compressionRatio);
+          val.setCompressibility(true, compressionRatio);
+
+          // set the output size accordingly
           mapOutputBytes /= compressionRatio;
         }
         reduces.add(new AvgRecordFactory(mapOutputBytes, reduceRecords[0],
@@ -387,9 +401,13 @@ class LoadJob extends GridmixJob {
     @Override
     public void cleanup(Context context) 
     throws IOException, InterruptedException {
+      LOG.info("Starting the cleanup phase.");
       for (RecordFactory factory : reduces) {
         key.setSeed(r.nextLong());
         while (factory.next(key, val)) {
+          // send the progress update (maybe make this a thread)
+          context.progress();
+          
           context.write(key, val);
           key.setSeed(r.nextLong());
           
@@ -462,7 +480,7 @@ class LoadJob extends GridmixJob {
           && FileOutputFormat.getCompressOutput(context)) {
         float compressionRatio = 
           CompressionEmulationUtil
-            .getReduceOutputCompressionEmulationRatio(conf);
+            .getJobOutputCompressionEmulationRatio(conf);
         LOG.info("GridMix is configured to use a compression ratio of " 
                  + compressionRatio + " for the reduce output data.");
         val.setCompressibility(true, compressionRatio);

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
Thu May  3 02:14:01 2012
@@ -322,10 +322,9 @@ public class TestCompressionEmulationUti
   public void testOutputCompressionRatioConfiguration() throws Exception {
     Configuration conf = new Configuration();
     float ratio = 0.567F;
-    CompressionEmulationUtil.setReduceOutputCompressionEmulationRatio(conf, 
-                                                                      ratio);
+    CompressionEmulationUtil.setJobOutputCompressionEmulationRatio(conf, ratio);
     assertEquals(ratio, 
-        CompressionEmulationUtil.getReduceOutputCompressionEmulationRatio(conf),
+        CompressionEmulationUtil.getJobOutputCompressionEmulationRatio(conf),
         0.0D);
   }
   

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1327719-1333290

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1327719-1333290

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1327719-1333290

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1327719-1333290

Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
(original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
Thu May  3 02:14:01 2012
@@ -287,7 +287,7 @@ public class JobHistory {
     FSDataOutputStream out = logDirFs.create(logFile, 
         new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
         true, defaultBufferSize, 
-        logDirFs.getDefaultReplication(), 
+        logDirFs.getDefaultReplication(logFile), 
         jobHistoryBlockSize, null);
   
     EventWriter writer = new EventWriter(out);
@@ -306,8 +306,8 @@ public class JobHistory {
           jobFileOut = logDirFs.create(logDirConfPath,
               new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
               true, defaultBufferSize,
-              logDirFs.getDefaultReplication(),
-              logDirFs.getDefaultBlockSize(), null);
+              logDirFs.getDefaultReplication(logDirConfPath),
+              logDirFs.getDefaultBlockSize(logDirConfPath), null);
           jobConf.writeXml(jobFileOut);
           jobFileOut.close();
         }

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1327719-1333290

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1327719-1333290

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1327719-1333290

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1327719-1333290

Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
  Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1327719-1333290



Mime
View raw message