Author: acmurthy Date: Mon Oct 24 21:03:22 2011 New Revision: 1188378 URL: http://svn.apache.org/viewvc?rev=1188378&view=rev Log: Merge -c 1188377 from trunk to branch-0.23 to complete fix for MAPREDUCE-3028. Added: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java - copied unchanged from r1188377, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/JobEndNotifier.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java - copied unchanged from r1188377, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestJobEndNotifier.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1188378&r1=1188377&r2=1188378&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon Oct 24 21:03:22 2011 @@ -1688,6 +1688,9 @@ Release 0.23.0 - Unreleased MAPREDUCE-3233. Fixed a bug in MR Job so as to be able to restart the application on AM crash. (Mahadev Konar via vinodkv) + MAPREDUCE-3028. Added job-end notification support. (Ravi Prakash via + acmurthy) + Release 0.22.0 - Unreleased INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1188378&r1=1188377&r2=1188378&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Mon Oct 24 21:03:22 2011 @@ -386,19 +386,34 @@ public class MRAppMaster extends Composi } catch (InterruptedException e) { e.printStackTrace(); } - LOG.info("Calling stop for all the services"); try { + // Stop all services + // This will also send the final report to the ResourceManager + LOG.info("Calling stop for all the services"); stop(); + + // Send job-end notification + try { + LOG.info("Job end notification started for jobID : " + + job.getReport().getJobId()); + JobEndNotifier notifier = new JobEndNotifier(); + notifier.setConf(getConfig()); + notifier.notify(job.getReport()); + } catch (InterruptedException ie) { + LOG.warn("Job end notification interrupted for jobID : " + + job.getReport().getJobId(), ie ); + } } catch (Throwable t) { LOG.warn("Graceful stop failed ", t); } + + // Cleanup staging directory try { cleanupStagingDir(); } catch(IOException io) { LOG.warn("Failed to delete staging dir"); } - //TODO: this is required because rpc server does not shut down - // in spite of calling server.stop(). + //Bring the process down by force. //Not needed after HADOOP-7140 LOG.info("Exiting MR AppMaster..GoodBye!"); Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1188378&r1=1188377&r2=1188378&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java Mon Oct 24 21:03:22 2011 @@ -1649,7 +1649,7 @@ public class JobConf extends Configurati * @see #setJobEndNotificationURI(String) */ public String getJobEndNotificationURI() { - return get(JobContext.END_NOTIFICATION_URL); + return get(JobContext.MR_JOB_END_NOTIFICATION_URL); } /** @@ -1669,7 +1669,7 @@ public class JobConf extends Configurati * JobCompletionAndChaining">Job Completion and Chaining */ public void setJobEndNotificationURI(String uri) { - set(JobContext.END_NOTIFICATION_URL, uri); + set(JobContext.MR_JOB_END_NOTIFICATION_URL, uri); } /** Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java?rev=1188378&r1=1188377&r2=1188378&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobEndNotifier.java Mon Oct 24 21:03:22 2011 @@ -102,8 +102,8 @@ public class JobEndNotifier { String uri = conf.getJobEndNotificationURI(); if (uri != null) { // +1 to make logic for first notification identical to a retry - int retryAttempts = conf.getInt(JobContext.END_NOTIFICATION_RETRIES, 0) + 1; - long retryInterval = conf.getInt(JobContext.END_NOTIFICATION_RETRIE_INTERVAL, 30000); + int retryAttempts = conf.getInt(JobContext.MR_JOB_END_RETRY_ATTEMPTS, 0) + 1; + long retryInterval = conf.getInt(JobContext.MR_JOB_END_RETRY_INTERVAL, 30000); if (uri.contains("$jobId")) { uri = uri.replace("$jobId", status.getJobID().toString()); } Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1188378&r1=1188377&r2=1188378&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Mon Oct 24 21:03:22 2011 @@ -91,12 +91,6 @@ public interface MRJobConfig { public static final String WORKING_DIR = "mapreduce.job.working.dir"; - public static final String END_NOTIFICATION_URL = "mapreduce.job.end-notification.url"; - - public static final String END_NOTIFICATION_RETRIES = "mapreduce.job.end-notification.retry.attempts"; - - public static final String END_NOTIFICATION_RETRIE_INTERVAL = "mapreduce.job.end-notification.retry.interval"; - public static final String CLASSPATH_ARCHIVES = "mapreduce.job.classpath.archives"; public static final String CLASSPATH_FILES = "mapreduce.job.classpath.files"; @@ -486,4 +480,23 @@ public interface MRJobConfig { public static final String APPLICATION_ATTEMPT_ID = "mapreduce.job.application.attempt.id"; + + /** + * Job end notification. + */ + public static final String MR_JOB_END_NOTIFICATION_URL = + "mapreduce.job.end-notification.url"; + + public static final String MR_JOB_END_RETRY_ATTEMPTS = + "mapreduce.job.end-notification.retry.attempts"; + + public static final String MR_JOB_END_RETRY_INTERVAL = + "mapreduce.job.end-notification.retry.interval"; + + public static final String MR_JOB_END_NOTIFICATION_MAX_ATTEMPTS = + "mapreduce.job.end-notification.max.attempts"; + + public static final String MR_JOB_END_NOTIFICATION_MAX_RETRY_INTERVAL = + "mapreduce.job.end-notification.max.retry.interval"; + } Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java?rev=1188378&r1=1188377&r2=1188378&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java Mon Oct 24 21:03:22 2011 @@ -177,11 +177,11 @@ public class ConfigUtil { Configuration.addDeprecation("tasktracker.contention.tracking", new String[] {TTConfig.TT_CONTENTION_TRACKING}); Configuration.addDeprecation("job.end.notification.url", - new String[] {MRJobConfig.END_NOTIFICATION_URL}); + new String[] {MRJobConfig.MR_JOB_END_NOTIFICATION_URL}); Configuration.addDeprecation("job.end.retry.attempts", - new String[] {MRJobConfig.END_NOTIFICATION_RETRIES}); + new String[] {MRJobConfig.MR_JOB_END_RETRY_ATTEMPTS}); Configuration.addDeprecation("job.end.retry.interval", - new String[] {MRJobConfig.END_NOTIFICATION_RETRIE_INTERVAL}); + new String[] {MRJobConfig.MR_JOB_END_RETRY_INTERVAL}); Configuration.addDeprecation("mapred.committer.job.setup.cleanup.needed", new String[] {MRJobConfig.SETUP_CLEANUP_NEEDED}); Configuration.addDeprecation("mapred.jar", Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1188378&r1=1188377&r2=1188378&view=diff ============================================================================== --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Mon Oct 24 21:03:22 2011 @@ -1179,4 +1179,49 @@ + + mapreduce.job.end-notification.max.attempts + 5 + true + The maximum number of times a URL will be read for providing job + end notification. Cluster administrators can set this to limit how long + after end of a job, the Application Master waits before exiting. Must be + marked as final to prevent users from overriding this. + + + + + mapreduce.job.end-notification.max.retry.interval + 5 + true + The maximum amount of time (in seconds) to wait before retrying + job end notification. Cluster administrators can set this to limit how long + the Application Master waits before exiting. Must be marked as final to + prevent users from overriding this. + + + + mapreduce.job.end-notification.url + + The URL to send job end notification. It may contain sentinels + $jobId and $jobStatus which will be replaced with jobId and jobStatus. + + + + + mapreduce.job.end-notification.retry.attempts + 5 + The number of times the submitter of the job wants to retry job + end notification if it fails. This is capped by + mapreduce.job.end-notification.max.attempts + + + + mapreduce.job.end-notification.retry.interval + 1 + The number of seconds the submitter of the job wants to wait + before job end notification is retried if it fails. This is capped by + mapreduce.job.end-notification.max.retry.interval + +