sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From cheol...@apache.org
Subject git commit: SQOOP-650 Add ability to name mapreduce job generated by Sqoop
Date Wed, 07 Nov 2012 00:30:01 GMT
Updated Branches:
  refs/heads/trunk c499f4909 -> ebeb93351


SQOOP-650 Add ability to name mapreduce job generated by Sqoop

(Jarek Jarcec Cecho via Cheolsoo Park)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ebeb9335
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ebeb9335
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ebeb9335

Branch: refs/heads/trunk
Commit: ebeb93351335cc73d6d77ef1fd0d7316655defa3
Parents: c499f49
Author: Cheolsoo Park <cheolsoo@apache.org>
Authored: Tue Nov 6 16:28:57 2012 -0800
Committer: Cheolsoo Park <cheolsoo@apache.org>
Committed: Tue Nov 6 16:28:57 2012 -0800

----------------------------------------------------------------------
 src/java/org/apache/sqoop/SqoopOptions.java        |   13 ++++++++++++-
 .../org/apache/sqoop/mapreduce/ExportJobBase.java  |    3 +++
 .../org/apache/sqoop/mapreduce/ImportJobBase.java  |    3 +++
 src/java/org/apache/sqoop/tool/BaseSqoopTool.java  |    1 +
 src/java/org/apache/sqoop/tool/ExportTool.java     |    8 ++++++++
 src/java/org/apache/sqoop/tool/ImportTool.java     |    8 ++++++++
 6 files changed, 35 insertions(+), 1 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 92c9996..2b5db29 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -87,6 +87,9 @@ public class SqoopOptions implements Cloneable {
   // are stored as constants in BaseSqoopTool.
 
   @StoredAsProperty("verbose") private boolean verbose;
+
+  @StoredAsProperty("mapreduce.job.name") private String mapreduceJobName;
+
   @StoredAsProperty("db.connect.string") private String connectString;
   @StoredAsProperty("db.table") private String tableName;
   private String [] columns; // Array stored as db.column.list.
@@ -1056,7 +1059,7 @@ public class SqoopOptions implements Cloneable {
       LOG.error("prompt the user to enter the password while being executed");
       LOG.error("as Oozie tasks. Please enable sqoop.metastore.client.record");
       LOG.error(".password in sqoop-site.xml, or provide the password");
-      LOG.error("explictly using --password in the command tag of the Oozie");
+      LOG.error("explicitly using --password in the command tag of the Oozie");
       LOG.error("workflow file.");
       return null;
     }
@@ -1070,6 +1073,14 @@ public class SqoopOptions implements Cloneable {
     this.password = securePasswordEntry();
   }
 
+  public String getMapreduceJobName() {
+    return mapreduceJobName;
+  }
+
+  public void setMapreduceJobName(String mapredJobName) {
+    this.mapreduceJobName = mapredJobName;
+  }
+
   public void setPassword(String pass) {
     this.password = pass;
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
index b9eaa83..d311ec1 100644
--- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
@@ -357,6 +357,9 @@ public class ExportJobBase extends JobBase {
     try {
       // Set the external jar to use for the job.
       job.getConfiguration().set("mapred.jar", ormJarFile);
+      if (options.getMapreduceJobName() != null) {
+        job.setJobName(options.getMapreduceJobName());
+      }
 
       propagateOptionsToJob(job);
       configureInputFormat(job, tableName, tableClassName, null);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
index efc2c37..057de84 100644
--- a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
@@ -195,6 +195,9 @@ public class ImportJobBase extends JobBase {
     try {
       // Set the external jar to use for the job.
       job.getConfiguration().set("mapred.jar", ormJarFile);
+      if (options.getMapreduceJobName() != null) {
+        job.setJobName(options.getMapreduceJobName());
+      }
 
       propagateOptionsToJob(job);
       configureInputFormat(job, tableName, tableClassName, splitByCol);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 0c9cda6..1ab3e5f 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -104,6 +104,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
   public static final String HIVE_PARTITION_VALUE_ARG = "hive-partition-value";
   public static final String CREATE_HIVE_TABLE_ARG =
       "create-hive-table";
+  public static final String MAPREDUCE_JOB_NAME = "mapreduce-job-name";
   public static final String NUM_MAPPERS_ARG = "num-mappers";
   public static final String NUM_MAPPERS_SHORT_ARG = "m";
   public static final String COMPRESS_ARG = "compress";

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/tool/ExportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ExportTool.java b/src/java/org/apache/sqoop/tool/ExportTool.java
index 494887d..6ac3a89 100644
--- a/src/java/org/apache/sqoop/tool/ExportTool.java
+++ b/src/java/org/apache/sqoop/tool/ExportTool.java
@@ -137,6 +137,10 @@ public class ExportTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
         .hasArg().withDescription("Columns to export to table")
         .withLongOpt(COLUMNS_ARG)
         .create());
+    exportOpts.addOption(OptionBuilder.withArgName("name")
+        .hasArg().withDescription("Set name for generated mapreduce job")
+        .withLongOpt(MAPREDUCE_JOB_NAME)
+        .create());
     exportOpts.addOption(OptionBuilder.withArgName("n")
         .hasArg().withDescription("Use 'n' map tasks to export in parallel")
         .withLongOpt(NUM_MAPPERS_ARG)
@@ -243,6 +247,10 @@ public class ExportTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
         out.setNumMappers(Integer.parseInt(in.getOptionValue(NUM_MAPPERS_ARG)));
       }
 
+      if (in.hasOption(MAPREDUCE_JOB_NAME)) {
+        out.setMapreduceJobName(in.getOptionValue(MAPREDUCE_JOB_NAME));
+      }
+
       if (in.hasOption(EXPORT_PATH_ARG)) {
         out.setExportDir(in.getOptionValue(EXPORT_PATH_ARG));
       }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ebeb9335/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index 415b315..932e39b 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -576,6 +576,10 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
         .hasArg().withDescription("Use 'n' map tasks to import in parallel")
         .withLongOpt(NUM_MAPPERS_ARG)
         .create(NUM_MAPPERS_SHORT_ARG));
+    importOpts.addOption(OptionBuilder.withArgName("name")
+        .hasArg().withDescription("Set name for generated mapreduce job")
+        .withLongOpt(MAPREDUCE_JOB_NAME)
+        .create());
     importOpts.addOption(OptionBuilder
         .withDescription("Enable compression")
         .withLongOpt(COMPRESS_ARG)
@@ -774,6 +778,10 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
         out.setNumMappers(Integer.parseInt(in.getOptionValue(NUM_MAPPERS_ARG)));
       }
 
+      if (in.hasOption(MAPREDUCE_JOB_NAME)) {
+        out.setMapreduceJobName(in.getOptionValue(MAPREDUCE_JOB_NAME));
+      }
+
       if (in.hasOption(COMPRESS_ARG)) {
         out.setUseCompression(true);
       }


Mime
View raw message