sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkatran...@apache.org
Subject git commit: SQOOP-1321: Add ability to serialize SqoopOption into JobConf (Jarek Jarcec Cecho via Venkat Ranganathan)
Date Mon, 19 May 2014 17:25:50 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 72e3bfdd6 -> 18f5b2a77


SQOOP-1321: Add ability to serialize SqoopOption into JobConf
  (Jarek Jarcec Cecho via Venkat Ranganathan)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/18f5b2a7
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/18f5b2a7
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/18f5b2a7

Branch: refs/heads/trunk
Commit: 18f5b2a77e95e412e806cabf4080a991f79aca92
Parents: 72e3bfd
Author: Venkat Ranganathan <venkat@hortonworks.com>
Authored: Mon May 19 10:23:34 2014 -0700
Committer: Venkat Ranganathan <venkat@hortonworks.com>
Committed: Mon May 19 10:23:34 2014 -0700

----------------------------------------------------------------------
 conf/sqoop-site-template.xml                    | 14 ++++++++
 .../apache/sqoop/mapreduce/ExportJobBase.java   |  2 +-
 .../apache/sqoop/mapreduce/ImportJobBase.java   |  2 +-
 .../org/apache/sqoop/mapreduce/JobBase.java     | 37 ++++++++++++++++++++
 .../org/apache/sqoop/mapreduce/MergeJob.java    |  2 +-
 5 files changed, 54 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/18f5b2a7/conf/sqoop-site-template.xml
----------------------------------------------------------------------
diff --git a/conf/sqoop-site-template.xml b/conf/sqoop-site-template.xml
index eccd168..368af26 100644
--- a/conf/sqoop-site-template.xml
+++ b/conf/sqoop-site-template.xml
@@ -127,6 +127,20 @@ under the License.
   </property>
   -->
 
+  <!--
+    Enabling this option will instruct Sqoop to put all options that
+    were used in the invocation into created mapreduce job(s). This
+    become handy when one needs to investigate what exact options were
+    used in the Sqoop invocation.
+  -->
+  <!--
+  <property>
+    <name>sqoop.jobbase.serialize.sqoopoptions</name>
+    <value>true</value>
+    <description>If true, then all options will be serialized into job.xml
+    </description>
+  </property>
+  -->
 
   <!--
     SERVER CONFIGURATION: If you plan to run a Sqoop metastore on this machine,

http://git-wip-us.apache.org/repos/asf/sqoop/blob/18f5b2a7/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
index 53de9c9..9f510b9 100644
--- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
@@ -379,7 +379,7 @@ public class ExportJobBase extends JobBase {
       }
     }
 
-    Job job = new Job(conf);
+    Job job = createJob(conf);
     try {
       // Set the external jar to use for the job.
       job.getConfiguration().set("mapred.jar", ormJarFile);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/18f5b2a7/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
index 8b1493d..0ac35de 100644
--- a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
@@ -227,7 +227,7 @@ public class ImportJobBase extends JobBase {
 
     loadJars(conf, ormJarFile, tableClassName);
 
-    Job job = new Job(conf);
+    Job job = createJob(conf);
     try {
       // Set the external jar to use for the job.
       job.getConfiguration().set("mapred.jar", ormJarFile);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/18f5b2a7/src/java/org/apache/sqoop/mapreduce/JobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java
index ddef421..92a78ac 100644
--- a/src/java/org/apache/sqoop/mapreduce/JobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java
@@ -22,6 +22,7 @@ import java.io.File;
 import java.io.IOException;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -48,6 +49,9 @@ public class JobBase {
 
   public static final Log LOG = LogFactory.getLog(JobBase.class.getName());
 
+  public static final String SERIALIZE_SQOOPOPTIONS = "sqoop.jobbase.serialize.sqoopoptions";
+  public static final boolean SERIALIZE_SQOOPOPTIONS_DEFAULT = false;
+
   protected SqoopOptions options;
   protected Class<? extends Mapper> mapperClass;
   protected Class<? extends InputFormat> inputFormatClass;
@@ -309,6 +313,39 @@ public class JobBase {
   }
 
   /**
+   * Create new Job object in unified way for all types of jobs.
+   *
+   * @param configuration Hadoop configuration that should be used
+   * @return New job object, created object won't be persisted in the instance
+   */
+  public Job createJob(Configuration configuration) throws IOException {
+    // Put the SqoopOptions into job if requested
+    if(configuration.getBoolean(SERIALIZE_SQOOPOPTIONS, SERIALIZE_SQOOPOPTIONS_DEFAULT))
{
+      putSqoopOptionsToConfiguration(options, configuration);
+    }
+
+    return new Job(configuration);
+  }
+
+  /**
+   * Iterates over serialized form of SqoopOptions and put them into Configuration
+   * object.
+   *
+   * @param opts SqoopOptions that should be serialized
+   * @param configuration Target configuration object
+   */
+  public void putSqoopOptionsToConfiguration(SqoopOptions opts, Configuration configuration)
{
+    for(Map.Entry<Object, Object> e : opts.writeProperties().entrySet()) {
+      String key = (String)e.getKey();
+      String value = (String)e.getValue();
+
+      // We don't need to do if(value is empty) because that is already done
+      // for us by the SqoopOptions.writeProperties() method.
+      configuration.set("sqoop.opt." + key, value);
+    }
+  }
+
+  /**
    * Actually run the MapReduce job.
    */
   protected boolean runJob(Job job) throws ClassNotFoundException, IOException,

http://git-wip-us.apache.org/repos/asf/sqoop/blob/18f5b2a7/src/java/org/apache/sqoop/mapreduce/MergeJob.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/MergeJob.java b/src/java/org/apache/sqoop/mapreduce/MergeJob.java
index 5f32127..4e2a916 100644
--- a/src/java/org/apache/sqoop/mapreduce/MergeJob.java
+++ b/src/java/org/apache/sqoop/mapreduce/MergeJob.java
@@ -59,7 +59,7 @@ public class MergeJob extends JobBase {
 
   public boolean runMergeJob() throws IOException {
     Configuration conf = options.getConf();
-    Job job = new Job(conf);
+    Job job = createJob(conf);
 
     String userClassName = options.getClassName();
     if (null == userClassName) {


Mime
View raw message