sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject svn commit: r1351503 - in /sqoop/trunk/src/java/org/apache/sqoop: ./ mapreduce/ tool/ util/
Date Mon, 18 Jun 2012 21:49:43 GMT
Author: blee
Date: Mon Jun 18 21:49:42 2012
New Revision: 1351503

URL: http://svn.apache.org/viewvc?rev=1351503&view=rev
Log:
SQOOP-436  Enable verbose logging for MapReduce jobs

Added:
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java   (with props)
Modified:
    sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java
    sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
    sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java

Modified: sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/SqoopOptions.java Mon Jun 18 21:49:42 2012
@@ -38,6 +38,7 @@ import com.cloudera.sqoop.lib.LargeObjec
 import com.cloudera.sqoop.tool.SqoopTool;
 import com.cloudera.sqoop.util.RandomHash;
 import com.cloudera.sqoop.util.StoredAsProperty;
+import org.apache.sqoop.util.LoggingUtils;
 
 /**
  * Configurable state used by Sqoop tools.
@@ -85,6 +86,7 @@ public class SqoopOptions implements Clo
   // arguments in the appropriate tools. The names of all command-line args
   // are stored as constants in BaseSqoopTool.
 
+  @StoredAsProperty("verbose") private boolean verbose;
   @StoredAsProperty("db.connect.string") private String connectString;
   @StoredAsProperty("db.table") private String tableName;
   private String [] columns; // Array stored as db.column.list.
@@ -560,6 +562,11 @@ public class SqoopOptions implements Clo
     // Delimiters were previously memoized; don't let the tool override
     // them with defaults.
     this.areDelimsManuallySet = true;
+
+    // If we loaded true verbose flag, we need to apply it
+    if (this.verbose) {
+      LoggingUtils.setDebugLevel();
+    }
   }
 
   /**
@@ -806,6 +813,9 @@ public class SqoopOptions implements Clo
     // Creating instances for user specific mapping
     this.mapColumnHive = new Properties();
     this.mapColumnJava = new Properties();
+
+    // We do not want to be verbose too much if not explicitly needed
+    this.verbose = false;
   }
 
   /**
@@ -893,6 +903,14 @@ public class SqoopOptions implements Clo
     }
   }
 
+  public boolean getVerbose() {
+    return verbose;
+  }
+
+  public void setVerbose(boolean beVerbose) {
+    this.verbose = beVerbose;
+  }
+
   /**
    * Get the temporary directory; guaranteed to end in File.separator
    * (e.g., '/').

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/AutoProgressMapper.java Mon Jun 18 21:49:42
2012
@@ -22,13 +22,12 @@ import java.io.IOException;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Mapper;
 
 /**
  * Identity mapper that continuously reports progress via a background thread.
  */
 public class AutoProgressMapper<KEYIN, VALIN, KEYOUT, VALOUT>
-    extends Mapper<KEYIN, VALIN, KEYOUT, VALOUT> {
+    extends SqoopMapper<KEYIN, VALIN, KEYOUT, VALOUT> {
 
   public static final Log LOG = LogFactory.getLog(
       AutoProgressMapper.class.getName());

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java Mon Jun 18 21:49:42
2012
@@ -351,6 +351,7 @@ public class ExportJobBase extends JobBa
       // Set the external jar to use for the job.
       job.getConfiguration().set("mapred.jar", ormJarFile);
 
+      propagateOptionsToJob(job);
       configureInputFormat(job, tableName, tableClassName, null);
       configureOutputFormat(job, tableName, tableClassName);
       configureMapper(job, tableName, tableClassName);

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java Mon Jun 18 21:49:42
2012
@@ -190,6 +190,7 @@ public class ImportJobBase extends JobBa
       // Set the external jar to use for the job.
       job.getConfiguration().set("mapred.jar", ormJarFile);
 
+      propagateOptionsToJob(job);
       configureInputFormat(job, tableName, tableClassName, splitByCol);
       configureOutputFormat(job, tableName, tableClassName);
       configureMapper(job, tableName, tableClassName);

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JobBase.java Mon Jun 18 21:49:42 2012
@@ -57,6 +57,8 @@ public class JobBase {
 
   private ClassLoader prevClassLoader = null;
 
+  public static final String PROPERTY_VERBOSE = "sqoop.verbose";
+
   public JobBase() {
     this(null);
   }
@@ -322,4 +324,18 @@ public class JobBase {
     log.info("A jobtracker restart is required for these settings");
     log.info("to take effect.");
   }
+
+  /**
+   * Save interesting options to constructed job. Goal here is to propagate some
+   * of them to the job itself, so that they can be easily accessed. We're
+   * propagating only interesting global options (like verbose flag).
+   *
+   * @param job Destination job to save options
+   */
+  protected void propagateOptionsToJob(Job job) {
+    Configuration configuration = job.getConfiguration();
+
+    // So far, propagate only verbose flag
+    configuration.setBoolean(PROPERTY_VERBOSE, options.getVerbose());
+  }
 }

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MergeJob.java Mon Jun 18 21:49:42 2012
@@ -102,6 +102,8 @@ public class MergeJob extends JobBase {
       oldPath = oldPath.makeQualified(fs);
       newPath = newPath.makeQualified(fs);
 
+      propagateOptionsToJob(job);
+
       FileInputFormat.addInputPath(job, oldPath);
       FileInputFormat.addInputPath(job, newPath);
 

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLDumpMapper.java Mon Jun 18 21:49:42
2012
@@ -30,7 +30,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.sqoop.util.AsyncSink;
 import org.apache.sqoop.util.JdbcUrl;
 import org.apache.sqoop.util.PerfCounters;
@@ -46,7 +45,7 @@ import com.cloudera.sqoop.util.LoggingAs
  * Mapper that opens up a pipe to mysqldump and pulls data directly.
  */
 public class MySQLDumpMapper
-    extends Mapper<String, NullWritable, String, NullWritable> {
+    extends SqoopMapper<String, NullWritable, String, NullWritable> {
 
   public static final Log LOG = LogFactory.getLog(
       MySQLDumpMapper.class.getName());
@@ -496,7 +495,9 @@ public class MySQLDumpMapper
   // CHECKSTYLE:ON
 
   @Override
-  protected void setup(Context context) {
+  protected void setup(Context context)
+    throws IOException, InterruptedException {
+    super.setup(context);
     this.conf = context.getConfiguration();
   }
 }

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLExportMapper.java Mon Jun 18 21:49:42
2012
@@ -28,7 +28,6 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.mapreduce.Mapper;
 import org.apache.sqoop.util.AsyncSink;
 import org.apache.sqoop.util.JdbcUrl;
 import org.apache.sqoop.util.LoggingAsyncSink;
@@ -48,7 +47,7 @@ import com.cloudera.sqoop.manager.MySQLU
  * used to interface with mysqlimport.
  */
 public class MySQLExportMapper<KEYIN, VALIN>
-    extends Mapper<KEYIN, VALIN, NullWritable, NullWritable> {
+    extends SqoopMapper<KEYIN, VALIN, NullWritable, NullWritable> {
 
   public static final Log LOG = LogFactory.getLog(
       MySQLExportMapper.class.getName());

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/MySQLTextExportMapper.java Mon Jun 18
21:49:42 2012
@@ -22,7 +22,7 @@ import java.io.IOException;
 import org.apache.hadoop.io.LongWritable;
 import org.apache.hadoop.io.Text;
 import com.cloudera.sqoop.manager.MySQLUtils;
-import com.cloudera.sqoop.mapreduce.MySQLExportMapper;;
+import com.cloudera.sqoop.mapreduce.MySQLExportMapper;
 
 /**
  * mysqlimport-based exporter which accepts lines of text from files

Added: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java?rev=1351503&view=auto
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java (added)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java Mon Jun 18 21:49:42 2012
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.mapreduce;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.sqoop.util.LoggingUtils;
+
+import java.io.IOException;
+
+/**
+ * Base sqoop mapper class that is convenient place for common functionality.
+ * Other specific mappers are highly encouraged to inherit from this class.
+ */
+public abstract class SqoopMapper<KI, VI, KO, VO>
+  extends Mapper<KI, VI, KO, VO> {
+
+  @Override
+  protected void setup(Context context)
+    throws IOException, InterruptedException {
+    super.setup(context);
+
+    Configuration configuration = context.getConfiguration();
+
+    // Propagate verbose flag if needed
+    if (configuration.getBoolean(JobBase.PROPERTY_VERBOSE, false)) {
+      LoggingUtils.setDebugLevel();
+    }
+  }
+}

Propchange: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/SqoopMapper.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain

Modified: sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/tool/BaseSqoopTool.java Mon Jun 18 21:49:42 2012
@@ -33,8 +33,6 @@ import org.apache.commons.cli.OptionGrou
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.util.StringUtils;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
 
 import com.cloudera.sqoop.ConnFactory;
 import com.cloudera.sqoop.Sqoop;
@@ -45,6 +43,7 @@ import com.cloudera.sqoop.cli.ToolOption
 import com.cloudera.sqoop.lib.DelimiterSet;
 import com.cloudera.sqoop.manager.ConnManager;
 import com.cloudera.sqoop.metastore.JobData;
+import org.apache.sqoop.util.LoggingUtils;
 
 /**
  * Layer on top of SqoopTool that provides some basic common code
@@ -630,8 +629,8 @@ public abstract class BaseSqoopTool exte
     // common options.
     if (in.hasOption(VERBOSE_ARG)) {
       // Immediately switch into DEBUG logging.
-      Logger.getLogger("org.apache.sqoop").setLevel(Level.DEBUG);
-      Logger.getLogger("com.cloudera.apache").setLevel(Level.DEBUG);
+      out.setVerbose(true);
+      LoggingUtils.setDebugLevel();
       LOG.debug("Enabled debug logging.");
     }
 

Modified: sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java?rev=1351503&r1=1351502&r2=1351503&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/util/LoggingUtils.java Mon Jun 18 21:49:42 2012
@@ -21,6 +21,8 @@ package org.apache.sqoop.util;
 import java.sql.SQLException;
 
 import org.apache.commons.logging.Log;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
 
 /**
  * A helper class for logging.
@@ -44,5 +46,9 @@ public final class LoggingUtils {
     }
   }
 
+  public static void setDebugLevel() {
+    Logger.getLogger("org.apache.sqoop").setLevel(Level.DEBUG);
+    Logger.getLogger("com.cloudera.apache").setLevel(Level.DEBUG);
+  }
 }
 



Mime
View raw message