sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2802: Sqoop2: Logging level should be debug for integration tests
Date Fri, 29 Jan 2016 20:52:30 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 66a86353c -> 0c20d1f06


SQOOP-2802: Sqoop2: Logging level should be debug for integration tests

(Abraham Fine via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0c20d1f0
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0c20d1f0
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0c20d1f0

Branch: refs/heads/sqoop2
Commit: 0c20d1f069ea77d3128f1bd05a890d2b6cb2c200
Parents: 66a8635
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Fri Jan 29 12:52:02 2016 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Fri Jan 29 12:52:02 2016 -0800

----------------------------------------------------------------------
 dist/src/main/conf/sqoop.properties                       |  3 +++
 .../main/java/org/apache/sqoop/job/MRJobConstants.java    |  2 ++
 .../org/apache/sqoop/job/mr/MRConfigurationUtils.java     |  3 +++
 .../src/main/resources/META-INF/log4j.properties          |  2 +-
 .../org/apache/sqoop/submission/mapreduce/Constants.java  |  3 +++
 .../submission/mapreduce/MapreduceSubmissionEngine.java   | 10 ++++++++++
 .../apache/sqoop/test/minicluster/SqoopMiniCluster.java   |  1 +
 7 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/dist/src/main/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/conf/sqoop.properties b/dist/src/main/conf/sqoop.properties
index 2895530..620146d 100755
--- a/dist/src/main/conf/sqoop.properties
+++ b/dist/src/main/conf/sqoop.properties
@@ -143,6 +143,9 @@ org.apache.sqoop.submission.engine=org.apache.sqoop.submission.mapreduce.Mapredu
 # Hadoop configuration directory
 org.apache.sqoop.submission.engine.mapreduce.configuration.directory=/etc/hadoop/conf/
 
+# Log level for Sqoop Mapper/Reducer
+org.apache.sqoop.submission.engine.mapreduce.configuration.loglevel=INFO
+
 #
 # Execution engine configuration
 #

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
index 8968198..6cbdac4 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
@@ -72,6 +72,8 @@ public final class MRJobConstants extends Constants {
    public static final String JOB_CONNECTOR_TO_NAME = PREFIX_JOB_CONFIG
       + "connector.to.name";
 
+  public static final String SYSPROP_MR_LOG_LEVEL = "sqoop.mr.loglevel";
+
   // Hadoop specific constants
   // We're using constants from Hadoop 1. Hadoop 2 has different names, but
   // provides backward compatibility layer for those names as well.

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
index 1e1b237..8410391 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
@@ -268,6 +268,9 @@ public final class MRConfigurationUtils {
   }
 
   public static void configureLogging() {
+    if (System.getProperty(MRJobConstants.SYSPROP_MR_LOG_LEVEL) == null) {
+      System.setProperty(MRJobConstants.SYSPROP_MR_LOG_LEVEL, "INFO");
+    }
     try (InputStream resourceAsStream =
                  SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties")) {
       Properties props = new Properties();

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/execution/mapreduce/src/main/resources/META-INF/log4j.properties
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/resources/META-INF/log4j.properties b/execution/mapreduce/src/main/resources/META-INF/log4j.properties
index 2a57714..e42c791 100644
--- a/execution/mapreduce/src/main/resources/META-INF/log4j.properties
+++ b/execution/mapreduce/src/main/resources/META-INF/log4j.properties
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 # Set root logger level to INFO and its only appender to A1.
-log4j.logger.org.apache.sqoop=INFO, A1
+log4j.logger.org.apache.sqoop=${sqoop.mr.loglevel}, A1
 
 # A1 is set to be a ConsoleAppender.
 log4j.appender.A1=org.apache.log4j.ConsoleAppender

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
index 93b0725..213da61 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/Constants.java
@@ -27,6 +27,9 @@ public class Constants {
   public static final String CONF_CONFIG_DIR =
     PREFIX_MAPREDUCE + "configuration.directory";
 
+  public static final String CONF_SQOOP_MR_LOG_LEVEL =
+    PREFIX_MAPREDUCE + "configuration.loglevel";
+
   public static final String SQOOP_JOB = "sqoop.job";
 
   private Constants() {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
index c03bf39..0dc4ef0 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.mapred.JobID;
 import org.apache.hadoop.mapred.JobStatus;
 import org.apache.hadoop.mapred.RunningJob;
 import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.common.Direction;
 import org.apache.sqoop.common.MapContext;
@@ -107,6 +108,15 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
       }
     }
 
+    // Log level for sqoop mapper and reducer
+    String mrLogLevel = context.getString(prefix + Constants.CONF_SQOOP_MR_LOG_LEVEL);
+    if (globalConfiguration.get(MRJobConfig.MR_AM_COMMAND_OPTS) == null) {
+      globalConfiguration.set(MRJobConfig.MR_AM_COMMAND_OPTS, MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS
+ " -Dsqoop.mr.loglevel=" + mrLogLevel);
+    } else {
+      globalConfiguration.set(MRJobConfig.MR_AM_COMMAND_OPTS, globalConfiguration.get(MRJobConfig.MR_AM_COMMAND_OPTS)
+ " -Dsqoop.mr.loglevel=" + mrLogLevel);
+    }
+
+
     // Save our own property inside the job to easily identify Sqoop jobs
     globalConfiguration.setBoolean(Constants.SQOOP_JOB, true);
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0c20d1f0/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index 4ff97e7..c8da7af 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -193,6 +193,7 @@ public abstract class SqoopMiniCluster {
 
     properties.put("org.apache.sqoop.submission.engine", "org.apache.sqoop.submission.mapreduce.MapreduceSubmissionEngine");
     properties.put("org.apache.sqoop.submission.engine.mapreduce.configuration.directory",
getConfigurationPath());
+    properties.put("org.apache.sqoop.submission.engine.mapreduce.configuration.loglevel",
"DEBUG");
 
     return properties;
   }


Mime
View raw message