sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2516: Sqoop2: Findbugs: Fix warning in execution module
Date Tue, 25 Aug 2015 16:07:45 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 b151ba384 -> 9f076edb0


SQOOP-2516: Sqoop2: Findbugs: Fix warning in execution module

(Colin Ma via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9f076edb
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9f076edb
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9f076edb

Branch: refs/heads/sqoop2
Commit: 9f076edb0265fda24705320e944511dd0fee772e
Parents: b151ba3
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Tue Aug 25 09:07:19 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Tue Aug 25 09:07:19 2015 -0700

----------------------------------------------------------------------
 .../mapreduce/MapreduceExecutionEngine.java     |  1 +
 .../sqoop/job/mr/MRConfigurationUtils.java      | 33 +++++++++++---------
 .../apache/sqoop/job/mr/SqoopInputFormat.java   |  3 +-
 .../org/apache/sqoop/job/mr/SqoopMapper.java    |  2 +-
 .../sqoop/job/mr/SqoopNullOutputFormat.java     |  2 +-
 .../org/apache/sqoop/job/mr/SqoopSplit.java     |  6 ++++
 6 files changed, 29 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
index 3f79325..c8d210e 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
@@ -46,6 +46,7 @@ public class MapreduceExecutionEngine extends ExecutionEngine {
   }
 
   public void prepareJob(JobRequest jobRequest) {
+    assert jobRequest instanceof MRJobRequest;
     MRJobRequest mrJobRequest = (MRJobRequest)jobRequest;
 
     // Add jar dependencies

http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
index e07ddd7..1e1b237 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java
@@ -31,7 +31,9 @@ import org.apache.sqoop.schema.Schema;
 import org.apache.sqoop.utils.ClassUtils;
 import org.json.simple.JSONObject;
 
+import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.util.Properties;
 
 /**
@@ -89,12 +91,12 @@ public final class MRConfigurationUtils {
     switch (type) {
       case FROM:
         job.getConfiguration().set(MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK, obj.getClass().getName());
-        job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes());
+        job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
         break;
 
       case TO:
         job.getConfiguration().set(MR_JOB_CONFIG_CLASS_TO_CONNECTOR_LINK, obj.getClass().getName());
-        job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes());
+        job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
         break;
     }
   }
@@ -109,12 +111,12 @@ public final class MRConfigurationUtils {
     switch (type) {
       case FROM:
         job.getConfiguration().set(MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, obj.getClass().getName());
-        job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
+        job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
         break;
 
       case TO:
         job.getConfiguration().set(MR_JOB_CONFIG_CLASS_TO_CONNECTOR_JOB, obj.getClass().getName());
-        job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
+        job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
         break;
     }
   }
@@ -128,7 +130,7 @@ public final class MRConfigurationUtils {
    */
   public static void setDriverConfig(Job job, Object obj) {
     job.getConfiguration().set(MR_JOB_CONFIG_DRIVER_CONFIG_CLASS, obj.getClass().getName());
-    job.getCredentials().addSecretKey(MR_JOB_CONFIG_DRIVER_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes());
+    job.getCredentials().addSecretKey(MR_JOB_CONFIG_DRIVER_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8")));
   }
 
   /**
@@ -142,10 +144,10 @@ public final class MRConfigurationUtils {
       String jsonSchema =  SchemaSerialization.extractSchema(schema).toJSONString();
       switch (type) {
         case FROM:
-          job.getCredentials().addSecretKey(SCHEMA_FROM_KEY,jsonSchema.getBytes());
+          job.getCredentials().addSecretKey(SCHEMA_FROM_KEY,jsonSchema.getBytes(Charset.forName("UTF-8")));
           return;
         case TO:
-          job.getCredentials().addSecretKey(SCHEMA_TO_KEY, jsonSchema.getBytes());
+          job.getCredentials().addSecretKey(SCHEMA_TO_KEY, jsonSchema.getBytes(Charset.forName("UTF-8")));
           return;
     }
   }
@@ -156,6 +158,7 @@ public final class MRConfigurationUtils {
    * @return Configuration object
    */
   public static Object getConnectorLinkConfig(Direction type, Configuration configuration)
{
+    assert configuration instanceof JobConf;
     switch (type) {
       case FROM:
         return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK,
MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY);
@@ -174,6 +177,7 @@ public final class MRConfigurationUtils {
    * @return Configuration object
    */
   public static Object getConnectorJobConfig(Direction type, Configuration configuration)
{
+    assert configuration instanceof JobConf;
     switch (type) {
       case FROM:
         return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB,
MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY);
@@ -192,6 +196,7 @@ public final class MRConfigurationUtils {
    * @return Configuration object
    */
   public static Object getDriverConfig(Configuration configuration) {
+    assert configuration instanceof JobConf;
     return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_DRIVER_CONFIG_CLASS,
MR_JOB_CONFIG_DRIVER_CONFIG_KEY);
   }
 
@@ -204,6 +209,7 @@ public final class MRConfigurationUtils {
    * @param configuration MapReduce configuration object
    */
   public static Schema getConnectorSchema(Direction type, Configuration configuration) {
+    assert configuration instanceof JobConf;
     switch (type) {
       case FROM:
         return getSchemaFromBytes(((JobConf) configuration).getCredentials().getSecretKey(SCHEMA_FROM_KEY));
@@ -228,7 +234,7 @@ public final class MRConfigurationUtils {
       return null;
     }
 
-    JSONObject jsonSchema = JSONUtils.parse(new String(bytes));
+    JSONObject jsonSchema = JSONUtils.parse(new String(bytes, Charset.forName("UTF-8")));
     return SchemaSerialization.restoreSchema(jsonSchema);
   }
 
@@ -248,7 +254,7 @@ public final class MRConfigurationUtils {
       return null;
     }
 
-    String json = new String(configuration.getCredentials().getSecretKey(valueProperty));
+    String json = new String(configuration.getCredentials().getSecretKey(valueProperty),
Charset.forName("UTF-8"));
 
     // Fill it with JSON data
     ConfigUtils.fillValues(json, object);
@@ -262,15 +268,14 @@ public final class MRConfigurationUtils {
   }
 
   public static void configureLogging() {
-    try {
+    try (InputStream resourceAsStream =
+                 SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties")) {
       Properties props = new Properties();
-      InputStream resourceAsStream =
-          SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties");
       props.load(resourceAsStream);
       PropertyConfigurator.configure(props);
-    } catch (Exception e) {
+    } catch (RuntimeException | IOException e) {
       System.err.println("Encountered exception while configuring logging " +
-        "for sqoop: " + e);
+              "for sqoop: " + e);
     }
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
index 5aef878..67189a1 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java
@@ -75,8 +75,7 @@ public class SqoopInputFormat extends InputFormat<SqoopSplit, NullWritable>
{
     List<InputSplit> splits = new LinkedList<InputSplit>();
     for (Partition partition : partitions) {
       LOG.debug("Partition: " + partition);
-      SqoopSplit split = new SqoopSplit();
-      split.setPartition(partition);
+      SqoopSplit split = new SqoopSplit(partition);
       splits.add(split);
     }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
index 14fdfdc..937ef5a 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java
@@ -149,7 +149,7 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable,
         toIDF.setObjectData(matcher.getMatchingData(fromIDF.getObjectData()));
         // NOTE: We do not use the reducer to do the writing (a.k.a LOAD in ETL). Hence the
mapper sets up the writable
         context.write(writable, NullWritable.get());
-      } catch (Exception e) {
+      } catch (RuntimeException | IOException | InterruptedException e) {
         throw new SqoopException(MRExecutionError.MAPRED_EXEC_0013, e);
       }
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
index 6134106..88ab98e 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java
@@ -55,7 +55,7 @@ public class SqoopNullOutputFormat extends OutputFormat<SqoopWritable,
NullWrita
     return new SqoopDestroyerOutputCommitter();
   }
 
-  class SqoopDestroyerOutputCommitter extends OutputCommitter {
+  private static class SqoopDestroyerOutputCommitter extends OutputCommitter {
     @Override
     public void setupJob(JobContext jobContext) {
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java
index b81f4d6..bb6b2b0 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java
@@ -35,6 +35,12 @@ public class SqoopSplit extends InputSplit implements Writable {
 
   private Partition partition;
 
+  public SqoopSplit() {}
+
+  public SqoopSplit(Partition partition) {
+    this.partition = partition;
+  }
+
   public void setPartition(Partition partition) {
     this.partition = partition;
   }


Mime
View raw message