sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject sqoop git commit: SQOOP-3346: Upgrade Hadoop version to 2.8.0
Date Fri, 20 Jul 2018 08:56:05 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 739bbce48 -> b3e941be0


SQOOP-3346: Upgrade Hadoop version to 2.8.0

(Boglarka Egyed via Szabolcs Vasas)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/b3e941be
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/b3e941be
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/b3e941be

Branch: refs/heads/trunk
Commit: b3e941be050d615f6aae5b4fafb9e0754c8c6a72
Parents: 739bbce
Author: Szabolcs Vasas <vasas@apache.org>
Authored: Fri Jul 20 10:54:40 2018 +0200
Committer: Szabolcs Vasas <vasas@apache.org>
Committed: Fri Jul 20 10:54:40 2018 +0200

----------------------------------------------------------------------
 ivy/libraries.properties                        |  2 +-
 .../sqoop/config/ConfigurationConstants.java    | 15 +++----------
 .../sqoop/config/ConfigurationHelper.java       | 23 +++++---------------
 .../org/apache/sqoop/mapreduce/JobBase.java     |  4 +---
 src/java/org/apache/sqoop/tool/ImportTool.java  |  5 ++---
 src/test/org/apache/sqoop/TestSqoopOptions.java |  1 +
 .../sqlserver/SQLServerParseMethodsTest.java    |  2 --
 .../org/apache/sqoop/orm/TestParseMethods.java  |  4 ----
 8 files changed, 13 insertions(+), 43 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/ivy/libraries.properties
----------------------------------------------------------------------
diff --git a/ivy/libraries.properties b/ivy/libraries.properties
index c506ca8..d679b12 100644
--- a/ivy/libraries.properties
+++ b/ivy/libraries.properties
@@ -55,7 +55,7 @@ accumulo.version=1.6.2
 
 slf4j.version=1.7.7
 
-hadoop.version=2.6.0
+hadoop.version=2.8.0
 hbase.version=1.2.4
 hcatalog.version=1.2.1
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/config/ConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/config/ConfigurationConstants.java b/src/java/org/apache/sqoop/config/ConfigurationConstants.java
index 666852c..3724f25 100644
--- a/src/java/org/apache/sqoop/config/ConfigurationConstants.java
+++ b/src/java/org/apache/sqoop/config/ConfigurationConstants.java
@@ -55,23 +55,14 @@ public final class ConfigurationConstants {
                                 "mapred.reduce.tasks.speculative.execution";
 
   /**
-   * The Configuration property identifying the job tracker address.
-   */
-  public static final String PROP_MAPRED_JOB_TRACKER_ADDRESS =
-                                "mapred.job.tracker";
-
-   /**
-   * The Configuration property identifying the job tracker address (new).
-   */
-  public static final String PROP_MAPREDUCE_JOB_TRACKER_ADDRESS =
-    "mapreduce.jobtracker.address";
-
-  /**
    * The Configuration property identifying the framework name. If set to YARN
    * then we will not be in local mode.
    */
   public static final String PROP_MAPREDUCE_FRAMEWORK_NAME =
     "mapreduce.framework.name";
+
+  public static final String MAPREDUCE_FRAMEWORK_LOCAL = "local";
+
   /**
    * The group name of task counters.
    */

http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/config/ConfigurationHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/config/ConfigurationHelper.java b/src/java/org/apache/sqoop/config/ConfigurationHelper.java
index fb2ab03..5a3d8ff 100644
--- a/src/java/org/apache/sqoop/config/ConfigurationHelper.java
+++ b/src/java/org/apache/sqoop/config/ConfigurationHelper.java
@@ -31,6 +31,9 @@ import org.apache.sqoop.mapreduce.db.DBConfiguration;
 
 import org.apache.hadoop.util.ReflectionUtils;
 
+import static org.apache.sqoop.config.ConfigurationConstants.MAPREDUCE_FRAMEWORK_LOCAL;
+import static org.apache.sqoop.config.ConfigurationConstants.PROP_MAPREDUCE_FRAMEWORK_NAME;
+
 /**
  * This class provides static helper methods that allow access and manipulation
  * of job configuration. It is convenient to keep such access in one place in
@@ -121,13 +124,6 @@ public final class ConfigurationHelper {
   }
 
   /**
-   * Sets the Jobtracker address to use for a job.
-   */
-  public static void setJobtrackerAddr(Configuration conf, String addr) {
-    conf.set(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS, addr);
-  }
-
-  /**
    * @return the Configuration property identifying a DBWritable to use.
    */
   public static String getDbInputClassProperty() {
@@ -233,17 +229,8 @@ public final class ConfigurationHelper {
       return config.getInt(ConfigurationConstants.PROP_SPLIT_LIMIT, -1);
   }
   public static boolean isLocalJobTracker(Configuration conf) {
-    // If framework is set to YARN, then we can't be running in local mode
-    if ("yarn".equalsIgnoreCase(conf
-      .get(ConfigurationConstants.PROP_MAPREDUCE_FRAMEWORK_NAME))) {
-      return false;
-    }
-    String jtAddr = conf
-      .get(ConfigurationConstants.PROP_MAPRED_JOB_TRACKER_ADDRESS);
-    String jtAddr2 = conf
-      .get(ConfigurationConstants.PROP_MAPREDUCE_JOB_TRACKER_ADDRESS);
-    return (jtAddr != null && jtAddr.equals("local"))
-      || (jtAddr2 != null && jtAddr2.equals("local"));
+    String frameworkName = conf.get(PROP_MAPREDUCE_FRAMEWORK_NAME);
+    return MAPREDUCE_FRAMEWORK_LOCAL.equals(frameworkName);
   }
 
   private ConfigurationHelper() {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/mapreduce/JobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java
index 6d1e049..f7b1b46 100644
--- a/src/java/org/apache/sqoop/mapreduce/JobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java
@@ -272,9 +272,7 @@ public class JobBase {
   protected void loadJars(Configuration conf, String ormJarFile,
       String tableClassName) throws IOException {
 
-    boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
-        || "local".equals(conf.get("mapred.job.tracker"));
-    if (isLocal) {
+    if (ConfigurationHelper.isLocalJobTracker(conf)) {
       // If we're using the LocalJobRunner, then instead of using the compiled
       // jar file as the job source, we're running in the current thread. Push
       // on another classloader that loads from that jar in addition to

http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index f7310b9..ccded65 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -42,6 +42,7 @@ import org.apache.sqoop.SqoopOptions;
 import org.apache.sqoop.SqoopOptions.InvalidOptionsException;
 import org.apache.sqoop.cli.RelatedOptions;
 import org.apache.sqoop.cli.ToolOptions;
+import org.apache.sqoop.config.ConfigurationHelper;
 import org.apache.sqoop.hive.HiveClient;
 import org.apache.sqoop.hive.HiveClientFactory;
 import org.apache.sqoop.manager.ImportJobContext;
@@ -118,9 +119,7 @@ public class ImportTool extends BaseSqoopTool {
   private void loadJars(Configuration conf, String ormJarFile,
                         String tableClassName) throws IOException {
 
-    boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
-        || "local".equals(conf.get("mapred.job.tracker"));
-    if (isLocal) {
+    if (ConfigurationHelper.isLocalJobTracker(conf)) {
       // If we're using the LocalJobRunner, then instead of using the compiled
       // jar file as the job source, we're running in the current thread. Push
       // on another classloader that loads from that jar in addition to

http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/TestSqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/TestSqoopOptions.java b/src/test/org/apache/sqoop/TestSqoopOptions.java
index ba4a4d4..d0591ad 100644
--- a/src/test/org/apache/sqoop/TestSqoopOptions.java
+++ b/src/test/org/apache/sqoop/TestSqoopOptions.java
@@ -858,6 +858,7 @@ public class TestSqoopOptions {
   @Test
   public void testSqoopOptionsCloneIsEqual() throws Exception {
     SqoopOptions options = createSqoopOptionsFilledWithRandomData();
+    options.getConf().setAllowNullValueProperties(false); // always false in cloned conf
     SqoopOptions clonedOptions = (SqoopOptions) options.clone();
     assertThat(options).isEqualToComparingFieldByFieldRecursively(clonedOptions);
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
index 833ebe8..fbd8d96 100644
--- a/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
+++ b/src/test/org/apache/sqoop/manager/sqlserver/SQLServerParseMethodsTest.java
@@ -172,8 +172,6 @@ public class SQLServerParseMethodsTest extends ImportJobTestCase {
       // Tell the job what class we're testing.
       job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
 
-      // use local mode in the same JVM.
-      ConfigurationHelper.setJobtrackerAddr(job, "local");
       job.set("fs.default.name", "file:///");
 
       String warehouseDir = getWarehouseDir();

http://git-wip-us.apache.org/repos/asf/sqoop/blob/b3e941be/src/test/org/apache/sqoop/orm/TestParseMethods.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/orm/TestParseMethods.java b/src/test/org/apache/sqoop/orm/TestParseMethods.java
index 46bb52d..0d34f5a 100644
--- a/src/test/org/apache/sqoop/orm/TestParseMethods.java
+++ b/src/test/org/apache/sqoop/orm/TestParseMethods.java
@@ -135,8 +135,6 @@ public class TestParseMethods extends ImportJobTestCase {
       // Tell the job what class we're testing.
       job.set(ReparseMapper.USER_TYPE_NAME_KEY, tableClassName);
 
-      // use local mode in the same JVM.
-      ConfigurationHelper.setJobtrackerAddr(job, "local");
       if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
         job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
       }
@@ -263,8 +261,6 @@ public class TestParseMethods extends ImportJobTestCase {
       job.set(ExplicitSetMapper.SET_COL_KEY, BASE_COL_NAME + "0");
       job.set(ExplicitSetMapper.SET_VAL_KEY, "this-is-a-test");
 
-      // use local mode in the same JVM.
-      ConfigurationHelper.setJobtrackerAddr(job, "local");
       if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
         job.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
       }


Mime
View raw message