sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2491: Sqoop2: Common external JAR handling in configuration
Date Mon, 17 Aug 2015 22:22:48 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 58b2d1c1c -> 094c9c241


SQOOP-2491: Sqoop2: Common external JAR handling in configuration

(Abraham Elmahrek via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/094c9c24
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/094c9c24
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/094c9c24

Branch: refs/heads/sqoop2
Commit: 094c9c241f728da8ee4d034c54bd0ed22b57ca38
Parents: 58b2d1c
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Mon Aug 17 15:21:45 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Mon Aug 17 15:21:45 2015 -0700

----------------------------------------------------------------------
 .../org/apache/sqoop/utils/ContextUtils.java    |  81 ++++++++++
 .../apache/sqoop/utils/TestContextUtils.java    | 147 +++++++++++++++++++
 .../apache/sqoop/core/SqoopConfiguration.java   |  14 +-
 3 files changed, 235 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java b/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java
new file mode 100644
index 0000000..25d6fcb
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/utils/ContextUtils.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.utils;
+
+import org.apache.commons.lang.StringUtils;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Utils to be used ubiquitously for contexts.
+ */
+public class ContextUtils {
+  /**
+   * Return an array of strings for given value or null by default.
+   *
+   * @param value Non-empty value
+   * @return Array of values for given key or null in case of unknown key
+   */
+  public static String[] getArrayOfStrings(String value) {
+    return getArrayOfStrings(value, ":");
+  }
+
+  /**
+   * Return an array of strings for given value or default value.
+   *
+   * @param value Non-empty value
+   * @param separator String separator
+   * @return Array of values for given key or default value in case of unknown key
+   */
+  public static String[] getArrayOfStrings(String value, String separator) {
+    assert !StringUtils.isBlank(value);
+
+    return value.split(separator);
+  }
+
+  private static Set<String> getUniqueStrings(String[] strings) {
+    if (strings == null) {
+      return new HashSet<String>();
+    }
+
+    return new HashSet(Arrays.asList(strings));
+  }
+
+  /**
+   * Return an array of unique strings for given value or null by default.
+   *
+   * @param value Non-empty value
+   * @return Set of unique strings for given value or null in case of unknown key
+   */
+  public static Set<String> getUniqueStrings(String value) {
+    return getUniqueStrings(getArrayOfStrings(value));
+  }
+
+  /**
+   * Return an array of unique strings for given value or default value.
+   *
+   * @param value Non-empty value
+   * @param separator String separator
+   * @return Set of unique strings for given value or default value in case of unknown key
+   */
+  public static Set<String> getUniqueStrings(String value, String separator) {
+    return getUniqueStrings(getArrayOfStrings(value, separator));
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java
new file mode 100644
index 0000000..779fc78
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/utils/TestContextUtils.java
@@ -0,0 +1,147 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.utils;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+import org.apache.sqoop.common.MapContext;
+import org.testng.annotations.Test;
+
+import static org.apache.sqoop.utils.ContextUtils.getArrayOfStrings;
+import static org.apache.sqoop.utils.ContextUtils.getUniqueStrings;
+import static org.testng.Assert.assertEquals;
+
+/**
+ * Test class for org.apache.sqoop.utils.ContextUtils
+ */
+public class TestContextUtils {
+  @Test
+  public void testGetArrayOfStrings() {
+    final String DEFAULT_SEPARATOR = ",";
+
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", "value1:value2");
+    options1.put("sqooptest2", "value1");
+
+    Map<String, String> options2 = new HashMap<String, String>();
+    options2.put("sqooptest1", "value1,value2");
+    options2.put("sqooptest2", "value1");
+
+    MapContext mc1 = new MapContext(options1);
+    MapContext mc2 = new MapContext(options2);
+
+    assertEquals(getArrayOfStrings(mc1.getString("sqooptest1")), new String[]{"value1", "value2"});
+    assertEquals(getArrayOfStrings(mc1.getString("sqooptest2")), new String[]{"value1"});
+
+    assertEquals(getArrayOfStrings(mc2.getString("sqooptest1"), DEFAULT_SEPARATOR), new String[]{"value1",
"value2"});
+    assertEquals(getArrayOfStrings(mc2.getString("sqooptest2"), DEFAULT_SEPARATOR), new String[]{"value1"});
+  }
+
+  @Test(expectedExceptions = {AssertionError.class})
+  public void testGetArrayOfStringsNullValue() {
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", null);
+    MapContext mc1 = new MapContext(options1);
+    getArrayOfStrings(mc1.getString("sqooptest1"));
+  }
+
+  @Test(expectedExceptions = {AssertionError.class})
+  public void testGetArrayOfStringsWithSeparatorNullValue() {
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", null);
+    MapContext mc1 = new MapContext(options1);
+    getArrayOfStrings(mc1.getString("sqooptest1"), ",");
+  }
+
+  @Test(expectedExceptions = {AssertionError.class})
+  public void testGetArrayOfStringsEmptyValue() {
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", "");
+    MapContext mc1 = new MapContext(options1);
+    getArrayOfStrings(mc1.getString("sqooptest1"));
+  }
+
+  @Test(expectedExceptions = {AssertionError.class})
+  public void testGetArrayOfStringsWithSeparatorEmptyValue() {
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", "");
+    MapContext mc1 = new MapContext(options1);
+    getArrayOfStrings(mc1.getString("sqooptest1"), ",");
+  }
+
+  @Test
+  public void testGetUniqueStrings() {
+    final String DEFAULT_SEPARATOR = ",";
+
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", "value1:value2");
+    options1.put("sqooptest2", "value1");
+    options1.put("sqooptest3", "value1:value2:value1");
+
+    Map<String, String> options2 = new HashMap<String, String>();
+    options2.put("sqooptest1", "value1,value2");
+    options2.put("sqooptest2", "value1");
+    options2.put("sqooptest3", "value1,value2,value1");
+
+    MapContext mc1 = new MapContext(options1);
+    MapContext mc2 = new MapContext(options2);
+
+    assertEquals(getUniqueStrings(mc1.getString("sqooptest1")), new HashSet<String>(Arrays.asList(new
String[]{"value1", "value2"})));
+    assertEquals(getUniqueStrings(mc1.getString("sqooptest2")), new HashSet<String>(Arrays.asList(new
String[]{"value1"})));
+    assertEquals(getUniqueStrings(mc1.getString("sqooptest3")), new HashSet<String>(Arrays.asList(new
String[]{"value1", "value2"})));
+
+    assertEquals(getUniqueStrings(mc2.getString("sqooptest1"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new
String[]{"value1", "value2"})));
+    assertEquals(getUniqueStrings(mc2.getString("sqooptest2"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new
String[]{"value1"})));
+    assertEquals(getUniqueStrings(mc2.getString("sqooptest3"), DEFAULT_SEPARATOR), new HashSet<String>(Arrays.asList(new
String[]{"value1", "value2"})));
+  }
+
+  @Test(expectedExceptions = {AssertionError.class})
+  public void testGetUniqueStringsNullValue() {
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", null);
+    MapContext mc1 = new MapContext(options1);
+    getUniqueStrings(mc1.getString("sqooptest1"));
+  }
+
+  @Test(expectedExceptions = {AssertionError.class})
+  public void testGetUniqueStringsWithSeparatorNullValue() {
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", null);
+    MapContext mc1 = new MapContext(options1);
+    getUniqueStrings(mc1.getString("sqooptest1"), ",");
+  }
+
+  @Test(expectedExceptions = {AssertionError.class})
+  public void testGetUniqueStringsEmptyValue() {
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", "");
+    MapContext mc1 = new MapContext(options1);
+    getUniqueStrings(mc1.getString("sqooptest1"));
+  }
+
+  @Test(expectedExceptions = {AssertionError.class})
+  public void testGetUniqueStringsWithSeparatorEmptyValue() {
+    Map<String, String> options1 = new HashMap<String, String>();
+    options1.put("sqooptest1", "");
+    MapContext mc1 = new MapContext(options1);
+    getUniqueStrings(mc1.getString("sqooptest1"), ",");
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/094c9c24/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
index 58c4c34..49f139c 100644
--- a/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
+++ b/core/src/main/java/org/apache/sqoop/core/SqoopConfiguration.java
@@ -24,8 +24,6 @@ import java.io.InputStream;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.net.URLClassLoader;
-import java.util.Arrays;
-import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
@@ -41,6 +39,8 @@ import org.apache.sqoop.common.MapContext;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.error.code.CoreError;
 
+import static org.apache.sqoop.utils.ContextUtils.getUniqueStrings;
+
 /**
  * Configuration manager that loads Sqoop configuration.
  */
@@ -230,9 +230,10 @@ public class SqoopConfiguration implements Reconfigurable {
   private synchronized void configureClassLoader(String classpathProperty) {
     LOG.info("Adding jars to current classloader from property: " + classpathProperty);
 
-    String classpath = getContext().getString(classpathProperty);
+    // CSV URL list separated by ":".
+    String paths = getContext().getString(classpathProperty);
 
-    if (StringUtils.isEmpty(classpath)) {
+    if (StringUtils.isEmpty(paths)) {
       LOG.debug("Property " + classpathProperty + " is null or empty. Not adding any extra
jars.");
       return;
     }
@@ -242,11 +243,10 @@ public class SqoopConfiguration implements Reconfigurable {
       throw new SqoopException(CoreError.CORE_0009, "No thread context classloader to override.");
     }
 
-    // CSV URL list separated by ":".
-    Set<String> paths = new HashSet(Arrays.asList(classpath.split(":")));
+    Set<String> pathSet = getUniqueStrings(paths);
     List<URL> urls = new LinkedList<URL>();
 
-    for (String path : paths) {
+    for (String path : pathSet) {
       try {
         LOG.debug("Found jar in path: " + path);
         URL url = new File(path).toURI().toURL();


Mime
View raw message