sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hshreedha...@apache.org
Subject git commit: SQOOP-1236. Sqoop2: Classpath generated by Submission engine should contain only unique elements.
Date Mon, 21 Jul 2014 19:14:56 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 e60fda87d -> 17c7219b9


SQOOP-1236. Sqoop2: Classpath generated by Submission engine should contain only unique elements.

(Jarcec via Hari)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/17c7219b
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/17c7219b
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/17c7219b

Branch: refs/heads/sqoop2
Commit: 17c7219b941240ddf21ecb52394d3dcfff846bd9
Parents: e60fda8
Author: Hari Shreedharan <harishreedharan@gmail.com>
Authored: Mon Jul 21 12:14:22 2014 -0700
Committer: Hari Shreedharan <harishreedharan@gmail.com>
Committed: Mon Jul 21 12:14:22 2014 -0700

----------------------------------------------------------------------
 .../sqoop/framework/SubmissionRequest.java      | 10 ++-
 .../sqoop/framework/TestSubmissionRequest.java  | 71 ++++++++++++++++++++
 .../mapreduce/MapreduceSubmissionEngine.java    | 14 +---
 3 files changed, 80 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/17c7219b/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java b/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java
index 53d0039..a138db5 100644
--- a/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java
+++ b/core/src/main/java/org/apache/sqoop/framework/SubmissionRequest.java
@@ -158,15 +158,19 @@ public class SubmissionRequest {
   }
 
   public void addJar(String jar) {
-    jars.add(jar);
+    if(!jars.contains(jar)) {
+      jars.add(jar);
+    }
   }
 
   public void addJarForClass(Class klass) {
-    jars.add(ClassUtils.jarForClass(klass));
+    addJar(ClassUtils.jarForClass(klass));
   }
 
   public void addJars(List<String> jars) {
-    this.jars.addAll(jars);
+    for(String j : jars) {
+      addJar(j);
+    }
   }
 
   public CallbackBase getConnectorCallbacks() {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/17c7219b/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java b/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java
new file mode 100644
index 0000000..3078ed2
--- /dev/null
+++ b/core/src/test/java/org/apache/sqoop/framework/TestSubmissionRequest.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.framework;
+
+import org.apache.sqoop.utils.ClassUtils;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ *
+ */
+public class TestSubmissionRequest {
+
+  private SubmissionRequest submissionRequest;
+
+  @Before
+  public void initializeSubmissionRequest() {
+    submissionRequest = new SubmissionRequest();
+  }
+
+  @Test
+  public void testAddJar() {
+    submissionRequest.addJar("A");
+    submissionRequest.addJar("B");
+    submissionRequest.addJar("A");
+
+    assertEquals(2, submissionRequest.getJars().size());
+    assertEquals("A", submissionRequest.getJars().get(0));
+    assertEquals("B", submissionRequest.getJars().get(1));
+  }
+
+  @Test
+  public void testAddJarForClass() {
+    submissionRequest.addJarForClass(TestSubmissionRequest.class);
+    submissionRequest.addJarForClass(TestFrameworkValidator.class);
+
+    assertEquals(1, submissionRequest.getJars().size());
+    assertTrue(submissionRequest.getJars().contains(ClassUtils.jarForClass(TestSubmissionRequest.class)));
+  }
+
+  @Test
+  public void testAddJars() {
+    submissionRequest.addJars(Arrays.asList("A", "B"));
+    submissionRequest.addJars(Arrays.asList("B", "C"));
+
+    assertEquals(3, submissionRequest.getJars().size());
+    assertEquals("A", submissionRequest.getJars().get(0));
+    assertEquals("B", submissionRequest.getJars().get(1));
+    assertEquals("C", submissionRequest.getJars().get(2));
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/17c7219b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
----------------------------------------------------------------------
diff --git a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
index 6fc485b..bfc28ef 100644
--- a/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
+++ b/submission/mapreduce/src/main/java/org/apache/sqoop/submission/mapreduce/MapreduceSubmissionEngine.java
@@ -17,6 +17,7 @@
  */
 package org.apache.sqoop.submission.mapreduce;
 
+import org.apache.commons.lang.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.JobClient;
@@ -187,18 +188,7 @@ public class MapreduceSubmissionEngine extends SubmissionEngine {
     configuration.setBoolean("mapred.reduce.tasks.speculative.execution", false);
 
     // Promote all required jars to the job
-    StringBuilder sb = new StringBuilder();
-    boolean first = true;
-    for(String jar : request.getJars()) {
-      if(first) {
-        first = false;
-      } else {
-        sb.append(",");
-      }
-      LOG.debug("Adding jar to the job: " + jar);
-      sb.append(jar);
-    }
-    configuration.set("tmpjars", sb.toString());
+    configuration.set("tmpjars", StringUtils.join(request.getJars(), ","));
 
     try {
       Job job = new Job(configuration);


Mime
View raw message