sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2720: Sqoop2: Allow the integration tests to be run against a real cluster
Date Tue, 08 Dec 2015 07:11:56 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 794adbcf3 -> 6c4743de1


SQOOP-2720: Sqoop2: Allow the integration tests to be run against a real cluster

(Abraham Fine via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/6c4743de
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/6c4743de
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/6c4743de

Branch: refs/heads/sqoop2
Commit: 6c4743de158e4ced43e57703a8bcee5fca6257c6
Parents: 794adbc
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Tue Dec 8 08:11:26 2015 +0100
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Tue Dec 8 08:11:26 2015 +0100

----------------------------------------------------------------------
 .../org/apache/sqoop/client/SqoopClient.java    | 17 +++++++
 pom.xml                                         | 51 +++++++++++++++-----
 .../test/minicluster/RealSqoopCluster.java      | 16 +++++-
 .../sqoop/test/testcases/JettyTestCase.java     | 21 +++++---
 .../connector/hive/FromRDBMSToKiteHiveTest.java |  2 +-
 .../connector/kafka/FromHDFSToKafkaTest.java    |  2 +-
 .../connector/kafka/FromRDBMSToKafkaTest.java   |  2 +-
 .../BlacklistedConnectorTest.java               |  1 +
 .../connectorloading/ClasspathTest.java         |  1 +
 .../derby/upgrade/Derby1_99_3UpgradeTest.java   |  1 +
 .../derby/upgrade/Derby1_99_4UpgradeTest.java   |  3 ++
 .../derby/upgrade/Derby1_99_5UpgradeTest.java   |  1 +
 .../derby/upgrade/Derby1_99_6UpgradeTest.java   |  1 +
 13 files changed, 94 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
index 9b9359b..4abee77 100644
--- a/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
+++ b/client/src/main/java/org/apache/sqoop/client/SqoopClient.java
@@ -508,6 +508,23 @@ public class SqoopClient {
     resourceRequests.deleteJob(String.valueOf(jobId));
   }
 
+  public void deleteAllLinks(){
+    for (MJob job : getJobs()) {
+      deleteJob(job.getName());
+    }
+  }
+
+  public void deleteAllJobs(){
+    for (MLink link : getLinks()) {
+      deleteLink(link.getName());
+    }
+  }
+
+  public void deleteAllLinksAndJobs(){
+    deleteAllLinks();
+    deleteAllJobs();
+  }
+
   /**
    * Start job with given name.
    *

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 91721ce..e66de4d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -230,6 +230,33 @@ limitations under the License.
       </build>
     </profile>
 
+    <profile>
+      <id>realcluster</id>
+      <activation>
+        <activeByDefault>false</activeByDefault>
+      </activation>
+      <build>
+        <plugins>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-surefire-plugin</artifactId>
+            <configuration>
+              <excludedGroups>no-real-cluster</excludedGroups>
+              <reportNameSuffix>real-cluster-integration</reportNameSuffix>
+            </configuration>
+          </plugin>
+          <plugin>
+            <groupId>org.apache.maven.plugins</groupId>
+            <artifactId>maven-failsafe-plugin</artifactId>
+            <configuration>
+              <excludedGroups>no-real-cluster</excludedGroups>
+              <reportNameSuffix>real-cluster-integration</reportNameSuffix>
+            </configuration>
+          </plugin>
+        </plugins>
+      </build>
+    </profile>
+
     <!-- Sign profile for releasing artifacts to Nexus repository -->
     <profile>
       <id>sign</id>
@@ -260,10 +287,10 @@ limitations under the License.
         <artifactId>sqoop-client</artifactId>
         <version>${project.version}</version>
       </dependency>
-	  <dependency>
-		<groupId>org.apache.sqoop</groupId>
-		<artifactId>sqoop-shell</artifactId>
-		<version>${project.version}</version>
+      <dependency>
+        <groupId>org.apache.sqoop</groupId>
+        <artifactId>sqoop-shell</artifactId>
+        <version>${project.version}</version>
       </dependency>
       <dependency>
         <groupId>org.apache.sqoop</groupId>
@@ -500,10 +527,10 @@ limitations under the License.
         <artifactId>commons-cli</artifactId>
         <version>${commons-cli.version}</version>
       </dependency>
-       <dependency>
-         <groupId>commons-io</groupId>
-         <artifactId>commons-io</artifactId>
-         <version>${commons-io.version}</version>
+      <dependency>
+        <groupId>commons-io</groupId>
+        <artifactId>commons-io</artifactId>
+        <version>${commons-io.version}</version>
       </dependency>
       <dependency>
         <groupId>commons-codec</groupId>
@@ -640,9 +667,9 @@ limitations under the License.
         <version>${hadoop.2.version}</version>
       </dependency>
       <dependency>
-       <groupId>org.apache.avro</groupId>
-       <artifactId>avro</artifactId>
-       <version>${avro.version}</version>
+        <groupId>org.apache.avro</groupId>
+        <artifactId>avro</artifactId>
+        <version>${avro.version}</version>
       </dependency>
       <dependency>
         <groupId>net.sourceforge.findbugs</groupId>
@@ -660,7 +687,7 @@ limitations under the License.
         <artifactId>jetty-servlet</artifactId>
         <version>${jetty.version}</version>
       </dependency>
-   </dependencies>
+    </dependencies>
   </dependencyManagement>
 
   <modules>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java
index 325a6a9..cf19730 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/RealSqoopCluster.java
@@ -18,6 +18,9 @@
 package org.apache.sqoop.test.minicluster;
 
 import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
 
 /**
  */
@@ -27,11 +30,15 @@ public class RealSqoopCluster extends SqoopMiniCluster {
 
   private String serverUrl;
 
+  private SqoopClient client;
+
   public RealSqoopCluster(String temporaryPath) throws Exception {
     super(temporaryPath);
 
     serverUrl = System.getProperty(SERVER_URL_KEY);
 
+    client = new SqoopClient(serverUrl);
+
     if(serverUrl == null) {
       throw new RuntimeException("Missing URL for real Sqoop 2 server: " + SERVER_URL_KEY);
     }
@@ -44,16 +51,21 @@ public class RealSqoopCluster extends SqoopMiniCluster {
 
   @Override
   public void start() throws Exception {
-    // Void operation
+    client.deleteAllLinksAndJobs();
   }
 
   @Override
   public void stop() throws Exception {
-    // Void operation
+    client.deleteAllLinksAndJobs();
   }
 
   @Override
   public String getServerUrl() {
     return serverUrl;
   }
+
+  @Override
+  public String getConfigurationPath() {
+    return "/etc/hadoop/conf/";
+  }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
index e78adc0..e3ee300 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/JettyTestCase.java
@@ -17,12 +17,9 @@
  */
 package org.apache.sqoop.test.testcases;
 
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Method;
-
 import org.apache.commons.io.FileUtils;
 import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.mapred.JobConf;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.client.SqoopClient;
@@ -31,7 +28,9 @@ import org.apache.sqoop.test.hadoop.HadoopMiniClusterRunner;
 import org.apache.sqoop.test.hadoop.HadoopRunner;
 import org.apache.sqoop.test.hadoop.HadoopRunnerFactory;
 import org.apache.sqoop.test.minicluster.JettySqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.RealSqoopCluster;
 import org.apache.sqoop.test.minicluster.SqoopMiniCluster;
+import org.apache.sqoop.test.minicluster.SqoopMiniClusterFactory;
 import org.apache.sqoop.test.utils.HdfsUtils;
 import org.testng.ITest;
 import org.testng.ITestContext;
@@ -39,6 +38,10 @@ import org.testng.annotations.AfterSuite;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.BeforeSuite;
 
+import java.io.File;
+import java.io.IOException;
+import java.lang.reflect.Method;
+
 /**
  * Basic test case that will bootstrap Sqoop server running in embedded Jetty
  * process.
@@ -82,7 +85,7 @@ abstract public class JettyTestCase implements ITest {
   /**
    * Jetty based Sqoop mini cluster
    */
-  private static JettySqoopMiniCluster cluster;
+  private static SqoopMiniCluster cluster;
 
   /**
    * Sqoop client API.
@@ -97,8 +100,9 @@ abstract public class JettyTestCase implements ITest {
   }
 
   @BeforeMethod(alwaysRun = true)
-  public void setMethodName(Method method) throws Exception {
+  public void setupMehodNameAndOutputPath(Method method) throws Exception {
     methodName = method.getName();
+    hdfsClient.delete(new Path(getMapreduceDirectory()), true);
   }
 
   @BeforeSuite(alwaysRun = true)
@@ -154,8 +158,9 @@ abstract public class JettyTestCase implements ITest {
    *
    * @return New instance of test mini cluster
    */
-  public JettySqoopMiniCluster createSqoopMiniCluster() throws Exception {
-    return new JettySqoopMiniCluster(getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration());
+  public SqoopMiniCluster createSqoopMiniCluster() throws Exception {
+    return SqoopMiniClusterFactory.getSqoopMiniCluster(System.getProperties(), JettySqoopMiniCluster.class,
+      getSqoopMiniClusterTemporaryPath(), hadoopCluster.getConfiguration());
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
index 0e46bf3..e964422 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
@@ -37,7 +37,7 @@ import org.testng.annotations.Test;
 import java.lang.reflect.Method;
 import java.util.List;
 
-@Test(groups = "slow")
+@Test(groups = {"slow", "no-real-cluster"})
 public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase implements ITest {
   private String testName;
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
index aa062fb..ea8c7b9 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromHDFSToKafkaTest.java
@@ -23,7 +23,7 @@ import org.apache.sqoop.model.MLink;
 import org.apache.sqoop.test.testcases.KafkaConnectorTestCase;
 import org.testng.annotations.Test;
 
-
+@Test(groups = "no-real-cluster")
 public class FromHDFSToKafkaTest extends KafkaConnectorTestCase {
 
   public static final String[] input = {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
index 6e78a13..4ecb018 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/kafka/FromRDBMSToKafkaTest.java
@@ -25,7 +25,7 @@ import org.apache.sqoop.model.MLink;
 import org.apache.sqoop.test.testcases.KafkaConnectorTestCase;
 import org.testng.annotations.Test;
 
-
+@Test(groups = "no-real-cluster")
 public class FromRDBMSToKafkaTest extends KafkaConnectorTestCase {
 
   private static final String[] input = {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
index 6228b0d..8b57d75 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/BlacklistedConnectorTest.java
@@ -30,6 +30,7 @@ import org.testng.annotations.Test;
 import java.util.HashMap;
 import java.util.Map;
 
+@Test(groups = "no-real-cluster")
 public class BlacklistedConnectorTest extends ConnectorTestCase {
   public static class DerbySqoopMiniCluster extends JettySqoopMiniCluster {
     public DerbySqoopMiniCluster(String temporaryPath, Configuration configuration) throws
Exception {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
index 4a2e7a4..e0879ed 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connectorloading/ClasspathTest.java
@@ -53,6 +53,7 @@ import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
 
+@Test(groups = "no-real-cluster")
 public class ClasspathTest extends ConnectorTestCase {
 
   private static final String TEST_CONNECTOR_JAR_NAME = "test-connector.jar";

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
index 8d02e24..2cfd43c 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
@@ -41,6 +41,7 @@ import static org.testng.Assert.assertEquals;
  * Job with id 3 has been disabled
  * Job with id 1 has been run 5 times
  */
+@Test(groups = "no-real-cluster")
 public class Derby1_99_3UpgradeTest extends DerbyRepositoryUpgradeTest {
 
   @Override

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
index b88940a..9f7b04b 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_4UpgradeTest.java
@@ -17,6 +17,8 @@
  */
 package org.apache.sqoop.integration.repository.derby.upgrade;
 
+import org.testng.annotations.Test;
+
 import java.util.HashMap;
 import java.util.Map;
 
@@ -42,6 +44,7 @@ import java.util.Map;
  * Job with id 4 has been disabled
  * Job with id 5 has been disabled
  */
+@Test(groups = "no-real-cluster")
 public class Derby1_99_4UpgradeTest extends DerbyRepositoryUpgradeTest {
 
   @Override

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
index 1f3563d..3b1e214 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_5UpgradeTest.java
@@ -51,6 +51,7 @@ import org.testng.annotations.Test;
  * Job with id 4 has been disabled
  * Job with id 5 has been disabled
  */
+@Test(groups = "no-real-cluster")
 public class Derby1_99_5UpgradeTest extends DerbyRepositoryUpgradeTest {
 
   @Override

http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c4743de/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
index a57a420..1081e50 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_6UpgradeTest.java
@@ -52,6 +52,7 @@ import static org.testng.Assert.assertNull;
  * Job with id 4 has been disabled
  * Job with id 5 has been disabled
  */
+@Test(groups = "no-real-cluster")
 public class Derby1_99_6UpgradeTest extends DerbyRepositoryUpgradeTest {
 
   @Override


Mime
View raw message