sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2365: Sqoop2: Create annotation and base class
Date Thu, 23 Jul 2015 00:48:11 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 749690679 -> 4d2db87b3


SQOOP-2365: Sqoop2: Create annotation and base class

(Abraham Elmahrek via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/4d2db87b
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/4d2db87b
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/4d2db87b

Branch: refs/heads/sqoop2
Commit: 4d2db87b3a124a3c2ff85a68092de32507c1be41
Parents: 7496906
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Wed Jul 22 17:47:47 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Wed Jul 22 17:47:47 2015 -0700

----------------------------------------------------------------------
 .../sqoop/connector/kite/KiteToDestroyer.java   |   2 +-
 .../test/infrastructure/Infrastructure.java     |  34 ++++
 .../test/infrastructure/SqoopTestCase.java      | 176 +++++++++++++++++++
 3 files changed, 211 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/4d2db87b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToDestroyer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToDestroyer.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToDestroyer.java
index 750559a..fb83f2b 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToDestroyer.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToDestroyer.java
@@ -87,4 +87,4 @@ public class KiteToDestroyer extends Destroyer<LinkConfiguration,
     return new KiteDatasetExecutor(dataset);
   }
 
-}
\ No newline at end of file
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/4d2db87b/test/src/main/java/org/apache/sqoop/test/infrastructure/Infrastructure.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/infrastructure/Infrastructure.java b/test/src/main/java/org/apache/sqoop/test/infrastructure/Infrastructure.java
new file mode 100644
index 0000000..f3db7ad
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/infrastructure/Infrastructure.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.infrastructure;
+
+import org.apache.sqoop.test.infrastructure.providers.InfrastructureProvider;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Look for infrastructure and load them.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE, ElementType.METHOD})
+public @interface Infrastructure {
+  Class<? extends InfrastructureProvider>[] dependencies() default {};
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/4d2db87b/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java b/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java
new file mode 100644
index 0000000..e60caf7
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/infrastructure/SqoopTestCase.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.infrastructure;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.sqoop.test.infrastructure.providers.HadoopInfrastructureProvider;
+import org.apache.sqoop.test.infrastructure.providers.InfrastructureProvider;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.testng.ITest;
+import org.testng.ITestContext;
+import org.testng.ITestNGMethod;
+import org.testng.annotations.AfterSuite;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeSuite;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Use Infrastructure annotation to boot up miniclusters.
+ * Order is built-in to code. Hadoop comes first, then
+ * the rest of the services.
+ */
+public class SqoopTestCase implements ITest {
+  private static final Logger LOG = Logger.getLogger(SqoopTestCase.class);
+
+  private static final String ROOT_PATH = System.getProperty("sqoop.integration.tmpdir",
System.getProperty("java.io.tmpdir", "/tmp")) + "/sqoop-cargo-tests";
+
+  private static final Map<String, InfrastructureProvider> PROVIDERS
+      = new HashMap<String, InfrastructureProvider>();
+
+  private static String suiteName;
+
+  private String methodName;
+
+  @BeforeSuite
+  public static void findSuiteName(ITestContext context) {
+    suiteName = context.getSuite().getName();
+  }
+
+  @BeforeMethod
+  public void findMethodName(Method method) {
+    methodName = method.getName();
+  }
+
+  @Override
+  public String getTestName() {
+    return methodName;
+  }
+
+  /**
+   * Create infrastructure components and start those services.
+   * @param context TestNG context that helps get all the test methods and classes.
+   */
+  @BeforeSuite(dependsOnMethods = "findSuiteName")
+  public static void startInfrastructureProviders(ITestContext context) {
+    // Find infrastructure provider classes to be used.
+    Set<Class<? extends InfrastructureProvider>> providers = new HashSet<Class<?
extends InfrastructureProvider>>();
+    for (ITestNGMethod method : context.getSuite().getAllMethods()) {
+      Infrastructure ann;
+
+      // If the method has an infrastructure annotation, process it.
+      if (method.getConstructorOrMethod().getMethod() != null) {
+        ann = method.getConstructorOrMethod().getMethod().getAnnotation(Infrastructure.class);
+        if (ann != null && ann.dependencies() != null) {
+          LOG.debug("Found dependencies on method ("
+              + method.getConstructorOrMethod().getDeclaringClass().getCanonicalName()
+              + "#" + method.getConstructorOrMethod().getMethod().getName()
+              + "): " + StringUtils.join(ann.dependencies(), ","));
+          providers.addAll(Arrays.asList(ann.dependencies()));
+        }
+      }
+
+      // Declaring class should be processed always.
+      ann = method.getConstructorOrMethod().getDeclaringClass().getAnnotation(Infrastructure.class);
+      if (ann != null && ann.dependencies() != null) {
+        LOG.debug("Found dependencies on class ("
+            + method.getConstructorOrMethod().getDeclaringClass().getCanonicalName()
+            + "): " + StringUtils.join(ann.dependencies(), ","));
+        providers.addAll(Arrays.asList(ann.dependencies()));
+      }
+    }
+
+    // Create/start infrastructure providers.
+    Configuration conf = new JobConf();
+
+    // Start hadoop first.
+    if (providers.contains(HadoopInfrastructureProvider.class)) {
+      InfrastructureProvider hadoopProviderObject = startInfrastructureProvider(HadoopInfrastructureProvider.class,
conf);
+
+      // Use the prepared hadoop configuration for the rest of the components.
+      if (hadoopProviderObject != null) {
+        conf = hadoopProviderObject.getHadoopConfiguration();
+      }
+    }
+
+    // Start the rest of the providers.
+    for (Class<? extends InfrastructureProvider> provider : providers) {
+      startInfrastructureProvider(provider, conf);
+    }
+  }
+
+  /**
+   * Start an infrastructure provider and add it to the PROVIDERS map
+   * for stopping in the future.
+   * @param providerClass
+   * @param hadoopConfiguration
+   * @param <T>
+   * @return
+   */
+  private static <T extends InfrastructureProvider> T startInfrastructureProvider(Class<T>
providerClass, Configuration hadoopConfiguration) {
+    T providerObject;
+
+    try {
+      providerObject = providerClass.newInstance();
+    } catch (Exception e) {
+      LOG.error("Could not instantiate new instance of InfrastructureProvider.", e);
+      return null;
+    }
+
+    providerObject.setRootPath(HdfsUtils.joinPathFragments(ROOT_PATH, suiteName, providerClass.getCanonicalName()));
+    providerObject.setHadoopConfiguration(hadoopConfiguration);
+    providerObject.start();
+
+    // Add for recall later.
+    PROVIDERS.put(providerClass.getCanonicalName(), providerObject);
+
+    System.out.println("Infrastructure Provider " + providerClass.getCanonicalName());
+
+    return providerObject;
+  }
+
+  /**
+   * Stop infrastructure components and services.
+   */
+  @AfterSuite
+  public static void stopInfrastructureProviders() {
+    // Hadoop infrastructure provider included in PROVIDERS.
+    for (InfrastructureProvider provider : PROVIDERS.values()) {
+      provider.stop();
+    }
+  }
+
+  /**
+   * Get the infrastructure provider from the PROVIDERS map.
+   * @param providerClass
+   * @param <T>
+   * @return T InfrastructureProvider
+   */
+  public static <T extends InfrastructureProvider> T getInfrastructureProvider(Class<T>
providerClass) {
+    InfrastructureProvider provider = PROVIDERS.get(providerClass.getCanonicalName());
+    return ((T) provider);
+  }
+}


Mime
View raw message