sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject sqoop git commit: SQOOP-1068: Sqoop2: Add integration tests for real cluster
Date Wed, 11 Mar 2015 21:13:46 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 1a4293df7 -> 3f618c917


SQOOP-1068: Sqoop2: Add integration tests for real cluster

(Syed Hashmi via Abraham Elmahrek)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3f618c91
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3f618c91
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3f618c91

Branch: refs/heads/sqoop2
Commit: 3f618c917695ab45e09e4e432c9e58f5df466e67
Parents: 1a4293d
Author: Abraham Elmahrek <abe@apache.org>
Authored: Wed Mar 11 14:12:19 2015 -0700
Committer: Abraham Elmahrek <abe@apache.org>
Committed: Wed Mar 11 14:12:19 2015 -0700

----------------------------------------------------------------------
 .../sqoop/test/hadoop/HadoopLocalRunner.java    |  9 ++-
 .../test/hadoop/HadoopMiniClusterRunner.java    | 19 +++--
 .../test/hadoop/HadoopRealClusterRunner.java    | 84 ++++++++++++++++++++
 .../apache/sqoop/test/hadoop/HadoopRunner.java  | 30 ++++---
 4 files changed, 117 insertions(+), 25 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/3f618c91/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java
index 44465b4..140d0f9 100644
--- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java
+++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopLocalRunner.java
@@ -21,14 +21,15 @@ import org.apache.hadoop.conf.Configuration;
 import org.apache.sqoop.test.utils.HdfsUtils;
 
 /**
- * Represents a local cluster.
- * It uses an unchanged Configuration object.
- * HadoopRunner implementation that is using LocalJobRunner for executing mapreduce jobs
and local filesystem instead of HDFS.
+ * Represents a local cluster. It uses an unchanged Configuration object.
+ * HadoopRunner implementation that is using LocalJobRunner for executing
+ * mapreduce jobs and local filesystem instead of HDFS.
  */
 public class HadoopLocalRunner extends HadoopRunner {
 
   @Override
-  public Configuration prepareConfiguration(Configuration conf) {
+  public Configuration prepareConfiguration(Configuration conf)
+      throws Exception {
     return conf;
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3f618c91/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
index b06dcab..2c0c4e6 100644
--- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
+++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopMiniClusterRunner.java
@@ -24,13 +24,13 @@ import org.apache.hadoop.mapred.MiniMRCluster;
 import org.apache.log4j.Logger;
 
 /**
- * Represents a minicluster setup.
- * It creates a configuration object and mutates it.
- * Clients that need to connect to the miniclusters should use
- * the provided configuration object.
+ * Represents a minicluster setup. It creates a configuration object and mutates
+ * it. Clients that need to connect to the miniclusters should use the provided
+ * configuration object.
  */
 public class HadoopMiniClusterRunner extends HadoopRunner {
-  private static final Logger LOG = Logger.getLogger(HadoopMiniClusterRunner.class);
+  private static final Logger LOG = Logger
+      .getLogger(HadoopMiniClusterRunner.class);
 
   /**
    * Hadoop HDFS cluster
@@ -43,7 +43,8 @@ public class HadoopMiniClusterRunner extends HadoopRunner {
   protected MiniMRCluster mrCluster;
 
   @Override
-  public Configuration prepareConfiguration(Configuration config) {
+  public Configuration prepareConfiguration(Configuration config)
+      throws Exception {
     config.set("dfs.block.access.token.enable", "false");
     config.set("dfs.permissions", "true");
     config.set("hadoop.security.authentication", "simple");
@@ -51,7 +52,8 @@ public class HadoopMiniClusterRunner extends HadoopRunner {
     config.set("mapred.tasktracker.reduce.tasks.maximum", "1");
     config.set("mapred.submit.replication", "1");
     config.set("yarn.resourcemanager.scheduler.class", "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler");
-    config.set("yarn.application.classpath", System.getProperty("java.class.path"));
+    config.set("yarn.application.classpath",
+        System.getProperty("java.class.path"));
     return config;
   }
 
@@ -75,7 +77,8 @@ public class HadoopMiniClusterRunner extends HadoopRunner {
 
     // Start MR server
     LOG.info("Starting MR cluster");
-    mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri().toString(),
1, null, null, null, new JobConf(config));
+    mrCluster = new MiniMRCluster(0, 0, 1, dfsCluster.getFileSystem().getUri()
+        .toString(), 1, null, null, null, new JobConf(config));
     LOG.info("Started MR cluster");
     config = prepareConfiguration(mrCluster.createJobConf());
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3f618c91/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java
b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java
new file mode 100644
index 0000000..eabdf0b
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRealClusterRunner.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.test.hadoop;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.Logger;
+
+/*
+* This class enables running tests against a real cluster.
+* From the command line, it expects "sqoop.hadoop.config.path"
+* variable to point to directory containing cluster config files,
+* otherwise it tries loading from default location: /etc/hadoop/conf.
+*/
+public class HadoopRealClusterRunner extends HadoopRunner {
+
+  private static final Logger LOG = Logger
+      .getLogger(HadoopRealClusterRunner.class);
+
+  /*
+  * This method loads config files for real cluster.
+  * core-site.xml, mapred-site.xml and hdfs-site.xml are mandatory
+  * while yarn-site.xml is optional to let tests execute against old
+  * (non-yarn based) M/R clusters.
+   */
+  @Override
+  public Configuration prepareConfiguration(Configuration config)
+      throws Exception {
+    String configPath = System.getProperty(
+        "sqoop.hadoop.config.path", "/etc/hadoop/conf");
+    LOG.debug("Config path is: " + configPath);
+
+    File dir = new File(configPath);
+    String [] files = dir.list(new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        return name.endsWith("-site.xml");
+      }
+    });
+
+    if(files == null) {
+      throw new FileNotFoundException("Hadoop config files not found: " + configPath);
+    }
+
+    // Add each config file to configuration object
+    for (String file : files) {
+      LOG.info("Found hadoop configuration file " + file);
+      config.addResource(new Path(configPath, file));
+    }
+    return config;
+  }
+
+  @Override
+  public void start() throws Exception {
+    // Do nothing
+
+  }
+
+  @Override
+  public void stop() throws Exception {
+    // Do nothing
+
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3f618c91/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java
index 2516ff1..cb4e384 100644
--- a/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java
+++ b/test/src/main/java/org/apache/sqoop/test/hadoop/HadoopRunner.java
@@ -22,15 +22,17 @@ import org.apache.sqoop.test.utils.HdfsUtils;
 
 /**
  * Hadoop cluster runner for testing purpose.
- *
- * Runner provides methods for bootstrapping and using Hadoop cluster.
- * This abstract implementation is agnostic about in what mode Hadoop is running.
- * Each mode will have it's own concrete implementation (for example LocalJobRunner, MiniCluster
or Real existing cluster).
+ * 
+ * Runner provides methods for bootstrapping and using Hadoop cluster. This
+ * abstract implementation is agnostic about in what mode Hadoop is running.
+ * Each mode will have it's own concrete implementation (for example
+ * LocalJobRunner, MiniCluster or Real existing cluster).
  */
 public abstract class HadoopRunner {
 
   /**
-   * Temporary path that can be used as a root for other directories storing various data
like logs or stored HDFS files.
+   * Temporary path that can be used as a root for other directories storing
+   * various data like logs or stored HDFS files.
    */
   private String temporaryPath;
 
@@ -40,12 +42,14 @@ public abstract class HadoopRunner {
   protected Configuration config = null;
 
   /**
-   * Prepare configuration object.
-   * This method should be called once before the start method is called.
+   * Prepare configuration object. This method should be called once before the
+   * start method is called.
    *
-   * @param config is the configuration object to prepare.
+   * @param config
+   *          is the configuration object to prepare.
    */
-  abstract public Configuration prepareConfiguration(Configuration config);
+  abstract public Configuration prepareConfiguration(Configuration config)
+      throws Exception;
 
   /**
    * Start hadoop cluster.
@@ -89,8 +93,8 @@ public abstract class HadoopRunner {
   }
 
   /**
-   * Return directory on local filesystem where logs and other
-   * data generated by the Hadoop Cluster should be stored.
+   * Return directory on local filesystem where logs and other data generated by
+   * the Hadoop Cluster should be stored.
    *
    * @return
    */
@@ -99,8 +103,8 @@ public abstract class HadoopRunner {
   }
 
   /**
-   * Return directory on local filesystem where logs and other
-   * data generated by the Hadoop Cluster should be stored.
+   * Return directory on local filesystem where logs and other data generated by
+   * the Hadoop Cluster should be stored.
    *
    * @return
    */


Mime
View raw message