sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject git commit: SQOOP-725: Enable HBase tests in sqoop 1
Date Thu, 29 Nov 2012 16:30:38 GMT
Updated Branches:
  refs/heads/trunk dc4a82102 -> 33a7a8141


SQOOP-725: Enable HBase tests in sqoop 1

(Hari Shreedharan via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/33a7a814
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/33a7a814
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/33a7a814

Branch: refs/heads/trunk
Commit: 33a7a81412d789371a235323ac15f875e1870894
Parents: dc4a821
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Thu Nov 29 08:29:44 2012 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Thu Nov 29 08:29:44 2012 -0800

----------------------------------------------------------------------
 build.xml                                          |    2 +-
 ivy/ivysettings.xml                                |    1 +
 .../com/cloudera/sqoop/hbase/HBaseTestCase.java    |  108 +++++++++++----
 .../com/cloudera/sqoop/hbase/TestHBaseImport.java  |   18 ---
 .../cloudera/sqoop/hbase/TestHBaseQueryImport.java |    6 -
 5 files changed, 81 insertions(+), 54 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/build.xml
----------------------------------------------------------------------
diff --git a/build.xml b/build.xml
index 12ba785..a0b6326 100644
--- a/build.xml
+++ b/build.xml
@@ -77,7 +77,7 @@
       <equals arg1="${hadoopversion}" arg2="200" />
       <then>
         <property name="hadoop.version" value="2.0.0-alpha" />
-        <property name="hbase.version" value="0.92.0" />
+        <property name="hbase.version" value="0.94.2" />
         <property name="zookeeper.version" value="3.4.2" />
         <property name="hadoop.version.full" value="2.0.0-alpha" />
       </then>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/ivy/ivysettings.xml
----------------------------------------------------------------------
diff --git a/ivy/ivysettings.xml b/ivy/ivysettings.xml
index 251131c..c4cc561 100644
--- a/ivy/ivysettings.xml
+++ b/ivy/ivysettings.xml
@@ -66,6 +66,7 @@ under the License.
 
     <chain name="default" dual="true" checkmodified="true"
         changingPattern=".*SNAPSHOT">
+      <resolver ref="fs"/>
       <resolver ref="apache-snapshot"/> 
       <resolver ref="cloudera-releases"/>
       <resolver ref="cloudera-staging"/> 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
index 3f5899e..65ff87b 100644
--- a/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
+++ b/src/test/com/cloudera/sqoop/hbase/HBaseTestCase.java
@@ -28,13 +28,17 @@ import org.apache.hadoop.conf.Configuration;
 
 import org.apache.hadoop.hbase.HBaseConfiguration;
 import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
+
 import org.apache.hadoop.hbase.client.Get;
 import org.apache.hadoop.hbase.client.HTable;
 import org.apache.hadoop.hbase.client.Result;
 import org.apache.hadoop.hbase.util.Bytes;
 
 import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.VersionInfo;
 
 import org.junit.After;
 import org.junit.Before;
@@ -42,6 +46,10 @@ import org.junit.Before;
 import com.cloudera.sqoop.testutil.CommonArgs;
 import com.cloudera.sqoop.testutil.HsqldbTestServer;
 import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import java.io.File;
+import java.lang.reflect.Method;
+import java.util.UUID;
+import org.apache.commons.io.FileUtils;
 
 /**
  * Utility methods that facilitate HBase import tests.
@@ -105,56 +113,94 @@ public abstract class HBaseTestCase extends ImportJobTestCase {
 
     return args.toArray(new String[0]);
   }
-
+  // Starts a mini hbase cluster in this process.
   // Starts a mini hbase cluster in this process.
   private HBaseTestingUtility hbaseTestUtil;
-
-  private void startMaster() throws Exception {
-    if (null == hbaseTestUtil) {
-      Configuration conf = new Configuration();
-      conf = HBaseConfiguration.addHbaseResources(conf);
-      hbaseTestUtil = new HBaseTestingUtility(conf);
-      hbaseTestUtil.startMiniCluster(1);
-    }
-  }
+  private String workDir = createTempDir().getAbsolutePath();
+  private MiniZooKeeperCluster zookeeperCluster;
+  private MiniHBaseCluster hbaseCluster;
 
   @Override
   @Before
   public void setUp() {
-    if (!isHadoop20()) {
-      return;
-    }
-    HBaseTestCase.recordTestBuildDataProperty();
     try {
-      startMaster();
-    } catch (Exception e) {
-      fail(e.toString());
+      HBaseTestCase.recordTestBuildDataProperty();
+      String hbaseDir = new File(workDir, "hbase").getAbsolutePath();
+      String hbaseRoot = "file://" + hbaseDir;
+      Configuration hbaseConf = HBaseConfiguration.create();
+      hbaseConf.set(HConstants.HBASE_DIR, hbaseRoot);
+      //Hbase 0.90 does not have HConstants.ZOOKEEPER_CLIENT_PORT
+      hbaseConf.setInt("hbase.zookeeper.property.clientPort", 21818);
+      hbaseConf.set(HConstants.ZOOKEEPER_QUORUM, "0.0.0.0");
+      hbaseConf.setInt("hbase.master.info.port", -1);
+      hbaseConf.setInt("hbase.zookeeper.property.maxClientCnxns", 500);
+      String zookeeperDir = new File(workDir, "zk").getAbsolutePath();
+      int zookeeperPort = 21818;
+      zookeeperCluster = new MiniZooKeeperCluster();
+      Method m;
+      Class<?> zkParam[] = {Integer.TYPE};
+      try {
+        m = MiniZooKeeperCluster.class.getDeclaredMethod("setDefaultClientPort",
+                zkParam);
+      } catch (NoSuchMethodException e) {
+        m = MiniZooKeeperCluster.class.getDeclaredMethod("setClientPort",
+                zkParam);
+      }
+      m.invoke(zookeeperCluster, new Object[]{new Integer(zookeeperPort)});
+      zookeeperCluster.startup(new File(zookeeperDir));
+      hbaseCluster = new MiniHBaseCluster(hbaseConf, 1);
+      HMaster master = hbaseCluster.getMaster();
+      Object serverName = master.getServerName();
+
+      String hostAndPort;
+      if (serverName instanceof String) {
+        System.out.println("Server name is string, using HServerAddress.");
+        m = HMaster.class.getDeclaredMethod("getMasterAddress",
+                new Class<?>[]{});
+        Class<?> clazz = Class.forName("org.apache.hadoop.hbase.HServerAddress");
+        /*
+         * Call method to get server address
+         */
+        Object serverAddr = clazz.cast(m.invoke(master, new Object[]{}));
+        //returns the address as hostname:port
+        hostAndPort = serverAddr.toString();
+      } else {
+        System.out.println("ServerName is org.apache.hadoop.hbase.ServerName,"
+                + "using getHostAndPort()");
+        Class<?> clazz = Class.forName("org.apache.hadoop.hbase.ServerName");
+        m = clazz.getDeclaredMethod("getHostAndPort", new Class<?>[]{});
+        hostAndPort = m.invoke(serverName, new Object[]{}).toString();
+      }
+      hbaseConf.set("hbase.master", hostAndPort);
+      hbaseTestUtil = new HBaseTestingUtility(hbaseConf);
+      hbaseTestUtil.setZkCluster(zookeeperCluster);
+      hbaseCluster.startMaster();
+      super.setUp();
+    } catch (Throwable e) {
+      throw new RuntimeException(e);
     }
-    super.setUp();
   }
 
-
   public void shutdown() throws Exception {
     LOG.info("In shutdown() method");
     if (null != hbaseTestUtil) {
       LOG.info("Shutting down HBase cluster");
-      hbaseTestUtil.shutdownMiniCluster();
-      this.hbaseTestUtil = null;
+      hbaseCluster.shutdown();
+      zookeeperCluster.shutdown();
+      hbaseTestUtil = null;
     }
+    FileUtils.deleteDirectory(new File(workDir));
     LOG.info("shutdown() method returning.");
   }
 
   @Override
   @After
   public void tearDown() {
-    if (!isHadoop20()) {
-      return;
-    }
     try {
       shutdown();
     } catch (Exception e) {
       LOG.warn("Error shutting down HBase minicluster: "
-          + StringUtils.stringifyException(e));
+              + StringUtils.stringifyException(e));
     }
     HBaseTestCase.restoreTestBuidlDataProperty();
     super.tearDown();
@@ -180,8 +226,12 @@ public abstract class HBaseTestCase extends ImportJobTestCase {
       table.close();
     }
   }
-
-  protected boolean isHadoop20() {
-    return VersionInfo.getVersion().startsWith("0.20");
+  public static File createTempDir() {
+    File baseDir = new File(System.getProperty("java.io.tmpdir"));
+    File tempDir = new File(baseDir, UUID.randomUUID().toString());
+    if (tempDir.mkdir()) {
+      return tempDir;
+    }
+    throw new IllegalStateException("Failed to create directory");
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java b/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java
index 2108e28..d5e15e8 100644
--- a/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java
+++ b/src/test/com/cloudera/sqoop/hbase/TestHBaseImport.java
@@ -29,9 +29,6 @@ public class TestHBaseImport extends HBaseTestCase {
 
   @Test
   public void testBasicUsage() throws IOException {
-    if (!isHadoop20()) {
-      return;
-    }
     // Create the HBase table in Sqoop as we run the job.
     String [] argv = getArgv(true, "BasicUsage", "BasicColFam", true, null);
     String [] types = { "INT", "INT" };
@@ -43,9 +40,6 @@ public class TestHBaseImport extends HBaseTestCase {
 
   @Test
   public void testMissingTableFails() throws IOException {
-    if (!isHadoop20()) {
-      return;
-    }
     // Test that if the table doesn't exist, we fail unless we
     // explicitly create the table.
     String [] argv = getArgv(true, "MissingTable", "MissingFam", false, null);
@@ -62,9 +56,6 @@ public class TestHBaseImport extends HBaseTestCase {
 
   @Test
   public void testOverwriteSucceeds() throws IOException {
-    if (!isHadoop20()) {
-      return;
-    }
     // Test that we can create a table and then import immediately
     // back on top of it without problem.
     String [] argv = getArgv(true, "OverwriteT", "OverwriteF", true, null);
@@ -80,9 +71,6 @@ public class TestHBaseImport extends HBaseTestCase {
 
   @Test
   public void testStrings() throws IOException {
-    if (!isHadoop20()) {
-      return;
-    }
     String [] argv = getArgv(true, "stringT", "stringF", true, null);
     String [] types = { "INT", "VARCHAR(32)" };
     String [] vals = { "0", "'abc'" };
@@ -93,9 +81,6 @@ public class TestHBaseImport extends HBaseTestCase {
 
   @Test
   public void testNulls() throws IOException {
-    if (!isHadoop20()) {
-      return;
-    }
     String [] argv = getArgv(true, "nullT", "nullF", true, null);
     String [] types = { "INT", "INT", "INT" };
     String [] vals = { "0", "42", "null" };
@@ -111,9 +96,6 @@ public class TestHBaseImport extends HBaseTestCase {
 
   @Test
   public void testExitFailure() throws IOException {
-    if (!isHadoop20()) {
-      return;
-    }
     String [] types = { "INT", "INT", "INT" };
     String [] vals = { "0", "42", "43" };
     createTableWithColTypes(types, vals);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/33a7a814/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java b/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java
index 705dcae..74eff50 100644
--- a/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java
+++ b/src/test/com/cloudera/sqoop/hbase/TestHBaseQueryImport.java
@@ -29,9 +29,6 @@ public class TestHBaseQueryImport extends HBaseTestCase {
 
   @Test
   public void testImportFromQuery() throws IOException {
-    if (!isHadoop20()) {
-      return;
-    }
     String [] types = { "INT", "INT", "INT" };
     String [] vals = { "0", "42", "43" };
     createTableWithColTypes(types, vals);
@@ -50,9 +47,6 @@ public class TestHBaseQueryImport extends HBaseTestCase {
 
   @Test
   public void testExitFailure() throws IOException {
-    if (!isHadoop20()) {
-      return;
-    }
     String [] types = { "INT", "INT", "INT" };
     String [] vals = { "0", "42", "43" };
     createTableWithColTypes(types, vals);


Mime
View raw message