sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2016: Sqoop2: Create integration test for JDBC to Hive
Date Wed, 25 Mar 2015 00:41:37 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 d615b39c1 -> 45d1c32d7


SQOOP-2016: Sqoop2: Create integration test for JDBC to Hive

(Abraham Elmahrek via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/45d1c32d
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/45d1c32d
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/45d1c32d

Branch: refs/heads/sqoop2
Commit: 45d1c32d7ed4811452c42626940d80a4b091f797
Parents: d615b39
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Tue Mar 24 17:41:13 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Tue Mar 24 17:41:13 2015 -0700

----------------------------------------------------------------------
 .../sqoop/common/test/db/HiveProvider.java      | 101 ++++++++++++++
 .../sqoop/common/test/utils/NetworkUtils.java   |  36 +++++
 .../connector/kite/KiteDatasetExecutor.java     |   2 +-
 .../sqoop/connector/kite/TestKiteExecutor.java  |  13 +-
 pom.xml                                         |  12 ++
 test/pom.xml                                    |  12 ++
 .../sqoop/test/hive/HiveServerRunner.java       | 135 ++++++++++++++++++
 .../test/hive/HiveServerRunnerFactory.java      |  56 ++++++++
 .../test/hive/InternalHiveServerRunner.java     |  55 ++++++++
 .../hive/InternalMetastoreServerRunner.java     |  71 ++++++++++
 .../sqoop/test/hive/MetastoreServerRunner.java  | 137 +++++++++++++++++++
 .../test/hive/MetastoreServerRunnerFactory.java |  56 ++++++++
 .../minicluster/TomcatSqoopMiniCluster.java     |   4 +
 .../test/testcases/HiveConnectorTestCase.java   |  88 ++++++++++++
 .../connector/hive/FromRDBMSToKiteHiveTest.java |  96 +++++++++++++
 .../connector/kite/FromRDBMSToKiteTest.java     |   4 +-
 16 files changed, 873 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java
----------------------------------------------------------------------
diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java b/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java
new file mode 100644
index 0000000..dfe0f43
--- /dev/null
+++ b/common-test/src/main/java/org/apache/sqoop/common/test/db/HiveProvider.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.common.test.db;
+
+/**
+ * MySQL Provider that will connect to remote MySQL server.
+ *
+ * JDBC can be configured via system properties. Default value is server running
+ * on the same box (localhost) that is access via sqoop/sqoop credentials.
+ */
+public class HiveProvider extends DatabaseProvider {
+
+  public static final String DRIVER = "org.apache.hive.jdbc.HiveDriver";
+
+  private static final String CONNECTION = System.getProperties().getProperty(
+      "sqoop.provider.hive.jdbc",
+      "jdbc:hive2://"
+  );
+
+  private static final String USERNAME = System.getProperties().getProperty(
+      "sqoop.provider.hive.username",
+      "sqoop"
+  );
+
+  private static final String PASSWORD = System.getProperties().getProperty(
+      "sqoop.provider.hive.password",
+      "sqoop"
+  );
+
+  private String jdbcUrl;
+
+  /**
+   * Use system properties to get JDBC URL.
+   */
+  public HiveProvider() {
+    this.jdbcUrl = CONNECTION;
+  }
+
+  /**
+   * Use JDBC URL provided.
+   *
+   * @param jdbcUrl hive server jdbc URL.
+   */
+  public HiveProvider(String jdbcUrl) {
+    this.jdbcUrl = jdbcUrl;
+  }
+
+  @Override
+  public String getConnectionUrl() {
+    return jdbcUrl;
+  }
+
+  @Override
+  public String getConnectionUsername() {
+    return USERNAME;
+  }
+
+  @Override
+  public String getConnectionPassword() {
+    return PASSWORD;
+  }
+
+  @Override
+  public String escapeColumnName(String columnName) {
+    return escape(columnName);
+  }
+
+  @Override
+  public String escapeTableName(String tableName) {
+    return escape(tableName);
+  }
+
+  @Override
+  public String escapeValueString(String value) {
+    return escape(value);
+  }
+
+  @Override
+  public String getJdbcDriver() {
+    return DRIVER;
+  }
+
+  public String escape(String entity) {
+    return entity;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
----------------------------------------------------------------------
diff --git a/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
b/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
index 87534c7..7f0f750 100644
--- a/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
+++ b/common-test/src/main/java/org/apache/sqoop/common/test/utils/NetworkUtils.java
@@ -17,14 +17,21 @@
  */
 package org.apache.sqoop.common.test.utils;
 
+import org.apache.log4j.Logger;
+
 import java.io.IOException;
+import java.net.InetAddress;
 import java.net.ServerSocket;
+import java.net.Socket;
+import java.util.concurrent.TimeoutException;
 
 /**
  * Network related utilities.
  */
 public class NetworkUtils {
 
+  private static final Logger LOG = Logger.getLogger(NetworkUtils.class);
+
   /**
    * Create port number that is available on this machine for
    * subsequent use.
@@ -58,6 +65,35 @@ public class NetworkUtils {
     }
   }
 
+  /**
+   * Create a socket and attempt to connect to ``hostname``:``port``
+   * ``numberOfAttempts`` amount of times with ``sleepTime`` milliseconds
+   * between each attempt.
+   *
+   * @param hostname host name to connect to
+   * @param port port to connect on
+   * @param numberOfAttempts number of tries to connect
+   * @param sleepTime time in between connection attempts in milliseconds
+   * @throws InterruptedException
+   * @throws TimeoutException
+   */
+  public static void waitForStartUp(String hostname, int port, int numberOfAttempts, long
sleepTime)
+      throws InterruptedException, TimeoutException {
+    for (int i = 0; i < numberOfAttempts; ++i) {
+      try {
+        LOG.debug("Attempt " + (i + 1) + " to access " + hostname + ":" + port);
+        new Socket(InetAddress.getByName(hostname), port).close();
+        return;
+      } catch (Exception e) {
+        LOG.debug("Failed to connect to " + hostname + ":" + port, e);
+      }
+
+      Thread.sleep(sleepTime);
+    }
+
+    throw new TimeoutException("Couldn't access new server: " + hostname + ":" + port);
+  }
+
   private NetworkUtils() {
     // Instantiation is prohibited
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
index db8d4e6..6aa28be 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
@@ -189,7 +189,7 @@ public class KiteDatasetExecutor {
           // Replace dataset name with temporary dataset name.
           uriParts[1] = uriParts[1].substring(0, uriParts[1].lastIndexOf("/")) + "/" + temporaryDatasetName;
         } else {
-          uriParts[1] = temporaryDatasetName;
+          uriParts[1] = ":" + temporaryDatasetName;
         }
       } else {
         uriParts[1] += "/" + temporaryDatasetName;

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
index 44721a8..0e797f8 100644
--- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
+++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
@@ -38,7 +38,8 @@ import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.verifyNoMoreInteractions;
 import static org.mockito.Mockito.when;
 import static org.mockito.MockitoAnnotations.initMocks;
-import static org.testng.AssertJUnit.assertTrue;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
 
 public class TestKiteExecutor {
 
@@ -173,6 +174,16 @@ public class TestKiteExecutor {
     assertTrue(suggestedUri.length() > subURI.length());
     assertTrue(suggestedUri.contains(subURI));
     assertTrue(suggestedUri.endsWith(endURI));
+
+    endURI = "auth:host=metastore&auth:port=9083";
+    uri = "dataset:hive:sqoop?auth:host=metastore&auth:port=9083";
+    subURI = "dataset:hive:";
+    suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri);
+    assertTrue(suggestedUri.length() > subURI.length());
+    assertTrue(suggestedUri.contains(subURI), suggestedUri);
+    assertTrue(suggestedUri.endsWith(endURI), suggestedUri);
+    assertFalse(suggestedUri.contains("sqoop"));
+    assertFalse(suggestedUri.contains("/"));
   }
 
   private static Schema createTwoFieldSchema() {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e3431bd..c608ca7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -368,6 +368,18 @@ limitations under the License.
       </dependency>
       <dependency>
         <groupId>org.apache.hive</groupId>
+        <artifactId>hive-jdbc</artifactId>
+        <version>${hive.version}</version>
+        <scope>test</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hive</groupId>
+        <artifactId>hive-service</artifactId>
+        <version>${hive.version}</version>
+        <scope>provided</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hive</groupId>
         <artifactId>hive-exec</artifactId>
         <version>${hive.version}</version>
         <scope>provided</scope>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index f743d25..d8fbfa2 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -113,6 +113,18 @@ limitations under the License.
     </dependency>
 
     <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-jdbc</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-service</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
       <groupId>org.codehaus.cargo</groupId>
       <artifactId>cargo-core-container-tomcat</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java
new file mode 100644
index 0000000..8b355bd
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunner.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+/**
+ * Hive server runner for testing purpose.
+ *
+ * Runner provides methods for bootstrapping and using HiveServer2. This
+ * should allow providing a HiveServer2 minicluster or using a real server.
+ */
+public abstract class HiveServerRunner {
+  private static final Logger LOG = Logger.getLogger(HiveServerRunner.class);
+
+  private final String hostname;
+  private final int port;
+
+  public HiveServerRunner(String hostname, int port) throws Exception {
+    this.hostname = hostname;
+
+    if (port == 0) {
+      this.port = NetworkUtils.findAvailablePort();
+    } else {
+      this.port = port;
+    }
+
+    LOG.info("Hive Server will bind to port " + getPort());
+  }
+
+  /**
+   * Configuration object.
+   */
+  protected HiveConf config = null;
+
+  /**
+   * Start Hive server.
+   *
+   * @throws Exception
+   */
+  abstract public void start() throws Exception;
+
+  /**
+   * Stop Hive server.
+   *
+   * @throws Exception
+   */
+  abstract public void stop() throws Exception;
+
+  /**
+   * Return JDBC URL to be used with HiveServer2 instance.
+   *
+   * @return String
+   */
+  public String getUrl() {
+    return "jdbc:hive2://" + hostname + ":" + port + "/default";
+  }
+
+  /**
+   * Prepare configuration object. This method should be called once before the
+   * start method is called.
+   *
+   * @param config is the configuration object to prepare.
+   */
+  public Configuration prepareConfiguration(Configuration config) throws Exception {
+    config.set(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_BIND_HOST.varname, getHostName());
+    config.setInt(HiveConf.ConfVars.HIVE_SERVER2_THRIFT_PORT.varname, getPort());
+    return config;
+  }
+
+  /**
+   * Get configuration.
+   *
+   * @return HiveConf
+   */
+  public HiveConf getConfiguration() {
+    return config;
+  }
+
+  /**
+   * Set the configuration object.
+   *
+   * @param config
+   */
+  public void setConfiguration(Configuration config) {
+    this.config = new HiveConf();
+    this.config.addResource(config);
+    this.printConfig();
+  }
+
+  /**
+   * Hostname used to start services on.
+   *
+   * @return String hostname
+   */
+  public String getHostName() {
+    return hostname;
+  }
+
+  /**
+   * Port hive service will be on.
+   *
+   * @return int port
+   */
+  public int getPort() {
+    return this.port;
+  }
+
+  private void printConfig() {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    config.logVars(new PrintStream(baos));
+    LOG.debug("Hive server runner configuration:\n" + baos.toString());
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java
new file mode 100644
index 0000000..f6a2fa6
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/HiveServerRunnerFactory.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import java.util.Properties;
+
+/**
+ * Create HiveServer2 runner.
+ */
+public class HiveServerRunnerFactory {
+
+  public static final String RUNNER_CLASS_PROPERTY = "sqoop.hive.runner.class";
+
+  public static final String HOSTNAME_PROPERTY = "sqoop.hive.server.hostname";
+
+  public static final String PORT_PROPERTY = "sqoop.hive.server.port";
+
+  public static final String DEFAULT_HOSTNAME = "127.0.0.1";
+
+  public static final String DEFAULT_PORT = "0";
+
+  public static HiveServerRunner getRunner(
+      Properties properties, Class<? extends HiveServerRunner> defaultRunner)
+          throws Exception {
+    Class<?> klass;
+
+    String hostname = properties.getProperty(HOSTNAME_PROPERTY, DEFAULT_HOSTNAME);
+    int port = Integer.parseInt(
+        properties.getProperty(PORT_PROPERTY, DEFAULT_PORT));
+
+    String className = properties.getProperty(RUNNER_CLASS_PROPERTY);
+    if(className == null) {
+      klass = defaultRunner;
+    } else {
+      klass = Class.forName(className);
+    }
+
+    return (HiveServerRunner)klass.getConstructor(String.class, int.class)
+        .newInstance(hostname, port);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java
new file mode 100644
index 0000000..3418525
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/InternalHiveServerRunner.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import org.apache.hive.service.server.HiveServer2;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+
+/**
+ * Use HiveServer2 JDBC client to run all operations.
+ *
+ * @see org.apache.sqoop.test.hive.HiveServerRunner
+ */
+public class InternalHiveServerRunner extends HiveServerRunner {
+  private static final Logger LOG = Logger.getLogger(InternalHiveServerRunner.class);
+
+  private final HiveServer2 hiveServer2;
+
+  public InternalHiveServerRunner(String hostname, int port) throws Exception {
+    super(hostname, port);
+    hiveServer2 = new HiveServer2();
+  }
+
+  @Override
+  public void start() throws Exception {
+    Long start = System.currentTimeMillis();
+    hiveServer2.init(getConfiguration());
+    hiveServer2.start();
+    NetworkUtils.waitForStartUp(getHostName(), getPort(), 5, 100);
+    Long end = System.currentTimeMillis();
+    LOG.debug("Hive service took " + (end - start)  + "ms to start");
+  }
+
+  @Override
+  public void stop() throws Exception {
+    if (hiveServer2 != null) {
+      hiveServer2.stop();
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java
b/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java
new file mode 100644
index 0000000..a8282ec
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/InternalMetastoreServerRunner.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * Start a Hive Metastore service on the specified hostname and port.
+ *
+ * @see org.apache.sqoop.test.hive.MetastoreServerRunner
+ */
+public class InternalMetastoreServerRunner extends MetastoreServerRunner {
+  private static final Logger LOG = Logger.getLogger(InternalMetastoreServerRunner.class);
+
+  private ExecutorService executor = Executors
+      .newSingleThreadExecutor();
+
+  public InternalMetastoreServerRunner(String hostname, int port) throws Exception {
+    super(hostname, port);
+  }
+
+  @Override
+  public void start() throws Exception {
+    Long start = System.currentTimeMillis();
+    final int metastorePort = getPort();
+    final HiveConf conf = getConfiguration();
+    Callable<Void> metastoreService = new Callable<Void>() {
+      public Void call() throws Exception {
+        try {
+          HiveMetaStore.startMetaStore(metastorePort,
+              ShimLoader.getHadoopThriftAuthBridge(), conf);
+          while(true);
+        } catch (Throwable e) {
+          throw new Exception("Error starting metastore", e);
+        }
+      }
+    };
+    executor.submit(metastoreService);
+    NetworkUtils.waitForStartUp(getHostName(), getPort(), 5, 1000);
+    Long end = System.currentTimeMillis();
+    LOG.debug("Metastore service took " + (end - start)  + "ms to start");
+  }
+
+  @Override
+  public void stop() throws Exception {
+    executor.shutdownNow();
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java
new file mode 100644
index 0000000..32a6b35
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunner.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.utils.NetworkUtils;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.util.UUID;
+
+/**
+ * Metastore server runner for testing purpose.
+ *
+ * Runner provides methods for bootstrapping and using the Hive metastore. This
+ * should allow providing a Hive metastore minicluster or using a real server.
+ */
+public abstract class MetastoreServerRunner {
+  private static final Logger LOG = Logger.getLogger(MetastoreServerRunner.class);
+
+  private final String hostname;
+  private final int port;
+  private final String warehouseDirectory;
+
+  public MetastoreServerRunner(String hostname, int port) throws Exception {
+    this.hostname = hostname;
+    this.warehouseDirectory = "/user/hive/" + UUID.randomUUID();
+
+    if (port == 0) {
+      port = NetworkUtils.findAvailablePort();
+    }
+    LOG.info("Hive Metastore will bind to port " + port);
+
+    this.port = port;
+  }
+
+  /**
+   * Configuration object.
+   */
+  protected HiveConf config = null;
+
+  /**
+   * Start Hive server.
+   *
+   * @throws Exception
+   */
+  abstract public void start() throws Exception;
+
+  /**
+   * Stop Hive server.
+   *
+   * @throws Exception
+   */
+  abstract public void stop() throws Exception;
+
+  /**
+   * Metastore URI authority (ex. hostname:port).
+   *
+   * @return String metastore authority.
+   */
+  public String getAuthority() {
+    return getHostName() + ":" + getPort();
+  }
+
+  /**
+   * Prepare configuration object. This method should be called once before the
+   * start method is called.
+   *
+   * @param config is the configuration object to prepare.
+   */
+  public Configuration prepareConfiguration(Configuration config) throws Exception {
+    config.set(HiveConf.ConfVars.METASTOREURIS.varname, "thrift://" + getHostName() + ":"
+ getPort());
+    config.set(HiveConf.ConfVars.METASTOREWAREHOUSE.varname, warehouseDirectory);
+    return config;
+  }
+
+  /**
+   * Get configuration.
+   *
+   * @return HiveConf
+   */
+  public HiveConf getConfiguration() {
+    return config;
+  }
+
+  /**
+   * Set the configuration object.
+   *
+   * @param config
+   */
+  public void setConfiguration(Configuration config) {
+    this.config = new HiveConf();
+    this.config.addResource(config);
+    this.printConfig();
+  }
+
+  /**
+   * Hostname used to start services on.
+   *
+   * @return String hostname
+   */
+  public String getHostName() {
+    return hostname;
+  }
+
+  /**
+   * Port meta store service will be on.
+   *
+   * @return int port
+   */
+  public int getPort() {
+    return this.port;
+  }
+
+  private void printConfig() {
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    config.logVars(new PrintStream(baos));
+    LOG.debug("Hive server runner configuration:\n" + baos.toString());
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java
b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java
new file mode 100644
index 0000000..be3720f
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/hive/MetastoreServerRunnerFactory.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.hive;
+
+import java.util.Properties;
+
+/**
+ * Create HiveServer2 runner.
+ */
+public class MetastoreServerRunnerFactory {
+
+  public static final String RUNNER_CLASS_PROPERTY = "sqoop.hive.metastore.runner.class";
+
+  public static final String HOSTNAME_PROPERTY = "sqoop.hive.metastore.server.hostname";
+
+  public static final String PORT_PROPERTY = "sqoop.hive.metastore.server.port";
+
+  public static final String DEFAULT_HOSTNAME = "127.0.0.1";
+
+  public static final String DEFAULT_PORT = "0";
+
+  public static MetastoreServerRunner getRunner(
+      Properties properties, Class<? extends MetastoreServerRunner> defaultRunner)
+      throws Exception {
+    Class<?> klass;
+
+    String hostname = properties.getProperty(HOSTNAME_PROPERTY, DEFAULT_HOSTNAME);
+    int port = Integer.parseInt(
+        properties.getProperty(PORT_PROPERTY, DEFAULT_PORT));
+
+    String className = properties.getProperty(RUNNER_CLASS_PROPERTY);
+    if(className == null) {
+      klass = defaultRunner;
+    } else {
+      klass = Class.forName(className);
+    }
+
+    return (MetastoreServerRunner)klass.getConstructor(String.class, int.class)
+        .newInstance(hostname, port);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
index 648e2f6..4d27886 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/TomcatSqoopMiniCluster.java
@@ -95,6 +95,7 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster {
     String []classpath = System.getProperty("java.class.path").split(":");
     for(String jar : classpath) {
       if(jar.contains("hadoop-")      || // Hadoop jars
+         jar.contains("hive-")        || // Hive jars
          jar.contains("commons-")     || // Apache Commons libraries
          jar.contains("httpcore-")    || // Apache Http Core libraries
          jar.contains("httpclient-")  || // Apache Http Client libraries
@@ -106,6 +107,7 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster {
          jar.contains("jackson-")     || // Jackson
          jar.contains("derby")        || // Derby drivers
          jar.contains("avro-")        || // Avro
+         jar.contains("parquet-")     || // Parquet
          jar.contains("mysql")        || // MySQL JDBC driver
          jar.contains("postgre")      || // PostgreSQL JDBC driver
          jar.contains("oracle")       || // Oracle driver
@@ -113,6 +115,8 @@ public class TomcatSqoopMiniCluster extends SqoopMiniCluster {
          jar.contains("tdgs")         || // Teradata driver
          jar.contains("nzjdbc")       || // Netezza driver
          jar.contains("sqljdbc")      || // Microsoft SQL Server driver
+         jar.contains("libfb303")     || // Facebook thrift lib
+         jar.contains("datanucleus-") || // Data nucleus libs
          jar.contains("google")          // Google libraries (guava, ...)
        ) {
         extraClassPath.add(jar);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java
b/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java
new file mode 100644
index 0000000..628f484
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/HiveConnectorTestCase.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.testcases;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.common.test.db.HiveProvider;
+import org.apache.sqoop.test.hive.InternalHiveServerRunner;
+import org.apache.sqoop.test.hive.HiveServerRunner;
+import org.apache.sqoop.test.hive.HiveServerRunnerFactory;
+import org.apache.sqoop.test.hive.InternalMetastoreServerRunner;
+import org.apache.sqoop.test.hive.MetastoreServerRunner;
+import org.apache.sqoop.test.hive.MetastoreServerRunnerFactory;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+
+public class HiveConnectorTestCase extends ConnectorTestCase {
+  private static final Logger LOG = Logger.getLogger(HiveConnectorTestCase.class);
+
+  protected HiveServerRunner hiveServerRunner;
+  protected MetastoreServerRunner metastoreServerRunner;
+  protected HiveProvider hiveProvider;
+
+  private void ensureWarehouseDirectory(Configuration conf) throws Exception {
+    String warehouseDirectory = conf.get(HiveConf.ConfVars.METASTOREWAREHOUSE.varname);
+    StringBuilder dir = new StringBuilder();
+    for (String part : warehouseDirectory.split("/")) {
+      dir.append(part).append("/");
+      Path path = new Path(dir.toString());
+      if (!hdfsClient.exists(path)) {
+        hdfsClient.mkdirs(path);
+      }
+    }
+    hdfsClient.setPermission(new Path(dir.toString()), new FsPermission((short)01777));
+  }
+
+  @BeforeMethod(alwaysRun = true)
+  public void startHive() throws Exception {
+    String databasePath = HdfsUtils.joinPathFragments(getTemporaryPath(), "metastore_db");
+    metastoreServerRunner = MetastoreServerRunnerFactory.getRunner(System.getProperties(),
InternalMetastoreServerRunner.class);
+    metastoreServerRunner.setConfiguration(metastoreServerRunner.prepareConfiguration(hadoopCluster.getConfiguration()));
+    metastoreServerRunner.getConfiguration().set(HiveConf.ConfVars.METASTORECONNECTURLKEY.varname,
+        "jdbc:derby:;databaseName=" + databasePath + ";create=true");
+    ensureWarehouseDirectory(metastoreServerRunner.getConfiguration());
+    LOG.info("Starting Metastore Server: " + metastoreServerRunner.getClass().getName());
+    metastoreServerRunner.start();
+
+    hiveServerRunner = HiveServerRunnerFactory.getRunner(System.getProperties(), InternalHiveServerRunner.class);
+    hiveServerRunner.setConfiguration(hiveServerRunner.prepareConfiguration(metastoreServerRunner.getConfiguration()));
+    LOG.info("Starting Hive Server: " + hiveServerRunner.getClass().getName());
+    hiveServerRunner.start();
+
+    LOG.info("Starting Hive Provider: " + provider.getClass().getName());
+    hiveProvider = new HiveProvider(hiveServerRunner.getUrl());
+    hiveProvider.start();
+  }
+
+  @AfterMethod(alwaysRun = true)
+  public void stopHive() throws Exception {
+    LOG.info("Stopping Hive Provider: " + provider.getClass().getName());
+    hiveProvider.stop();
+
+    LOG.info("Stopping Hive Server: " + hiveServerRunner.getClass().getName());
+    hiveServerRunner.stop();
+
+    LOG.info("Stopping Metastore Server: " + metastoreServerRunner.getClass().getName());
+    metastoreServerRunner.stop();
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
new file mode 100644
index 0000000..e789ce0
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/hive/FromRDBMSToKiteHiveTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.integration.connector.hive;
+
+import org.apache.sqoop.common.Direction;
+import org.apache.sqoop.common.test.asserts.ProviderAsserts;
+import org.apache.sqoop.common.test.db.TableName;
+import org.apache.sqoop.connector.common.FileFormat;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.test.testcases.HiveConnectorTestCase;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ */
+public class FromRDBMSToKiteHiveTest extends HiveConnectorTestCase {
+  private MLink rdbmsLink;
+  private MLink kiteLink;
+
+  @BeforeMethod(alwaysRun = true)
+  public void createTable() {
+    createAndLoadTableCities();
+  }
+
+  @AfterMethod(alwaysRun = true)
+  public void dropTable() {
+    super.dropTable();
+  }
+
+  @BeforeMethod(alwaysRun = true)
+  public void createLinks() {
+    // RDBMS link
+    rdbmsLink = getClient().createLink("generic-jdbc-connector");
+    fillRdbmsLinkConfig(rdbmsLink);
+    saveLink(rdbmsLink);
+
+    // Kite link
+    kiteLink = getClient().createLink("kite-connector");
+    kiteLink.getConnectorLinkConfig().getStringInput("linkConfig.authority")
+        .setValue(metastoreServerRunner.getAuthority());
+    saveLink(kiteLink);
+  }
+
+  @Test
+  public void testCities() throws Exception {
+    // Job creation
+    MJob job = getClient().createJob(rdbmsLink.getPersistenceId(), kiteLink.getPersistenceId());
+
+    // Set rdbms "FROM" config
+    MConfigList fromConfig = job.getJobConfig(Direction.FROM);
+    fillRdbmsFromConfig(job, "id");
+    // TODO: Kite have troubles with some data types, so we're limiting the columns to int
only
+    fromConfig.getStringInput("fromJobConfig.columns").setValue(provider.escapeColumnName("id"));
+
+    // Fill the Kite "TO" config
+    MConfigList toConfig = job.getJobConfig(Direction.TO);
+    toConfig.getStringInput("toJobConfig.uri").setValue("dataset:hive:testtable");
+    toConfig.getEnumInput("toJobConfig.fileFormat").setValue(FileFormat.AVRO);
+
+    // driver config
+    MDriverConfig driverConfig = job.getDriverConfig();
+    driverConfig.getIntegerInput("throttlingConfig.numExtractors").setValue(1);
+
+    saveJob(job);
+
+    executeJob(job);
+
+    // Assert correct output
+    ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id",
1}, "1");
+    ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id",
2}, "2");
+    ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id",
3}, "3");
+    ProviderAsserts.assertRow(hiveProvider, new TableName("testtable"), new Object[]{"id",
4}, "4");
+
+    hiveProvider.dropTable(new TableName("testtable"));
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/45d1c32d/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
index 8ca1c3a..828d244 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/kite/FromRDBMSToKiteTest.java
@@ -29,8 +29,7 @@ import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
-/**
- */
+@Test
 public class FromRDBMSToKiteTest extends ConnectorTestCase {
   @BeforeMethod(alwaysRun = true)
   public void createTable() {
@@ -96,5 +95,4 @@ public class FromRDBMSToKiteTest extends ConnectorTestCase {
       "\"4\""
     );
   }
-
 }


Mime
View raw message