sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-1998: Sqoop2: Kite Connector: Hive support
Date Fri, 20 Mar 2015 01:25:17 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 0c06e8b0b -> 7a2285c3c


SQOOP-1998: Sqoop2: Kite Connector: Hive support

(Abraham Elmahrek via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/7a2285c3
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/7a2285c3
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/7a2285c3

Branch: refs/heads/sqoop2
Commit: 7a2285c3c329927e89c1d06ebe4340a20092e0d4
Parents: 0c06e8b
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Thu Mar 19 18:24:21 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Thu Mar 19 18:24:21 2015 -0700

----------------------------------------------------------------------
 connector/connector-kite/pom.xml                | 12 +++++++++
 .../connector/kite/KiteDatasetExecutor.java     | 26 +++++++++++++++-----
 .../connector/kite/KiteDatasetPartitioner.java  |  5 +++-
 .../sqoop/connector/kite/KiteExtractor.java     |  4 ++-
 .../connector/kite/KiteFromInitializer.java     | 24 ++++++++++++++++--
 .../apache/sqoop/connector/kite/KiteLoader.java | 12 ++++-----
 .../sqoop/connector/kite/KiteToInitializer.java | 16 +++++++++++-
 .../kite/configuration/ConfigUtil.java          |  8 ++++++
 .../sqoop/connector/kite/TestKiteExecutor.java  | 19 ++++++++++++--
 .../connector/kite/TestKiteFromInitializer.java |  5 ++--
 .../sqoop/connector/kite/TestKiteLoader.java    |  2 +-
 dist/src/main/server/conf/catalina.properties   |  4 +--
 pom.xml                                         | 21 +++++++++++++---
 13 files changed, 131 insertions(+), 27 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/pom.xml
----------------------------------------------------------------------
diff --git a/connector/connector-kite/pom.xml b/connector/connector-kite/pom.xml
index 11ec534..8593899 100644
--- a/connector/connector-kite/pom.xml
+++ b/connector/connector-kite/pom.xml
@@ -41,6 +41,14 @@ limitations under the License.
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-common</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-exec</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.hive</groupId>
+      <artifactId>hive-serde</artifactId>
+    </dependency>
 
     <!-- Testing specified modules -->
     <dependency>
@@ -70,6 +78,10 @@ limitations under the License.
       <artifactId>kite-data-core</artifactId>
     </dependency>
     <dependency>
+      <groupId>org.kitesdk</groupId>
+      <artifactId>kite-data-hive</artifactId>
+    </dependency>
+    <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
index 883925b..db8d4e6 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
@@ -25,6 +25,8 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.common.FileFormat;
 import org.apache.sqoop.connector.common.AvroDataTypeUtil;
+import org.apache.sqoop.connector.kite.configuration.ConfigUtil;
+import org.apache.sqoop.connector.kite.configuration.LinkConfig;
 import org.apache.sqoop.connector.kite.util.KiteDataTypeUtil;
 import org.apache.sqoop.error.code.KiteConnectorError;
 import org.kitesdk.data.Dataset;
@@ -147,7 +149,6 @@ public class KiteDatasetExecutor {
     FileSystemDataset<GenericRecord> update = Datasets.load(uri);
     if (dataset instanceof FileSystemDataset) {
       ((FileSystemDataset<GenericRecord>) dataset).merge(update);
-
       // And let's completely drop the temporary dataset
       Datasets.delete(uri);
     } else {
@@ -156,14 +157,15 @@ public class KiteDatasetExecutor {
     }
   }
 
-  private static final String TEMPORARY_DATASET_PREFIX = "/temp_";
+  private static final String TEMPORARY_DATASET_PREFIX = "temp_";
 
   /**
    * Workaround for managing temporary datasets.
    */
-  public static String suggestTemporaryDatasetUri(String uri) {
-    if (uri.startsWith("dataset:hdfs:")) {
-      int pathStart = uri.lastIndexOf(":") + 1;
+  public static String suggestTemporaryDatasetUri(LinkConfig linkConfig, String uri) {
+    if (uri.startsWith("dataset:hdfs:") || uri.startsWith("dataset:hive:")) {
+      int pathStart = uri.indexOf(":") + 1;
+      pathStart = uri.indexOf(":", pathStart);
       int pathEnd = uri.lastIndexOf("?");
       String[] uriParts = null;
 
@@ -179,7 +181,19 @@ public class KiteDatasetExecutor {
       uriParts[0] = uri.substring(0, pathStart);
 
       // Add to path
-      uriParts[1] += TEMPORARY_DATASET_PREFIX + UUID.randomUUID().toString().replace("-",
"");
+      String temporaryDatasetName = TEMPORARY_DATASET_PREFIX + UUID.randomUUID().toString().replace("-",
"");
+      if (uri.startsWith("dataset:hive:")) {
+        if (uriParts[1].lastIndexOf("/") > -1) {
+          // Kite creates databases from namespace names in hive.
+          // Re-use entire path except for dataset name (table name in hive).
+          // Replace dataset name with temporary dataset name.
+          uriParts[1] = uriParts[1].substring(0, uriParts[1].lastIndexOf("/")) + "/" + temporaryDatasetName;
+        } else {
+          uriParts[1] = temporaryDatasetName;
+        }
+      } else {
+        uriParts[1] += "/" + temporaryDatasetName;
+      }
 
       return StringUtils.join(uriParts, "");
     } else {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java
index 41fe7e1..e03bd60 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetPartitioner.java
@@ -18,6 +18,7 @@
 
 package org.apache.sqoop.connector.kite;
 
+import org.apache.sqoop.connector.kite.configuration.ConfigUtil;
 import org.apache.sqoop.connector.kite.configuration.FromJobConfiguration;
 import org.apache.sqoop.connector.kite.configuration.LinkConfiguration;
 import org.apache.sqoop.job.etl.Partition;
@@ -41,7 +42,9 @@ public class KiteDatasetPartitioner extends Partitioner<LinkConfiguration,
     // There is no way to create partitions of an un-partitioned dataset.
     // TODO: SQOOP-1942 will create partitions, if dataset is partitioned.
     KiteDatasetPartition partition = new KiteDatasetPartition();
-    partition.setUri(fromJobConfig.fromJobConfig.uri);
+    String uri = ConfigUtil.buildDatasetUri(
+        linkConfiguration.linkConfig, fromJobConfig.fromJobConfig.uri);
+    partition.setUri(uri);
 
     List<Partition> partitions = new LinkedList<Partition>();
     partitions.add(partition);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java
index d4a8a77..d93f9b5 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteExtractor.java
@@ -20,6 +20,7 @@ package org.apache.sqoop.connector.kite;
 import com.google.common.annotations.VisibleForTesting;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.log4j.Logger;
+import org.apache.sqoop.connector.kite.configuration.ConfigUtil;
 import org.apache.sqoop.connector.kite.configuration.FromJobConfiguration;
 import org.apache.sqoop.connector.kite.configuration.LinkConfiguration;
 import org.apache.sqoop.etl.io.DataWriter;
@@ -48,7 +49,8 @@ public class KiteExtractor extends Extractor<LinkConfiguration,
   @Override
   public void extract(ExtractorContext context, LinkConfiguration linkConfig,
       FromJobConfiguration fromJobConfig, KiteDatasetPartition partition) {
-    String uri = partition.getUri();
+    String uri = ConfigUtil.buildDatasetUri(
+        linkConfig.linkConfig, partition.getUri());
     LOG.info("Loading data from " + uri);
 
     KiteDatasetExecutor executor = getExecutor(uri);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
index 292510a..06f18e7 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
@@ -19,6 +19,7 @@ package org.apache.sqoop.connector.kite;
 
 import org.apache.log4j.Logger;
 import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.connector.kite.configuration.ConfigUtil;
 import org.apache.sqoop.connector.kite.configuration.FromJobConfiguration;
 import org.apache.sqoop.connector.kite.configuration.LinkConfiguration;
 import org.apache.sqoop.connector.common.AvroDataTypeUtil;
@@ -43,7 +44,10 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration,
   @Override
   public void initialize(InitializerContext context,
       LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
-    if (!Datasets.exists(fromJobConfig.fromJobConfig.uri)) {
+    String uri = ConfigUtil.buildDatasetUri(
+        linkConfig.linkConfig, fromJobConfig.fromJobConfig.uri);
+    LOG.debug("Constructed dataset URI: " + uri);
+    if (!Datasets.exists(uri)) {
       LOG.error("Dataset does not exist");
       throw new SqoopException(KiteConnectorError.GENERIC_KITE_CONNECTOR_0002);
     }
@@ -55,13 +59,29 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration,
     jars.add(ClassUtils.jarForClass("org.kitesdk.data.Datasets"));
     jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.databind.JsonNode"));
     jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.core.TreeNode"));
+    if (fromJobConfig.fromJobConfig.uri.startsWith("dataset:hive")) {
+      // @TODO(Abe): Remove a deps that aren't used?
+      jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.conf.HiveConf"));
+      jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.serde2.SerDe"));
+      jars.add(ClassUtils.jarForClass("org.kitesdk.data.spi.hive.MetaStoreUtil"));
+      jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors"));
+      jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse"));
+      jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat"));
+      jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory"));
+      jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService"));
+      jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
+      jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
+      jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
+    }
     return jars;
   }
 
   @Override
   public Schema getSchema(InitializerContext context,
       LinkConfiguration linkConfig, FromJobConfiguration fromJobConfig) {
-    Dataset dataset = Datasets.load(fromJobConfig.fromJobConfig.uri);
+    String uri = ConfigUtil.buildDatasetUri(
+        linkConfig.linkConfig, fromJobConfig.fromJobConfig.uri);
+    Dataset dataset = Datasets.load(uri);
     org.apache.avro.Schema avroSchema = dataset.getDescriptor().getSchema();
     return AvroDataTypeUtil.createSqoopSchema(avroSchema);
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java
index 57790a1..ca0a5c7 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteLoader.java
@@ -40,27 +40,27 @@ public class KiteLoader extends Loader<LinkConfiguration, ToJobConfiguration>
{
   private long rowsWritten = 0;
 
   @VisibleForTesting
-  KiteDatasetExecutor getExecutor(String uri, Schema schema,
+  KiteDatasetExecutor getExecutor(LinkConfiguration linkConfiguration, String uri, Schema
schema,
       FileFormat format) {
     // Note that instead of creating a dataset at destination, we create a
     // temporary dataset by every KiteLoader instance. They will be merged when
     // all data portions are written successfully. Unfortunately, KiteLoader is
     // not able to pass the temporary dataset uri to KiteToDestroyer. So we
     // delegate KiteDatasetExecutor to manage name convention for datasets.
-    uri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri);
-    LOG.info("Generated temporary dataset URI: " + uri);
+    uri = KiteDatasetExecutor.suggestTemporaryDatasetUri(linkConfiguration.linkConfig, uri);
+    LOG.info("Constructed temporary dataset URI: " + uri);
     Dataset<GenericRecord> dataset =
         KiteDatasetExecutor.createDataset(uri, schema, format);
     return new KiteDatasetExecutor(dataset);
   }
 
   @Override
-  public void load(LoaderContext context, LinkConfiguration linkConfig,
+  public void load(LoaderContext context, LinkConfiguration linkConfiguration,
       ToJobConfiguration toJobConfig) throws Exception {
     String uri = ConfigUtil.buildDatasetUri(
-        linkConfig.linkConfig, toJobConfig.toJobConfig);
+        linkConfiguration.linkConfig, toJobConfig.toJobConfig);
     KiteDatasetExecutor executor = getExecutor(
-        uri, context.getSchema(), toJobConfig.toJobConfig.fileFormat);
+        linkConfiguration, uri, context.getSchema(), toJobConfig.toJobConfig.fileFormat);
     LOG.info("Temporary dataset created.");
 
     DataReader reader = context.getDataReader();

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
index 187ad21..39a2989 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
@@ -48,7 +48,7 @@ public class KiteToInitializer extends Initializer<LinkConfiguration,
       LinkConfiguration linkConfig, ToJobConfiguration toJobConfig) {
     String uri = ConfigUtil.buildDatasetUri(
         linkConfig.linkConfig, toJobConfig.toJobConfig);
-    LOG.info("Generated final dataset URI: " + uri);
+    LOG.debug("Constructed dataset URI: " + uri);
     if (Datasets.exists(uri)) {
       LOG.error("Overwrite an existing dataset is not expected in new create mode.");
       throw new SqoopException(KiteConnectorError.GENERIC_KITE_CONNECTOR_0001);
@@ -65,6 +65,20 @@ public class KiteToInitializer extends Initializer<LinkConfiguration,
     if (FileFormat.CSV.equals(toJobConfig.toJobConfig.fileFormat)) {
       jars.add(ClassUtils.jarForClass("au.com.bytecode.opencsv.CSVWriter"));
     }
+    if (toJobConfig.toJobConfig.uri.startsWith("dataset:hive")) {
+      // @TODO(Abe): Remove a deps that aren't used?
+      jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.conf.HiveConf"));
+      jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.serde2.SerDe"));
+      jars.add(ClassUtils.jarForClass("org.kitesdk.data.spi.hive.MetaStoreUtil"));
+      jars.add(ClassUtils.jarForClass("org.kitesdk.compat.DynConstructors"));
+      jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.metastore.Warehouse"));
+      jars.add(ClassUtils.jarForClass("org.apache.hive.common.HiveCompat"));
+      jars.add(ClassUtils.jarForClass("parquet.hive.HiveBindingFactory"));
+      jars.add(ClassUtils.jarForClass("com.facebook.fb303.FacebookService"));
+      jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
+      jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
+      jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
+    }
     return jars;
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
index 3052a87..e63bccf 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
@@ -48,6 +48,14 @@ public class ConfigUtil {
    * Returns a dataset uri, including the filesystem location part, if it is
    * provided separated,
    */
+  public static String buildDatasetUri(LinkConfig linkConfig, String uri) {
+    return buildDatasetUri(linkConfig.authority, uri);
+  }
+
+  /**
+   * Returns a dataset uri, including the filesystem location part, if it is
+   * provided separated,
+   */
   public static String buildDatasetUri(LinkConfig linkConfig,
       ToJobConfig toJobConfig) {
     return buildDatasetUri(linkConfig.authority, toJobConfig.uri);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
index 7173c18..44721a8 100644
--- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
+++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
@@ -20,6 +20,7 @@ package org.apache.sqoop.connector.kite;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericRecord;
 import org.apache.avro.generic.GenericRecordBuilder;
+import org.apache.sqoop.connector.kite.configuration.LinkConfig;
 import org.kitesdk.data.Dataset;
 import org.kitesdk.data.DatasetDescriptor;
 import org.kitesdk.data.DatasetReader;
@@ -150,14 +151,28 @@ public class TestKiteExecutor {
   @Test
   public void testSuggestTemporaryDatasetUri() {
     String uri = "dataset:hdfs:/tmp/sqoop/test";
-    String suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri);
+    String suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(),
uri);
     assertTrue(suggestedUri.length() > uri.length());
     assertTrue(suggestedUri.contains(uri));
 
     uri = "dataset:hdfs://namenode:8020/tmp/sqoop/test";
-    suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri);
+    suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri);
     assertTrue(suggestedUri.length() > uri.length());
     assertTrue(suggestedUri.contains(uri));
+
+    String subURI = "dataset:hive://metastore:9083/tmp/sqoop";
+    uri = "dataset:hive://metastore:9083/tmp/sqoop/test";
+    suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri);
+    assertTrue(suggestedUri.length() > subURI.length());
+    assertTrue(suggestedUri.contains(subURI));
+
+    String endURI = "auth:host=metastore&auth:port=9083";
+    uri = "dataset:hive:tmp/sqoop?auth:host=metastore&auth:port=9083";
+    subURI = "dataset:hive:tmp";
+    suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(new LinkConfig(), uri);
+    assertTrue(suggestedUri.length() > subURI.length());
+    assertTrue(suggestedUri.contains(subURI));
+    assertTrue(suggestedUri.endsWith(endURI));
   }
 
   private static Schema createTwoFieldSchema() {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java
b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java
index a60b2fc..6df5d83 100644
--- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java
+++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteFromInitializer.java
@@ -20,6 +20,7 @@ package org.apache.sqoop.connector.kite;
 
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.kite.configuration.FromJobConfiguration;
+import org.apache.sqoop.connector.kite.configuration.LinkConfiguration;
 import org.kitesdk.data.Datasets;
 import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.core.classloader.annotations.PrepareForTest;
@@ -53,7 +54,7 @@ public class TestKiteFromInitializer extends PowerMockTestCase {
     when(Datasets.exists(jobConfig.fromJobConfig.uri)).thenReturn(true);
 
     // exercise
-    initializer.initialize(null, null, jobConfig);
+    initializer.initialize(null, new LinkConfiguration(), jobConfig);
   }
 
   @Test(expectedExceptions = SqoopException.class)
@@ -64,7 +65,7 @@ public class TestKiteFromInitializer extends PowerMockTestCase {
     when(Datasets.exists(jobConfig.fromJobConfig.uri)).thenReturn(false);
 
     // exercise
-    initializer.initialize(null, null, jobConfig);
+    initializer.initialize(null, new LinkConfiguration(), jobConfig);
   }
 
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java
b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java
index b18c91e..533b8c3 100644
--- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java
+++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteLoader.java
@@ -46,7 +46,7 @@ public class TestKiteLoader {
 
     loader = new KiteLoader() {
       @Override
-      protected KiteDatasetExecutor getExecutor(String uri, Schema schema,
+      protected KiteDatasetExecutor getExecutor(LinkConfiguration linkConfiguration, String
uri, Schema schema,
           FileFormat format) {
         return executorMock;
       }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/dist/src/main/server/conf/catalina.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/server/conf/catalina.properties b/dist/src/main/server/conf/catalina.properties
index 260bfd6..5e201be 100644
--- a/dist/src/main/server/conf/catalina.properties
+++ b/dist/src/main/server/conf/catalina.properties
@@ -50,9 +50,9 @@ package.definition=sun.,java.,org.apache.catalina.,org.apache.coyote.,org.apache
 # Sqoop installation guide for more details.
 #
 # Example (based on default BigTop locations):
-#common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${catalina.home}/../lib/*.jar,/usr/lib/hadoop/*.jar,/usr/lib/hadoop/lib/*.jar,/usr/lib/hadoop-hdfs/*.jar,/usr/lib/hadoop-hdfs/lib/*.jar,/usr/lib/hadoop-mapreduce/*.jar,/usr/lib/hadoop-mapreduce/lib/*.jar,/usr/lib/hadoop-yarn/*.jar,/usr/lib/hadoop-yarn/lib/*.jar
+#common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${catalina.home}/../lib/*.jar,/usr/lib/hadoop/*.jar,/usr/lib/hadoop/lib/*.jar,/usr/lib/hadoop-hdfs/*.jar,/usr/lib/hadoop-hdfs/lib/*.jar,/usr/lib/hadoop-mapreduce/*.jar,/usr/lib/hadoop-mapreduce/lib/*.jar,/usr/lib/hadoop-yarn/*.jar,/usr/lib/hadoop-yarn/lib/*.jar,/usr/lib/hive/lib/*.jar
 #
-common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${catalina.home}/../lib/*.jar,/usr/lib/hadoop/*.jar,/usr/lib/hadoop/lib/*.jar,/usr/lib/hadoop-hdfs/*.jar,/usr/lib/hadoop-hdfs/lib/*.jar,/usr/lib/hadoop-mapreduce/*.jar,/usr/lib/hadoop-mapreduce/lib/*.jar,/usr/lib/hadoop-yarn/*.jar,/usr/lib/hadoop-yarn/lib/*.jar
+common.loader=${catalina.base}/lib,${catalina.base}/lib/*.jar,${catalina.home}/lib,${catalina.home}/lib/*.jar,${catalina.home}/../lib/*.jar,/usr/lib/hadoop/*.jar,/usr/lib/hadoop/lib/*.jar,/usr/lib/hadoop-hdfs/*.jar,/usr/lib/hadoop-hdfs/lib/*.jar,/usr/lib/hadoop-mapreduce/*.jar,/usr/lib/hadoop-mapreduce/lib/*.jar,/usr/lib/hadoop-yarn/*.jar,/usr/lib/hadoop-yarn/lib/*.jar,/usr/lib/hive/lib/*.jar
 
 #
 # List of comma-separated paths defining the contents of the "server" 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/7a2285c3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3d7434e..e3431bd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -97,6 +97,7 @@ limitations under the License.
     <commons-codec.version>1.9</commons-codec.version>
     <derby.version>10.8.2.2</derby.version>
     <hadoop.2.version>2.6.0</hadoop.2.version>
+    <hive.version>0.13.1</hive.version>
     <guava.version>11.0.2</guava.version>
     <json-simple.version>1.1</json-simple.version>
     <testng.version>6.8.13</testng.version>
@@ -349,26 +350,35 @@ limitations under the License.
         <version>${hadoop.2.version}</version>
         <scope>provided</scope>
       </dependency>
-
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
         <version>${hadoop.2.version}</version>
         <scope>provided</scope>
       </dependency>
-
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-minicluster</artifactId>
         <version>${hadoop.2.version}</version>
       </dependency>
-
       <dependency>
         <groupId>org.apache.hadoop</groupId>
         <artifactId>hadoop-auth</artifactId>
         <version>${hadoop.2.version}</version>
       </dependency>
       <dependency>
+        <groupId>org.apache.hive</groupId>
+        <artifactId>hive-exec</artifactId>
+        <version>${hive.version}</version>
+        <scope>provided</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.apache.hive</groupId>
+        <artifactId>hive-serde</artifactId>
+        <version>${hive.version}</version>
+        <scope>provided</scope>
+      </dependency>
+      <dependency>
         <groupId>com.googlecode.json-simple</groupId>
         <artifactId>json-simple</artifactId>
         <version>${json-simple.version}</version>
@@ -532,6 +542,11 @@ limitations under the License.
         <version>${kitesdk.version}</version>
       </dependency>
       <dependency>
+        <groupId>org.kitesdk</groupId>
+        <artifactId>kite-data-hive</artifactId>
+        <version>${kitesdk.version}</version>
+      </dependency>
+      <dependency>
         <groupId>org.apache.tomcat</groupId>
         <artifactId>catalina</artifactId>
         <version>${tomcat.version}</version>


Mime
View raw message