sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2230: Sqoop2: Kite connector can use URIBuilder
Date Thu, 19 Mar 2015 01:36:40 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 700d55ac7 -> 598460f4a


SQOOP-2230: Sqoop2: Kite connector can use URIBuilder

(Abraham Elmahrek via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/598460f4
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/598460f4
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/598460f4

Branch: refs/heads/sqoop2
Commit: 598460f4a5f81cd2dded2b2d3c99af99934de50f
Parents: 700d55a
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Wed Mar 18 18:35:56 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Wed Mar 18 18:35:56 2015 -0700

----------------------------------------------------------------------
 .../connector/kite/KiteDatasetExecutor.java     | 21 +++++++++++++++++++-
 .../kite/configuration/ConfigUtil.java          | 16 ++++++++++++---
 .../sqoop/connector/kite/TestKiteExecutor.java  | 13 ++++++++++++
 3 files changed, 46 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/598460f4/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
index b111b5c..883925b 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteDatasetExecutor.java
@@ -21,6 +21,7 @@ import com.google.common.annotations.VisibleForTesting;
 import com.google.common.io.Closeables;
 import org.apache.avro.Schema;
 import org.apache.avro.generic.GenericRecord;
+import org.apache.commons.lang.StringUtils;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.common.FileFormat;
 import org.apache.sqoop.connector.common.AvroDataTypeUtil;
@@ -162,7 +163,25 @@ public class KiteDatasetExecutor {
    */
   public static String suggestTemporaryDatasetUri(String uri) {
     if (uri.startsWith("dataset:hdfs:")) {
-      return uri + TEMPORARY_DATASET_PREFIX + UUID.randomUUID().toString().replace("-", "");
+      int pathStart = uri.lastIndexOf(":") + 1;
+      int pathEnd = uri.lastIndexOf("?");
+      String[] uriParts = null;
+
+      // Get URI parts
+      if (pathEnd > -1) {
+        uriParts = new String[3];
+        uriParts[2] = uri.substring(pathEnd, uri.length());
+      } else {
+        pathEnd = uri.length();
+        uriParts = new String[2];
+      }
+      uriParts[1] = uri.substring(pathStart, pathEnd);
+      uriParts[0] = uri.substring(0, pathStart);
+
+      // Add to path
+      uriParts[1] += TEMPORARY_DATASET_PREFIX + UUID.randomUUID().toString().replace("-",
"");
+
+      return StringUtils.join(uriParts, "");
     } else {
       throw new SqoopException(
           KiteConnectorError.GENERIC_KITE_CONNECTOR_0000, uri);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/598460f4/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
index d48e250..1df1859 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/configuration/ConfigUtil.java
@@ -18,6 +18,7 @@
 package org.apache.sqoop.connector.kite.configuration;
 
 import com.google.common.base.Strings;
+import org.kitesdk.data.URIBuilder;
 
 public class ConfigUtil {
 
@@ -27,10 +28,19 @@ public class ConfigUtil {
    */
   public static String buildDatasetUri(String fsLocation, String uri) {
     if (!Strings.isNullOrEmpty(fsLocation) && !uri.contains("://")) {
-      // Add fsLocation after the second colon
-      int p = uri.indexOf(":", uri.indexOf(":") + 1);
-      return uri.substring(0, p + 1) + "//" + fsLocation + uri.substring(p + 1);
+      URIBuilder builder = new URIBuilder(uri);
+
+      String[] parts = fsLocation.split(":");
+      if (parts.length > 0) {
+        builder.with("auth:host", parts[0]);
+      }
+      if (parts.length > 1) {
+        builder.with("auth:port", parts[1]);
+      }
+
+      return builder.build().toString().replaceFirst("view:", "dataset:");
     }
+
     return uri;
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/598460f4/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
index eed6ee0..7173c18 100644
--- a/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
+++ b/connector/connector-kite/src/test/java/org/apache/sqoop/connector/kite/TestKiteExecutor.java
@@ -147,6 +147,19 @@ public class TestKiteExecutor {
     assertTrue(executor.isReaderClosed());
   }
 
+  @Test
+  public void testSuggestTemporaryDatasetUri() {
+    String uri = "dataset:hdfs:/tmp/sqoop/test";
+    String suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri);
+    assertTrue(suggestedUri.length() > uri.length());
+    assertTrue(suggestedUri.contains(uri));
+
+    uri = "dataset:hdfs://namenode:8020/tmp/sqoop/test";
+    suggestedUri = KiteDatasetExecutor.suggestTemporaryDatasetUri(uri);
+    assertTrue(suggestedUri.length() > uri.length());
+    assertTrue(suggestedUri.contains(uri));
+  }
+
   private static Schema createTwoFieldSchema() {
     return new Schema.Parser().parse("{" +
         "\"name\":\"test\",\"type\":\"record\"," +


Mime
View raw message