sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mau...@apache.org
Subject sqoop git commit: SQOOP-3152: Fix SqoopOptions#parseColumnMapping, thus Sqoop will support mappings with precisison (e.g. DECIMAL(10, 5)) in the future properly
Date Mon, 20 Mar 2017 15:41:14 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 6d8a670d3 -> 803edc279


SQOOP-3152: Fix SqoopOptions#parseColumnMapping, thus Sqoop will support
mappings with precisison (e.g. DECIMAL(10,5)) in the future properly

(Eric Lin via Attila Szabo)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/803edc27
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/803edc27
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/803edc27

Branch: refs/heads/trunk
Commit: 803edc27925ec6ee950b2e7fa2ffea5727248fa6
Parents: 6d8a670
Author: Attila Szabo <maugli@apache.org>
Authored: Mon Mar 20 16:37:49 2017 +0100
Committer: Attila Szabo <maugli@apache.org>
Committed: Mon Mar 20 16:37:49 2017 +0100

----------------------------------------------------------------------
 src/docs/web/images/Thumbs.db                   | Bin 0 -> 4608 bytes
 src/java/org/apache/sqoop/SqoopOptions.java     |   9 +++--
 .../com/cloudera/sqoop/hive/TestHiveImport.java |  17 ++++++++
 .../cloudera/sqoop/hive/TestTableDefWriter.java |  39 ++++++++++++++++++-
 testdata/hive/scripts/decimalMapImport.q        |  17 ++++++++
 5 files changed, 78 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/docs/web/images/Thumbs.db
----------------------------------------------------------------------
diff --git a/src/docs/web/images/Thumbs.db b/src/docs/web/images/Thumbs.db
new file mode 100644
index 0000000..bc9428c
Binary files /dev/null and b/src/docs/web/images/Thumbs.db differ

http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index b33b54b..caf95f6 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -1322,11 +1322,14 @@ public class SqoopOptions implements Cloneable {
   public void setPasswordAlias(String alias) {
     this.passwordAlias = alias;
   }
+
   protected void parseColumnMapping(String mapping,
           Properties output) {
     output.clear();
 
-    String[] maps = mapping.split(",");
+    // replace (xx,xx) with (xx#xx), so that we can just split by "," afterwards
+    String[] maps = mapping.replaceAll("\\(([0-9]+),([0-9]+)\\)", "($1#$2)").split(",");
+
     for(String map : maps) {
       String[] details = map.split("=");
       if (details.length != 2) {
@@ -1336,8 +1339,8 @@ public class SqoopOptions implements Cloneable {
 
       try {
         output.put(
-            URLDecoder.decode(details[0], "UTF-8"),
-            URLDecoder.decode(details[1], "UTF-8"));
+            URLDecoder.decode(details[0].replace("#", ","), "UTF-8"),
+            URLDecoder.decode(details[1].replace("#", ","), "UTF-8"));
       } catch (UnsupportedEncodingException e) {
         throw new IllegalArgumentException("Encoding not supported. "
             + "Column mapping should be UTF-8 encoding.");

http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
index 6f13fe2..33e0cc4 100644
--- a/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
+++ b/src/test/com/cloudera/sqoop/hive/TestHiveImport.java
@@ -499,6 +499,23 @@ public class TestHiveImport extends ImportJobTestCase {
         getArgv(false, null), new ImportTool());
   }
 
+  /** Test that DECIMALS using --map-column-hive option maps can run without issues. */
+  @Test
+  public void testDecimalMapColumnHive() throws IOException {
+    final String TABLE_NAME = "DECIMAL_MAP_HIVE_IMPORT";
+    setCurTableName(TABLE_NAME);
+    setNumCols(2);
+    String [] types = { "NUMERIC", "CHAR(64)" };
+    String [] vals = { "12343.14159", "'foo'" };
+
+    ArrayList<String> args = new ArrayList<String>();
+    args.add("--map-column-hive");
+    args.add(BASE_COL_NAME  + "0=DECIMAL(10,10)");
+
+    runImportTest(TABLE_NAME, types, vals, "decimalMapImport.q",
+        getArgv(false, args.toArray(new String[args.size()])), new ImportTool());
+  }
+
   /** If bin/hive returns an error exit status, we should get an IOException. */
   @Test
   public void testHiveExitFails() throws IOException {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
index 6af12da..dbf0dde 100644
--- a/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
+++ b/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
@@ -180,7 +180,7 @@ public class TestTableDefWriter {
   }
 
   @Test
-  public void testUserMapping() throws Exception {
+  public void testUserMappingNoDecimal() throws Exception {
     String[] args = {
         "--map-column-hive", "id=STRING,value=INTEGER",
     };
@@ -207,6 +207,43 @@ public class TestTableDefWriter {
   }
 
   @Test
+  public void testUserMappingWithDecimal() throws Exception {
+    String[] args = {
+        "--map-column-hive", "id=STRING,value2=DECIMAL(13,5),value1=INTEGER," +
+                             "value3=DECIMAL(4,5),value4=VARCHAR(255)",
+    };
+    Configuration conf = new Configuration();
+    SqoopOptions options =
+        new ImportTool().parseArguments(args, null, null, false);
+    TableDefWriter writer = new TableDefWriter(options,
+        null, HsqldbTestServer.getTableName(), "outputTable", conf, false);
+
+    Map<String, Integer> colTypes = new SqlTypeMap<String, Integer>();
+    colTypes.put("id", Types.INTEGER);
+    colTypes.put("value1", Types.VARCHAR);
+    colTypes.put("value2", Types.DOUBLE);
+    colTypes.put("value3", Types.FLOAT);
+    colTypes.put("value4", Types.CHAR);
+    writer.setColumnTypes(colTypes);
+
+    String createTable = writer.getCreateTableStmt();
+
+    assertNotNull(createTable);
+
+    assertTrue(createTable.contains("`id` STRING"));
+    assertTrue(createTable.contains("`value1` INTEGER"));
+    assertTrue(createTable.contains("`value2` DECIMAL(13,5)"));
+    assertTrue(createTable.contains("`value3` DECIMAL(4,5)"));
+    assertTrue(createTable.contains("`value4` VARCHAR(255)"));
+
+    assertFalse(createTable.contains("`id` INTEGER"));
+    assertFalse(createTable.contains("`value1` STRING"));
+    assertFalse(createTable.contains("`value2` DOUBLE"));
+    assertFalse(createTable.contains("`value3` FLOAT"));
+    assertFalse(createTable.contains("`value4` CHAR"));
+  }
+
+  @Test
   public void testUserMappingFailWhenCantBeApplied() throws Exception {
     String[] args = {
         "--map-column-hive", "id=STRING,value=INTEGER",

http://git-wip-us.apache.org/repos/asf/sqoop/blob/803edc27/testdata/hive/scripts/decimalMapImport.q
----------------------------------------------------------------------
diff --git a/testdata/hive/scripts/decimalMapImport.q b/testdata/hive/scripts/decimalMapImport.q
new file mode 100644
index 0000000..8f05d58
--- /dev/null
+++ b/testdata/hive/scripts/decimalMapImport.q
@@ -0,0 +1,17 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE TABLE IF NOT EXISTS `DECIMAL_MAP_HIVE_IMPORT` ( `DATA_COL0` DECIMAL(10, 10), `DATA_COL1`
STRING) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED
AS TEXTFILE;
+LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/DECIMAL_MAP_HIVE_IMPORT' INTO TABLE `DECIMAL_MAP_HIVE_IMPORT`;


Mime
View raw message