sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject sqoop git commit: SQOOP-3233: SqoopHCatImportHelper.convertNumberTypes check for Varchar instead of Char
Date Fri, 08 Dec 2017 10:45:28 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk a50869437 -> 855c56fef


SQOOP-3233: SqoopHCatImportHelper.convertNumberTypes check for Varchar instead of Char

(Ferenc Szabo via Szabolcs Vasas)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/855c56fe
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/855c56fe
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/855c56fe

Branch: refs/heads/trunk
Commit: 855c56fef8ab833e63587f17f70030e2c7764f4a
Parents: a508694
Author: Szabolcs Vasas <vasas@apache.org>
Authored: Fri Dec 8 11:39:24 2017 +0100
Committer: Szabolcs Vasas <vasas@apache.org>
Committed: Fri Dec 8 11:39:24 2017 +0100

----------------------------------------------------------------------
 .../mapreduce/hcat/SqoopHCatImportHelper.java   | 78 +++++++++++---------
 .../apache/sqoop/hcat/HCatalogImportTest.java   | 12 +++
 2 files changed, 56 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/855c56fe/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java
index 1c1ed1e..efaef17 100644
--- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java
+++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java
@@ -335,9 +335,7 @@ public class SqoopHCatImportHelper {
 
   private Object convertStringTypes(Object val, HCatFieldSchema hfs) {
     HCatFieldSchema.Type hfsType = hfs.getType();
-    if (hfsType == HCatFieldSchema.Type.STRING
-        || hfsType == HCatFieldSchema.Type.VARCHAR
-        || hfsType == HCatFieldSchema.Type.CHAR) {
+    if (isStringType(hfsType)) {
       String str = val.toString();
       if (doHiveDelimsReplacement) {
         str = FieldFormatter.hiveStringReplaceDelims(str,
@@ -400,59 +398,71 @@ public class SqoopHCatImportHelper {
       return null;
     }
     if (val instanceof BigDecimal
-        && hfsType == HCatFieldSchema.Type.STRING
-        || hfsType == HCatFieldSchema.Type.VARCHAR
-        || hfsType == HCatFieldSchema.Type.CHAR) {
+        && isStringType(hfsType)) {
       BigDecimal bd = (BigDecimal) val;
-      String bdStr = null;
-      if (bigDecimalFormatString) {
-        bdStr = bd.toPlainString();
-      } else {
-        bdStr = bd.toString();
-      }
-      if (hfsType == HCatFieldSchema.Type.VARCHAR) {
-        VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
-        HiveVarchar hvc = new HiveVarchar(bdStr, vti.getLength());
-        return hvc;
-      } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
-        CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
-        HiveChar hChar = new HiveChar(bdStr, cti.getLength());
-        return hChar;
-      } else {
-        return bdStr;
-      }
+      return convertBigDecimalToTextTypes(hfs, hfsType, bd);
     }
     Number n = (Number) val;
+    return convertNumberToAnyType(hfs, hfsType, n);
+  }
+
+  private boolean isStringType(HCatFieldSchema.Type hfsType) {
+    return hfsType == HCatFieldSchema.Type.STRING
+        || hfsType == HCatFieldSchema.Type.VARCHAR
+        || hfsType == HCatFieldSchema.Type.CHAR;
+  }
+
+  private Object convertNumberToAnyType(HCatFieldSchema hfs, HCatFieldSchema.Type hfsType,
Number number) {
     if (hfsType == HCatFieldSchema.Type.TINYINT) {
-      return n.byteValue();
+      return number.byteValue();
     } else if (hfsType == HCatFieldSchema.Type.SMALLINT) {
-      return n.shortValue();
+      return number.shortValue();
     } else if (hfsType == HCatFieldSchema.Type.INT) {
-      return n.intValue();
+      return number.intValue();
     } else if (hfsType == HCatFieldSchema.Type.BIGINT) {
-      return n.longValue();
+      return number.longValue();
     } else if (hfsType == HCatFieldSchema.Type.FLOAT) {
-      return n.floatValue();
+      return number.floatValue();
     } else if (hfsType == HCatFieldSchema.Type.DOUBLE) {
-      return n.doubleValue();
+      return number.doubleValue();
     } else if (hfsType == HCatFieldSchema.Type.BOOLEAN) {
-      return n.byteValue() == 0 ? Boolean.FALSE : Boolean.TRUE;
+      return number.byteValue() == 0 ? Boolean.FALSE : Boolean.TRUE;
     } else if (hfsType == HCatFieldSchema.Type.STRING) {
-      return n.toString();
+      return number.toString();
     } else if (hfsType == HCatFieldSchema.Type.VARCHAR) {
       VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
-      HiveVarchar hvc = new HiveVarchar(val.toString(), vti.getLength());
+      HiveVarchar hvc = new HiveVarchar(number.toString(), vti.getLength());
       return hvc;
     } else if (hfsType == HCatFieldSchema.Type.CHAR) {
       CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
-      HiveChar hChar = new HiveChar(val.toString(), cti.getLength());
+      HiveChar hChar = new HiveChar(number.toString(), cti.getLength());
       return hChar;
     } else if (hfsType == HCatFieldSchema.Type.DECIMAL) {
-      return convertNumberIntoHiveDecimal(n);
+      return convertNumberIntoHiveDecimal(number);
     }
     return null;
   }
 
+  private Object convertBigDecimalToTextTypes(HCatFieldSchema hfs, HCatFieldSchema.Type hfsType,
BigDecimal bigDecimal) {
+    String bdStr = null;
+    if (bigDecimalFormatString) {
+      bdStr = bigDecimal.toPlainString();
+    } else {
+      bdStr = bigDecimal.toString();
+    }
+    if (hfsType == HCatFieldSchema.Type.VARCHAR) {
+      VarcharTypeInfo vti = (VarcharTypeInfo) hfs.getTypeInfo();
+      HiveVarchar hvc = new HiveVarchar(bdStr, vti.getLength());
+      return hvc;
+    } else if (hfsType == HCatFieldSchema.Type.CHAR) {
+      CharTypeInfo cti = (CharTypeInfo) hfs.getTypeInfo();
+      HiveChar hChar = new HiveChar(bdStr, cti.getLength());
+      return hChar;
+    } else {
+      return bdStr;
+    }
+  }
+
   HiveDecimal convertNumberIntoHiveDecimal(Number number) {
     BigDecimal bigDecimal = null;
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/855c56fe/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
index 4686493..b6741f4 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
@@ -446,6 +446,18 @@ public class HCatalogImportTest extends ImportJobTestCase {
           "decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.DECIMAL, 18, 2,
           HiveDecimal.create(new BigDecimal("2000")),
           new BigDecimal("2000"), KeyType.NOT_A_KEY),
+      HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(3),
+              "decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.VARCHAR, 20, 0,
+              new HiveVarchar("1999", 20), new BigDecimal("1999"), KeyType.NOT_A_KEY),
+      HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(4),
+              "decimal(18,2)", Types.DECIMAL, HCatFieldSchema.Type.CHAR, 20, 0,
+              new HiveChar("1998", 20), new BigDecimal("1998"), KeyType.NOT_A_KEY),
+      HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(5),
+              "int", Types.INTEGER, HCatFieldSchema.Type.VARCHAR, 20, 0,
+              new HiveVarchar("2001", 20 ), new BigDecimal("2001"), KeyType.NOT_A_KEY),
+      HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(6),
+              "int", Types.INTEGER, HCatFieldSchema.Type.CHAR, 20, 0,
+              new HiveChar("2002", 20), new BigDecimal("2002"), KeyType.NOT_A_KEY),
     };
     List<String> addlArgsArray = new ArrayList<String>();
     setExtraArgs(addlArgsArray);


Mime
View raw message