sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From arv...@apache.org
Subject svn commit: r1182523 - in /incubator/sqoop/trunk/src: docs/man/ docs/user/ java/com/cloudera/sqoop/ java/com/cloudera/sqoop/hive/ java/com/cloudera/sqoop/orm/ java/com/cloudera/sqoop/tool/ test/com/cloudera/sqoop/ test/com/cloudera/sqoop/hive/ test/com...
Date Wed, 12 Oct 2011 18:54:32 GMT
Author: arvind
Date: Wed Oct 12 18:54:31 2011
New Revision: 1182523

URL: http://svn.apache.org/viewvc?rev=1182523&view=rev
Log:
SQOOP-342. Allow user to override Sqoop type mapping.

(Jarek Jarcec Checho via Arvind Prabhakar)

Modified:
    incubator/sqoop/trunk/src/docs/man/codegen-args.txt
    incubator/sqoop/trunk/src/docs/man/hive-args.txt
    incubator/sqoop/trunk/src/docs/man/import-args.txt
    incubator/sqoop/trunk/src/docs/user/codegen-args.txt
    incubator/sqoop/trunk/src/docs/user/codegen.txt
    incubator/sqoop/trunk/src/docs/user/hive-args.txt
    incubator/sqoop/trunk/src/docs/user/import.txt
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/SqoopOptions.java
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/hive/TableDefWriter.java
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/orm/ClassWriter.java
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/ImportTool.java
    incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestSqoopOptions.java
    incubator/sqoop/trunk/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
    incubator/sqoop/trunk/src/test/com/cloudera/sqoop/orm/TestClassWriter.java

Modified: incubator/sqoop/trunk/src/docs/man/codegen-args.txt
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/docs/man/codegen-args.txt?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/docs/man/codegen-args.txt (original)
+++ incubator/sqoop/trunk/src/docs/man/codegen-args.txt Wed Oct 12 18:54:31 2011
@@ -39,3 +39,5 @@ Code generation options
 --package-name (package)::
   Puts auto-generated classes in the named Java package
 
+--map-column-java (mapping)::
+	Override default mapping from SQL type to Java type for configured columns

Modified: incubator/sqoop/trunk/src/docs/man/hive-args.txt
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/docs/man/hive-args.txt?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/docs/man/hive-args.txt (original)
+++ incubator/sqoop/trunk/src/docs/man/hive-args.txt Wed Oct 12 18:54:31 2011
@@ -38,3 +38,5 @@ Hive options
 --hive-table (table-name)::
   When used with --hive-import, overrides the destination table name
 
+--map-column-hive (mapping)::
+	Override default mapping for SQL types into Hive types for configured columns

Modified: incubator/sqoop/trunk/src/docs/man/import-args.txt
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/docs/man/import-args.txt?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/docs/man/import-args.txt (original)
+++ incubator/sqoop/trunk/src/docs/man/import-args.txt Wed Oct 12 18:54:31 2011
@@ -86,4 +86,3 @@ Import control options
 
 --null-non-string::
   The string to be written for a null value for non-string columns
-

Modified: incubator/sqoop/trunk/src/docs/user/codegen-args.txt
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/docs/user/codegen-args.txt?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/docs/user/codegen-args.txt (original)
+++ incubator/sqoop/trunk/src/docs/user/codegen-args.txt Wed Oct 12 18:54:31 2011
@@ -32,5 +32,7 @@ Argument                 Description
 +\--jar-file <file>+     Disable code generation; use specified jar
 +\--outdir <dir>+        Output directory for generated code
 +\--package-name <name>+ Put auto-generated classes in this package
++\--map-column-java <m>+ Override default mapping from SQL type to\
+                         Java type for configured columns.
 -------------------------------------------------------------------------
 

Modified: incubator/sqoop/trunk/src/docs/user/codegen.txt
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/docs/user/codegen.txt?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/docs/user/codegen.txt (original)
+++ incubator/sqoop/trunk/src/docs/user/codegen.txt Wed Oct 12 18:54:31 2011
@@ -44,18 +44,7 @@ another.
 
 include::common-args.txt[]
 
-.Code generation arguments:
-[grid="all"]
-`------------------------`-----------------------------------------------
-Argument                 Description
--------------------------------------------------------------------------
-+\--bindir <dir>+        Output directory for compiled objects
-+\--class-name <name>+   Sets the generated class name. This overrides\
-                         +\--package-name+.
-+\--outdir <dir>+        Output directory for generated code
-+\--package-name <name>+ Put auto-generated classes in this package
-+\--table <table-name>+  Name of the table to generate code for.
--------------------------------------------------------------------------
+include::codegen-args.txt[]
 
 include::output-args.txt[]
 

Modified: incubator/sqoop/trunk/src/docs/user/hive-args.txt
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/docs/user/hive-args.txt?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/docs/user/hive-args.txt (original)
+++ incubator/sqoop/trunk/src/docs/user/hive-args.txt Wed Oct 12 18:54:31 2011
@@ -41,5 +41,7 @@ Argument                      Descriptio
 			      sharded on
 +\--hive-partition-value <v>+ String-value that serves as partition key\
 			      for this imported into hive in this job.
++\--map-column-hive <map>+    Override default mapping from SQL type to\
+                              Hive type for configured columns.
 --------------------------------------------------------------------------
 

Modified: incubator/sqoop/trunk/src/docs/user/import.txt
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/docs/user/import.txt?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/docs/user/import.txt (original)
+++ incubator/sqoop/trunk/src/docs/user/import.txt Wed Oct 12 18:54:31 2011
@@ -262,6 +262,33 @@ are expected to be present in the shell 
 the utilities +mysqldump+ and +mysqlimport+ are required, whereas for
 PostgreSQL the utility +psql+ is required.
 
+Controlling type mapping
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+Sqoop is preconfigured to map most SQL types to appropriate Java or Hive
+representatives. However the default mapping might not be suitable for
+everyone and might be overridden by +--map-column-java+ (for changing
+mapping to Java) or +--map-column-hive+ (for changing Hive mapping).
+
+.Parameters for overriding mapping
+[grid="all"]
+`---------------------------------`--------------------------------------
+Argument                          Description
+-------------------------------------------------------------------------
++\--map-column-java <mapping>+    Override mapping from SQL to Java type\
+                                  for configured columns.
++\--map-column-hive <mapping>+    Override mapping from SQL to Hive type\
+                                  for configured columns.
+-------------------------------------------------------------------------
+
+Sqoop is expecting comma separated list of mapping in form <name of column>=<new
type>. For example:
+
+----
+$ sqoop import ... --map-column-java id=String,value=Integer
+----
+
+Sqoop will rise exception in case that some configured mapping will not be used.
+
 Incremental Imports
 ^^^^^^^^^^^^^^^^^^^
 

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/SqoopOptions.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/SqoopOptions.java?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/SqoopOptions.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/SqoopOptions.java Wed Oct 12 18:54:31
2011
@@ -159,6 +159,10 @@ public class SqoopOptions implements Clo
   @StoredAsProperty("hive.partition.key") private String hivePartitionKey;
   @StoredAsProperty("hive.partition.value") private String hivePartitionValue;
 
+  // User explicit mapping of types
+  private Properties mapColumnJava; // stored as map.colum.java
+  private Properties mapColumnHive; // stored as map.column.hive
+
   // An ordered list of column names denoting what order columns are
   // serialized to a PreparedStatement from a generated record type.
   // Not serialized to metastore.
@@ -584,6 +588,12 @@ public class SqoopOptions implements Clo
     this.connectionParams =
         getPropertiesAsNetstedProperties(props, "db.connect.params");
 
+    // Loading user mapping
+    this.mapColumnHive =
+            getPropertiesAsNetstedProperties(props, "map.column.hive");
+    this.mapColumnJava =
+            getPropertiesAsNetstedProperties(props, "map.column.java");
+
     // Delimiters were previously memoized; don't let the tool override
     // them with defaults.
     this.areDelimsManuallySet = true;
@@ -656,6 +666,10 @@ public class SqoopOptions implements Clo
     setPropertiesAsNestedProperties(props,
             "db.connect.params", this.connectionParams);
 
+    setPropertiesAsNestedProperties(props,
+            "map.column.hive", this.mapColumnHive);
+    setPropertiesAsNestedProperties(props,
+            "map.column.java", this.mapColumnJava);
     return props;
   }
 
@@ -691,6 +705,14 @@ public class SqoopOptions implements Clo
         other.setConnectionParams(this.connectionParams);
       }
 
+      if (null != mapColumnHive) {
+        other.mapColumnHive = (Properties) this.mapColumnHive.clone();
+      }
+
+      if (null != mapColumnJava) {
+        other.mapColumnJava = (Properties) this.mapColumnJava.clone();
+      }
+
       return other;
     } catch (CloneNotSupportedException cnse) {
       // Shouldn't happen.
@@ -817,6 +839,10 @@ public class SqoopOptions implements Clo
     this.incrementalMode = IncrementalMode.None;
 
     this.updateMode = UpdateMode.UpdateOnly;
+
+    // Creating instances for user specific mapping
+    this.mapColumnHive = new Properties();
+    this.mapColumnJava = new Properties();
   }
 
   /**
@@ -1008,6 +1034,32 @@ public class SqoopOptions implements Clo
     return password;
   }
 
+  protected void parseColumnMapping(String mapping,
+          Properties output) {
+    output.clear();
+    String[] maps = mapping.split(",");
+    for(String map : maps) {
+      String[] details = map.split("=");
+      output.put(details[0], details[1]);
+    }
+  }
+
+  public void setMapColumnHive(String mapColumn) {
+    parseColumnMapping(mapColumn, mapColumnHive);
+  }
+
+  public void setMapColumn(String mapColumn) {
+    parseColumnMapping(mapColumn, mapColumnJava);
+  }
+
+  public Properties getMapColumnHive() {
+    return mapColumnHive;
+  }
+
+  public Properties getMapColumnJava() {
+    return mapColumnJava;
+  }
+
   /**
    * Allow the user to enter his password on the console without printing
    * characters.

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/hive/TableDefWriter.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/hive/TableDefWriter.java?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/hive/TableDefWriter.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/hive/TableDefWriter.java Wed Oct 12
18:54:31 2011
@@ -36,6 +36,7 @@ import java.util.Date;
 import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 
+import java.util.Properties;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
@@ -117,6 +118,7 @@ public class TableDefWriter {
    */
   public String getCreateTableStmt() throws IOException {
     Map<String, Integer> columnTypes;
+    Properties userMapping = options.getMapColumnHive();
 
     if (externalColTypes != null) {
       // Use pre-defined column types.
@@ -139,6 +141,22 @@ public class TableDefWriter {
       sb.append(outputTableName).append("` ( ");
     }
 
+    // Check that all explicitly mapped columns are present in result set
+    for(Object column : userMapping.keySet()) {
+      boolean found = false;
+      for(String c : colNames) {
+        if(c.equals(column)) {
+          found = true;
+          break;
+        }
+      }
+
+      if(!found) {
+        throw new IllegalArgumentException("No column by the name " + column
+                + "found while importing data");
+      }
+    }
+
     boolean first = true;
     for (String col : colNames) {
       if (!first) {
@@ -148,7 +166,8 @@ public class TableDefWriter {
       first = false;
 
       Integer colType = columnTypes.get(col);
-      String hiveColType = connManager.toHiveType(colType);
+      String hiveColType = userMapping.getProperty(col);
+      if(hiveColType == null) { hiveColType = connManager.toHiveType(colType); }
       if (null == hiveColType) {
         throw new IOException("Hive does not support the SQL type for column "
             + col);

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/orm/ClassWriter.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/orm/ClassWriter.java?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/orm/ClassWriter.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/orm/ClassWriter.java Wed Oct 12 18:54:31
2011
@@ -43,6 +43,7 @@ import java.io.OutputStreamWriter;
 import java.io.Writer;
 import java.util.HashSet;
 import java.util.Map;
+import java.util.Properties;
 import java.util.Set;
 
 import org.apache.commons.logging.Log;
@@ -229,6 +230,20 @@ public class ClassWriter {
     return output;
   }
 
+  private String toJavaType(String columnName, int sqlType) {
+    Properties mapping = options.getMapColumnJava();
+
+    if(mapping.containsKey(columnName)) {
+      String type = mapping.getProperty(columnName);
+      if(LOG.isDebugEnabled()) {
+        LOG.info("Overriding type of column " + columnName + " to " + type);
+      }
+      return type;
+    }
+
+    return connManager.toJavaType(sqlType);
+  }
+
   /**
    * @param javaType
    * @return the name of the method of JdbcWritableBridge to read an entry
@@ -453,7 +468,7 @@ public class ClassWriter {
 
     for (String col : colNames) {
       int sqlType = columnTypes.get(col);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(col, sqlType);
       if (null == javaType) {
         LOG.error("Cannot resolve SQL type " + sqlType);
         continue;
@@ -496,7 +511,7 @@ public class ClassWriter {
     sb.append("    boolean equal = true;\n");
     for (String col : colNames) {
       int sqlType = columnTypes.get(col);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(col, sqlType);
       if (null == javaType) {
         LOG.error("Cannot resolve SQL type " + sqlType);
         continue;
@@ -530,7 +545,7 @@ public class ClassWriter {
       fieldNum++;
 
       int sqlType = columnTypes.get(col);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(col, sqlType);
       if (null == javaType) {
         LOG.error("No Java type for SQL type " + sqlType
                   + " for column " + col);
@@ -570,7 +585,7 @@ public class ClassWriter {
       fieldNum++;
 
       int sqlType = columnTypes.get(col);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(col, sqlType);
       if (null == javaType) {
         LOG.error("No Java type for SQL type " + sqlType
                   + " for column " + col);
@@ -614,7 +629,7 @@ public class ClassWriter {
       fieldNum++;
 
       int sqlType = columnTypes.get(col);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(col, sqlType);
       if (null == javaType) {
         LOG.error("No Java type for SQL type " + sqlType
                   + " for column " + col);
@@ -650,7 +665,7 @@ public class ClassWriter {
 
     for (String col : colNames) {
       int sqlType = columnTypes.get(col);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(col, sqlType);
       if (null == javaType) {
         LOG.error("No Java type for SQL type " + sqlType
                   + " for column " + col);
@@ -687,7 +702,7 @@ public class ClassWriter {
     // For each field that is mutable, we need to perform the deep copy.
     for (String colName : colNames) {
       int sqlType = columnTypes.get(colName);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(colName, sqlType);
       if (null == javaType) {
         continue;
       } else if (javaType.equals("java.sql.Date")
@@ -721,7 +736,7 @@ public class ClassWriter {
     boolean first = true;
     for (String colName : colNames) {
       int sqlType = columnTypes.get(colName);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(colName, sqlType);
       if (null == javaType) {
         continue;
       } else {
@@ -806,7 +821,7 @@ public class ClassWriter {
     boolean first = true;
     for (String col : colNames) {
       int sqlType = columnTypes.get(col);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(col, sqlType);
       if (null == javaType) {
         LOG.error("No Java type for SQL type " + sqlType
                   + " for column " + col);
@@ -896,7 +911,7 @@ public class ClassWriter {
     // assume that we have __it and __cur_str vars, based on
     // __loadFromFields() code.
     sb.append("    __cur_str = __it.next();\n");
-    String javaType = connManager.toJavaType(colType);
+    String javaType = toJavaType(colName, colType);
 
     parseNullVal(javaType, colName, sb);
     if (javaType.equals("String")) {
@@ -991,7 +1006,7 @@ public class ClassWriter {
 
     for (String col : colNames) {
       int sqlType = columnTypes.get(col);
-      String javaType = connManager.toJavaType(sqlType);
+      String javaType = toJavaType(col, sqlType);
       if (null == javaType) {
         LOG.error("No Java type for SQL type " + sqlType
                   + " for column " + col);
@@ -1071,6 +1086,17 @@ public class ClassWriter {
       columnTypes.put(identifier, type);
     }
 
+    // Check that all explicitly mapped columns are present in result set
+    Properties mapping = options.getMapColumnJava();
+    if(mapping != null && !mapping.isEmpty()) {
+      for(Object column : mapping.keySet()) {
+        if(!uniqColNames.contains((String)column)) {
+        throw new IllegalArgumentException("No column by the name " + column
+                + "found while importing data");
+        }
+      }
+    }
+
     // The db write() method may use column names in a different
     // order. If this is set in the options, pull it out here and
     // make sure we format the column names to identifiers in the same way

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/BaseSqoopTool.java Wed Oct 12 18:54:31
2011
@@ -92,6 +92,8 @@ public abstract class BaseSqoopTool exte
   public static final String INPUT_NULL_STRING = "input-null-string";
   public static final String NULL_NON_STRING = "null-non-string";
   public static final String INPUT_NULL_NON_STRING = "input-null-non-string";
+  public static final String MAP_COLUMN_JAVA = "map-column-java";
+  public static final String MAP_COLUMN_HIVE = "map-column-hive";
 
   public static final String FMT_SEQUENCEFILE_ARG = "as-sequencefile";
   public static final String FMT_TEXTFILE_ARG = "as-textfile";
@@ -449,6 +451,13 @@ public abstract class BaseSqoopTool exte
             + "to hive")
         .withLongOpt(HIVE_PARTITION_VALUE_ARG)
         .create());
+    hiveOpts.addOption(OptionBuilder
+        .hasArg()
+        .withDescription("Override mapping for specific column to hive"
+          + " types.")
+        .withLongOpt(MAP_COLUMN_HIVE)
+        .create());
+
     return hiveOpts;
   }
 
@@ -550,25 +559,31 @@ public abstract class BaseSqoopTool exte
         .withLongOpt(PACKAGE_NAME_ARG)
         .create());
     codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
-            .hasArg()
-            .withDescription("Null string representation")
-            .withLongOpt(NULL_STRING)
-            .create());
+        .hasArg()
+        .withDescription("Null string representation")
+        .withLongOpt(NULL_STRING)
+        .create());
     codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
-            .hasArg()
-            .withDescription("Input null string representation")
-            .withLongOpt(INPUT_NULL_STRING)
-            .create());
+        .hasArg()
+        .withDescription("Input null string representation")
+        .withLongOpt(INPUT_NULL_STRING)
+        .create());
     codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
-            .hasArg()
-            .withDescription("Null non-string representation")
-            .withLongOpt(NULL_NON_STRING)
-            .create());
+        .hasArg()
+        .withDescription("Null non-string representation")
+        .withLongOpt(NULL_NON_STRING)
+        .create());
     codeGenOpts.addOption(OptionBuilder.withArgName("null-str")
-            .hasArg()
-            .withDescription("Input null non-string representation")
-            .withLongOpt(INPUT_NULL_NON_STRING)
-            .create());
+        .hasArg()
+        .withDescription("Input null non-string representation")
+        .withLongOpt(INPUT_NULL_NON_STRING)
+        .create());
+    codeGenOpts.addOption(OptionBuilder
+        .hasArg()
+        .withDescription("Override mapping for specific columns to java types")
+        .withLongOpt(MAP_COLUMN_JAVA)
+        .create());
+
     if (!multiTable) {
       codeGenOpts.addOption(OptionBuilder.withArgName("name")
           .hasArg()
@@ -753,6 +768,10 @@ public abstract class BaseSqoopTool exte
     if (in.hasOption(HIVE_PARTITION_VALUE_ARG)) {
       out.setHivePartitionValue(in.getOptionValue(HIVE_PARTITION_VALUE_ARG));
     }
+
+   if (in.hasOption(MAP_COLUMN_HIVE)) {
+      out.setMapColumnHive(in.getOptionValue(MAP_COLUMN_HIVE));
+    }
   }
 
   protected void applyOutputFormatOptions(CommandLine in, SqoopOptions out)
@@ -843,6 +862,10 @@ public abstract class BaseSqoopTool exte
       out.setPackageName(in.getOptionValue(PACKAGE_NAME_ARG));
     }
 
+    if (in.hasOption(MAP_COLUMN_JAVA)) {
+      out.setMapColumn(in.getOptionValue(MAP_COLUMN_JAVA));
+    }
+
     if (!multiTable && in.hasOption(CLASS_NAME_ARG)) {
       out.setClassName(in.getOptionValue(CLASS_NAME_ARG));
     }

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/ImportTool.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/ImportTool.java?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/ImportTool.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/tool/ImportTool.java Wed Oct 12 18:54:31
2011
@@ -40,6 +40,7 @@ import org.apache.hadoop.util.StringUtil
 
 import com.cloudera.sqoop.Sqoop;
 import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.FileLayout;
 import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
 import com.cloudera.sqoop.cli.RelatedOptions;
 import com.cloudera.sqoop.cli.ToolOptions;
@@ -839,6 +840,10 @@ public class ImportTool extends BaseSqoo
             "MySQL direct export currently supports only text output format."
              + "Parameters --as-sequencefile and --as-avrodatafile are not "
              + "supported with --direct params in MySQL case.");
+    } else if (!options.getMapColumnJava().isEmpty()
+            && options.getFileLayout() == FileLayout.AvroDataFile) {
+      throw new InvalidOptionsException(
+              "Overriding column types is currently not supported with avro.");
     }
   }
 

Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestSqoopOptions.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestSqoopOptions.java?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestSqoopOptions.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/TestSqoopOptions.java Wed Oct 12 18:54:31
2011
@@ -257,6 +257,28 @@ public class TestSqoopOptions extends Te
     assertEquals("select 1, 2", opts.getBoundaryQuery());
   }
 
+  public void testMapColumnHiveParams() throws Exception {
+    String[] args = {
+      "--map-column-hive", "id=STRING",
+    };
+
+    SqoopOptions opts = parse(args);
+    Properties mapping = opts.getMapColumnHive();
+    assertTrue(mapping.containsKey("id"));
+    assertEquals("STRING", mapping.get("id"));
+  }
+
+  public void testMapColumnJavaParams() throws Exception {
+    String[] args = {
+      "--map-column-java", "id=String",
+    };
+
+    SqoopOptions opts = parse(args);
+    Properties mapping = opts.getMapColumnJava();
+    assertTrue(mapping.containsKey("id"));
+    assertEquals("String", mapping.get("id"));
+  }
+
   public void testPropertySerialization1() {
     // Test that if we write a SqoopOptions out to a Properties,
     // and then read it back in, we get all the same results.

Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/hive/TestTableDefWriter.java Wed Oct
12 18:54:31 2011
@@ -31,6 +31,8 @@ import org.apache.hadoop.conf.Configurat
 
 import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.tool.ImportTool;
+import com.cloudera.sqoop.testutil.HsqldbTestServer;
+import java.sql.Types;
 
 /**
  * Test Hive DDL statement generation.
@@ -136,4 +138,52 @@ public class TestTableDefWriter extends 
         + "'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'",
         createTable);
   }
+
+  public void testUserMapping() throws Exception {
+    String[] args = {
+        "--map-column-hive", "id=STRING,value=INTEGER",
+    };
+    Configuration conf = new Configuration();
+    SqoopOptions options =
+      new ImportTool().parseArguments(args, null, null, false);
+    TableDefWriter writer = new TableDefWriter(options,
+        null, HsqldbTestServer.getTableName(), "outputTable", conf, false);
+
+    Map<String, Integer> colTypes = new HashMap<String, Integer>();
+    colTypes.put("id", Types.INTEGER);
+    colTypes.put("value", Types.VARCHAR);
+    writer.setColumnTypes(colTypes);
+
+    String createTable = writer.getCreateTableStmt();
+
+    assertNotNull(createTable);
+
+    assertTrue(createTable.contains("`id` STRING"));
+    assertTrue(createTable.contains("`value` INTEGER"));
+
+    assertFalse(createTable.contains("`id` INTEGER"));
+    assertFalse(createTable.contains("`value` STRING"));
+  }
+
+  public void testUserMappingFailWhenCantBeApplied() throws Exception {
+    String[] args = {
+        "--map-column-hive", "id=STRING,value=INTEGER",
+    };
+    Configuration conf = new Configuration();
+    SqoopOptions options =
+      new ImportTool().parseArguments(args, null, null, false);
+    TableDefWriter writer = new TableDefWriter(options,
+        null, HsqldbTestServer.getTableName(), "outputTable", conf, false);
+
+    Map<String, Integer> colTypes = new HashMap<String, Integer>();
+    colTypes.put("id", Types.INTEGER);
+    writer.setColumnTypes(colTypes);
+
+    try {
+      String createTable = writer.getCreateTableStmt();
+      fail("Expected failure on non applied mapping.");
+    } catch(IllegalArgumentException iae) {
+      // Expected, ok
+    }
+  }
 }

Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/orm/TestClassWriter.java?rev=1182523&r1=1182522&r2=1182523&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/orm/TestClassWriter.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/orm/TestClassWriter.java Wed Oct 12
18:54:31 2011
@@ -46,6 +46,7 @@ import com.cloudera.sqoop.testutil.Hsqld
 import com.cloudera.sqoop.testutil.ImportJobTestCase;
 import com.cloudera.sqoop.tool.ImportTool;
 import com.cloudera.sqoop.util.ClassLoaderStack;
+import java.lang.reflect.Field;
 
 /**
  * Test that the ClassWriter generates Java classes based on the given table,
@@ -425,4 +426,44 @@ public class TestClassWriter extends Tes
     }
   }
 
+  private static final String USERMAPPING_CLASS_AND_PACKAGE_NAME =
+      "usermapping.pkg.prefix.classname";
+
+  @Test
+  public void testUserMapping() throws IOException, ClassNotFoundException,
+      InstantiationException, IllegalAccessException, NoSuchMethodException,
+      InvocationTargetException {
+
+    // Set the option strings in an "argv" to redirect our srcdir and bindir
+    String [] argv = {
+      "--bindir", JAR_GEN_DIR,
+      "--outdir", CODE_GEN_DIR,
+      "--class-name", USERMAPPING_CLASS_AND_PACKAGE_NAME,
+      "--map-column-java", "INTFIELD1=String",
+    };
+
+    File ormJarFile = runGenerationTest(argv,
+            USERMAPPING_CLASS_AND_PACKAGE_NAME);
+    ClassLoader prevClassLoader = ClassLoaderStack.addJarFile(
+        ormJarFile.getCanonicalPath(),
+        USERMAPPING_CLASS_AND_PACKAGE_NAME);
+    Class tableClass = Class.forName(
+        USERMAPPING_CLASS_AND_PACKAGE_NAME,
+        true,
+        Thread.currentThread().getContextClassLoader());
+
+    try {
+      Field intfield = tableClass.getDeclaredField("INTFIELD1");
+
+      assertEquals(String.class, intfield.getType());
+    } catch (NoSuchFieldException ex) {
+      fail("Can't find field for INTFIELD1");
+    } catch (SecurityException ex) {
+      fail("Can't find field for INTFIELD1");
+    }
+
+    if (null != prevClassLoader) {
+      ClassLoaderStack.setCurrentClassLoader(prevClassLoader);
+    }
+  }
 }



Mime
View raw message