sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From arv...@apache.org
Subject svn commit: r1297782 - in /incubator/sqoop/trunk/src: java/org/apache/sqoop/hive/ java/org/apache/sqoop/manager/ java/org/apache/sqoop/mapreduce/ java/org/apache/sqoop/orm/ test/com/cloudera/sqoop/manager/
Date Tue, 06 Mar 2012 23:01:03 GMT
Author: arvind
Date: Tue Mar  6 23:01:02 2012
New Revision: 1297782

URL: http://svn.apache.org/viewvc?rev=1297782&view=rev
Log:
SQOOP-352. Export of avro data fails on year values.

(Bilung Lee via Arvind Prabhakar)

Modified:
    incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java
    incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java
    incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java
    incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java
    incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java
    incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
    incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
    incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java
    incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java

Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java Tue Mar  6 23:01:02
2012
@@ -165,7 +165,7 @@ public class TableDefWriter {
       Integer colType = columnTypes.get(col);
       String hiveColType = userMapping.getProperty(col);
       if (hiveColType == null) {
-        hiveColType = connManager.toHiveType(colType);
+        hiveColType = connManager.toHiveType(col, colType);
       }
       if (null == hiveColType) {
         throw new IOException("Hive does not support the SQL type for column "

Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java Tue Mar  6 23:01:02
2012
@@ -23,6 +23,7 @@ import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Timestamp;
+import java.sql.Types;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.LinkedHashSet;
@@ -31,10 +32,16 @@ import java.util.Map;
 import java.util.Set;
 import java.util.StringTokenizer;
 
+import org.apache.avro.Schema.Type;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.io.BytesWritable;
 
 import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.hive.HiveTypes;
+import com.cloudera.sqoop.lib.BlobRef;
+import com.cloudera.sqoop.lib.ClobRef;
+import com.cloudera.sqoop.manager.SqlManager;
 import com.cloudera.sqoop.util.ExportException;
 import com.cloudera.sqoop.util.ImportException;
 
@@ -76,18 +83,154 @@ public abstract class ConnManager {
   public abstract String getPrimaryKey(String tableName);
 
   /**
+   * Resolve a database-specific type to the Java type that should contain it.
+   * @param sqlType     sql type
+   * @return the name of a Java type to hold the sql datatype, or null if none.
+   */
+  public String toJavaType(int sqlType) {
+    // Mappings taken from:
+    // http://java.sun.com/j2se/1.3/docs/guide/jdbc/getstart/mapping.html
+    if (sqlType == Types.INTEGER) {
+      return "Integer";
+    } else if (sqlType == Types.VARCHAR) {
+      return "String";
+    } else if (sqlType == Types.CHAR) {
+      return "String";
+    } else if (sqlType == Types.LONGVARCHAR) {
+      return "String";
+    } else if (sqlType == Types.NVARCHAR) {
+      return "String";
+    } else if (sqlType == Types.NCHAR) {
+      return "String";
+    } else if (sqlType == Types.LONGNVARCHAR) {
+      return "String";
+    } else if (sqlType == Types.NUMERIC) {
+      return "java.math.BigDecimal";
+    } else if (sqlType == Types.DECIMAL) {
+      return "java.math.BigDecimal";
+    } else if (sqlType == Types.BIT) {
+      return "Boolean";
+    } else if (sqlType == Types.BOOLEAN) {
+      return "Boolean";
+    } else if (sqlType == Types.TINYINT) {
+      return "Integer";
+    } else if (sqlType == Types.SMALLINT) {
+      return "Integer";
+    } else if (sqlType == Types.BIGINT) {
+      return "Long";
+    } else if (sqlType == Types.REAL) {
+      return "Float";
+    } else if (sqlType == Types.FLOAT) {
+      return "Double";
+    } else if (sqlType == Types.DOUBLE) {
+      return "Double";
+    } else if (sqlType == Types.DATE) {
+      return "java.sql.Date";
+    } else if (sqlType == Types.TIME) {
+      return "java.sql.Time";
+    } else if (sqlType == Types.TIMESTAMP) {
+      return "java.sql.Timestamp";
+    } else if (sqlType == Types.BINARY
+        || sqlType == Types.VARBINARY) {
+      return BytesWritable.class.getName();
+    } else if (sqlType == Types.CLOB) {
+      return ClobRef.class.getName();
+    } else if (sqlType == Types.BLOB
+        || sqlType == Types.LONGVARBINARY) {
+      return BlobRef.class.getName();
+    } else {
+      // TODO(aaron): Support DISTINCT, ARRAY, STRUCT, REF, JAVA_OBJECT.
+      // Return null indicating database-specific manager should return a
+      // java data type if it can find one for any nonstandard type.
+      return null;
+    }
+  }
+
+  /**
+   * Resolve a database-specific type to Hive data type.
+   * @param sqlType     sql type
+   * @return            hive type
+   */
+  public String toHiveType(int sqlType) {
+    return HiveTypes.toHiveType(sqlType);
+  }
+
+  /**
+   * Resolve a database-specific type to Avro data type.
+   * @param sqlType     sql type
+   * @return            avro type
+   */
+  public Type toAvroType(int sqlType) {
+    switch (sqlType) {
+    case Types.TINYINT:
+    case Types.SMALLINT:
+    case Types.INTEGER:
+      return Type.INT;
+    case Types.BIGINT:
+      return Type.LONG;
+    case Types.BIT:
+    case Types.BOOLEAN:
+      return Type.BOOLEAN;
+    case Types.REAL:
+      return Type.FLOAT;
+    case Types.FLOAT:
+    case Types.DOUBLE:
+      return Type.DOUBLE;
+    case Types.NUMERIC:
+    case Types.DECIMAL:
+      return Type.STRING;
+    case Types.CHAR:
+    case Types.VARCHAR:
+    case Types.LONGVARCHAR:
+    case Types.LONGNVARCHAR:
+    case Types.NVARCHAR:
+    case Types.NCHAR:
+      return Type.STRING;
+    case Types.DATE:
+    case Types.TIME:
+    case Types.TIMESTAMP:
+      return Type.LONG;
+    case Types.BINARY:
+    case Types.VARBINARY:
+      return Type.BYTES;
+    default:
+      throw new IllegalArgumentException("Cannot convert SQL type "
+          + sqlType);
+    }
+  }
+
+  /**
    * Return java type for SQL type.
+   * @param columnName  column name
    * @param sqlType     sql type
    * @return            java type
    */
-  public abstract String toJavaType(int sqlType);
+  public String toJavaType(String columnName, int sqlType) {
+    // ignore column name by default.
+    return toJavaType(sqlType);
+  }
 
     /**
      * Return hive type for SQL type.
+     * @param columnName  column name
      * @param sqlType   sql type
      * @return          hive type
      */
-  public abstract String toHiveType(int sqlType);
+  public String toHiveType(String columnName, int sqlType) {
+    // ignore column name by default.
+    return toHiveType(sqlType);
+  }
+
+  /**
+   * Return avro type for SQL type.
+   * @param columnName  column name
+   * @param sqlType   sql type
+   * @return          avro type
+   */
+  public Type toAvroType(String columnName, int sqlType) {
+    // ignore column name by default.
+    return toAvroType(sqlType);
+  }
 
   /**
    * Return an unordered mapping from colname to sqltype for
@@ -126,6 +269,51 @@ public abstract class ConnManager {
   }
 
   /**
+   * Return an unordered mapping from colname to sql type name for
+   * all columns in a table.
+   */
+  public Map<String, String> getColumnTypeNamesForTable(String tableName) {
+    LOG.error("This database does not support column type names.");
+    return null;
+  }
+
+  /**
+   * Return an unordered mapping from colname to sql type name for
+   * all columns in a query.
+   */
+  public Map<String, String> getColumnTypeNamesForQuery(String query) {
+    LOG.error("This database does not support free-form query"
+        + " column type names.");
+    return null;
+  }
+
+  /**
+   * Return an unordered mapping from colname to sql type name for
+   * all columns in a table or query.
+   *
+   * @param tableName the name of the table
+   * @param sqlQuery the SQL query to use if tableName is null
+   */
+  public Map<String, String> getColumnTypeNames(String tableName,
+      String sqlQuery) {
+    Map<String, String> columnTypeNames;
+    if (null != tableName) {
+      // We're generating a class based on a table import.
+      columnTypeNames = getColumnTypeNamesForTable(tableName);
+    } else {
+      // This is based on an arbitrary query.
+      String query = sqlQuery;
+      if (query.indexOf(SqlManager.SUBSTITUTE_TOKEN) == -1) {
+        throw new RuntimeException("Query [" + query + "] must contain '"
+            + SqlManager.SUBSTITUTE_TOKEN + "' in WHERE clause.");
+      }
+
+      columnTypeNames = getColumnTypeNamesForQuery(query);
+    }
+    return columnTypeNames;
+  }
+
+  /**
    * This method allows various connection managers to indicate if they support
    * staging data for export jobs. The managers that do support this must
    * override this method and return <tt>true</tt>.

Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java Tue Mar  6 23:01:02
2012
@@ -24,11 +24,14 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.sql.Types;
+import java.util.Map;
 
+import org.apache.avro.Schema.Type;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.util.StringUtils;
+
 import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.util.ImportException;
 
@@ -224,5 +227,50 @@ public class MySQLManager
   protected String getSchemaQuery() {
     return "SELECT SCHEMA()";
   }
+
+  private Map<String, String> colTypeNames;
+  private static final int YEAR_TYPE_OVERWRITE = Types.SMALLINT;
+
+  @Override
+  public String toJavaType(String columnName, int sqlType) {
+    if (colTypeNames == null) {
+      colTypeNames = getColumnTypeNames(options.getTableName(),
+          options.getSqlQuery());
+    }
+
+    if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
+      sqlType = YEAR_TYPE_OVERWRITE;
+    }
+
+    return super.toJavaType(columnName, sqlType);
+  }
+
+  @Override
+  public String toHiveType(String columnName, int sqlType) {
+    if (colTypeNames == null) {
+      colTypeNames = getColumnTypeNames(options.getTableName(),
+          options.getSqlQuery());
+    }
+
+    if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
+      sqlType = YEAR_TYPE_OVERWRITE;
+    }
+
+    return super.toHiveType(columnName, sqlType);
+  }
+
+  @Override
+  public Type toAvroType(String columnName, int sqlType) {
+    if (colTypeNames == null) {
+      colTypeNames = getColumnTypeNames(options.getTableName(),
+          options.getSqlQuery());
+    }
+
+    if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
+      sqlType = YEAR_TYPE_OVERWRITE;
+    }
+
+    return super.toAvroType(columnName, sqlType);
+  }
 }
 

Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java Tue Mar  6
23:01:02 2012
@@ -474,8 +474,8 @@ public class OracleManager
    * @param sqlType
    * @return the name of a Java type to hold the sql datatype, or null if none.
    */
-  public String toJavaType(int sqlType) {
-    String defaultJavaType = super.toJavaType(sqlType);
+  public String toJavaType(String columnName, int sqlType) {
+    String defaultJavaType = super.toJavaType(columnName, sqlType);
     return (defaultJavaType == null) ? dbToJavaType(sqlType) : defaultJavaType;
   }
 
@@ -511,8 +511,8 @@ public class OracleManager
    * @param sqlType     sql data type
    * @return            hive data type
    */
-  public String toHiveType(int sqlType) {
-    String defaultHiveType = super.toHiveType(sqlType);
+  public String toHiveType(String columnName, int sqlType) {
+    String defaultHiveType = super.toHiveType(columnName, sqlType);
     return (defaultHiveType == null) ? dbToHiveType(sqlType) : defaultHiveType;
   }
 

Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/manager/SqlManager.java Tue Mar  6 23:01:02
2012
@@ -22,14 +22,12 @@ import java.sql.Timestamp;
 
 import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.hbase.HBaseUtil;
-import com.cloudera.sqoop.hive.HiveTypes;
-import com.cloudera.sqoop.lib.BlobRef;
-import com.cloudera.sqoop.lib.ClobRef;
 import com.cloudera.sqoop.mapreduce.DataDrivenImportJob;
 import com.cloudera.sqoop.mapreduce.HBaseImportJob;
 import com.cloudera.sqoop.mapreduce.ImportJobBase;
 import com.cloudera.sqoop.mapreduce.JdbcExportJob;
 import com.cloudera.sqoop.mapreduce.JdbcUpdateExportJob;
+import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
 import com.cloudera.sqoop.util.ExportException;
 import com.cloudera.sqoop.util.ImportException;
 import com.cloudera.sqoop.util.ResultSetPrinter;
@@ -52,9 +50,7 @@ import java.util.Properties;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.BytesWritable;
 import org.apache.hadoop.util.StringUtils;
-import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
 
 /**
  * ConnManager implementation for generic SQL-compliant database.
@@ -236,6 +232,61 @@ public abstract class SqlManager
   }
 
   @Override
+  public Map<String, String> getColumnTypeNamesForTable(String tableName) {
+    String stmt = getColTypesQuery(tableName);
+    return getColumnTypeNamesForRawQuery(stmt);
+  }
+
+  @Override
+  public Map<String, String> getColumnTypeNamesForQuery(String query) {
+    // Manipulate the query to return immediately, with zero rows.
+    String rawQuery = query.replace(SUBSTITUTE_TOKEN, " (1 = 0) ");
+    return getColumnTypeNamesForRawQuery(rawQuery);
+  }
+
+  protected Map<String, String> getColumnTypeNamesForRawQuery(String stmt) {
+    ResultSet results;
+    try {
+      results = execute(stmt);
+    } catch (SQLException sqlE) {
+      LOG.error("Error executing statement: " + sqlE.toString(), sqlE);
+      release();
+      return null;
+    }
+
+    try {
+      Map<String, String> colTypeNames = new HashMap<String, String>();
+
+      int cols = results.getMetaData().getColumnCount();
+      ResultSetMetaData metadata = results.getMetaData();
+      for (int i = 1; i < cols + 1; i++) {
+        String colTypeName = metadata.getColumnTypeName(i);
+
+        String colName = metadata.getColumnName(i);
+        if (colName == null || colName.equals("")) {
+          colName = metadata.getColumnLabel(i);
+        }
+
+        colTypeNames.put(colName, colTypeName);
+      }
+
+      return colTypeNames;
+    } catch (SQLException sqlException) {
+      LOG.error("Error reading from database: " + sqlException.toString());
+      return null;
+    } finally {
+      try {
+        results.close();
+        getConnection().commit();
+      } catch (SQLException sqlE) {
+        LOG.warn("SQLException closing ResultSet: " + sqlE.toString());
+      }
+
+      release();
+    }
+  }
+
+  @Override
   public ResultSet readTable(String tableName, String[] columns)
       throws SQLException {
     if (columns == null) {
@@ -496,79 +547,6 @@ public abstract class SqlManager
     return execute(stmt, options.getFetchSize(), args);
   }
 
-  /**
-   * Resolve a database-specific type to the Java type that should contain it.
-   * @param sqlType
-   * @return the name of a Java type to hold the sql datatype, or null if none.
-   */
-  public String toJavaType(int sqlType) {
-    // Mappings taken from:
-    // http://java.sun.com/j2se/1.3/docs/guide/jdbc/getstart/mapping.html
-    if (sqlType == Types.INTEGER) {
-      return "Integer";
-    } else if (sqlType == Types.VARCHAR) {
-      return "String";
-    } else if (sqlType == Types.CHAR) {
-      return "String";
-    } else if (sqlType == Types.LONGVARCHAR) {
-      return "String";
-    } else if (sqlType == Types.NVARCHAR) {
-      return "String";
-    } else if (sqlType == Types.NCHAR) {
-      return "String";
-    } else if (sqlType == Types.LONGNVARCHAR) {
-      return "String";
-    } else if (sqlType == Types.NUMERIC) {
-      return "java.math.BigDecimal";
-    } else if (sqlType == Types.DECIMAL) {
-      return "java.math.BigDecimal";
-    } else if (sqlType == Types.BIT) {
-      return "Boolean";
-    } else if (sqlType == Types.BOOLEAN) {
-      return "Boolean";
-    } else if (sqlType == Types.TINYINT) {
-      return "Integer";
-    } else if (sqlType == Types.SMALLINT) {
-      return "Integer";
-    } else if (sqlType == Types.BIGINT) {
-      return "Long";
-    } else if (sqlType == Types.REAL) {
-      return "Float";
-    } else if (sqlType == Types.FLOAT) {
-      return "Double";
-    } else if (sqlType == Types.DOUBLE) {
-      return "Double";
-    } else if (sqlType == Types.DATE) {
-      return "java.sql.Date";
-    } else if (sqlType == Types.TIME) {
-      return "java.sql.Time";
-    } else if (sqlType == Types.TIMESTAMP) {
-      return "java.sql.Timestamp";
-    } else if (sqlType == Types.BINARY
-        || sqlType == Types.VARBINARY) {
-      return BytesWritable.class.getName();
-    } else if (sqlType == Types.CLOB) {
-      return ClobRef.class.getName();
-    } else if (sqlType == Types.BLOB
-        || sqlType == Types.LONGVARBINARY) {
-      return BlobRef.class.getName();
-    } else {
-      // TODO(aaron): Support DISTINCT, ARRAY, STRUCT, REF, JAVA_OBJECT.
-      // Return null indicating database-specific manager should return a
-      // java data type if it can find one for any nonstandard type.
-      return null;
-    }
-  }
-
-  /**
-   * Resolve a database-specific type to Hive data type.
-   * @param sqlType     sql type
-   * @return            hive type
-   */
-  public String toHiveType(int sqlType) {
-    return HiveTypes.toHiveType(sqlType);
-  }
-
   public void close() throws SQLException {
     release();
   }

Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java Tue Mar 
6 23:01:02 2012
@@ -73,7 +73,8 @@ public class JdbcExportJob extends Expor
       MapWritable columnTypes = new MapWritable();
       for (Map.Entry<String, Integer> e : columnTypeInts.entrySet()) {
         Text columnName = new Text(e.getKey());
-        Text columnText = new Text(connManager.toJavaType(e.getValue()));
+        Text columnText = new Text(
+            connManager.toJavaType(e.getKey(), e.getValue()));
         columnTypes.put(columnName, columnText);
       }
       DefaultStringifier.store(job.getConfiguration(), columnTypes,

Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java Tue Mar 
6 23:01:02 2012
@@ -19,24 +19,28 @@
 package org.apache.sqoop.orm;
 
 import java.io.IOException;
-import java.sql.Types;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 
 import org.apache.avro.Schema;
 import org.apache.avro.Schema.Field;
 import org.apache.avro.Schema.Type;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.manager.ConnManager;
-import java.util.Properties;
 
 /**
  * Creates an Avro schema to represent a table from a database.
  */
 public class AvroSchemaGenerator {
 
+  public static final Log LOG =
+      LogFactory.getLog(AvroSchemaGenerator.class.getName());
+
   private final SqoopOptions options;
   private final ConnManager connManager;
   private final String tableName;
@@ -78,58 +82,6 @@ public class AvroSchemaGenerator {
     return schema;
   }
 
-  private Type toAvroType(int sqlType) {
-    switch (sqlType) {
-      case Types.TINYINT:
-      case Types.SMALLINT:
-      case Types.INTEGER:
-        return Type.INT;
-      case Types.BIGINT:
-        return Type.LONG;
-      case Types.BIT:
-      case Types.BOOLEAN:
-        return Type.BOOLEAN;
-      case Types.REAL:
-        return Type.FLOAT;
-      case Types.FLOAT:
-      case Types.DOUBLE:
-        return Type.DOUBLE;
-      case Types.NUMERIC:
-      case Types.DECIMAL:
-        return Type.STRING;
-      case Types.CHAR:
-      case Types.VARCHAR:
-      case Types.LONGVARCHAR:
-      case Types.LONGNVARCHAR:
-      case Types.NVARCHAR:
-      case Types.NCHAR:
-        return Type.STRING;
-      case Types.DATE:
-      case Types.TIME:
-      case Types.TIMESTAMP:
-        return Type.LONG;
-      case Types.BINARY:
-      case Types.VARBINARY:
-        return Type.BYTES;
-      default:
-        throw new IllegalArgumentException("Cannot convert SQL type "
-            + sqlType);
-    }
-  }
-
-  private Type toAvroType(String type) {
-    if (type.equalsIgnoreCase("INTEGER")) { return Type.INT; }
-    if (type.equalsIgnoreCase("LONG")) { return Type.LONG; }
-    if (type.equalsIgnoreCase("BOOLEAN")) { return Type.BOOLEAN; }
-    if (type.equalsIgnoreCase("FLOAT")) { return Type.FLOAT; }
-    if (type.equalsIgnoreCase("DOUBLE")) { return Type.DOUBLE; }
-    if (type.equalsIgnoreCase("STRING")) { return Type.STRING; }
-    if (type.equalsIgnoreCase("BYTES")) { return Type.BYTES; }
-
-    // Mapping was not found
-    throw new IllegalArgumentException("Cannot convert to AVRO type " + type);
-  }
-
   /**
    * Will create union, because each type is assumed to be nullable.
    *
@@ -138,18 +90,8 @@ public class AvroSchemaGenerator {
    * @return Schema
    */
   public Schema toAvroSchema(int sqlType, String columnName) {
-    Properties mappingJava = options.getMapColumnJava();
-
-    // Try to apply any user specified mapping
-    Type targetType;
-    if (columnName != null && mappingJava.containsKey(columnName)) {
-        targetType = toAvroType((String)mappingJava.get(columnName));
-    } else {
-      targetType = toAvroType(sqlType);
-    }
-
     List<Schema> childSchemas = new ArrayList<Schema>();
-    childSchemas.add(Schema.create(targetType));
+    childSchemas.add(Schema.create(toAvroType(columnName, sqlType)));
     childSchemas.add(Schema.create(Schema.Type.NULL));
     return Schema.createUnion(childSchemas);
   }
@@ -157,4 +99,28 @@ public class AvroSchemaGenerator {
   public Schema toAvroSchema(int sqlType) {
     return toAvroSchema(sqlType, null);
   }
+
+  private Type toAvroType(String columnName, int sqlType) {
+    Properties mapping = options.getMapColumnJava();
+
+    if (mapping.containsKey(columnName)) {
+      String type = mapping.getProperty(columnName);
+      if (LOG.isDebugEnabled()) {
+        LOG.info("Overriding type of column " + columnName + " to " + type);
+      }
+
+      if (type.equalsIgnoreCase("INTEGER")) { return Type.INT; }
+      if (type.equalsIgnoreCase("LONG")) { return Type.LONG; }
+      if (type.equalsIgnoreCase("BOOLEAN")) { return Type.BOOLEAN; }
+      if (type.equalsIgnoreCase("FLOAT")) { return Type.FLOAT; }
+      if (type.equalsIgnoreCase("DOUBLE")) { return Type.DOUBLE; }
+      if (type.equalsIgnoreCase("STRING")) { return Type.STRING; }
+      if (type.equalsIgnoreCase("BYTES")) { return Type.BYTES; }
+
+      // Mapping was not found
+      throw new IllegalArgumentException("Cannot convert to AVRO type " + type);
+    }
+
+    return connManager.toAvroType(columnName, sqlType);
+  }
 }

Modified: incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java (original)
+++ incubator/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java Tue Mar  6 23:01:02
2012
@@ -250,7 +250,7 @@ public class ClassWriter {
       return type;
     }
 
-    return connManager.toJavaType(sqlType);
+    return connManager.toJavaType(columnName, sqlType);
   }
 
   /**

Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java?rev=1297782&r1=1297781&r2=1297782&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLCompatTest.java Tue Mar
 6 23:01:02 2012
@@ -27,6 +27,8 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 
 import org.apache.hadoop.conf.Configuration;
+import org.junit.Test;
+
 import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.testutil.ManagerCompatTestCase;
 
@@ -189,5 +191,10 @@ public class MySQLCompatTest extends Man
   protected String getDecimalDbOutput(String numAsInserted) {
     return getNumericDbOutput(numAsInserted);
   }
+
+  @Test
+  public void testYear() {
+    verifyType("YEAR", "2012", "2012-01-01", "2012");
+  }
 }
 



Mime
View raw message