sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From arv...@apache.org
Subject svn commit: r1178661 - in /incubator/sqoop/trunk/src: java/com/cloudera/sqoop/manager/SqlManager.java java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java test/com/cloudera/sqoop/manager/DirectMySQLTest.java
Date Tue, 04 Oct 2011 01:12:13 GMT
Author: arvind
Date: Tue Oct  4 01:12:13 2011
New Revision: 1178661

URL: http://svn.apache.org/viewvc?rev=1178661&view=rev
Log:
SQOOP-341. Support for unsigned integers with MySQL.

(Alex Newman via Arvind Prabhakar)

Modified:
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
    incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
    incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java?rev=1178661&r1=1178660&r2=1178661&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java (original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/manager/SqlManager.java Tue Oct  4 01:12:13
2011
@@ -206,6 +206,12 @@ public abstract class SqlManager extends
       ResultSetMetaData metadata = results.getMetaData();
       for (int i = 1; i < cols + 1; i++) {
         int typeId = metadata.getColumnType(i);
+        // If we have an unsigned int we need to make extra room by
+        // plopping it into a bigint
+        if (typeId == Types.INTEGER &&  !metadata.isSigned(i)){
+            typeId = Types.BIGINT;
+        }
+
         String colName = metadata.getColumnName(i);
         if (colName == null || colName.equals("")) {
           colName = metadata.getColumnLabel(i);

Modified: incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java?rev=1178661&r1=1178660&r2=1178661&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
(original)
+++ incubator/sqoop/trunk/src/java/com/cloudera/sqoop/mapreduce/db/DataDrivenDBInputFormat.java
Tue Oct  4 01:12:13 2011
@@ -199,6 +199,13 @@ public class DataDrivenDBInputFormat<T e
       // for interpolating split points (i.e., numeric splits, text splits,
       // dates, etc.)
       int sqlDataType = results.getMetaData().getColumnType(1);
+      boolean isSigned = results.getMetaData().isSigned(1);
+
+      // MySQL has an unsigned integer which we need to allocate space for
+      if (sqlDataType == Types.INTEGER && !isSigned){
+          sqlDataType = Types.BIGINT;
+      }
+
       DBSplitter splitter = getSplitter(sqlDataType);
       if (null == splitter) {
         throw new IOException("Unknown SQL data type: " + sqlDataType);

Modified: incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java
URL: http://svn.apache.org/viewvc/incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java?rev=1178661&r1=1178660&r2=1178661&view=diff
==============================================================================
--- incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java (original)
+++ incubator/sqoop/trunk/src/test/com/cloudera/sqoop/manager/DirectMySQLTest.java Tue Oct
 4 01:12:13 2011
@@ -105,16 +105,17 @@ public class DirectMySQLTest extends Imp
       st.executeUpdate("CREATE TABLE " + getTableName() + " ("
           + "id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, "
           + "name VARCHAR(24) NOT NULL, "
+          + "overly_large_number INT UNSIGNED,"
           + "start_date DATE, "
           + "salary FLOAT, "
           + "dept VARCHAR(32))");
 
       st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
-          + "NULL,'Aaron','2009-05-14',1000000.00,'engineering')");
+          + "NULL,'Aaron',0,'2009-05-14',1000000.00,'engineering')");
       st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
-          + "NULL,'Bob','2009-04-20',400.00,'sales')");
+          + "NULL,'Bob',100,'2009-04-20',400.00,'sales')");
       st.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
-          + "NULL,'Fred','2009-01-23',15.00,'marketing')");
+          + "NULL,'Fred',4000000000,'2009-01-23',15.00,'marketing')");
       connection.commit();
     } catch (SQLException sqlE) {
       LOG.error("Encountered SQL Exception: " + sqlE);
@@ -231,8 +232,8 @@ public class DirectMySQLTest extends Imp
   public void testDirectBulkImportWithDefaultDelims() throws IOException {
     // no quoting of strings allowed.
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
+      "2,Bob,100,2009-04-20,400,sales",
+      "3,Fred,4000000000,2009-01-23,15,marketing",
     };
 
     doImport(false, true, getTableName(), expectedResults, null);
@@ -242,8 +243,8 @@ public class DirectMySQLTest extends Imp
   public void testWithExtraParams() throws IOException {
     // no quoting of strings allowed.
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
+      "2,Bob,100,2009-04-20,400,sales",
+      "3,Fred,4000000000,2009-01-23,15,marketing",
     };
 
     String [] extraArgs = { "--", "--lock-tables" };
@@ -255,8 +256,8 @@ public class DirectMySQLTest extends Imp
   public void testMultiMappers() throws IOException {
     // no quoting of strings allowed.
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400,sales",
-      "3,Fred,2009-01-23,15,marketing",
+      "2,Bob,100,2009-04-20,400,sales",
+      "3,Fred,4000000000,2009-01-23,15,marketing",
     };
 
     String [] extraArgs = { "-m", "2" };
@@ -297,8 +298,8 @@ public class DirectMySQLTest extends Imp
   public void testDirectBulkImportWithMySQLQuotes() throws IOException {
     // mysql quotes all string-based output.
     String [] expectedResults = {
-      "2,'Bob','2009-04-20',400,'sales'",
-      "3,'Fred','2009-01-23',15,'marketing'",
+      "2,'Bob',100,'2009-04-20',400,'sales'",
+      "3,'Fred',4000000000,'2009-01-23',15,'marketing'",
     };
 
     doImport(true, true, getTableName(), expectedResults, null);
@@ -307,8 +308,8 @@ public class DirectMySQLTest extends Imp
   @Test
   public void testMySQLJdbcImport() throws IOException {
     String [] expectedResults = {
-      "2,Bob,2009-04-20,400.0,sales",
-      "3,Fred,2009-01-23,15.0,marketing",
+      "2,Bob,100,2009-04-20,400.0,sales",
+      "3,Fred,4000000000,2009-01-23,15.0,marketing",
     };
 
     doImport(false, false, getTableName(), expectedResults, null);



Mime
View raw message