sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject svn commit: r1309163 - in /sqoop/trunk/src: java/org/apache/sqoop/hive/ java/org/apache/sqoop/manager/ java/org/apache/sqoop/mapreduce/ java/org/apache/sqoop/orm/ test/com/cloudera/sqoop/ test/com/cloudera/sqoop/manager/
Date Tue, 03 Apr 2012 21:47:16 GMT
Author: blee
Date: Tue Apr  3 21:47:15 2012
New Revision: 1309163

URL: http://svn.apache.org/viewvc?rev=1309163&view=rev
Log:
SQOOP-473 Sqoop import-all-tables fails with MySQL connector

Added:
    sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java   (with props)
Modified:
    sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java
    sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java
    sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java
    sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java
    sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
    sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
    sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java
    sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java

Modified: sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/hive/TableDefWriter.java Tue Apr  3 21:47:15 2012
@@ -165,7 +165,7 @@ public class TableDefWriter {
       Integer colType = columnTypes.get(col);
       String hiveColType = userMapping.getProperty(col);
       if (hiveColType == null) {
-        hiveColType = connManager.toHiveType(col, colType);
+        hiveColType = connManager.toHiveType(inputTableName, col, colType);
       }
       if (null == hiveColType) {
         throw new IOException("Hive does not support the SQL type for column "

Modified: sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/manager/ConnManager.java Tue Apr  3 21:47:15 2012
@@ -203,34 +203,37 @@ public abstract class ConnManager {
 
   /**
    * Return java type for SQL type.
+   * @param tableName   table name
    * @param columnName  column name
    * @param sqlType     sql type
    * @return            java type
    */
-  public String toJavaType(String columnName, int sqlType) {
-    // ignore column name by default.
+  public String toJavaType(String tableName, String columnName, int sqlType) {
+    // ignore table name and column name by default.
     return toJavaType(sqlType);
   }
 
-    /**
-     * Return hive type for SQL type.
-     * @param columnName  column name
-     * @param sqlType   sql type
-     * @return          hive type
-     */
-  public String toHiveType(String columnName, int sqlType) {
-    // ignore column name by default.
+  /**
+   * Return hive type for SQL type.
+   * @param tableName   table name
+   * @param columnName  column name
+   * @param sqlType     sql type
+   * @return            hive type
+   */
+  public String toHiveType(String tableName, String columnName, int sqlType) {
+    // ignore table name and column name by default.
     return toHiveType(sqlType);
   }
 
   /**
    * Return avro type for SQL type.
+   * @param tableName   table name
    * @param columnName  column name
-   * @param sqlType   sql type
-   * @return          avro type
+   * @param sqlType     sql type
+   * @return            avro type
    */
-  public Type toAvroType(String columnName, int sqlType) {
-    // ignore column name by default.
+  public Type toAvroType(String tableName, String columnName, int sqlType) {
+    // ignore table name and column name by default.
     return toAvroType(sqlType);
   }
 

Modified: sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/manager/MySQLManager.java Tue Apr  3 21:47:15 2012
@@ -232,45 +232,42 @@ public class MySQLManager
   private static final int YEAR_TYPE_OVERWRITE = Types.SMALLINT;
 
   @Override
-  public String toJavaType(String columnName, int sqlType) {
+  public String toJavaType(String tableName, String columnName, int sqlType) {
     if (colTypeNames == null) {
-      colTypeNames = getColumnTypeNames(options.getTableName(),
-          options.getSqlQuery());
+      colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
     }
 
     if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
       sqlType = YEAR_TYPE_OVERWRITE;
     }
 
-    return super.toJavaType(columnName, sqlType);
+    return super.toJavaType(tableName, columnName, sqlType);
   }
 
   @Override
-  public String toHiveType(String columnName, int sqlType) {
+  public String toHiveType(String tableName, String columnName, int sqlType) {
     if (colTypeNames == null) {
-      colTypeNames = getColumnTypeNames(options.getTableName(),
-          options.getSqlQuery());
+      colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
     }
 
     if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
       sqlType = YEAR_TYPE_OVERWRITE;
     }
 
-    return super.toHiveType(columnName, sqlType);
+    return super.toHiveType(tableName, columnName, sqlType);
   }
 
   @Override
-  public Type toAvroType(String columnName, int sqlType) {
+  public Type toAvroType(String tableName, String columnName, int sqlType) {
     if (colTypeNames == null) {
-      colTypeNames = getColumnTypeNames(options.getTableName(),
-          options.getSqlQuery());
+      colTypeNames = getColumnTypeNames(tableName, options.getSqlQuery());
     }
 
     if ("YEAR".equalsIgnoreCase(colTypeNames.get(columnName))) {
       sqlType = YEAR_TYPE_OVERWRITE;
     }
 
-    return super.toAvroType(columnName, sqlType);
+    return super.toAvroType(tableName, columnName, sqlType);
   }
 }
 

Modified: sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/manager/OracleManager.java Tue Apr  3 21:47:15 2012
@@ -471,11 +471,14 @@ public class OracleManager
 
   /**
    * Resolve a database-specific type to the Java type that should contain it.
-   * @param sqlType
+   * @param tableName   table name
+   * @param columnName  column name
+   * @param sqlType     sql data type
    * @return the name of a Java type to hold the sql datatype, or null if none.
    */
-  public String toJavaType(String columnName, int sqlType) {
-    String defaultJavaType = super.toJavaType(columnName, sqlType);
+  @Override
+  public String toJavaType(String tableName, String columnName, int sqlType) {
+    String defaultJavaType = super.toJavaType(tableName, columnName, sqlType);
     return (defaultJavaType == null) ? dbToJavaType(sqlType) : defaultJavaType;
   }
 
@@ -508,11 +511,13 @@ public class OracleManager
 
   /**
    * Attempt to map sql type to hive type.
+   * @param tableName   table name
+   * @param columnName  column name
    * @param sqlType     sql data type
    * @return            hive data type
    */
-  public String toHiveType(String columnName, int sqlType) {
-    String defaultHiveType = super.toHiveType(columnName, sqlType);
+  public String toHiveType(String tableName, String columnName, int sqlType) {
+    String defaultHiveType = super.toHiveType(tableName, columnName, sqlType);
     return (defaultHiveType == null) ? dbToHiveType(sqlType) : defaultHiveType;
   }
 

Modified: sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java Tue Apr  3 21:47:15
2012
@@ -74,7 +74,7 @@ public class JdbcExportJob extends Expor
       for (Map.Entry<String, Integer> e : columnTypeInts.entrySet()) {
         Text columnName = new Text(e.getKey());
         Text columnText = new Text(
-            connManager.toJavaType(e.getKey(), e.getValue()));
+            connManager.toJavaType(tableName, e.getKey(), e.getValue()));
         columnTypes.put(columnName, columnText);
       }
       DefaultStringifier.store(job.getConfiguration(), columnTypes,

Modified: sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java Tue Apr  3 21:47:15
2012
@@ -121,6 +121,6 @@ public class AvroSchemaGenerator {
       throw new IllegalArgumentException("Cannot convert to AVRO type " + type);
     }
 
-    return connManager.toAvroType(columnName, sqlType);
+    return connManager.toAvroType(tableName, columnName, sqlType);
   }
 }

Modified: sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java (original)
+++ sqoop/trunk/src/java/org/apache/sqoop/orm/ClassWriter.java Tue Apr  3 21:47:15 2012
@@ -250,7 +250,7 @@ public class ClassWriter {
       return type;
     }
 
-    return connManager.toJavaType(columnName, sqlType);
+    return connManager.toJavaType(tableName, columnName, sqlType);
   }
 
   /**

Modified: sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java?rev=1309163&r1=1309162&r2=1309163&view=diff
==============================================================================
--- sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java (original)
+++ sqoop/trunk/src/test/com/cloudera/sqoop/TestAllTables.java Tue Apr  3 21:47:15 2012
@@ -30,7 +30,6 @@ import org.apache.hadoop.io.IOUtils;
 import org.junit.Before;
 
 import com.cloudera.sqoop.testutil.CommonArgs;
-import com.cloudera.sqoop.testutil.HsqldbTestServer;
 import com.cloudera.sqoop.testutil.ImportJobTestCase;
 import com.cloudera.sqoop.tool.ImportAllTablesTool;
 
@@ -53,7 +52,7 @@ public class TestAllTables extends Impor
     args.add("--warehouse-dir");
     args.add(getWarehouseDir());
     args.add("--connect");
-    args.add(HsqldbTestServer.getUrl());
+    args.add(getConnectString());
     args.add("--num-mappers");
     args.add("1");
     args.add("--escaped-by");
@@ -73,11 +72,13 @@ public class TestAllTables extends Impor
     // start the server
     super.setUp();
 
-    // throw away TWOINTTABLE and things we don't care about.
-    try {
-      this.getTestServer().dropExistingSchema();
-    } catch (SQLException sqlE) {
-      fail(sqlE.toString());
+    if (useHsqldbTestServer()) {
+      // throw away TWOINTTABLE and things we don't care about.
+      try {
+        this.getTestServer().dropExistingSchema();
+      } catch (SQLException sqlE) {
+        fail(sqlE.toString());
+      }
     }
 
     this.tableNames = new ArrayList<String>();

Added: sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
URL: http://svn.apache.org/viewvc/sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java?rev=1309163&view=auto
==============================================================================
--- sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java (added)
+++ sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java Tue Apr  3 21:47:15
2012
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.sqoop.manager;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.hadoop.conf.Configuration;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.TestAllTables;
+
+/**
+ * Test the --all-tables functionality with MySQL.
+ */
+public class MySQLAllTablesTest extends TestAllTables {
+
+  @Override
+  protected boolean useHsqldbTestServer() {
+    return false;
+  }
+
+  @Override
+  protected String getConnectString() {
+    return MySQLTestUtils.CONNECT_STRING;
+  }
+
+  @Override
+  protected SqoopOptions getSqoopOptions(Configuration conf) {
+    SqoopOptions opts = new SqoopOptions(conf);
+    opts.setUsername(MySQLTestUtils.getCurrentUser());
+    return opts;
+  }
+
+  @Override
+  protected void dropTableIfExists(String table) throws SQLException {
+    Connection conn = getManager().getConnection();
+    PreparedStatement statement = conn.prepareStatement(
+        "DROP TABLE IF EXISTS " + table,
+        ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+    try {
+      statement.executeUpdate();
+      conn.commit();
+    } finally {
+      statement.close();
+    }
+  }
+}

Propchange: sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Propchange: sqoop/trunk/src/test/com/cloudera/sqoop/manager/MySQLAllTablesTest.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain



Mime
View raw message