sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kathl...@apache.org
Subject git commit: SQOOP-1071: Avro import is failing when columns are starting with underscore (Jarek Jarcec Cecho via Kate Ting)
Date Mon, 10 Jun 2013 22:53:47 GMT
Updated Branches:
  refs/heads/trunk 658c15848 -> 5d767e796


SQOOP-1071: Avro import is failing when columns are starting with underscore
(Jarek Jarcec Cecho via Kate Ting)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/5d767e79
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/5d767e79
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/5d767e79

Branch: refs/heads/trunk
Commit: 5d767e796d70830657496558514c349074fbb3eb
Parents: 658c158
Author: Kate Ting <kathleen@apache.org>
Authored: Mon Jun 10 17:17:28 2013 -0400
Committer: Kate Ting <kathleen@apache.org>
Committed: Mon Jun 10 17:17:28 2013 -0400

----------------------------------------------------------------------
 .../apache/sqoop/orm/AvroSchemaGenerator.java   |  2 +-
 src/test/com/cloudera/sqoop/TestAvroImport.java | 21 +++++++++++++
 .../sqoop/testutil/BaseSqoopTestCase.java       | 33 +++++++++++++++-----
 3 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
index 864fa0d..806bace 100644
--- a/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
+++ b/src/java/org/apache/sqoop/orm/AvroSchemaGenerator.java
@@ -60,7 +60,7 @@ public class AvroSchemaGenerator {
 
     List<Field> fields = new ArrayList<Field>();
     for (String columnName : columnNames) {
-      String cleanedCol = ClassWriter.toIdentifier(columnName);
+      String cleanedCol = ClassWriter.toJavaIdentifier(columnName);
       int sqlType = columnTypes.get(columnName);
       Schema avroSchema = toAvroSchema(sqlType, columnName);
       Field field = new Field(cleanedCol, avroSchema, null, null);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/test/com/cloudera/sqoop/TestAvroImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAvroImport.java b/src/test/com/cloudera/sqoop/TestAvroImport.java
index 32d1bbe..34a7d41 100644
--- a/src/test/com/cloudera/sqoop/TestAvroImport.java
+++ b/src/test/com/cloudera/sqoop/TestAvroImport.java
@@ -181,6 +181,27 @@ public class TestAvroImport extends ImportJobTestCase {
     assertEquals("DATA_COL0", new Utf8("10"), record1.get("DATA_COL0"));
   }
 
+  public void testFirstUnderscoreInColumnName() throws IOException {
+    String [] names = { "_NAME" };
+    String [] types = { "INT" };
+    String [] vals = { "1987" };
+    createTableWithColTypesAndNames(names, types, vals);
+
+    runImport(getOutputArgv(true, null));
+
+    Path outputFile = new Path(getTablePath(), "part-m-00000.avro");
+    DataFileReader<GenericRecord> reader = read(outputFile);
+    Schema schema = reader.getSchema();
+    assertEquals(Schema.Type.RECORD, schema.getType());
+    List<Field> fields = schema.getFields();
+    assertEquals(types.length, fields.size());
+
+    checkField(fields.get(0), "__NAME", Type.INT);
+
+    GenericRecord record1 = reader.next();
+    assertEquals("__NAME", 1987, record1.get("__NAME"));
+  }
+
   private void checkField(Field field, String name, Type type) {
     assertEquals(name, field.name());
     assertEquals(Schema.Type.UNION, field.schema().getType());

http://git-wip-us.apache.org/repos/asf/sqoop/blob/5d767e79/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
index d6afbc8..877d7f8 100644
--- a/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
+++ b/src/test/com/cloudera/sqoop/testutil/BaseSqoopTestCase.java
@@ -290,18 +290,24 @@ public abstract class BaseSqoopTestCase extends TestCase {
   }
 
   /**
-   * Create a table with a set of columns and add a row of values.
+   * Create a table with a set of columns with their names and add a row of values.
+   * @param colNames Column names
    * @param colTypes the types of the columns to make
    * @param vals the SQL text for each value to insert
    */
-  protected void createTableWithColTypes(String [] colTypes, String [] vals) {
+  protected void createTableWithColTypesAndNames(String[] colNames,
+                                                 String[] colTypes,
+                                                 String[] vals) {
+    assert colNames != null;
+    assert colTypes != null;
+    assert colNames.length == colTypes.length;
+
     Connection conn = null;
     PreparedStatement statement = null;
     String createTableStr = null;
     String columnDefStr = "";
     String columnListStr = "";
     String valueListStr = "";
-    String [] myColNames = new String[colTypes.length];
 
     try {
       try {
@@ -310,9 +316,7 @@ public abstract class BaseSqoopTestCase extends TestCase {
         conn = getManager().getConnection();
 
         for (int i = 0; i < colTypes.length; i++) {
-          String colName = BASE_COL_NAME + Integer.toString(i);
-          columnDefStr += colName + " " + colTypes[i];
-          myColNames[i] = colName;
+          columnDefStr += colNames[i] + " " + colTypes[i];
           if (i < colTypes.length - 1) {
             columnDefStr += ", ";
           }
@@ -342,7 +346,7 @@ public abstract class BaseSqoopTestCase extends TestCase {
 
       if (vals!=null) {
         for (int i = 0; i < colTypes.length; i++) {
-          columnListStr += myColNames[i];
+          columnListStr += colNames[i];
           valueListStr += vals[i];
           if (i < colTypes.length - 1) {
             columnListStr += ", ";
@@ -375,7 +379,7 @@ public abstract class BaseSqoopTestCase extends TestCase {
       }
 
       conn.commit();
-      this.colNames = myColNames;
+      this.colNames = colNames;
     } catch (SQLException se) {
       if (null != conn) {
         try {
@@ -389,6 +393,19 @@ public abstract class BaseSqoopTestCase extends TestCase {
   }
 
   /**
+   * Create a table with a set of columns and add a row of values.
+   * @param colTypes the types of the columns to make
+   * @param vals the SQL text for each value to insert
+   */
+  protected void createTableWithColTypes(String [] colTypes, String [] vals) {
+    String[] colNames = new String[colTypes.length];
+    for( int i = 0; i < colTypes.length; i++) {
+      colNames[i] = BASE_COL_NAME + Integer.toString(i);
+    }
+    createTableWithColTypesAndNames(colNames, colTypes, vals);
+  }
+
+  /**
    * Create a table with a single column and put a data element in it.
    * @param colType the type of the column to create
    * @param val the value to insert (reformatted as a string)


Mime
View raw message