sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From annaszo...@apache.org
Subject sqoop git commit: SQOOP-3159: Sqoop (export + --table) with Oracle table_name having '$' fails with error (ORA-00942 or java.lang.NoClassDefFoundError)
Date Mon, 27 Mar 2017 14:19:43 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 690541533 -> afd6a8610


SQOOP-3159: Sqoop (export + --table) with Oracle table_name having '$' fails with error (ORA-00942
or java.lang.NoClassDefFoundError)

(Szabolcs Vasas via Anna Szonyi)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/afd6a861
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/afd6a861
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/afd6a861

Branch: refs/heads/trunk
Commit: afd6a8610e8ac4291e52c2c167fe7b8d6f706be8
Parents: 6905415
Author: Anna Szonyi <annaszonyi@apache.org>
Authored: Mon Mar 27 16:15:02 2017 +0200
Committer: Anna Szonyi <annaszonyi@apache.org>
Committed: Mon Mar 27 16:15:02 2017 +0200

----------------------------------------------------------------------
 .../apache/sqoop/orm/CompilationManager.java    |  54 +++++--
 src/test/com/cloudera/sqoop/TestExport.java     |  10 ++
 .../sqoop/manager/OracleExportTest.java         |  16 +++
 .../OracleSpecialCharacterTableImportTest.java  | 130 +++++++++++++++++
 .../sqoop/orm/TestCompilationManager.java       | 139 +++++++++++++++++++
 5 files changed, 334 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/java/org/apache/sqoop/orm/CompilationManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java
index c1a656b..3322c8b 100644
--- a/src/java/org/apache/sqoop/orm/CompilationManager.java
+++ b/src/java/org/apache/sqoop/orm/CompilationManager.java
@@ -45,6 +45,8 @@ import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.util.FileListing;
 import com.cloudera.sqoop.util.Jars;
 
+import static org.apache.commons.lang3.StringUtils.substringBeforeLast;
+
 /**
  * Manages the compilation of a bunch of .java files into .class files
  * and eventually a jar.
@@ -60,6 +62,9 @@ public class CompilationManager {
 
   public static final Log LOG = LogFactory.getLog(
       CompilationManager.class.getName());
+  private static final String INNER_CLASS_SEPARATOR = "$";
+  private static final String CLASS_EXTENSION = ".class";
+  private static final String JAVA_EXTENSION = ".java";
 
   private SqoopOptions options;
   private List<String> sources;
@@ -299,26 +304,17 @@ public class CompilationManager {
         // with the base directory where class files were put;
         // we only record the subdir parts in the zip entry.
         String fullPath = entry.getAbsolutePath();
-        String chompedPath = fullPath.substring(baseDirName.length());
-        int indexOfDollarSign = chompedPath.indexOf("$");
-        String innerTypesChompedPath = chompedPath
-            .substring(0, indexOfDollarSign == -1 ? chompedPath.length() : indexOfDollarSign);
-
-        boolean include = chompedPath.endsWith(".class")
-            && (sources.contains(
-            chompedPath.substring(0, chompedPath.length() - ".class".length())
-                    + ".java")
-                || sources.contains(innerTypesChompedPath + ".java"));
-
-        if (include) {
+        String classFileName = fullPath.substring(baseDirName.length());
+
+        if (includeFileInJar(classFileName)) {
           // include this file.
           if (Shell.WINDOWS) {
             // In Windows OS, elements in jar files still need to have a path
             // separator of '/' rather than the default File.separator which is '\'
-            chompedPath = chompedPath.replace(File.separator, "/");
+            classFileName = classFileName.replace(File.separator, "/");
           }
-          LOG.debug("Got classfile: " + entry.getPath() + " -> " + chompedPath);
-          ZipEntry ze = new ZipEntry(chompedPath);
+          LOG.debug("Got classfile: " + entry.getPath() + " -> " + classFileName);
+          ZipEntry ze = new ZipEntry(classFileName);
           jstream.putNextEntry(ze);
           copyFileToStream(entry, jstream);
           jstream.closeEntry();
@@ -327,6 +323,34 @@ public class CompilationManager {
     }
   }
 
+  boolean includeFileInJar(String classFileName) {
+    if (!classFileName.endsWith(CLASS_EXTENSION)) {
+      return false;
+    }
+
+    String className = substringBeforeLast(classFileName, CLASS_EXTENSION);
+    String sourceFileOfClass = className + JAVA_EXTENSION;
+
+    return isInnerClass(sourceFileOfClass) || isOuterClass(sourceFileOfClass);
+  }
+
+  boolean isInnerClass(String sourceFileName) {
+    String potentialOuterClassName = potentialOuterClassNameOf(sourceFileName);
+
+    String sourceFileOfPotentialOuterClass = potentialOuterClassName + JAVA_EXTENSION;
+
+    return isOuterClass(sourceFileOfPotentialOuterClass);
+  }
+
+  boolean isOuterClass(String sourceFileOfClass) {
+    return sources.contains(sourceFileOfClass);
+  }
+
+
+  String potentialOuterClassNameOf(String className) {
+    return substringBeforeLast(className, INNER_CLASS_SEPARATOR);
+  }
+
   /**
    * Create an output jar file to use when executing MapReduce jobs.
    */

http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/test/com/cloudera/sqoop/TestExport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestExport.java b/src/test/com/cloudera/sqoop/TestExport.java
index df5a663..b2edc53 100644
--- a/src/test/com/cloudera/sqoop/TestExport.java
+++ b/src/test/com/cloudera/sqoop/TestExport.java
@@ -885,4 +885,14 @@ public class TestExport extends ExportJobTestCase {
     assertColMinAndMax(forIdx(2), gen2);
     assertColMinAndMax(forIdx(1), genNull);
   }
+
+  protected void testExportToTableWithName(String tableName) throws IOException, SQLException
{
+    final int TOTAL_RECORDS = 10;
+    setCurTableName(tableName);
+    createTextFile(0, TOTAL_RECORDS, false);
+    createTable();
+    runExport(getArgv(true, 10, 10));
+    verifyExport(TOTAL_RECORDS);
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/test/com/cloudera/sqoop/manager/OracleExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/OracleExportTest.java b/src/test/com/cloudera/sqoop/manager/OracleExportTest.java
index ec56cbe..fe2e265 100644
--- a/src/test/com/cloudera/sqoop/manager/OracleExportTest.java
+++ b/src/test/com/cloudera/sqoop/manager/OracleExportTest.java
@@ -294,4 +294,20 @@ public class OracleExportTest extends TestExport {
         "--update-key", "ID", "--update-mode", "allowinsert", "--oracle-escaping-disabled",
"true")));
     verifyExport(TOTAL_RECORDS);
   }
+
+  @Test
+  public void testExportToTableWithNameEndingWithDollarSign() throws IOException, SQLException
{
+    testExportToTableWithName("DOLLAR$");
+  }
+
+  @Test
+  public void testExportToTableWithNameContainingDollarSign() throws IOException, SQLException
{
+    testExportToTableWithName("FOO$BAR");
+  }
+
+  @Test
+  public void testExportToTableWithNameContainingHashtag() throws IOException, SQLException
{
+    testExportToTableWithName("FOO#BAR");
+  }
+
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java
b/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java
new file mode 100644
index 0000000..907be49
--- /dev/null
+++ b/src/test/org/apache/sqoop/manager/oracle/OracleSpecialCharacterTableImportTest.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.manager.oracle;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.manager.OracleUtils;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.google.common.base.Charsets;
+import com.google.common.io.Files;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import static org.junit.Assert.assertEquals;
+
+@RunWith(Parameterized.class)
+public class OracleSpecialCharacterTableImportTest extends ImportJobTestCase {
+
+  @Parameterized.Parameters(name = "tableName = {0}")
+  public static Iterable<? extends Object> tableNameParameters() {
+    return Arrays.asList("DOLLAR$","FOO$BAR", "T#1");
+  }
+
+  public static final Log LOG = LogFactory.getLog(
+      OracleSpecialCharacterTableImportTest.class.getName());
+
+  private final String tableName;
+
+  @Override
+  protected boolean useHsqldbTestServer() {
+    return false;
+  }
+
+  @Override
+  protected String getConnectString() {
+    return OracleUtils.CONNECT_STRING;
+  }
+
+  @Override
+  protected SqoopOptions getSqoopOptions(Configuration conf) {
+    SqoopOptions opts = new SqoopOptions(conf);
+    OracleUtils.setOracleAuth(opts);
+    return opts;
+  }
+
+  @Override
+  protected void dropTableIfExists(String table) throws SQLException {
+    OracleUtils.dropTable(table, getManager());
+  }
+
+  @After
+  public void tearDown() {
+    try {
+      OracleUtils.dropTable(getTableName(), getManager());
+    } catch (SQLException e) {
+      LOG.error("Test table could not be dropped", e);
+    }
+    super.tearDown();
+  }
+
+  protected String [] getArgv() {
+    ArrayList<String> args = new ArrayList<String>();
+
+    CommonArgs.addHadoopFlags(args);
+
+    args.add("--connect");
+    args.add(getConnectString());
+    args.add("--username");
+    args.add(OracleUtils.ORACLE_USER_NAME);
+    args.add("--password");
+    args.add(OracleUtils.ORACLE_USER_PASS);
+    args.add("--target-dir");
+    args.add(getWarehouseDir());
+    args.add("--num-mappers");
+    args.add("1");
+    args.add("--table");
+    args.add(getTableName());
+
+    return args.toArray(new String[0]);
+  }
+
+  public OracleSpecialCharacterTableImportTest(String tableName) {
+    this.tableName = tableName;
+  }
+
+  @Test
+  public void testImportWithTableNameContainingSpecialCharacters() throws IOException {
+    String [] types = { "VARCHAR(50)"};
+    String [] vals = { "'hello, world!'"};
+    setCurTableName(tableName);
+    createTableWithColTypes(types, vals);
+    String[] args = getArgv();
+    runImport(args);
+
+    Path warehousePath = new Path(this.getWarehouseDir());
+    Path filePath = new Path(warehousePath, "part-m-00000");
+    String output = Files.toString(new File(filePath.toString()), Charsets.UTF_8);
+
+    assertEquals("hello, world!", output.trim());
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/afd6a861/src/test/org/apache/sqoop/orm/TestCompilationManager.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/orm/TestCompilationManager.java b/src/test/org/apache/sqoop/orm/TestCompilationManager.java
new file mode 100644
index 0000000..6e91d31
--- /dev/null
+++ b/src/test/org/apache/sqoop/orm/TestCompilationManager.java
@@ -0,0 +1,139 @@
+package org.apache.sqoop.orm;
+
+import com.cloudera.sqoop.SqoopOptions;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+
+public class TestCompilationManager {
+
+  private CompilationManager compilationManager;
+
+  private SqoopOptions sqoopOptions;
+
+  @Before
+  public void before() {
+    sqoopOptions = mock(SqoopOptions.class);
+    compilationManager = new CompilationManager(sqoopOptions);
+  }
+
+  @Test
+  public void testPotentialOuterClassNameOfWithPlainClassName() {
+    String inputClassName = "FooBar";
+    String expectedOutput = inputClassName;
+
+    assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName));
+  }
+
+  @Test
+  public void testPotentialOuterClassNameOfWithClassNameEndingWithDollar() {
+    String inputClassName = "FooBar$";
+    String expectedOutput = "FooBar";
+
+    assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName));
+  }
+
+  @Test
+  public void testPotentialOuterClassNameOfWithInnerClassName() {
+    String inputClassName = "Foo$Bar";
+    String expectedOutput = "Foo";
+
+    assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName));
+  }
+
+  @Test
+  public void testPotentialOuterClassNameOfWithOuterClassNameContainingDollarSign() {
+    String inputClassName = "Foo$Bar$1";
+    String expectedOutput = "Foo$Bar";
+
+    assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName));
+  }
+
+  @Test
+  public void testPotentialOuterClassNameOfWithOuterClassNameEndingWithDollarSign() {
+    String inputClassName = "FooBar$$1";
+    String expectedOutput = "FooBar$";
+
+    assertEquals(expectedOutput, compilationManager.potentialOuterClassNameOf(inputClassName));
+  }
+
+  @Test
+  public void testIsInnerClassWithOuterClass() {
+    String inputSourceFileName = "Foo$Bar.java";
+    compilationManager.addSourceFile(inputSourceFileName);
+    boolean expectedOutput = false;
+
+    assertEquals(expectedOutput, compilationManager.isInnerClass(inputSourceFileName));
+  }
+
+  @Test
+  public void testIsInnerClassWithNonExistingClass() {
+    String inputSourceFileName = "ThisDoesNotExist.java";
+    boolean expectedOutput = false;
+
+    assertEquals(expectedOutput, compilationManager.isInnerClass(inputSourceFileName));
+  }
+
+  @Test
+  public void testIsInnerClassWithInnerClass() {
+    String inputSourceFileName = "Foo$Bar$1.java";
+    compilationManager.addSourceFile("Foo$Bar.java");
+    boolean expectedOutput = true;
+
+    assertEquals(expectedOutput, compilationManager.isInnerClass(inputSourceFileName));
+  }
+
+  @Test
+  public void testIsOuterClassWithExistingSourceFile() {
+    String inputSourceFileName = "OuterClass.java";
+    compilationManager.addSourceFile(inputSourceFileName);
+    boolean expectedOutput = true;
+
+    assertEquals(expectedOutput, compilationManager.isOuterClass(inputSourceFileName));
+  }
+
+  @Test
+  public void testIsOuterClassWithNonExistingSourceFile() {
+    String inputSourceFileName = "OuterClass.java";
+    boolean expectedOutput = false;
+
+    assertEquals(expectedOutput, compilationManager.isOuterClass(inputSourceFileName));
+  }
+
+  @Test
+  public void testIncludeFileInJarWithNonClassFile() {
+    String inputClassFileName = "FooBar.txt";
+    boolean expectedOutput = false;
+
+    assertEquals(expectedOutput, compilationManager.includeFileInJar(inputClassFileName));
+  }
+
+  @Test
+  public void testIncludeFileInJarWithSourceFilePresent() {
+    String inputClassFileName = "FooBar.class";
+    boolean expectedOutput = true;
+    compilationManager.addSourceFile("FooBar.java");
+
+    assertEquals(expectedOutput, compilationManager.includeFileInJar(inputClassFileName));
+  }
+
+  @Test
+  public void testIncludeFileInJarWithSourceFileNotPresent() {
+    String inputClassFileName = "FooBar.class";
+    boolean expectedOutput = false;
+
+    assertEquals(expectedOutput, compilationManager.includeFileInJar(inputClassFileName));
+  }
+
+  @Test
+  public void testIncludeFileInJarWithOuterClassSourceFilePresent() {
+    String inputClassFileName = "FooBar$1.class";
+    boolean expectedOutput = true;
+    compilationManager.addSourceFile("FooBar.java");
+
+    assertEquals(expectedOutput, compilationManager.includeFileInJar(inputClassFileName));
+  }
+
+}
\ No newline at end of file


Mime
View raw message