sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject git commit: SQOOP-937: Dont generate ORM files for Direct mode connectors
Date Wed, 20 Mar 2013 21:31:04 GMT
Updated Branches:
  refs/heads/trunk 86812b853 -> f2ba7eaae


SQOOP-937: Dont generate ORM files for Direct mode connectors

(Venkat Ranganathan via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f2ba7eaa
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f2ba7eaa
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f2ba7eaa

Branch: refs/heads/trunk
Commit: f2ba7eaae25ac23c88651e1d14b9624aa5ded6ca
Parents: 86812b8
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Wed Mar 20 14:30:04 2013 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Wed Mar 20 14:30:04 2013 -0700

----------------------------------------------------------------------
 src/java/org/apache/sqoop/manager/ConnManager.java |   15 +++++
 .../apache/sqoop/manager/DirectNetezzaManager.java |    5 ++
 .../org/apache/sqoop/manager/ExportJobContext.java |    7 ++
 .../org/apache/sqoop/manager/ImportJobContext.java |    7 ++
 .../org/apache/sqoop/mapreduce/ExportJobBase.java  |   10 +++-
 .../org/apache/sqoop/mapreduce/ImportJobBase.java  |    9 ++-
 src/java/org/apache/sqoop/mapreduce/JobBase.java   |    1 +
 src/java/org/apache/sqoop/orm/ClassWriter.java     |   15 ++++-
 src/java/org/apache/sqoop/tool/CodeGenTool.java    |   16 +++++-
 .../com/cloudera/sqoop/orm/TestClassWriter.java    |   48 +++++++++++++++
 10 files changed, 125 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ConnManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/ConnManager.java b/src/java/org/apache/sqoop/manager/ConnManager.java
index 1b32dc9..a1ac38e 100644
--- a/src/java/org/apache/sqoop/manager/ConnManager.java
+++ b/src/java/org/apache/sqoop/manager/ConnManager.java
@@ -639,5 +639,20 @@ public abstract class ConnManager {
   public String getInputBoundsQuery(String splitByCol, String sanitizedQuery) {
       return null;
   }
+
+  /**
+   * This method allows the ConnManager to override the generation of ORM
+   * classes if the SQOOP generated classes are not used by it.
+   * A return value of false from this method means that the SQOOP ORM
+   * classes are needed to use with the connector.
+   * A return value of true indicates that the connection manager does not
+   * use the SQOOP ORM classes.   For example, in the Direct mode of some of
+   * the connectors, the text files are directly processed by DB specific
+   * facilities without even being passed through the SQOOP process and
+   * in those circumstances, it makes sense to disable the ORM generation.
+   */
+  public boolean isORMFacilitySelfManaged() {
+    return false;
+  }
 }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
index 0a1e605..ef98936 100644
--- a/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
+++ b/src/java/org/apache/sqoop/manager/DirectNetezzaManager.java
@@ -246,4 +246,9 @@ public class DirectNetezzaManager extends NetezzaManager {
   public boolean supportsStagingForExport() {
     return false;
   }
+
+  @Override
+  public boolean isORMFacilitySelfManaged() {
+    return true;
+  }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ExportJobContext.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/ExportJobContext.java b/src/java/org/apache/sqoop/manager/ExportJobContext.java
index 5699e2f..2a6f2b5 100644
--- a/src/java/org/apache/sqoop/manager/ExportJobContext.java
+++ b/src/java/org/apache/sqoop/manager/ExportJobContext.java
@@ -18,6 +18,9 @@
 
 package org.apache.sqoop.manager;
 
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.util.Jars;
+
 import com.cloudera.sqoop.SqoopOptions;
 
 /**
@@ -35,6 +38,10 @@ public class ExportJobContext {
       final SqoopOptions opts) {
     this.tableName = table;
     this.jarFile = jar;
+    if (this.jarFile == null) {
+      // Set the jarFile to the hadoop core jar file.
+      this.jarFile = Jars.getJarPathForClass(Configuration.class);
+    }
     this.options = opts;
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/manager/ImportJobContext.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/manager/ImportJobContext.java b/src/java/org/apache/sqoop/manager/ImportJobContext.java
index 09a7abe..354cd15 100644
--- a/src/java/org/apache/sqoop/manager/ImportJobContext.java
+++ b/src/java/org/apache/sqoop/manager/ImportJobContext.java
@@ -21,7 +21,10 @@ package org.apache.sqoop.manager;
 import org.apache.hadoop.mapreduce.InputFormat;
 import com.cloudera.sqoop.mapreduce.db.DataDrivenDBInputFormat;
 import com.cloudera.sqoop.SqoopOptions;
+
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.util.Jars;
 
 /**
  * A set of parameters describing an import operation; this is passed to
@@ -40,6 +43,10 @@ public class ImportJobContext {
       final SqoopOptions opts, final Path destination) {
     this.tableName = table;
     this.jarFile = jar;
+    if (this.jarFile == null) {
+      // Set the jarFile to the hadoop core jar file.
+      this.jarFile = Jars.getJarPathForClass(Configuration.class);
+    }
     this.options = opts;
     this.inputFormatClass = DataDrivenDBInputFormat.class;
     this.destination = destination;

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
index ff84974..1065d0b 100644
--- a/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ExportJobBase.java
@@ -321,8 +321,14 @@ public class ExportJobBase extends JobBase {
       }
     }
 
-    String tableClassName =
-        new TableClassName(options).getClassForTable(outputTableName);
+
+    String tableClassName = null;
+    if (!cmgr.isORMFacilitySelfManaged()) {
+        tableClassName =
+            new TableClassName(options).getClassForTable(outputTableName);
+    }
+    // For ORM self managed, we leave the tableClassName to null so that
+    // we don't check for non-existing classes.
     String ormJarFile = context.getJarFile();
 
     LOG.info("Beginning export of " + outputTableName);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
index f766532..2465f3f 100644
--- a/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/ImportJobBase.java
@@ -196,9 +196,14 @@ public class ImportJobBase extends JobBase {
     } else {
       LOG.info("Beginning query import.");
     }
+    String tableClassName = null;
+    if (!getContext().getConnManager().isORMFacilitySelfManaged()) {
+        tableClassName =
+            new TableClassName(options).getClassForTable(tableName);
+    }
+    // For ORM self managed, we leave the tableClassName to null so that
+    // we don't check for non-existing classes.
 
-    String tableClassName =
-        new TableClassName(options).getClassForTable(tableName);
     loadJars(conf, ormJarFile, tableClassName);
 
     Job job = new Job(conf);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/mapreduce/JobBase.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/JobBase.java b/src/java/org/apache/sqoop/mapreduce/JobBase.java
index 4e7723f..0df1156 100644
--- a/src/java/org/apache/sqoop/mapreduce/JobBase.java
+++ b/src/java/org/apache/sqoop/mapreduce/JobBase.java
@@ -220,6 +220,7 @@ public class JobBase {
    */
   protected void loadJars(Configuration conf, String ormJarFile,
       String tableClassName) throws IOException {
+ 
     boolean isLocal = "local".equals(conf.get("mapreduce.jobtracker.address"))
         || "local".equals(conf.get("mapred.job.tracker"));
     if (isLocal) {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/orm/ClassWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java
index 982e444..0202f7f 100644
--- a/src/java/org/apache/sqoop/orm/ClassWriter.java
+++ b/src/java/org/apache/sqoop/orm/ClassWriter.java
@@ -1064,13 +1064,24 @@ public class ClassWriter {
     return cleanedColNames;
   }
 
+  /**
+   * Made this a separate method to overcome the 150 line limit of checkstyle.
+   */
+  private void logORMSelfGenerationMessage() {
+    LOG.info("The connection manager declares that it self manages mapping"
+        + " between records & fields and rows & columns.  No class will"
+        + " will be generated.");
+  }
 
   /**
    * Generate the ORM code for the class.
    */
   public void generate() throws IOException {
     Map<String, Integer> columnTypes = getColumnTypes();
-
+    if (connManager.isORMFacilitySelfManaged()) {
+      logORMSelfGenerationMessage();
+      return;
+    }
     if (columnTypes == null) {
       throw new IOException("No columns to generate for ClassWriter");
     }
@@ -1110,7 +1121,6 @@ public class ClassWriter {
       }
       columnTypes.put(identifier, type);
     }
-
     // Check that all explicitly mapped columns are present in result set
     Properties mapping = options.getMapColumnJava();
     if (mapping != null && !mapping.isEmpty()) {
@@ -1207,7 +1217,6 @@ public class ClassWriter {
           // ignored because we're closing.
         }
       }
-
       if (null != ostream) {
         try {
           ostream.close();

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/java/org/apache/sqoop/tool/CodeGenTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/CodeGenTool.java b/src/java/org/apache/sqoop/tool/CodeGenTool.java
index 8a4aa42..dd34a97 100644
--- a/src/java/org/apache/sqoop/tool/CodeGenTool.java
+++ b/src/java/org/apache/sqoop/tool/CodeGenTool.java
@@ -71,10 +71,24 @@ public class CodeGenTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
       // This code generator is being invoked as part of an import or export
       // process, and the user has pre-specified a jar and class to use.
       // Don't generate.
+      if (manager.isORMFacilitySelfManaged()) {
+        // No need to generated any ORM.  Ignore any jar file given on
+        // command line also.
+        LOG.info("The connection manager declares that it self manages mapping"
+            + " between records & fields and rows & columns.  The jar file "
+            + " provided will have no effect");
+      }
       LOG.info("Using existing jar: " + existingJar);
       return existingJar;
     }
-
+    if (manager.isORMFacilitySelfManaged()) {
+      // No need to generated any ORM.  Ignore any jar file given on
+      // command line also.
+      LOG.info("The connection manager declares that it self manages mapping"
+          + " between records & fields and rows & columns.  No class will"
+          + " will be generated.");
+      return null;
+    }
     LOG.info("Beginning code generation");
     CompilationManager compileMgr = new CompilationManager(options);
     ClassWriter classWriter = new ClassWriter(options, manager, tableName,

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f2ba7eaa/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
index 3b77571..bfb25b0 100644
--- a/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
+++ b/src/test/com/cloudera/sqoop/orm/TestClassWriter.java
@@ -38,6 +38,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.TestConnFactory.DummyManager;
 import com.cloudera.sqoop.manager.ConnManager;
 import com.cloudera.sqoop.testutil.DirUtil;
 import com.cloudera.sqoop.testutil.HsqldbTestServer;
@@ -485,4 +486,51 @@ public class TestClassWriter extends TestCase {
     }
     fail("we shouldn't successfully generate code");
   }
+
+  private void runFailedGenerationTest(String [] argv,
+      String classNameToCheck) {
+    File codeGenDirFile = new File(CODE_GEN_DIR);
+    File classGenDirFile = new File(JAR_GEN_DIR);
+
+    try {
+      options = new ImportTool().parseArguments(argv,
+          null, options, true);
+    } catch (Exception e) {
+      LOG.error("Could not parse options: " + e.toString());
+    }
+
+    CompilationManager compileMgr = new CompilationManager(options);
+    ClassWriter writer = new ClassWriter(options, manager,
+        HsqldbTestServer.getTableName(), compileMgr);
+
+    try {
+      writer.generate();
+      compileMgr.compile();
+      fail("ORM class file generation succeeded when it was expected to fail");
+    } catch (IOException ioe) {
+      LOG.error("Got IOException from ORM generation as expected : "
+        + ioe.toString());
+    }
+  }
+  /**
+   * A dummy manager that declares that it ORM is self managed.
+   */
+  public static class DummyDirectManager extends DummyManager {
+    @Override
+    public boolean isORMFacilitySelfManaged() {
+      return true;
+    }
+  }
+
+  @Test
+  public void testNoClassGeneration() throws Exception {
+    manager = new DummyDirectManager();
+    String [] argv = {
+      "--bindir",
+      JAR_GEN_DIR,
+      "--outdir",
+      CODE_GEN_DIR,
+    };
+    runFailedGenerationTest(argv, HsqldbTestServer.getTableName());
+  }
 }


Mime
View raw message