sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2647: Add option for drop-if-exists when using sqoop hcat import.
Date Fri, 04 Mar 2016 16:51:04 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk c2351511e -> 2eecf6801


SQOOP-2647: Add option for drop-if-exists when using sqoop hcat import.

(Muhammad Ehsan ul Haque via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/2eecf680
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/2eecf680
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/2eecf680

Branch: refs/heads/trunk
Commit: 2eecf68010df83b49eb3965a9a99de0cfbfb5828
Parents: c235151
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Fri Mar 4 08:50:15 2016 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Fri Mar 4 08:50:15 2016 -0800

----------------------------------------------------------------------
 src/docs/user/hcatalog.txt                      |  5 +++
 src/java/org/apache/sqoop/SqoopOptions.java     | 10 ++++++
 .../mapreduce/hcat/SqoopHCatUtilities.java      | 14 ++++++--
 .../org/apache/sqoop/tool/BaseSqoopTool.java    | 16 +++++++++
 .../apache/sqoop/hcat/HCatalogImportTest.java   | 38 ++++++++++++++++++++
 .../apache/sqoop/hcat/TestHCatalogBasic.java    | 35 ++++++++++++++++++
 6 files changed, 116 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/docs/user/hcatalog.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/hcatalog.txt b/src/docs/user/hcatalog.txt
index 99ae4f5..2ae1d54 100644
--- a/src/docs/user/hcatalog.txt
+++ b/src/docs/user/hcatalog.txt
@@ -79,6 +79,11 @@ to exist. The table name will be the same as the database table name
 translated to lower case. Further described in +Automatic Table Creation+
 below.
 
++--drop-and-create-hcatalog-table+::
+
+Same as +--create-hcatalog-table+, but does a +drop if exists+ before creating
+the table.
+
 +--hcatalog-storage-stanza+::
 
 This option specifies the storage stanza to be appended to the table.

http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 4d30007..17751c7 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -173,6 +173,8 @@ public class SqoopOptions implements Cloneable {
   private String hCatDatabaseName;
   @StoredAsProperty("hcatalog.create.table")
   private boolean hCatCreateTable;
+  @StoredAsProperty("hcatalog.drop.and.create.table")
+  private boolean hCatDropAndCreateTable;
   @StoredAsProperty("hcatalog.storage.stanza")
   private String hCatStorageStanza;
   private String hCatHome; // not serialized to metastore.
@@ -1477,6 +1479,14 @@ public class SqoopOptions implements Cloneable {
     this.hCatCreateTable = create;
   }
 
+  public boolean doDropAndCreateHCatalogTable() {
+    return hCatDropAndCreateTable;
+  }
+
+  public void setDropAndCreateHCatalogTable(boolean dropAndCreate) {
+    this.hCatDropAndCreateTable = dropAndCreate;
+  }
+
   public void setHCatStorageStanza(String stanza) {
     this.hCatStorageStanza = stanza;
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
index 9d5a9ee..ebe7209 100644
--- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
+++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
@@ -66,6 +66,7 @@ import org.apache.sqoop.config.ConfigurationConstants;
 import org.apache.sqoop.config.ConfigurationHelper;
 import org.apache.sqoop.hive.HiveTypes;
 import org.apache.sqoop.manager.ConnManager;
+import org.apache.sqoop.tool.BaseSqoopTool;
 import org.apache.sqoop.util.Executor;
 import org.apache.sqoop.util.LoggingAsyncSink;
 import org.apache.sqoop.util.SubprocessSecurityManager;
@@ -337,7 +338,11 @@ public final class SqoopHCatUtilities {
     if (options.doCreateHCatalogTable()) {
       LOG.info("Creating HCatalog table " + hCatQualifiedTableName
         + " for import");
-      createHCatTable();
+      createHCatTable(false);
+    } else if (options.doDropAndCreateHCatalogTable()) {
+      LOG.info("Dropping and Creating HCatalog table "
+        + hCatQualifiedTableName + " for import");
+      createHCatTable(true);
     }
     // For serializing the schema to conf
     HCatInputFormat hif = HCatInputFormat.setInput(hCatJob, hCatDatabaseName,
@@ -555,8 +560,13 @@ public final class SqoopHCatUtilities {
     return sb;
   }
 
-  private void createHCatTable() throws IOException {
+  private void createHCatTable(boolean dropIfExists) throws IOException {
     StringBuilder sb = new StringBuilder();
+    if (dropIfExists) {
+      sb.append("drop table ").
+              append(escHCatObj(hCatDatabaseName)).append('.').
+              append(escHCatObj(hCatTableName)).append(";\n");
+    }
     sb.append("create table ").
       append(escHCatObj(hCatDatabaseName)).append('.');
     sb.append(escHCatObj(hCatTableName)).append(" (\n\t");

http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 9c810a9..50dd67d 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -122,6 +122,8 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
   public static final String HCATALOG_DATABASE_ARG = "hcatalog-database";
   public static final String CREATE_HCATALOG_TABLE_ARG =
     "create-hcatalog-table";
+  public static final String DROP_AND_CREATE_HCATALOG_TABLE =
+    "drop-and-create-hcatalog-table";
   public static final String HCATALOG_STORAGE_STANZA_ARG =
     "hcatalog-storage-stanza";
   public static final String HCATALOG_HOME_ARG = "hcatalog-home";
@@ -606,6 +608,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
       .withLongOpt(CREATE_HCATALOG_TABLE_ARG)
       .create());
     hCatOptions.addOption(OptionBuilder
+      .withDescription("Drop and Create HCatalog before import")
+      .withLongOpt(DROP_AND_CREATE_HCATALOG_TABLE)
+      .create());
+    hCatOptions.addOption(OptionBuilder
       .hasArg()
       .withDescription("HCatalog storage stanza for table creation")
       .withLongOpt(HCATALOG_STORAGE_STANZA_ARG)
@@ -1140,6 +1146,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
       out.setCreateHCatalogTable(true);
     }
 
+    if (in.hasOption(DROP_AND_CREATE_HCATALOG_TABLE)) {
+      out.setDropAndCreateHCatalogTable(true);
+    }
+
     if (in.hasOption(HCATALOG_HOME_ARG)) {
       out.setHCatHome(in.getOptionValue(HCATALOG_HOME_ARG));
     }
@@ -1621,6 +1631,12 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
             + "these options should be omitted");
       }
     }
+    if (options.doCreateHCatalogTable() &&
+            options.doDropAndCreateHCatalogTable()) {
+      throw new InvalidOptionsException("Options --create-hcatalog-table" +
+              " and --drop-and-create-hcatalog-table are mutually exclusive." +
+              " Use any one of them");
+    }
   }
 
   protected void validateHBaseOptions(SqoopOptions options)

http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
index 8db06d1..67b7a78 100644
--- a/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
+++ b/src/test/org/apache/sqoop/hcat/HCatalogImportTest.java
@@ -854,6 +854,44 @@ public class HCatalogImportTest extends ImportJobTestCase {
         + " create-hcatalog-table with pre-existing table test", e);
     }
   }
+
+  public void testDropAndCreateWithPreExistingTable() throws Exception {
+    final int TOTAL_RECORDS = 1 * 10;
+    String table = getTableName().toUpperCase();
+    ColumnGenerator[] cols = new ColumnGenerator[] {
+      HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+        "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+        new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
+      HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+        "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+        new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY), };
+    List<String> addlArgsArray = new ArrayList<String>();
+    addlArgsArray.add("--drop-and-create-hcatalog-table");
+    setExtraArgs(addlArgsArray);
+    // Precreate table
+    utils.createHCatTable(CreateMode.CREATE, TOTAL_RECORDS, table, cols);
+    runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
+      null, true, false);
+  }
+
+  public void testDropAndCreateWithoutPreExistingTable() throws Exception {
+    final int TOTAL_RECORDS = 1 * 10;
+    String table = getTableName().toUpperCase();
+    ColumnGenerator[] cols = new ColumnGenerator[] {
+      HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(0),
+        "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+        new HiveVarchar("1", 20), "1", KeyType.STATIC_KEY),
+      HCatalogTestUtils.colGenerator(HCatalogTestUtils.forIdx(1),
+        "varchar(20)", Types.VARCHAR, HCatFieldSchema.Type.STRING, 0, 0,
+        new HiveVarchar("2", 20), "2", KeyType.DYNAMIC_KEY),
+    };
+    List<String> addlArgsArray = new ArrayList<String>();
+    addlArgsArray.add("--drop-and-create-hcatalog-table");
+    setExtraArgs(addlArgsArray);
+    runHCatImport(addlArgsArray, TOTAL_RECORDS, table, cols,
+      null, true, false);
+  }
+
   public void testTableWithNonIdentColChars() throws Exception {
     final int TOTAL_RECORDS = 1 * 10;
     String table = getTableName().toUpperCase();

http://git-wip-us.apache.org/repos/asf/sqoop/blob/2eecf680/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java
index d33d456..5cd4c26 100644
--- a/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java
+++ b/src/test/org/apache/sqoop/hcat/TestHCatalogBasic.java
@@ -211,6 +211,41 @@ public class TestHCatalogBasic extends TestCase {
     SqoopOptions opts = parseImportArgs(args);
   }
 
+  public void testHCatImportWithDropAndCreateTable() throws Exception {
+    String[] args = {
+            "--connect",
+            "jdbc:db:url",
+            "--table",
+            "dbtable",
+            "--hcatalog-table",
+            "table",
+            "--drop-and-create-hcatalog-table",
+    };
+    SqoopOptions opts = parseImportArgs(args);
+    importTool.validateOptions(opts);
+  }
+
+  public void testHCatImportWithCreateTableAndDropAndCreateTable()
+    throws Exception {
+    String[] args = {
+            "--connect",
+            "jdbc:db:url",
+            "--table",
+            "dbtable",
+            "--hcatalog-table",
+            "table",
+            "--create-hcatalog-table",
+            "--drop-and-create-hcatalog-table",
+    };
+    SqoopOptions opts = parseImportArgs(args);
+    try {
+      importTool.validateOptions(opts);
+      fail("Expected InvalidOptionsException");
+    } catch (SqoopOptions.InvalidOptionsException ioe) {
+      // expected.
+    }
+  }
+
   public void testHCatImportWithStorageStanza() throws Exception {
     String[] args = {
       "--hcatalog-table",


Mime
View raw message