sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mau...@apache.org
Subject sqoop git commit: SQOOP-2986: Add validation check for --hive-import and --incremental lastmodified
Date Thu, 13 Oct 2016 13:23:09 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk b4afcf417 -> 14754342d


SQOOP-2986: Add validation check for --hive-import and --incremental lastmodified

(Szabolcs Vasas via Attila Szabo)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/14754342
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/14754342
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/14754342

Branch: refs/heads/trunk
Commit: 14754342d3a9bd6e146b9628b2e103ff30f310d8
Parents: b4afcf4
Author: Attila Szabo <maugli@apache.org>
Authored: Thu Oct 13 15:21:56 2016 +0200
Committer: Attila Szabo <maugli@apache.org>
Committed: Thu Oct 13 15:21:56 2016 +0200

----------------------------------------------------------------------
 .../org/apache/sqoop/tool/BaseSqoopTool.java    |  8 ++
 .../tool/ImportToolValidateOptionsTest.java     | 92 ++++++++++++++++++++
 2 files changed, 100 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/14754342/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index b71bc5e..13a9697 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -230,6 +230,9 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
 
   public static final String AUTORESET_TO_ONE_MAPPER = "autoreset-to-one-mapper";
 
+  static final String HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED = "--incremental lastmodified
option for hive imports is not "
+      + "supported. Please remove the parameter --incremental lastmodified.";
+
 
   public BaseSqoopTool() {
   }
@@ -1430,6 +1433,11 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
       }
 
     if (options.doHiveImport()
+        && options.getIncrementalMode().equals(IncrementalMode.DateLastModified))
{
+      throw new InvalidOptionsException(HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED);
+    }
+
+    if (options.doHiveImport()
         && options.isAppendMode()
         && !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) {
       throw new InvalidOptionsException("Append mode for hive imports is not "

http://git-wip-us.apache.org/repos/asf/sqoop/blob/14754342/src/test/org/apache/sqoop/tool/ImportToolValidateOptionsTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/tool/ImportToolValidateOptionsTest.java b/src/test/org/apache/sqoop/tool/ImportToolValidateOptionsTest.java
new file mode 100644
index 0000000..503a863
--- /dev/null
+++ b/src/test/org/apache/sqoop/tool/ImportToolValidateOptionsTest.java
@@ -0,0 +1,92 @@
+package org.apache.sqoop.tool;
+
+import com.cloudera.sqoop.SqoopOptions;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class ImportToolValidateOptionsTest {
+
+  private static final String TABLE_NAME = "testTableName";
+  private static final String CONNECT_STRING = "testConnectString";
+  private static final String CHECK_COLUMN_NAME = "checkColumnName";
+
+  @Rule
+  public ExpectedException thrown = ExpectedException.none();
+
+  private ImportTool importTool;
+
+  @Before
+  public void setup() {
+    importTool = new ImportTool();
+    importTool.extraArguments = new String[0];
+  }
+
+  @Test
+  public void testValidationFailsWithHiveImportAndIncrementalLastmodified() throws Exception
{
+    SqoopOptions options = buildBaseSqoopOptions();
+    options.setHiveImport(true);
+    options.setIncrementalTestColumn(CHECK_COLUMN_NAME);
+    options.setIncrementalMode(SqoopOptions.IncrementalMode.DateLastModified);
+
+    thrown.expect(SqoopOptions.InvalidOptionsException.class);
+    thrown.expectMessage(BaseSqoopTool.HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED);
+
+    importTool.validateOptions(options);
+  }
+
+  /**
+   * Note that append mode (--append) is designed to be used with HDFS import and not Hive
import.
+   * However this test case is added to make sure that the error message generated is correct
even if --append is used.
+   *
+   */
+  @Test
+  public void testValidationFailsWithHiveImportAndAppendModeIncrementalLastmodified() throws
Exception {
+    SqoopOptions options = buildBaseSqoopOptions();
+    options.setHiveImport(true);
+    options.setIncrementalTestColumn(CHECK_COLUMN_NAME);
+    options.setIncrementalMode(SqoopOptions.IncrementalMode.DateLastModified);
+    options.setAppendMode(true);
+
+    thrown.expect(SqoopOptions.InvalidOptionsException.class);
+    thrown.expectMessage(BaseSqoopTool.HIVE_IMPORT_WITH_LASTMODIFIED_NOT_SUPPORTED);
+
+    importTool.validateOptions(options);
+  }
+
+  @Test
+  public void testValidationSucceedsWithHiveImportAndIncrementalAppendRows() throws Exception
{
+    SqoopOptions options = buildBaseSqoopOptions();
+    options.setHiveImport(true);
+    options.setIncrementalTestColumn(CHECK_COLUMN_NAME);
+    options.setIncrementalMode(SqoopOptions.IncrementalMode.AppendRows);
+
+    importTool.validateOptions(options);
+  }
+
+  /**
+   * Note that append mode (--append) is designed to be used with HDFS import and not Hive
import.
+   * However this test case is added to make sure that SQOOP-2986 does not break the already
existing validation.
+   *
+   */
+  @Test
+  public void testValidationSucceedsWithHiveImportAndAppendModeAndIncrementalAppendRows()
throws Exception {
+    SqoopOptions options = buildBaseSqoopOptions();
+    options.setHiveImport(true);
+    options.setIncrementalTestColumn(CHECK_COLUMN_NAME);
+    options.setIncrementalMode(SqoopOptions.IncrementalMode.AppendRows);
+    options.setAppendMode(true);
+
+    importTool.validateOptions(options);
+  }
+
+  private SqoopOptions buildBaseSqoopOptions() {
+    SqoopOptions result = new SqoopOptions();
+    result.setTableName(TABLE_NAME);
+    result.setConnectString(CONNECT_STRING);
+    return result;
+  }
+
+}
+


Mime
View raw message