sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2470: Incremental Hive import with append not working after validation check for --hive-import and --import
Date Fri, 14 Aug 2015 00:11:03 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 73ef0c133 -> de6fa8279


SQOOP-2470: Incremental Hive import with append not working after validation check for --hive-import
and --import

(Aaditya Raj via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/de6fa827
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/de6fa827
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/de6fa827

Branch: refs/heads/trunk
Commit: de6fa8279d8125afb4fd2d7f1e8c7e731d70451d
Parents: 73ef0c1
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Thu Aug 13 17:10:42 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Thu Aug 13 17:10:42 2015 -0700

----------------------------------------------------------------------
 .../org/apache/sqoop/tool/BaseSqoopTool.java    |  4 +-
 .../cloudera/sqoop/TestIncrementalImport.java   | 63 +++++++++++++++++++-
 testdata/hive/scripts/incrementalHiveAppend10.q | 17 ++++++
 testdata/hive/scripts/incrementalHiveAppend20.q | 17 ++++++
 .../hive/scripts/incrementalHiveAppendEmpty.q   | 17 ++++++
 5 files changed, 114 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 4e2e66d..a7f6aec 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -40,6 +40,7 @@ import org.apache.sqoop.util.password.CredentialProviderHelper;
 import com.cloudera.sqoop.ConnFactory;
 import com.cloudera.sqoop.Sqoop;
 import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.SqoopOptions.IncrementalMode;
 import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
 import com.cloudera.sqoop.cli.RelatedOptions;
 import com.cloudera.sqoop.cli.ToolOptions;
@@ -1398,7 +1399,8 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
     }
 
     if (options.doHiveImport()
-        && options.isAppendMode()) {
+        && options.isAppendMode()
+        && !options.getIncrementalMode().equals(IncrementalMode.AppendRows)) {
       throw new InvalidOptionsException("Append mode for hive imports is not "
           + " yet supported. Please remove the parameter --append-mode");
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/src/test/com/cloudera/sqoop/TestIncrementalImport.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestIncrementalImport.java b/src/test/com/cloudera/sqoop/TestIncrementalImport.java
index a6680e8..3e3f929 100644
--- a/src/test/com/cloudera/sqoop/TestIncrementalImport.java
+++ b/src/test/com/cloudera/sqoop/TestIncrementalImport.java
@@ -19,6 +19,7 @@
 package com.cloudera.sqoop;
 
 import java.io.BufferedReader;
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.sql.Connection;
@@ -29,9 +30,7 @@ import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
-
 import junit.framework.TestCase;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
@@ -39,7 +38,7 @@ import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.util.StringUtils;
-
+import org.apache.sqoop.hive.HiveImport;
 import com.cloudera.sqoop.manager.ConnManager;
 import com.cloudera.sqoop.manager.HsqldbManager;
 import com.cloudera.sqoop.manager.ManagerFactory;
@@ -1216,6 +1215,64 @@ public class TestIncrementalImport extends TestCase {
     runJob(TABLE_NAME);
     assertDirOfNumbers(TABLE_NAME, 20);
   }
+  
+	public void testIncrementalHiveAppendEmptyThenFull() throws Exception {
+		// This is to test Incremental Hive append feature. SQOOP-2470
+		final String TABLE_NAME = "incrementalHiveAppendEmptyThenFull";
+		Configuration conf = newConf();
+		conf.set(ConnFactory.FACTORY_CLASS_NAMES_KEY,
+				InstrumentHsqldbManagerFactory.class.getName());
+		clearDir(TABLE_NAME);
+		createIdTable(TABLE_NAME, 0);
+		List<String> args = new ArrayList<String>();
+		args.add("--connect");
+		args.add(SOURCE_DB_URL);
+		args.add("--table");
+		args.add(TABLE_NAME);
+		args.add("--warehouse-dir");
+		args.add(BaseSqoopTestCase.LOCAL_WAREHOUSE_DIR);
+		args.add("--hive-import");
+		args.add("--hive-table");
+		args.add(TABLE_NAME + "hive");
+		args.add("--incremental");
+		args.add("append");
+		args.add("--check-column");
+		args.add("id");
+		args.add("-m");
+		args.add("1");
+		createJob(TABLE_NAME, args, conf);
+		HiveImport.setTestMode(true);
+		String hiveHome = org.apache.sqoop.SqoopOptions.getHiveHomeDefault();
+		assertNotNull("hive.home was not set", hiveHome);
+		String testDataPath = new Path(new Path(hiveHome), "scripts/"
+				+ "incrementalHiveAppendEmpty.q").toString();
+		System.clearProperty("expected.script");
+		System.setProperty("expected.script",
+				new File(testDataPath).getAbsolutePath());
+		runJob(TABLE_NAME);
+		assertDirOfNumbers(TABLE_NAME, 0);
+		// Now add some rows.
+		insertIdRows(TABLE_NAME, 0, 10);
+		String testDataPath10 = new Path(new Path(hiveHome), "scripts/"
+				+ "incrementalHiveAppend10.q").toString();
+		System.clearProperty("expected.script");
+		System.setProperty("expected.script",
+				new File(testDataPath10).getAbsolutePath());
+		System.getProperty("expected.script");
+		// Running the job a second time should import 10 rows.
+		runJob(TABLE_NAME);
+		assertDirOfNumbers(TABLE_NAME, 10);
+		// Add some more rows.
+		insertIdRows(TABLE_NAME, 10, 20);
+		String testDataPath20 = new Path(new Path(hiveHome), "scripts/"
+				+ "incrementalHiveAppend20.q").toString();
+		System.clearProperty("expected.script");
+		System.setProperty("expected.script",
+				new File(testDataPath20).getAbsolutePath());
+		// Import only those rows.
+		runJob(TABLE_NAME);
+		assertDirOfNumbers(TABLE_NAME, 20);
+	}
 
   // SQOOP-1890
   public void testTableNameWithSpecialCharacters() throws Exception {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppend10.q
----------------------------------------------------------------------
diff --git a/testdata/hive/scripts/incrementalHiveAppend10.q b/testdata/hive/scripts/incrementalHiveAppend10.q
new file mode 100644
index 0000000..383c7b4
--- /dev/null
+++ b/testdata/hive/scripts/incrementalHiveAppend10.q
@@ -0,0 +1,17 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT
DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
+LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO
TABLE `incrementalHiveAppendEmptyThenFullhive`;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppend20.q
----------------------------------------------------------------------
diff --git a/testdata/hive/scripts/incrementalHiveAppend20.q b/testdata/hive/scripts/incrementalHiveAppend20.q
new file mode 100644
index 0000000..383c7b4
--- /dev/null
+++ b/testdata/hive/scripts/incrementalHiveAppend20.q
@@ -0,0 +1,17 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT
DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
+LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO
TABLE `incrementalHiveAppendEmptyThenFullhive`;
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sqoop/blob/de6fa827/testdata/hive/scripts/incrementalHiveAppendEmpty.q
----------------------------------------------------------------------
diff --git a/testdata/hive/scripts/incrementalHiveAppendEmpty.q b/testdata/hive/scripts/incrementalHiveAppendEmpty.q
new file mode 100644
index 0000000..383c7b4
--- /dev/null
+++ b/testdata/hive/scripts/incrementalHiveAppendEmpty.q
@@ -0,0 +1,17 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements.  See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership.  The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+CREATE TABLE IF NOT EXISTS `incrementalHiveAppendEmptyThenFullhive` ( `ID` INT) ROW FORMAT
DELIMITED FIELDS TERMINATED BY '\001' LINES TERMINATED BY '\012' STORED AS TEXTFILE;
+LOAD DATA INPATH 'file:BASEPATH/sqoop/warehouse/incrementalHiveAppendEmptyThenFull' INTO
TABLE `incrementalHiveAppendEmptyThenFullhive`;
\ No newline at end of file


Mime
View raw message