sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2910: Add capability to Sqoop to require an explicit option to be specified with --split-by for a String column
Date Tue, 19 Apr 2016 22:55:36 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk dc2568431 -> 660f3e8ad


SQOOP-2910: Add capability to Sqoop to require an explicit option to be specified with --split-by
for a String column

(Attila Szabo via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/660f3e8a
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/660f3e8a
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/660f3e8a

Branch: refs/heads/trunk
Commit: 660f3e8ad07758aabf0a9b6ede3accdfac5fb1be
Parents: dc25684
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Tue Apr 19 15:54:58 2016 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Tue Apr 19 15:54:58 2016 -0700

----------------------------------------------------------------------
 .../apache/sqoop/mapreduce/db/TextSplitter.java |  6 ++
 .../TextSplitterHadoopConfIntegrationTest.java  | 59 ++++++++++++++++++++
 2 files changed, 65 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/660f3e8a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java b/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
index 9896d95..8c98d14 100644
--- a/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
+++ b/src/java/org/apache/sqoop/mapreduce/db/TextSplitter.java
@@ -38,6 +38,8 @@ import org.apache.sqoop.validation.ValidationException;
  */
 public class TextSplitter extends BigDecimalSplitter {
 
+  public static final String ALLOW_TEXT_SPLITTER_PROPERTY = "org.apache.sqoop.splitter.allow_text_splitter";
+
   private static final Log LOG = LogFactory.getLog(TextSplitter.class);
 
   private boolean useNCharStrings = false;
@@ -61,6 +63,10 @@ public class TextSplitter extends BigDecimalSplitter {
    */
   public List<InputSplit> split(Configuration conf, ResultSet results,
       String colName) throws SQLException, ValidationException {
+    if (!conf.getBoolean(ALLOW_TEXT_SPLITTER_PROPERTY, false)) {
+      throw new ValidationException("Generating splits for a textual index column " + "allowed
only in case of \"-D"
+          + ALLOW_TEXT_SPLITTER_PROPERTY + "=true\" property " + "passed as a parameter");
+    }
 
     LOG.warn("Generating splits for a textual index column.");
     LOG.warn("If your database sorts in a case-insensitive order, "

http://git-wip-us.apache.org/repos/asf/sqoop/blob/660f3e8a/src/test/org/apache/sqoop/mapreduce/db/TextSplitterHadoopConfIntegrationTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/mapreduce/db/TextSplitterHadoopConfIntegrationTest.java
b/src/test/org/apache/sqoop/mapreduce/db/TextSplitterHadoopConfIntegrationTest.java
new file mode 100644
index 0000000..043130e
--- /dev/null
+++ b/src/test/org/apache/sqoop/mapreduce/db/TextSplitterHadoopConfIntegrationTest.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.mapreduce.db;
+
+import java.sql.ResultSet;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.sqoop.validation.ValidationException;
+
+import com.cloudera.sqoop.Sqoop;
+import com.cloudera.sqoop.testutil.MockResultSet;
+
+import junit.framework.TestCase;
+
+public class TextSplitterHadoopConfIntegrationTest extends TestCase {
+  private static final String TEXT_COL_NAME = "text_col_name";
+
+  public void testDefaultValueOfUnsetBooleanParam() throws Exception {
+    Configuration conf = Job.getInstance().getConfiguration();
+    TextSplitter splitter = new TextSplitter();
+    ResultSet rs = new MockResultSet();
+    try {
+      splitter.split(conf, rs, TEXT_COL_NAME);
+      fail();
+    } catch (ValidationException e) {
+      // expected to throw ValidationException with the a message about the
+      // "i-know-what-i-am-doing" prop
+      assertTrue(e.getMessage().contains(TextSplitter.ALLOW_TEXT_SPLITTER_PROPERTY));
+    }
+  }
+
+  public void testBooleanParamValue() throws Exception {
+    Configuration conf = Job.getInstance().getConfiguration();
+    conf.set(TextSplitter.ALLOW_TEXT_SPLITTER_PROPERTY, "true");
+    TextSplitter splitter = new TextSplitter();
+    ResultSet rs = new MockResultSet();
+    List<InputSplit> splits = splitter.split(conf, rs, TEXT_COL_NAME);
+    assertFalse(splits.isEmpty());
+  }
+}
+


Mime
View raw message