sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkatran...@apache.org
Subject git commit: SQOOP-1154: Sqoop2: Text partitioner might miss or include edge values
Date Wed, 31 Jul 2013 05:48:28 GMT
Updated Branches:
  refs/heads/sqoop2 81e71dc45 -> 9c7adb4e1


SQOOP-1154: Sqoop2: Text partitioner might miss or include edge values

(Jarek Jarcec Cecho via Venkat Ranganathan)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9c7adb4e
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9c7adb4e
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9c7adb4e

Branch: refs/heads/sqoop2
Commit: 9c7adb4e112e57e5d2c4720be71b8a5278ef7638
Parents: 81e71dc
Author: Venkat Ranganathan <n.r.v@live.com>
Authored: Tue Jul 30 22:47:44 2013 -0700
Committer: Venkat Ranganathan <n.r.v@live.com>
Committed: Tue Jul 30 22:47:44 2013 -0700

----------------------------------------------------------------------
 .../jdbc/GenericJdbcImportPartitioner.java      | 25 +++++++-------------
 .../connector/jdbc/TestImportPartitioner.java   |  5 ++++
 2 files changed, 13 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/9c7adb4e/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
index 8d0c4ab..d103223 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
@@ -250,7 +250,8 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
     // Having one single value means that we can create only one single split
     if(minStringBD.equals(maxStringBD)) {
       GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
-      partition.setConditions(constructTextConditions(prefix, maxStringBD));
+      partition.setConditions(constructTextConditions(prefix, 0, 0,
+        partitionMinValue, partitionMaxValue, true, true));
       partitions.add(partition);
       return partitions;
     }
@@ -294,8 +295,8 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
       BigDecimal end = splitPoints.get(i);
 
       GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
-      partition.setConditions(constructTextConditions(prefix, start,
-          end, i == splitPoints.size() - 1));
+      partition.setConditions(constructTextConditions(prefix, start, end,
+        partitionMinValue, partitionMaxValue, i == 1, i == splitPoints.size() - 1));
       partitions.add(partition);
 
       start = end;
@@ -521,31 +522,21 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
     return conditions.toString();
   }
 
-  protected String constructTextConditions(String prefix,
-      Object lowerBound, Object upperBound, boolean lastOne) {
+  protected String constructTextConditions(String prefix, Object lowerBound, Object upperBound,
+      String lowerStringBound, String upperStringBound, boolean firstOne, boolean lastOne)
{
     StringBuilder conditions = new StringBuilder();
     String lbString = prefix + bigDecimalToText((BigDecimal)lowerBound);
     String ubString = prefix + bigDecimalToText((BigDecimal)upperBound);
-    conditions.append('\'').append(lbString).append('\'');
+    conditions.append('\'').append(firstOne ? lowerStringBound : lbString).append('\'');
     conditions.append(" <= ");
     conditions.append(partitionColumnName);
     conditions.append(" AND ");
     conditions.append(partitionColumnName);
     conditions.append(lastOne ? " <= " : " < ");
-    conditions.append('\'').append(ubString).append('\'');
+    conditions.append('\'').append(lastOne ? upperStringBound : ubString).append('\'');
     return conditions.toString();
   }
 
-  protected String constructTextConditions(String prefix, Object value) {
-    return new StringBuilder()
-      .append(partitionColumnName)
-      .append(" = ").append('\'')
-      .append(prefix + bigDecimalToText((BigDecimal)value))
-      .append('\'').toString()
-     ;
-  }
-
-
   /**
    *  Converts a string to a BigDecimal representation in Base 2^21 format.
    *  The maximum Unicode code point value defined is 10FFFF.  Although

http://git-wip-us.apache.org/repos/asf/sqoop/blob/9c7adb4e/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
index a33dd6c..679accf 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
@@ -409,6 +409,7 @@ public class TestImportPartitioner extends TestCase {
         "'Y' <= VCCOL AND VCCOL <= 'Z'",
     });
   }
+
   public void testVarcharPartition2() throws Exception {
     MutableContext context = new MutableMapContext();
     context.setString(GenericJdbcConnectorConstants
@@ -426,6 +427,10 @@ public class TestImportPartitioner extends TestCase {
     PartitionerContext partitionerContext = new PartitionerContext(context, 5, null);
     List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf,
jobConf);
     assertEquals(partitions.size(), 5);
+    // First partition needs to contain entire upper bound
+    assertTrue(partitions.get(0).toString().contains("Breezy Badger"));
+    // Last partition needs to contain entire lower bound
+    assertTrue(partitions.get(4).toString().contains("Warty Warthog"));
   }
 
   public void testVarcharPartitionWithCommonPrefix() throws Exception {


Mime
View raw message