sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject git commit: SQOOP-1080: Sqoop2: Investigate Jenkins test failures
Date Sun, 16 Jun 2013 13:45:03 GMT
Updated Branches:
  refs/heads/sqoop2 66a296ead -> 05a73ef16


SQOOP-1080: Sqoop2: Investigate Jenkins test failures

(Venkat Ranganathan via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/05a73ef1
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/05a73ef1
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/05a73ef1

Branch: refs/heads/sqoop2
Commit: 05a73ef168f65fea155befb9dbbac01b61febed5
Parents: 66a296e
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Sun Jun 16 06:40:13 2013 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Sun Jun 16 06:40:35 2013 -0700

----------------------------------------------------------------------
 .../jdbc/GenericJdbcImportPartitioner.java      | 31 ++++++++++++++++----
 .../connector/jdbc/TestImportPartitioner.java   |  6 ++--
 2 files changed, 28 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/05a73ef1/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
index eef18f2..0bb979e 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcImportPartitioner.java
@@ -22,8 +22,11 @@ import java.sql.Date;
 import java.sql.Time;
 import java.sql.Timestamp;
 import java.sql.Types;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.TimeZone;
 
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.jdbc.configuration.ConnectionConfiguration;
@@ -106,27 +109,39 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
 
     long minDateValue = 0;
     long maxDateValue = 0;
-
+    SimpleDateFormat sdf = null;
     switch(partitionColumnType) {
       case Types.DATE:
+        sdf = new SimpleDateFormat("yyyy-MM-dd");
         minDateValue = Date.valueOf(partitionMinValue).getTime();
         maxDateValue = Date.valueOf(partitionMaxValue).getTime();
         break;
       case Types.TIME:
+        sdf = new SimpleDateFormat("HH:mm:ss");
         minDateValue = Time.valueOf(partitionMinValue).getTime();
         maxDateValue = Time.valueOf(partitionMaxValue).getTime();
         break;
       case Types.TIMESTAMP:
+        sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS");
         minDateValue = Timestamp.valueOf(partitionMinValue).getTime();
         maxDateValue = Timestamp.valueOf(partitionMaxValue).getTime();
         break;
     }
+
+    long tzOffset = TimeZone.getDefault().getRawOffset();
+
+    minDateValue += tzOffset;
+    maxDateValue += tzOffset;
+
+    sdf.setTimeZone(TimeZone.getTimeZone("GMT"));
+
     long interval =  (maxDateValue - minDateValue) / numberPartitions;
     long remainder = (maxDateValue - minDateValue) % numberPartitions;
 
     if (interval == 0) {
       numberPartitions = (int)remainder;
     }
+
     long lowerBound;
     long upperBound = minDateValue;
 
@@ -146,6 +161,7 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
         case Types.TIME:
           objLB = new Time(lowerBound);
           objUB = new Time(upperBound);
+
           break;
         case Types.TIMESTAMP:
           objLB = new Timestamp(lowerBound);
@@ -155,9 +171,10 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
 
       GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
       partition.setConditions(
-          constructDateConditions(objLB, objUB, false));
+          constructDateConditions(sdf, objLB, objUB, false));
       partitions.add(partition);
     }
+
     switch(partitionColumnType) {
       case Types.DATE:
         objLB = new Date(upperBound);
@@ -172,9 +189,11 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
         objUB = new Timestamp(maxDateValue);
         break;
     }
+
+
     GenericJdbcImportPartition partition = new GenericJdbcImportPartition();
     partition.setConditions(
-        constructDateConditions(objLB, objUB, true));
+        constructDateConditions(sdf, objLB, objUB, true));
     partitions.add(partition);
     return partitions;
   }
@@ -461,16 +480,16 @@ public class GenericJdbcImportPartitioner extends Partitioner<ConnectionConfigur
      ;
   }
 
-  protected String constructDateConditions(
+  protected String constructDateConditions(SimpleDateFormat sdf,
       Object lowerBound, Object upperBound, boolean lastOne) {
     StringBuilder conditions = new StringBuilder();
-    conditions.append('\'').append(lowerBound.toString()).append('\'');
+    conditions.append('\'').append(sdf.format((java.util.Date)lowerBound)).append('\'');
     conditions.append(" <= ");
     conditions.append(partitionColumnName);
     conditions.append(" AND ");
     conditions.append(partitionColumnName);
     conditions.append(lastOne ? " <= " : " < ");
-    conditions.append('\'').append(upperBound.toString()).append('\'');
+    conditions.append('\'').append(sdf.format((java.util.Date)upperBound)).append('\'');
     return conditions.toString();
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/05a73ef1/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
index 522a515..1a3601a 100644
--- a/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
+++ b/connector/connector-generic-jdbc/src/test/java/org/apache/sqoop/connector/jdbc/TestImportPartitioner.java
@@ -333,9 +333,9 @@ public class TestImportPartitioner extends TestCase {
         3);
     List<Partition> partitions = partitioner.getPartitions(partitionerContext, connConf,
jobConf);
     verifyResult(partitions, new String[]{
-        "'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 13:14:17.634'",
-        "'2013-05-02 13:14:17.634' <= TSCOL AND TSCOL < '2013-09-01 00:27:34.144'",
-        "'2013-09-01 00:27:34.144' <= TSCOL AND TSCOL <= '2013-12-31 10:40:50.654'",
+        "'2013-01-01 01:01:01.123' <= TSCOL AND TSCOL < '2013-05-02 12:14:17.634'",
+        "'2013-05-02 12:14:17.634' <= TSCOL AND TSCOL < '2013-08-31 23:27:34.144'",
+        "'2013-08-31 23:27:34.144' <= TSCOL AND TSCOL <= '2013-12-31 10:40:50.654'",
     });
   }
 


Mime
View raw message