sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2632: Sqoop2: add partition class to FROM
Date Fri, 23 Oct 2015 20:21:52 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 6f113c094 -> 0773c10f8


SQOOP-2632: Sqoop2: add partition class to FROM

(Dian Fu via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0773c10f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0773c10f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0773c10f

Branch: refs/heads/sqoop2
Commit: 0773c10f83d2714cd05ce37393ec968f940ad302
Parents: 6f113c0
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Fri Oct 23 13:21:30 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Fri Oct 23 13:21:30 2015 -0700

----------------------------------------------------------------------
 .../apache/sqoop/connector/jdbc/GenericJdbcConnector.java    | 1 +
 .../java/org/apache/sqoop/connector/hdfs/HdfsConnector.java  | 1 +
 .../java/org/apache/sqoop/connector/kite/KiteConnector.java  | 1 +
 .../src/main/java/org/apache/sqoop/job/etl/From.java         | 8 ++++++++
 .../sqoop/execution/mapreduce/MapreduceExecutionEngine.java  | 1 +
 .../src/main/java/org/apache/sqoop/job/MRJobConstants.java   | 3 +++
 6 files changed, 15 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
index 113465a..2083244 100644
--- a/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
+++ b/connector/connector-generic-jdbc/src/main/java/org/apache/sqoop/connector/jdbc/GenericJdbcConnector.java
@@ -35,6 +35,7 @@ public class GenericJdbcConnector extends SqoopConnector {
   private static final From FROM = new From(
       GenericJdbcFromInitializer.class,
       GenericJdbcPartitioner.class,
+      GenericJdbcPartition.class,
       GenericJdbcExtractor.class,
       GenericJdbcFromDestroyer.class);
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
index 7e7c022..ecd2b68 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsConnector.java
@@ -38,6 +38,7 @@ public class HdfsConnector extends SqoopConnector {
   private static final From FROM = new From(
           HdfsFromInitializer.class,
           HdfsPartitioner.class,
+          HdfsPartition.class,
           HdfsExtractor.class,
           HdfsFromDestroyer.class);
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java
index ca860b1..87a5464 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteConnector.java
@@ -46,6 +46,7 @@ public class KiteConnector extends SqoopConnector {
   private static final From FROM = new From(
       KiteFromInitializer.class,
       KiteDatasetPartitioner.class,
+      KiteDatasetPartition.class,
       KiteExtractor.class,
       KiteFromDestroyer.class);
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java
----------------------------------------------------------------------
diff --git a/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java b/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java
index e9d2abe..1d2a3ed 100644
--- a/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java
+++ b/connector/connector-sdk/src/main/java/org/apache/sqoop/job/etl/From.java
@@ -34,14 +34,17 @@ import org.apache.sqoop.classification.InterfaceStability;
 public class From extends Transferable {
 
   private Class<? extends Partitioner> partitioner;
+  private Class<? extends Partition> partition;
   private Class<? extends Extractor> extractor;
 
   public From(Class<? extends Initializer> initializer,
               Class<? extends Partitioner> partitioner,
+              Class<? extends Partition> partition,
               Class<? extends Extractor> extractor,
               Class<? extends Destroyer> destroyer) {
     super(initializer, destroyer);
     this.partitioner = partitioner;
+    this.partition = partition;
     this.extractor = extractor;
   }
 
@@ -49,6 +52,10 @@ public class From extends Transferable {
     return partitioner;
   }
 
+  public Class<? extends Partition> getPartition() {
+    return partition;
+  }
+
   public Class<? extends Extractor> getExtractor() {
     return extractor;
   }
@@ -57,6 +64,7 @@ public class From extends Transferable {
   public String toString() {
     return "From{" + super.toString() +
       ", partitioner=" + partitioner.getName() +
+      ", partition=" + partition.getName() +
       ", extractor=" + extractor.getName() +
       '}';
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
index c8d210e..403f213 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java
@@ -67,6 +67,7 @@ public class MapreduceExecutionEngine extends ExecutionEngine {
     To to = (To) mrJobRequest.getTo();
     MutableMapContext context = mrJobRequest.getDriverContext();
     context.setString(MRJobConstants.JOB_ETL_PARTITIONER, from.getPartitioner().getName());
+    context.setString(MRJobConstants.JOB_ETL_PARTITION, from.getPartition().getName());
     context.setString(MRJobConstants.JOB_ETL_EXTRACTOR, from.getExtractor().getName());
     context.setString(MRJobConstants.JOB_ETL_LOADER, to.getLoader().getName());
     context.setString(MRJobConstants.JOB_ETL_FROM_DESTROYER, from.getDestroyer().getName());

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0773c10f/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
----------------------------------------------------------------------
diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
index b7aa8c6..df767e6 100644
--- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
+++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/MRJobConstants.java
@@ -31,6 +31,9 @@ public final class MRJobConstants extends Constants {
   public static final String JOB_ETL_PARTITIONER = PREFIX_JOB_CONFIG
       + "etl.partitioner";
 
+  public static final String JOB_ETL_PARTITION = PREFIX_JOB_CONFIG
+      + "etl.partition";
+
   public static final String JOB_ETL_EXTRACTOR = PREFIX_JOB_CONFIG
       + "etl.extractor";
 


Mime
View raw message