sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2238: Sqoop2: Parquet jars missing
Date Sun, 29 Mar 2015 20:20:47 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 3ba34e250 -> a18d27e37


SQOOP-2238: Sqoop2: Parquet jars missing

(Abraham Elmahrek via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/a18d27e3
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/a18d27e3
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/a18d27e3

Branch: refs/heads/sqoop2
Commit: a18d27e3711cbf4a00c70492a3bbca19bf12e3c9
Parents: 3ba34e2
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Sun Mar 29 13:20:22 2015 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Sun Mar 29 13:20:22 2015 -0700

----------------------------------------------------------------------
 .../apache/sqoop/connector/kite/KiteFromInitializer.java |  9 +++++++++
 .../apache/sqoop/connector/kite/KiteToInitializer.java   | 11 +++++++++++
 2 files changed, 20 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/a18d27e3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
index 06f18e7..fd3017b 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteFromInitializer.java
@@ -59,6 +59,14 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration,
     jars.add(ClassUtils.jarForClass("org.kitesdk.data.Datasets"));
     jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.databind.JsonNode"));
     jars.add(ClassUtils.jarForClass("com.fasterxml.jackson.core.TreeNode"));
+    jars.add(ClassUtils.jarForClass("parquet.hadoop.metadata.CompressionCodecName"));
+    jars.add(ClassUtils.jarForClass("parquet.format.CompressionCodec"));
+    jars.add(ClassUtils.jarForClass("parquet.avro.AvroParquetWriter"));
+    jars.add(ClassUtils.jarForClass("parquet.column.ParquetProperties"));
+    jars.add(ClassUtils.jarForClass("parquet.Version"));
+    jars.add(ClassUtils.jarForClass("parquet.org.codehaus.jackson.type.TypeReference"));
+    jars.add(ClassUtils.jarForClass("parquet.bytes.CapacityByteArrayOutputStream"));
+    jars.add(ClassUtils.jarForClass("parquet.encoding.Generator"));
     if (fromJobConfig.fromJobConfig.uri.startsWith("dataset:hive")) {
       // @TODO(Abe): Remove a deps that aren't used?
       jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.conf.HiveConf"));
@@ -72,6 +80,7 @@ public class KiteFromInitializer extends Initializer<LinkConfiguration,
       jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
+      jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe"));
     }
     return jars;
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a18d27e3/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
index 39a2989..e27c0b8 100644
--- a/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
+++ b/connector/connector-kite/src/main/java/org/apache/sqoop/connector/kite/KiteToInitializer.java
@@ -65,6 +65,16 @@ public class KiteToInitializer extends Initializer<LinkConfiguration,
     if (FileFormat.CSV.equals(toJobConfig.toJobConfig.fileFormat)) {
       jars.add(ClassUtils.jarForClass("au.com.bytecode.opencsv.CSVWriter"));
     }
+    if (FileFormat.PARQUET.equals(toJobConfig.toJobConfig.fileFormat)) {
+      jars.add(ClassUtils.jarForClass("parquet.hadoop.metadata.CompressionCodecName"));
+      jars.add(ClassUtils.jarForClass("parquet.format.CompressionCodec"));
+      jars.add(ClassUtils.jarForClass("parquet.avro.AvroParquetWriter"));
+      jars.add(ClassUtils.jarForClass("parquet.column.ParquetProperties"));
+      jars.add(ClassUtils.jarForClass("parquet.Version"));
+      jars.add(ClassUtils.jarForClass("parquet.org.codehaus.jackson.type.TypeReference"));
+      jars.add(ClassUtils.jarForClass("parquet.bytes.CapacityByteArrayOutputStream"));
+      jars.add(ClassUtils.jarForClass("parquet.encoding.Generator"));
+    }
     if (toJobConfig.toJobConfig.uri.startsWith("dataset:hive")) {
       // @TODO(Abe): Remove a deps that aren't used?
       jars.add(ClassUtils.jarForClass("org.apache.hadoop.hive.conf.HiveConf"));
@@ -78,6 +88,7 @@ public class KiteToInitializer extends Initializer<LinkConfiguration,
       jars.add(ClassUtils.jarForClass("org.datanucleus.query.compiler.JavaQueryCompiler"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.query.typesafe.TypesafeSubquery"));
       jars.add(ClassUtils.jarForClass("org.datanucleus.store.rdbms.sql.SQLStatement"));
+      jars.add(ClassUtils.jarForClass("parquet.hive.serde.ParquetHiveSerDe"));
     }
     return jars;
   }


Mime
View raw message