sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject sqoop git commit: SQOOP-2505: Sqoop2: Add ability to specify custom configuration properties for HDFS connector
Date Mon, 24 Aug 2015 21:42:12 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 09a21649d -> 3668aea7d


SQOOP-2505: Sqoop2: Add ability to specify custom configuration properties for HDFS connector

(Jarek Jarcec Cecho via Abraham Elmahrek)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3668aea7
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3668aea7
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3668aea7

Branch: refs/heads/sqoop2
Commit: 3668aea7db65c63b539deea9518267890dd8c8bc
Parents: 09a2164
Author: Abraham Elmahrek <abe@apache.org>
Authored: Mon Aug 24 14:41:53 2015 -0700
Committer: Abraham Elmahrek <abe@apache.org>
Committed: Mon Aug 24 14:41:53 2015 -0700

----------------------------------------------------------------------
 .../apache/sqoop/common/MutableMapContext.java  |  4 ++
 .../sqoop/common/TestMutableMapContext.java     |  3 ++
 .../connector/hdfs/HdfsFromInitializer.java     |  1 +
 .../sqoop/connector/hdfs/HdfsToInitializer.java |  2 +
 .../hdfs/configuration/LinkConfig.java          |  8 ++++
 .../resources/hdfs-connector-config.properties  |  3 ++
 .../connector/hdfs/TestFromInitializer.java     | 47 ++++++++++++++++++++
 7 files changed, 68 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java b/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java
index 06ea7a9..7ff7269 100644
--- a/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java
+++ b/common/src/main/java/org/apache/sqoop/common/MutableMapContext.java
@@ -64,6 +64,10 @@ public class MutableMapContext extends MapContext implements MutableContext
{
 
   @Override
   public void setAll(Map<String, String> map) {
+    if(map == null) {
+      return;
+    }
+
     getOptions().putAll(map);
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java b/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java
index 9aab7f8..db7fa34 100644
--- a/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java
+++ b/common/src/test/java/org/apache/sqoop/common/TestMutableMapContext.java
@@ -95,6 +95,9 @@ public class TestMutableMapContext {
     assertEquals(context.getLong("long", -1), 1L);
     assertEquals(context.getInt("integer", -1), 13);
     assertEquals(context.getBoolean("boolean", false), true);
+
+    // Verify that we're resilient against null
+    context.setAll(null);
   }
 
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java
b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java
index 902549d..f5d9e1f 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsFromInitializer.java
@@ -38,5 +38,6 @@ public class HdfsFromInitializer extends Initializer<LinkConfiguration,
FromJobC
   public void initialize(InitializerContext context, LinkConfiguration linkConfig, FromJobConfiguration
jobConfig) {
     Configuration configuration = HdfsUtils.createConfiguration(linkConfig);
     HdfsUtils.configurationToContext(configuration, context.getContext());
+    context.getContext().setAll(linkConfig.linkConfig.configOverrides);
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java
b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java
index 234bb71..5bb0928 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/HdfsToInitializer.java
@@ -48,6 +48,8 @@ public class HdfsToInitializer extends Initializer<LinkConfiguration,
ToJobConfi
 
     Configuration configuration = HdfsUtils.createConfiguration(linkConfig);
     HdfsUtils.configurationToContext(configuration, context.getContext());
+    context.getContext().setAll(linkConfig.linkConfig.configOverrides);
+
     boolean appendMode = Boolean.TRUE.equals(jobConfig.toJobConfig.appendMode);
 
     // Verification that given HDFS directory either don't exists or is empty

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
index b54ad15..39f3752 100644
--- a/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
+++ b/connector/connector-hdfs/src/main/java/org/apache/sqoop/connector/hdfs/configuration/LinkConfig.java
@@ -24,6 +24,8 @@ import org.apache.sqoop.validation.Status;
 import org.apache.sqoop.validation.validators.AbstractValidator;
 import org.apache.sqoop.validation.validators.DirectoryExistsValidator;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -34,6 +36,12 @@ public class LinkConfig {
   @Input(size = 255, validators = { @Validator(DirectoryExistsValidator.class)})
   public String confDir;
 
+  @Input public Map<String, String> configOverrides;
+
+  public LinkConfig() {
+    configOverrides = new HashMap<>();
+  }
+
   public static class ConfigValidator extends AbstractValidator<LinkConfig> {
     private static final Pattern URI_PATTERN = Pattern.compile("((?<=\\()[A-Za-z][A-Za-z0-9\\+\\.\\-]*:([A-Za-z0-9\\.\\-_~:/\\?#\\[\\]@!\\$&'\\(\\)\\*\\+,;=]|%[A-Fa-f0-9]{2})+(?=\\)))|([A-Za-z][A-Za-z0-9\\+\\.\\-]*:([A-Za-z0-9\\.\\-_~:/\\?#\\[\\]@!\\$&'\\(\\)\\*\\+,;=]|%[A-Fa-f0-9]{2})+)");
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
index 5feae54..db23a95 100644
--- a/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
+++ b/connector/connector-hdfs/src/main/resources/hdfs-connector-config.properties
@@ -27,6 +27,9 @@ linkConfig.uri.help = HDFS URI used to connect to HDFS
 linkConfig.confDir.label = Hadoop conf directory
 linkConfig.confDir.help = Directory with Hadoop configuration files. The connector will load
all -site.xml files.
 
+linkConfig.configOverrides.label = Override configuration
+linkConfig.configOverrides.help = Map of properties that that should be set for the Hadoop's
configuration object on top of the files loaded from configuration directory.
+
 # To Job Config
 #
 toJobConfig.label = To HDFS configuration

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3668aea7/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestFromInitializer.java
----------------------------------------------------------------------
diff --git a/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestFromInitializer.java
b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestFromInitializer.java
new file mode 100644
index 0000000..5215901
--- /dev/null
+++ b/connector/connector-hdfs/src/test/java/org/apache/sqoop/connector/hdfs/TestFromInitializer.java
@@ -0,0 +1,47 @@
+/**
+ * Created by jarcec on 8/18/15.
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.connector.hdfs;
+
+import org.apache.sqoop.common.MutableMapContext;
+import org.apache.sqoop.connector.hdfs.configuration.FromJobConfiguration;
+import org.apache.sqoop.connector.hdfs.configuration.LinkConfiguration;
+import org.apache.sqoop.job.etl.Initializer;
+import org.apache.sqoop.job.etl.InitializerContext;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+public class TestFromInitializer {
+
+  @Test
+  public void testConfigOverrides() {
+    LinkConfiguration linkConfig = new LinkConfiguration();
+    FromJobConfiguration jobConfig = new FromJobConfiguration();
+
+    linkConfig.linkConfig.uri = "file:///";
+    linkConfig.linkConfig.configOverrides.put("key", "value");
+
+    InitializerContext initializerContext = new InitializerContext(new MutableMapContext());
+
+    Initializer initializer = new HdfsFromInitializer();
+    initializer.initialize(initializerContext, linkConfig, jobConfig);
+
+    assertEquals(initializerContext.getString("key"), "value");
+  }
+}


Mime
View raw message