sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2880: Provide argument for overriding temporary directory
Date Mon, 14 Mar 2016 16:22:38 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 152fe9300 -> a0b730c77


SQOOP-2880: Provide argument for overriding temporary directory

(Attila Szabo via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/a0b730c7
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/a0b730c7
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/a0b730c7

Branch: refs/heads/trunk
Commit: a0b730c77e297a62909063289ef37a2b993ff5e1
Parents: 152fe93
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Mon Mar 14 09:21:16 2016 -0700
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Mon Mar 14 09:21:16 2016 -0700

----------------------------------------------------------------------
 src/java/org/apache/sqoop/SqoopOptions.java     | 16 +++++++++
 .../org/apache/sqoop/tool/BaseSqoopTool.java    | 11 ++++++
 src/java/org/apache/sqoop/tool/ImportTool.java  |  2 +-
 src/java/org/apache/sqoop/util/AppendUtils.java |  7 ++--
 .../com/cloudera/sqoop/TestSqoopOptions.java    | 36 ++++++++++++++++++++
 .../cloudera/sqoop/orm/TestParseMethods.java    | 10 ++++++
 6 files changed, 76 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 17751c7..ff96280 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -55,6 +55,8 @@ import com.cloudera.sqoop.util.StoredAsProperty;
  */
 public class SqoopOptions implements Cloneable {
 
+  private static final String OLD_SQOOP_TEST_IMPORT_ROOT_DIR = "sqoop.test.import.rootDir";
+
   public static final Log LOG = LogFactory.getLog(SqoopOptions.class.getName());
 
   /**
@@ -106,6 +108,8 @@ public class SqoopOptions implements Cloneable {
 
   @StoredAsProperty("verbose") private boolean verbose;
 
+  @StoredAsProperty("temporary.dirRoot") private String tempRootDir;
+
   @StoredAsProperty("mapreduce.job.name") private String mapreduceJobName;
 
   @StoredAsProperty("db.connect.string") private String connectString;
@@ -1007,6 +1011,10 @@ public class SqoopOptions implements Cloneable {
 
     // We do not want to be verbose too much if not explicitly needed
     this.verbose = false;
+    //This name of the system property is intentionally OLD_SQOOP_TEST_IMPORT_ROOT_DIR
+    //to support backward compatibility. Do not exchange it with
+    //org.apache.sqoop.tool.BaseSqoopTool#TEMP_ROOTDIR_ARG
+    this.tempRootDir = System.getProperty(OLD_SQOOP_TEST_IMPORT_ROOT_DIR, "_sqoop");
     this.isValidationEnabled = false; // validation is disabled by default
     this.validatorClass = RowCountValidator.class;
     this.validationThresholdClass = AbsoluteValidationThreshold.class;
@@ -1110,6 +1118,14 @@ public class SqoopOptions implements Cloneable {
     this.verbose = beVerbose;
   }
 
+  public String getTempRootDir() {
+    return tempRootDir;
+  }
+
+  public void setTempRootDir(String tempRootDir) {
+    this.tempRootDir = tempRootDir;
+  }
+
   /**
    * Get the temporary directory; guaranteed to end in File.separator
    * (e.g., '/').

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 50dd67d..fecdf43 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -162,6 +162,7 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
   public static final String SQL_QUERY_SHORT_ARG = "e";
   public static final String VERBOSE_ARG = "verbose";
   public static final String HELP_ARG = "help";
+  public static final String TEMP_ROOTDIR_ARG = "temporary-rootdir";
   public static final String UPDATE_KEY_ARG = "update-key";
   public static final String UPDATE_MODE_ARG = "update-mode";
   public static final String CALL_ARG = "call";
@@ -467,6 +468,12 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
         .withDescription("Print usage instructions")
         .withLongOpt(HELP_ARG)
         .create());
+    commonOpts.addOption(OptionBuilder
+        .withDescription("Defines the temporary root directory for the import")
+        .withLongOpt(TEMP_ROOTDIR_ARG)
+        .hasArg()
+        .withArgName("rootdir")
+        .create());
     // relax isolation requirements
     commonOpts.addOption(OptionBuilder
         .withDescription("Use read-uncommitted isolation for imports")
@@ -939,6 +946,10 @@ public abstract class BaseSqoopTool extends com.cloudera.sqoop.tool.SqoopTool
{
       throw new InvalidOptionsException("");
     }
 
+    if (in.hasOption(TEMP_ROOTDIR_ARG)) {
+      out.setTempRootDir(in.getOptionValue(TEMP_ROOTDIR_ARG));
+    }
+
     if (in.hasOption(CONNECT_STRING_ARG)) {
       out.setConnectString(in.getOptionValue(CONNECT_STRING_ARG));
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index ad1d48b..ff7b822 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -571,7 +571,7 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
       if(salt == null && options.getSqlQuery() != null) {
         salt = Integer.toHexString(options.getSqlQuery().hashCode());
       }
-      outputPath = AppendUtils.getTempAppendDir(salt);
+      outputPath = AppendUtils.getTempAppendDir(salt, options);
       LOG.debug("Using temporary folder: " + outputPath.getName());
     } else {
       // Try in this order: target-dir or warehouse-dir

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/java/org/apache/sqoop/util/AppendUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java
index 0102ee7..a3082c4 100644
--- a/src/java/org/apache/sqoop/util/AppendUtils.java
+++ b/src/java/org/apache/sqoop/util/AppendUtils.java
@@ -39,9 +39,6 @@ public class AppendUtils {
 
   public static final Log LOG = LogFactory.getLog(AppendUtils.class.getName());
 
-  private static final String TEMP_IMPORT_ROOT =
-          System.getProperty("sqoop.test.import.rootDir", "_sqoop");
-
   private static final int PARTITION_DIGITS = 5;
   private static final String FILEPART_SEPARATOR = "-";
   private static final String FILEEXT_SEPARATOR = ".";
@@ -273,9 +270,9 @@ public class AppendUtils {
    *             Can be arbitrary string, for example table name or query checksum.
    * @return a path pointing to the temporary directory
    */
-  public static Path getTempAppendDir(String salt) {
+  public static Path getTempAppendDir(String salt, SqoopOptions options) {
     String uuid = UUID.randomUUID().toString().replace("-", "");
-    String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + uuid + "_" + salt;
+    String tempDir = options.getTempRootDir() + Path.SEPARATOR + uuid + "_" + salt;
     return new Path(tempDir);
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/test/com/cloudera/sqoop/TestSqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestSqoopOptions.java b/src/test/com/cloudera/sqoop/TestSqoopOptions.java
index f4660e5..f9d1d54 100644
--- a/src/test/com/cloudera/sqoop/TestSqoopOptions.java
+++ b/src/test/com/cloudera/sqoop/TestSqoopOptions.java
@@ -390,6 +390,42 @@ public class TestSqoopOptions extends TestCase {
             connParams, in.getConnectionParams());
   }
 
+  public void testDefaultTempRootDir() {
+    SqoopOptions opts = new SqoopOptions();
+
+    assertEquals("_sqoop", opts.getTempRootDir());
+  }
+
+  public void testDefaultLoadedTempRootDir() {
+    SqoopOptions out = new SqoopOptions();
+    Properties props = out.writeProperties();
+    SqoopOptions opts = new SqoopOptions();
+    opts.loadProperties(props);
+
+    assertEquals("_sqoop", opts.getTempRootDir());
+  }
+
+  public void testLoadedTempRootDir() {
+    SqoopOptions out = new SqoopOptions();
+    final String tempRootDir = "customRoot";
+    out.setTempRootDir(tempRootDir);
+    Properties props = out.writeProperties();
+    SqoopOptions opts = new SqoopOptions();
+    opts.loadProperties(props);
+
+    assertEquals(tempRootDir, opts.getTempRootDir());
+  }
+
+  public void testNulledTempRootDir() {
+    SqoopOptions out = new SqoopOptions();
+    out.setTempRootDir(null);
+    Properties props = out.writeProperties();
+    SqoopOptions opts = new SqoopOptions();
+    opts.loadProperties(props);
+
+    assertEquals("_sqoop", opts.getTempRootDir());
+  }
+
   // test that hadoop-home is accepted as an option
   public void testHadoopHome() throws Exception {
     String [] args = {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a0b730c7/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
index f222dd7..cec7614 100644
--- a/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
+++ b/src/test/com/cloudera/sqoop/orm/TestParseMethods.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.mapred.FileInputFormat;
 import org.apache.hadoop.mapred.FileOutputFormat;
 import org.apache.hadoop.mapred.JobClient;
 import org.apache.hadoop.mapred.JobConf;
+import org.apache.sqoop.tool.BaseSqoopTool;
 
 import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.SqoopOptions.InvalidOptionsException;
@@ -86,6 +87,15 @@ public class TestParseMethods extends ImportJobTestCase {
     return args.toArray(new String[0]);
   }
 
+  public void testTemporaryRootDirParse() throws Exception {
+    String customRoot = "customroot";
+    String[] args = new String[] {"--"+BaseSqoopTool.TEMP_ROOTDIR_ARG, customRoot};
+
+    SqoopOptions opts = new ImportTool().parseArguments(args, null, null, true);
+
+    assertEquals(customRoot, opts.getTempRootDir());
+  }
+
   public void runParseTest(String fieldTerminator, String lineTerminator,
       String encloser, String escape, boolean encloseRequired)
       throws IOException {


Mime
View raw message