sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From venkatran...@apache.org
Subject git commit: SQOOP-1273: Multiple append jobs can easily end up sharing directories (Jarek Jarcec Cecho via Venkat Ranganathan)
Date Wed, 29 Jan 2014 19:30:19 GMT
Updated Branches:
  refs/heads/trunk da5ab302e -> ad12695b5


SQOOP-1273: Multiple append jobs can easily end up sharing directories
(Jarek Jarcec Cecho via Venkat Ranganathan)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ad12695b
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ad12695b
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ad12695b

Branch: refs/heads/trunk
Commit: ad12695b59e7f0af09e27da8dca08e9a2be9b6a2
Parents: da5ab30
Author: Venkat Ranganathan <venkat@hortonworks.com>
Authored: Wed Jan 29 11:29:47 2014 -0800
Committer: Venkat Ranganathan <venkat@hortonworks.com>
Committed: Wed Jan 29 11:29:47 2014 -0800

----------------------------------------------------------------------
 src/java/org/apache/sqoop/tool/ImportTool.java  |  6 +++-
 src/java/org/apache/sqoop/util/AppendUtils.java | 11 ++++---
 .../com/cloudera/sqoop/TestAppendUtils.java     | 32 +++++++++++++++-----
 3 files changed, 37 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/java/org/apache/sqoop/tool/ImportTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/ImportTool.java b/src/java/org/apache/sqoop/tool/ImportTool.java
index 59279b6..6cbb873 100644
--- a/src/java/org/apache/sqoop/tool/ImportTool.java
+++ b/src/java/org/apache/sqoop/tool/ImportTool.java
@@ -458,7 +458,11 @@ public class ImportTool extends com.cloudera.sqoop.tool.BaseSqoopTool
{
     Path outputPath = null;
     if (options.isAppendMode()) {
       // Use temporary path, later removed when appending
-      outputPath = AppendUtils.getTempAppendDir(tableName);
+      String salt = tableName;
+      if(salt == null && options.getSqlQuery() != null) {
+        salt = Integer.toHexString(options.getSqlQuery().hashCode());
+      }
+      outputPath = AppendUtils.getTempAppendDir(salt);
       LOG.debug("Using temporary folder: " + outputPath.getName());
     } else {
       // Try in this order: target-dir or warehouse-dir

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/java/org/apache/sqoop/util/AppendUtils.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/AppendUtils.java b/src/java/org/apache/sqoop/util/AppendUtils.java
index d7cd6c5..f5a9a60 100644
--- a/src/java/org/apache/sqoop/util/AppendUtils.java
+++ b/src/java/org/apache/sqoop/util/AppendUtils.java
@@ -19,6 +19,7 @@
 package org.apache.sqoop.util;
 
 import java.io.IOException;
+import java.lang.management.ManagementFactory;
 import java.text.NumberFormat;
 import java.text.SimpleDateFormat;
 import java.util.Date;
@@ -41,7 +42,7 @@ public class AppendUtils {
   public static final Log LOG = LogFactory.getLog(AppendUtils.class.getName());
 
   private static final SimpleDateFormat DATE_FORM = new SimpleDateFormat(
-      "ddHHmmssSSS");
+      "ddHHmmssSSSSSSSSS");
   private static final String TEMP_IMPORT_ROOT =
           System.getProperty("sqoop.test.import.rootDir", "_sqoop");
 
@@ -263,12 +264,14 @@ public class AppendUtils {
   /**
    * Creates a unique path object inside the sqoop temporary directory.
    *
-   * @param tableName
+   * @param salt Salt that will be appended at the end of the generated directory.
+   *             Can be arbitrary string, for example table name or query checksum.
    * @return a path pointing to the temporary directory
    */
-  public static Path getTempAppendDir(String tableName) {
+  public static Path getTempAppendDir(String salt) {
     String timeId = DATE_FORM.format(new Date(System.currentTimeMillis()));
-    String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + timeId + tableName;
+    String jvmName = ManagementFactory.getRuntimeMXBean().getName().replaceAll("@", "_");
+    String tempDir = TEMP_IMPORT_ROOT + Path.SEPARATOR + timeId + "_"  + jvmName + "_" +
salt;
     return new Path(tempDir);
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ad12695b/src/test/com/cloudera/sqoop/TestAppendUtils.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/TestAppendUtils.java b/src/test/com/cloudera/sqoop/TestAppendUtils.java
index ed66b4a..b09dfc6 100644
--- a/src/test/com/cloudera/sqoop/TestAppendUtils.java
+++ b/src/test/com/cloudera/sqoop/TestAppendUtils.java
@@ -58,7 +58,7 @@ public class TestAppendUtils extends ImportJobTestCase {
    *
    * @return the argv as an array of strings.
    */
-  protected ArrayList getOutputlessArgv(boolean includeHadoopFlags,
+  protected ArrayList getOutputlessArgv(boolean includeHadoopFlags, boolean queryBased,
       String[] colNames, Configuration conf) {
     if (null == colNames) {
       colNames = getColNames();
@@ -76,8 +76,13 @@ public class TestAppendUtils extends ImportJobTestCase {
       CommonArgs.addHadoopFlags(args);
     }
 
-    args.add("--table");
-    args.add(getTableName());
+    if(queryBased) {
+      args.add("--query");
+      args.add("SELECT * FROM " + getTableName() + " WHERE $CONDITIONS");
+    } else {
+      args.add("--table");
+      args.add(getTableName());
+    }
     args.add("--columns");
     args.add(columnsString);
     args.add("--split-by");
@@ -241,8 +246,7 @@ public class TestAppendUtils extends ImportJobTestCase {
 
   /** independent to target-dir. */
   public void testAppend() throws IOException {
-    ArrayList args = getOutputlessArgv(false, HsqldbTestServer.getFieldNames(),
-        getConf());
+    ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf());
     args.add("--warehouse-dir");
     args.add(getWarehouseDir());
 
@@ -252,8 +256,7 @@ public class TestAppendUtils extends ImportJobTestCase {
 
   /** working with target-dir. */
   public void testAppendToTargetDir() throws IOException {
-    ArrayList args = getOutputlessArgv(false, HsqldbTestServer.getFieldNames(),
-        getConf());
+    ArrayList args = getOutputlessArgv(false, false, HsqldbTestServer.getFieldNames(), getConf());
     String targetDir = getWarehouseDir() + "/tempTargetDir";
     args.add("--target-dir");
     args.add(targetDir);
@@ -265,6 +268,21 @@ public class TestAppendUtils extends ImportJobTestCase {
   }
 
   /**
+   * Query based import should also work in append mode.
+   *
+   * @throws IOException
+   */
+  public void testAppendWithQuery() throws IOException {
+    ArrayList args = getOutputlessArgv(false, true, HsqldbTestServer.getFieldNames(), getConf());
+    String targetDir = getWarehouseDir() + "/tempTargetDir";
+    args.add("--target-dir");
+    args.add(targetDir);
+
+    Path output = new Path(targetDir);
+    runAppendTest(args, output);
+  }
+
+  /**
    * If the append source does not exist, don't crash.
    */
   public void testAppendSrcDoesNotExist() throws IOException {


Mime
View raw message