sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject sqoop git commit: SQOOP-3042: Sqoop does not clear compile directory under /tmp/sqoop-<username>/compile automatically
Date Thu, 12 Jul 2018 14:54:56 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk b790c606a -> 0cfbf5671


SQOOP-3042: Sqoop does not clear compile directory under /tmp/sqoop-<username>/compile
automatically

(Eric Lin via Szabolcs Vasas)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/0cfbf567
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/0cfbf567
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/0cfbf567

Branch: refs/heads/trunk
Commit: 0cfbf56713f7574568ea3754f6854e82f5540954
Parents: b790c60
Author: Szabolcs Vasas <vasas@apache.org>
Authored: Thu Jul 12 16:26:33 2018 +0200
Committer: Szabolcs Vasas <vasas@apache.org>
Committed: Thu Jul 12 16:26:33 2018 +0200

----------------------------------------------------------------------
 src/docs/man/codegen-args.txt                   |  3 ++
 src/docs/user/common-args.txt                   |  2 +
 src/java/org/apache/sqoop/SqoopOptions.java     | 14 ++++++
 src/java/org/apache/sqoop/orm/ClassWriter.java  |  7 +++
 .../apache/sqoop/orm/CompilationManager.java    | 13 ++---
 .../org/apache/sqoop/tool/BaseSqoopTool.java    |  9 ++++
 .../org/apache/sqoop/util/DirCleanupHook.java   | 52 ++++++++++++++++++++
 .../apache/sqoop/util/TestDirCleanupHook.java   | 48 ++++++++++++++++++
 8 files changed, 142 insertions(+), 6 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/docs/man/codegen-args.txt
----------------------------------------------------------------------
diff --git a/src/docs/man/codegen-args.txt b/src/docs/man/codegen-args.txt
index c1afaef..adee7d9 100644
--- a/src/docs/man/codegen-args.txt
+++ b/src/docs/man/codegen-args.txt
@@ -34,6 +34,9 @@ Code generation options
 --outdir (dir)::
   Output directory for generated code
 
+--delete-compile-dir::
+  Remove temporarily generated class Jar files after job finishes
+
 --package-name (package)::
   Puts auto-generated classes in the named Java package
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/docs/user/common-args.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/common-args.txt b/src/docs/user/common-args.txt
index 98f19be..6d7560a 100644
--- a/src/docs/user/common-args.txt
+++ b/src/docs/user/common-args.txt
@@ -34,6 +34,8 @@ Argument                                  Description
 +-P+                                      Read password from console
 +\--password <password>+                  Set authentication password
 +\--username <username>+                  Set authentication username
++\--delete-compile-dir+                   Remove temporarily generated class Jar\
+                                          files after job finishes
 +\--verbose+                              Print more information while working
 +\--connection-param-file <filename>+     Optional properties file that\
                                           provides connection parameters

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 3a19aea..cc1b752 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -204,6 +204,7 @@ public class SqoopOptions implements Cloneable {
 
   @StoredAsProperty("codegen.output.dir") private String codeOutputDir;
   @StoredAsProperty("codegen.compile.dir") private String jarOutputDir;
+  @StoredAsProperty("codegen.delete.compile.dir") private boolean deleteJarOutputDir;
   // Boolean specifying whether jarOutputDir is a nonce tmpdir (true), or
   // explicitly set by the user (false). If the former, disregard any value
   // for jarOutputDir saved in the metastore.
@@ -1086,6 +1087,8 @@ public class SqoopOptions implements Cloneable {
     this.jarOutputDir = getNonceJarDir(tmpDir + "sqoop-" + localUsername
         + "/compile");
     this.jarDirIsAuto = true;
+    // Default to false, so behaviour does not change
+    this.deleteJarOutputDir = false;
     this.layout = FileLayout.TextFile;
 
     this.areOutputDelimsManuallySet = false;
@@ -1707,6 +1710,17 @@ public class SqoopOptions implements Cloneable {
   }
 
   /**
+   * @return boolean - whether or not to delete the compile JAR directory
+   */
+  public Boolean getDeleteJarOutputDir() {
+    return this.deleteJarOutputDir;
+  }
+
+  public void setDeleteJarOutputDir(Boolean delete) {
+    this.deleteJarOutputDir = delete;
+  }
+
+  /**
    * Return the value of $HADOOP_MAPRED_HOME.
    * @return $HADOOP_MAPRED_HOME, or null if it's not set.
    */

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/orm/ClassWriter.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/ClassWriter.java b/src/java/org/apache/sqoop/orm/ClassWriter.java
index a4a768a..46d0698 100644
--- a/src/java/org/apache/sqoop/orm/ClassWriter.java
+++ b/src/java/org/apache/sqoop/orm/ClassWriter.java
@@ -51,6 +51,7 @@ import org.apache.sqoop.lib.LobSerializer;
 import org.apache.sqoop.lib.RecordParser;
 import org.apache.sqoop.lib.SqoopRecord;
 import org.apache.sqoop.manager.ConnManager;
+import org.apache.sqoop.util.DirCleanupHook;
 
 /**
  * Creates an ORM class to represent a table from a database.
@@ -1785,6 +1786,12 @@ public class ClassWriter {
       LOG.debug("sourceFilename is " + sourceFilename);
     }
 
+    // SQOOP-3042 - Sqoop does not clear compile directory under /tmp/sqoop-<username>/compile
+    // Add shutdown hook so that all files under the jar output dir can be cleared
+    if(options.getDeleteJarOutputDir()) {
+      Runtime.getRuntime().addShutdownHook(new DirCleanupHook(options.getJarOutputDir()));
+    }
+
     compileManager.addSourceFile(sourceFilename);
 
     // Create any missing parent directories.

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/orm/CompilationManager.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/orm/CompilationManager.java b/src/java/org/apache/sqoop/orm/CompilationManager.java
index 6590cac..f0f5de7 100644
--- a/src/java/org/apache/sqoop/orm/CompilationManager.java
+++ b/src/java/org/apache/sqoop/orm/CompilationManager.java
@@ -245,13 +245,14 @@ public class CompilationManager {
         }
       }
       try {
-          FileUtils.moveFile(fOrig, fDest);
+        LOG.debug("moving file from " + fOrig.getAbsolutePath() + " to " + fDest.getAbsolutePath());
+        FileUtils.moveFile(fOrig, fDest);
       } catch (IOException e) {
-    	  /*Removed the exception being thrown
-    	   *even if the .java file can not be renamed
-    	   *or can not be moved a "dest" directory for
-    	   *any reason.*/
-          LOG.debug("Could not rename " + orig + " to " + dest); 
+        /*Removed the exception being thrown
+         *even if the .java file can not be renamed
+         *or can not be moved a "dest" directory for
+         *any reason.*/
+        LOG.error("Could not rename " + orig + " to " + dest + ". Error: " + e.getMessage());
       }
     }
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
index 8d31832..e505c26 100644
--- a/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
+++ b/src/java/org/apache/sqoop/tool/BaseSqoopTool.java
@@ -102,6 +102,7 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool
{
   public static final String TARGET_DIR_ARG = "target-dir";
   public static final String APPEND_ARG = "append";
   public static final String DELETE_ARG = "delete-target-dir";
+  public static final String DELETE_COMPILE_ARG = "delete-compile-dir";
   public static final String NULL_STRING = "null-string";
   public static final String INPUT_NULL_STRING = "input-null-string";
   public static final String NULL_NON_STRING = "null-non-string";
@@ -805,6 +806,10 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool
{
         .withDescription("Output directory for compiled objects")
         .withLongOpt(BIN_OUT_DIR_ARG)
         .create());
+    codeGenOpts.addOption(OptionBuilder
+        .withDescription("Delete compile directory")
+        .withLongOpt(DELETE_COMPILE_ARG)
+        .create());
     codeGenOpts.addOption(OptionBuilder.withArgName("name")
         .hasArg()
         .withDescription("Put auto-generated classes in this package")
@@ -1423,6 +1428,10 @@ public abstract class BaseSqoopTool extends org.apache.sqoop.tool.SqoopTool
{
       out.setJarOutputDir(in.getOptionValue(BIN_OUT_DIR_ARG));
     }
 
+    if (in.hasOption(DELETE_COMPILE_ARG)) {
+      out.setDeleteJarOutputDir(true);
+    }
+
     if (in.hasOption(PACKAGE_NAME_ARG)) {
       out.setPackageName(in.getOptionValue(PACKAGE_NAME_ARG));
     }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/java/org/apache/sqoop/util/DirCleanupHook.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/util/DirCleanupHook.java b/src/java/org/apache/sqoop/util/DirCleanupHook.java
new file mode 100644
index 0000000..120bf0d
--- /dev/null
+++ b/src/java/org/apache/sqoop/util/DirCleanupHook.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.util;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * This Hook is currently used to clean up the temp directory that is used
+ * to hold the generated JAR files for each table class under /tmp/sqoop-<username>/compile.
+ *
+ * But it is generic so it can also be used to clean up other directories if needed
+ */
+public class DirCleanupHook extends Thread {
+
+  private File dir;
+
+  public static final Log LOG = LogFactory.getLog(DirCleanupHook.class.getName());
+
+  public DirCleanupHook(String dirPath) {
+    dir = new File(dirPath);
+  }
+
+  public void run() {
+    try {
+      LOG.debug("Removing directory: " + dir + " in the clean up hook.");
+      FileUtils.deleteDirectory(dir);
+    } catch (IOException e) {
+      LOG.error("Unable to remove directory: " + dir + ". Error was: " + e.getMessage());
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0cfbf567/src/test/org/apache/sqoop/util/TestDirCleanupHook.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/util/TestDirCleanupHook.java b/src/test/org/apache/sqoop/util/TestDirCleanupHook.java
new file mode 100644
index 0000000..41146fd
--- /dev/null
+++ b/src/test/org/apache/sqoop/util/TestDirCleanupHook.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.util;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.junit.Assert.assertFalse;
+
+public class TestDirCleanupHook {
+  @Rule
+  public TemporaryFolder tmpFolder = new TemporaryFolder();
+
+  private DirCleanupHook dirCleanupHook;
+
+  @Before
+  public void before() throws Exception {
+    // Make sure the directory is not empty.
+    tmpFolder.newFile();
+
+    dirCleanupHook = new DirCleanupHook(tmpFolder.getRoot().getAbsolutePath());
+  }
+
+  @Test
+  public void testDirCleanup() {
+    dirCleanupHook.run();
+
+    assertFalse(tmpFolder.getRoot().exists());
+  }
+}


Mime
View raw message