sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From b...@apache.org
Subject [1/5] git commit: SQOOP-658 Solve hadoop dependency distribution/loading (Jarek Jarcec Cecho)
Date Tue, 06 Nov 2012 20:53:35 GMT
Updated Branches:
  refs/heads/sqoop2 0976713f0 -> 6c20aa4e6


SQOOP-658 Solve hadoop dependency distribution/loading
(Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/6c20aa4e
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/6c20aa4e
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/6c20aa4e

Branch: refs/heads/sqoop2
Commit: 6c20aa4e641bf384c44b0749939213f4c4ae8e47
Parents: 06e054b
Author: Bilung Lee <blee@apache.org>
Authored: Tue Nov 6 12:45:01 2012 -0800
Committer: Bilung Lee <blee@apache.org>
Committed: Tue Nov 6 12:45:01 2012 -0800

----------------------------------------------------------------------
 dist/src/main/bin/addtowar.sh |  308 ++++++++++++++++++++++++++++++++++++
 1 files changed, 308 insertions(+), 0 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/6c20aa4e/dist/src/main/bin/addtowar.sh
----------------------------------------------------------------------
diff --git a/dist/src/main/bin/addtowar.sh b/dist/src/main/bin/addtowar.sh
new file mode 100644
index 0000000..e5c4a85
--- /dev/null
+++ b/dist/src/main/bin/addtowar.sh
@@ -0,0 +1,308 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Creating temporary directory
+function prepare() {
+  tmpDir=/tmp/sqoop-war-packing-$$
+  rm -rf ${tmpDir}
+  mkdir ${tmpDir}
+  tmpWarDir=${tmpDir}/sqoop-war
+  mkdir ${tmpWarDir}
+  checkExec "Creating staging directory ${tmpDir}"
+}
+
+# Cleans up temporary directory
+function cleanUp() {
+  if [ ! "${tmpDir}" = "" ]; then
+    rm -rf ${tmpDir}
+    checkExec "Deleting staging directory ${tmpDir}"
+  fi
+}
+
+# Check execution of command
+function checkExec() {
+  if [ $? -ne 0 ]
+  then
+    echo
+    echo "Failed: $1"
+    echo
+    cleanUp
+    exit -1;
+  fi
+}
+
+# Check that a file/path exists
+function checkFileExists() {
+  if [ ! -e ${1} ]; then
+    echo
+    echo "File/Dir does no exist: ${1}"
+    echo
+    cleanUp
+    exit -1
+  fi
+}
+
+# Check that a file/path does not exist
+function checkFileDoesNotExist() {
+  if [ -e ${1} ]; then
+    echo
+    echo "File/Dir already exists: ${1}"
+    echo
+    cleanUp
+    exit -1
+  fi
+}
+
+# Finds a file under a directory any depth, file returns in variable RET
+function findFile() {
+   RET=`find -H ${1} -name ${2} | grep -e "[0-9.a${hadoopJarsSuffix}].jar"`
+   RET=`echo ${RET} | sed "s/ .*//"`
+   if [ "${RET}" = "" ]; then
+     echo
+     echo "File '${2}' not found in '${1}'"
+     echo
+     cleanUp
+     exit -1;
+   fi
+}
+
+function checkOption() {
+  if [ "$2" = "" ]; then
+    echo
+    echo "Missing option: ${1}"
+    echo
+    printUsage
+    exit -1
+  fi
+}
+
+# Get the list of hadoop jars that will be injected based on the hadoop version
+# TODO(jarcec): Add configuration specific to Hadoop 1.x
+function getHadoopJars() {
+  version=$1
+#   Commented distributions are not tested
+#  if [ "${version}" = "0.20.1" ]; then
+#    #List is separated by ":"
+#    hadoopJars="hadoop-core*.jar"
+#  elif [ "${version}" = "0.20.2" ]; then
+#    #List is separated by ":"
+#    hadoopJars="hadoop-core*.jar"
+#  elif [ "${version}" = "0.20.104" ]; then
+#    #List is separated by ":"
+#    hadoopJars="hadoop-core*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar"
+#  elif [ "${version}" = "0.20.200" ]; then
+#    #List is separated by ":"
+#    hadoopJars="hadoop-core*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar"
+#  elif [[ "${version}" =~ .*23 ]]; then
+#    suffix="-[0-9.]*"
+#    #List is separated by ":"
+#    hadoopJars="hadoop-mapreduce-client-core${suffix}.jar:hadoop-mapreduce-client-common${suffix}.jar:hadoop-mapreduce-client-jobclient${suffix}.jar:hadoop-mapreduce-client-app${suffix}.jar:hadoop-yarn-common${suffix}.jar:hadoop-yarn-api${suffix}.jar:hadoop-hdfs${suffix}.jar:hadoop-common${suffix}.jar:hadoop-auth${suffix}.jar:guava*.jar:protobuf-*.jar:avro-ipc-*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar"
+#  elif [[ "${version}" =~ 2.* ]]; then
+  if [[ "${version}" =~ 2.* ]]; then
+    suffix="-[0-9.]*"
+    # List is separated by ":"
+    # Removed hadoop-yarn-client${suffix}.jar, my distribution do not have such artifact?
+    hadoopJars="hadoop-mapreduce-client-core${suffix}.jar:hadoop-mapreduce-client-common${suffix}.jar:hadoop-mapreduce-client-jobclient${suffix}.jar:hadoop-mapreduce-client-app${suffix}.jar:hadoop-yarn-common${suffix}.jar:hadoop-yarn-api${suffix}.jar:hadoop-hdfs${suffix}.jar:hadoop-common${suffix}.jar:hadoop-auth${suffix}.jar:guava*.jar:protobuf-*.jar:jackson-core-asl-*.jar:jackson-mapper-asl-*.jar:commons-configuration-*.jar:commons-cli-*.jar:commons-logging-*.jar:slf4j-api-*.jar:slf4j-log4j*.jar:avro-*.jar"
+  else
+    echo
+    echo "Exiting: Unsupported Hadoop version '${hadoopVer}', supported versions: 2.x"
+#    echo "Exiting: Unsupported Hadoop version '${hadoopVer}', supported versions: 0.20.1,
0.20.2, 0.20.104, 0.20.200, 0.23.x and 2.x"
+    echo
+    cleanUp
+    exit -1;
+  fi
+}
+
+function printUsage() {
+  echo " Usage  : addtowar.sh <OPTIONS>"
+  echo " Options: -hadoop HADOOP_VERSION HADOOP_PATH"
+  echo "          [-jars JARS_PATH] (multiple JAR path separated by ':')"
+  echo "          [-war SQOOP_WAR]"
+  echo
+}
+
+# We need at least some arguments
+if [ $# -eq 0 ]; then
+  echo
+  echo "Missing options"
+  echo
+  printUsage
+  exit -1
+fi
+
+# Variables that will be populated by our command line arguments
+addHadoop=""
+addJars=""
+hadoopVersion=""
+hadoopHome=""
+jarsPath=""
+warPath="`dirname $0`/../server/webapps/sqoop.war"
+
+# Parse command line arguments
+while [ $# -gt 0 ]
+do
+  if [ "$1" = "-hadoop" ]; then
+    shift
+    if [ $# -eq 0 ]; then
+      echo
+      echo "Missing option value, Hadoop version"
+      echo
+      printUsage
+      exit -1
+    fi
+    hadoopVersion=$1
+    shift
+    if [ $# -eq 0 ]; then
+      echo
+      echo "Missing option value, Hadoop path"
+      echo
+      printUsage
+      exit -1
+    fi
+    hadoopHome=$1
+    addHadoop=true
+  elif [ "$1" = "-jars" ]; then
+    shift
+    if [ $# -eq 0 ]; then
+      echo
+      echo "Missing option value, JARs path"
+      echo
+      printUsage
+      exit -1
+    fi
+    jarsPath=$1
+    addJars=true
+  elif [ "$1" = "-war" ]; then
+    shift
+    if [ $# -eq 0 ]; then
+      echo
+      echo "Missing option value, Input Sqoop WAR path"
+      echo
+      printUsage
+      exit -1
+    fi
+    warPath=$1
+  fi
+
+  shift
+done
+
+# Check that we have something to do
+if [ "${addHadoop}${addJars}" == "" ]; then
+  echo
+  echo "Nothing to do"
+  echo
+  printUsage
+  exit -1
+fi
+
+prepare
+
+checkOption "-war" ${warPath}
+checkFileExists ${warPath}
+
+if [ "${addHadoop}" = "true" ]; then
+  checkFileExists ${hadoopHome}
+  getHadoopJars ${hadoopVersion}
+fi
+
+if [ "${addJars}" = "true" ]; then
+  for jarPath in ${jarsPath//:/$'\n'}
+  do
+    checkFileExists ${jarPath}
+  done
+fi
+
+# Unpacking original war
+unzip ${warPath} -d ${tmpWarDir} > /dev/null
+checkExec "Unzipping Sqoop WAR"
+
+components=""
+
+# Adding hadoop binaries to WAR file
+if [ "${addHadoop}" = "true" ]; then
+  components="Hadoop JARs";
+  found=`ls ${tmpWarDir}/WEB-INF/lib/hadoop*core*jar 2> /dev/null | wc -l`
+  checkExec "Looking for Hadoop JARs in WAR file"
+  if [ ! $found = 0 ]; then
+    echo
+    echo "Specified Sqoop WAR '${warPath}' already contains Hadoop JAR files"
+    echo
+    cleanUp
+    exit -1
+  fi
+  ## adding hadoop
+  echo "Injecting following Hadoop JARs"
+  echo
+  for jar in ${hadoopJars//:/$'\n'}
+  do
+    findFile ${hadoopHome} ${jar}
+    jar=${RET}
+    echo ${jar}
+    cp ${jar} ${tmpWarDir}/WEB-INF/lib/
+    checkExec "Copying jar ${jar} to staging"
+  done
+fi
+
+# Adding new jars to WAR file
+if [ "${addJars}" = "true" ]; then
+  if [ ! "${components}" = "" ];then
+    components="${components}, "
+  fi
+  components="${components}JARs"
+
+  for jarPath in ${jarsPath//:/$'\n'}
+  do
+    found=`ls ${tmpWarDir}/WEB-INF/lib/${jarPath} 2> /dev/null | wc -l`
+    checkExec "Looking for JAR ${jarPath} in WAR path"
+    if [ ! $found = 0 ]; then
+      echo
+      echo "Specified Sqoop WAR '${inputWar}' already contains JAR ${jarPath}"
+      echo
+      cleanUp
+      exit -1
+    fi
+    cp ${jarPath} ${tmpWarDir}/WEB-INF/lib/
+    checkExec "Copying jar ${jarPath} to staging"
+  done
+fi
+
+# Creating new Sqoop WAR
+currentDir=`pwd`
+cd ${tmpWarDir}
+zip -r sqoop.war * > /dev/null
+checkExec "Creating new Sqoop WAR"
+cd ${currentDir}
+
+# Save original WAR file as a backup in case that something went wrong
+backupPath="${warPath}_`date +%Y-%m-%d_%H:%M:%S.%N`"
+echo
+echo "Backing up original WAR file to $backupPath"
+mv $warPath $backupPath
+checkExec "Backing up original WAR file to $backupPath"
+
+# Move our jar to new position
+mv ${tmpWarDir}/sqoop.war ${warPath}
+checkExec "Moving generated WAR file to original location"
+
+echo
+echo "New Sqoop WAR file with added '${components}' at ${warPath}"
+echo
+cleanUp
+exit 0


Mime
View raw message