sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-2635: Sqoop2: Improve ConnectorClassLoader to support loading classes from the dependencies inside the connector jar
Date Wed, 16 Dec 2015 19:50:06 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 5329c1b33 -> 857432246


SQOOP-2635: Sqoop2: Improve ConnectorClassLoader to support loading classes from the dependencies inside the connector jar

(Dian Fu via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/85743224
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/85743224
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/85743224

Branch: refs/heads/sqoop2
Commit: 857432246790c2791e7cd3a10ee21d174dfa12f9
Parents: 5329c1b
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Wed Dec 16 20:48:54 2015 +0100
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Wed Dec 16 20:48:54 2015 +0100

----------------------------------------------------------------------
 .../sqoop/classloader/ConnectorClassLoader.java | 657 +++++++++++++++++++
 .../sqoop/classloader/ConnectorURLFactory.java  |  74 +++
 .../apache/sqoop/classloader/URLFactory.java    |  29 +
 .../sqoop/utils/ConnectorClassLoader.java       | 257 --------
 ...pache.sqoop.connector-classloader.properties |   2 +
 .../classloader/TestConnectorClassLoader.java   | 311 +++++++++
 .../org/apache/sqoop/utils/TestClassUtils.java  |   1 +
 .../sqoop/utils/TestConnectorClassLoader.java   | 149 -----
 .../resources/TestConnectorClassLoader/A.java   |  20 +
 .../TestConnectorClassLoader/lib/B.java         |  20 +
 10 files changed, 1114 insertions(+), 406 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/java/org/apache/sqoop/classloader/ConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/classloader/ConnectorClassLoader.java b/common/src/main/java/org/apache/sqoop/classloader/ConnectorClassLoader.java
new file mode 100644
index 0000000..370de2a
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/classloader/ConnectorClassLoader.java
@@ -0,0 +1,657 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.classloader;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.security.CodeSource;
+import java.security.ProtectionDomain;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+import java.util.jar.Attributes;
+import java.util.jar.Attributes.Name;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarInputStream;
+import java.util.jar.Manifest;
+
+import org.apache.log4j.Logger;
+
+import sun.misc.CompoundEnumeration;
+
+/**
+ * A {@link URLClassLoader} for connector isolation. Classes from the
+ * connector JARs are loaded in preference to the parent loader.
+ */
+public class ConnectorClassLoader extends URLClassLoader {
+  /**
+   * Default value of the system classes if the user did not override them.
+   * JDK classes, sqoop classes and resources, and some select third-party
+   * classes are considered system classes, and are not loaded by the
+   * connector classloader.
+   */
+  public static final String SYSTEM_CLASSES_DEFAULT;
+
+  private static final String PROPERTIES_FILE =
+      "org.apache.sqoop.connector-classloader.properties";
+  private static final String SYSTEM_CLASSES_DEFAULT_KEY =
+      "system.classes.default";
+
+  private static final String MANIFEST = "META-INF/MANIFEST.MF";
+  private static final String CLASS = ".class";
+  private static final String LIB_PREFIX = "lib/";
+
+  private Map<String, ByteCode> byteCodeCache = new HashMap<String, ByteCode>();
+  private Set<String> jarNames = new LinkedHashSet<String>();
+  private Map<String, ProtectionDomain> pdCache = new HashMap<String, ProtectionDomain>();
+  private URLFactory urlFactory;
+
+  private static final Logger LOG = Logger.getLogger(ConnectorClassLoader.class);
+
+  private static final FilenameFilter JAR_FILENAME_FILTER =
+    new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        return name.endsWith(".jar") || name.endsWith(".JAR");
+      }
+  };
+
+  private static class ByteCode {
+    public byte[] bytes;
+    public String codebase;
+    public Manifest manifest;
+
+    public ByteCode(byte[] bytes,
+        String codebase, Manifest manifest) {
+      this.bytes = bytes;
+      this.codebase = codebase;
+      this.manifest = manifest;
+    }
+  }
+
+  static {
+    try (InputStream is = ConnectorClassLoader.class.getClassLoader()
+        .getResourceAsStream(PROPERTIES_FILE);) {
+      if (is == null) {
+        throw new ExceptionInInitializerError("properties file " +
+            PROPERTIES_FILE + " is not found");
+      }
+      Properties props = new Properties();
+      props.load(is);
+      // get the system classes default
+      String systemClassesDefault =
+          props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY);
+      if (systemClassesDefault == null) {
+        throw new ExceptionInInitializerError("property " +
+            SYSTEM_CLASSES_DEFAULT_KEY + " is not found");
+      }
+      SYSTEM_CLASSES_DEFAULT = systemClassesDefault;
+    } catch (IOException e) {
+      throw new ExceptionInInitializerError(e);
+    }
+  }
+
+  private final ClassLoader parent;
+  private final List<String> systemClasses;
+
+  public ConnectorClassLoader(URL[] urls, ClassLoader parent,
+      List<String> systemClasses, boolean overrideDefaultSystemClasses) throws IOException {
+    super(urls, parent);
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("urls: " + Arrays.toString(urls));
+      LOG.debug("system classes: " + systemClasses);
+    }
+    this.parent = parent;
+    if (parent == null) {
+      throw new IllegalArgumentException("No parent classloader!");
+    }
+    // if the caller-specified system classes are null or empty, use the default
+    this.systemClasses = new ArrayList<String>();
+    if (systemClasses != null && !systemClasses.isEmpty()) {
+      this.systemClasses.addAll(systemClasses);
+    }
+    if (!overrideDefaultSystemClasses || this.systemClasses.isEmpty()) {
+      this.systemClasses.addAll(Arrays.asList(SYSTEM_CLASSES_DEFAULT.split("\\s*,\\s*")));
+    }
+    LOG.info("system classes: " + this.systemClasses);
+
+    urlFactory = new ConnectorURLFactory(this);
+    load(urls);
+  }
+
+  public ConnectorClassLoader(String classpath, ClassLoader parent,
+      List<String> systemClasses) throws IOException {
+    this(constructUrlsFromClasspath(classpath), parent, systemClasses, true);
+  }
+
+  public ConnectorClassLoader(String classpath, ClassLoader parent,
+      List<String> systemClasses, boolean overrideDefaultSystemClasses) throws IOException {
+    this(constructUrlsFromClasspath(classpath), parent, systemClasses, overrideDefaultSystemClasses);
+  }
+
+  static URL[] constructUrlsFromClasspath(String classpath)
+      throws MalformedURLException {
+    List<URL> urls = new ArrayList<URL>();
+    for (String element : classpath.split(File.pathSeparator)) {
+      if (element.endsWith("/*")) {
+        String dir = element.substring(0, element.length() - 1);
+        File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
+        if (files != null) {
+          for (File file : files) {
+            urls.add(file.toURI().toURL());
+          }
+        }
+      } else {
+        File file = new File(element);
+        if (file.exists()) {
+          urls.add(new File(element).toURI().toURL());
+        }
+      }
+    }
+    return urls.toArray(new URL[urls.size()]);
+  }
+
+  @Override
+  public URL getResource(String name) {
+    URL url = null;
+
+    if (!isSystemClass(name, systemClasses)) {
+      url= findResource(name);
+      if (url == null && name.startsWith("/")) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Remove leading / off " + name);
+        }
+        url= findResource(name.substring(1));
+      }
+    }
+
+    if (url == null) {
+      url= parent.getResource(name);
+    }
+
+    if (url != null) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("getResource("+name+")=" + url);
+      }
+    }
+
+    return url;
+  }
+
+  @SuppressWarnings("rawtypes")
+  @Override
+  public Enumeration<URL> getResources(String name) throws IOException {
+    Enumeration[] tmp = new Enumeration[2];
+
+    if (!isSystemClass(name, systemClasses)) {
+      tmp[0]= findResources(name);
+      if (!tmp[0].hasMoreElements() && name.startsWith("/")) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug("Remove leading / off " + name);
+        }
+        tmp[0]= findResources(name.substring(1));
+      }
+    }
+
+    tmp[1]= parent.getResources(name);
+
+    return new CompoundEnumeration<>(tmp);
+  }
+
+  @Override
+  public Class<?> loadClass(String name) throws ClassNotFoundException {
+    return this.loadClass(name, false);
+  }
+
+  @Override
+  protected synchronized Class<?> loadClass(String name, boolean resolve)
+      throws ClassNotFoundException {
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("Loading class: " + name);
+    }
+
+    Class<?> c = findLoadedClass(name);
+    ClassNotFoundException ex = null;
+
+    if (c == null && !isSystemClass(name, systemClasses)) {
+      // Try to load class from this classloader's URLs. Note that this is like
+      // the servlet spec, not the usual Java 2 behaviour where we ask the
+      // parent to attempt to load first.
+      try {
+        c = findClass(name);
+        if (LOG.isDebugEnabled() && c != null) {
+          LOG.debug("Loaded class: " + name + " ");
+        }
+      } catch (ClassNotFoundException e) {
+        if (LOG.isDebugEnabled()) {
+          LOG.debug(e);
+        }
+        ex = e;
+      }
+    }
+
+    if (c == null) { // try parent
+      c = parent.loadClass(name);
+      if (LOG.isDebugEnabled() && c != null) {
+        LOG.debug("Loaded class from parent: " + name + " ");
+      }
+    }
+
+    if (c == null) {
+      throw ex != null ? ex : new ClassNotFoundException(name);
+    }
+
+    if (resolve) {
+      resolveClass(c);
+    }
+
+    return c;
+  }
+
+  @Override
+  protected Class<?> findClass(String name) throws ClassNotFoundException {
+    // Make sure not to load duplicate classes.
+    Class<?> cls = findLoadedClass(name);
+    if (cls != null) {
+      return cls;
+    }
+
+    // Look up the class in the byte codes.
+    String cache = name.replace('.', '/') + CLASS;
+    cache = resolve(cache);
+    if (cache != null) {
+      ByteCode bytecode = byteCodeCache.get(cache);
+      // Use a protectionDomain to associate the codebase with the class.
+      ProtectionDomain pd = pdCache.get(bytecode.codebase);
+      if (pd == null) {
+        try {
+          URL url = urlFactory.getCodeBase(bytecode.codebase);
+          CodeSource source = new CodeSource(url, (java.security.cert.Certificate[]) null);
+          pd = new ProtectionDomain(source, null, this, null);
+          pdCache.put(bytecode.codebase, pd);
+        } catch (MalformedURLException mux) {
+          throw new ClassNotFoundException(name, mux);
+        }
+      }
+
+      byte bytes[] = bytecode.bytes;
+      int i = name.lastIndexOf('.');
+      if (i != -1) {
+        String pkgname = name.substring(0, i);
+        // Check if package already loaded.
+        Package pkg = getPackage(pkgname);
+        Manifest man = bytecode.manifest;
+        if (pkg != null) {
+          // Package found, so check package sealing.
+          if (pkg.isSealed()) {
+            // Verify that code source URL is the same.
+            if (!pkg.isSealed(pd.getCodeSource().getLocation())) {
+              throw new SecurityException("sealing violation: package " + pkgname + " is sealed");
+            }
+          } else {
+            // Make sure we are not attempting to seal the package at this code source URL.
+            if ((man != null) && isSealed(pkgname, man)) {
+              throw new SecurityException("sealing violation: can't seal package " + pkgname + ": already loaded");
+            }
+          }
+        } else {
+          if (man != null) {
+            definePackage(pkgname, man, pd.getCodeSource().getLocation());
+          } else {
+            definePackage(pkgname, null, null, null, null, null, null, null);
+          }
+        }
+      }
+
+      return defineClass(name, bytes, 0, bytes.length, pd);
+    }
+
+    return null;
+  }
+
+  @Override
+  public URL findResource(final String name) {
+    LOG.debug("Finding resource: " + name);
+
+    try {
+      // Do we have the named resource in our cache? If so, construct a 'sqoopconnector:' URL.
+      String resource = resolve(name);
+      if (resource != null) {
+        ByteCode entry = byteCodeCache.get(resource);
+        return urlFactory.getURL(entry.codebase, name);
+      }
+    } catch (MalformedURLException mux) {
+      LOG.debug("Unable to find resource: " + name + " due to " + mux);
+    }
+
+    return null;
+  }
+
+  @Override
+  @edu.umd.cs.findbugs.annotations.SuppressWarnings("SIC_INNER_SHOULD_BE_STATIC_ANON")
+  public Enumeration<URL> findResources(final String name) throws IOException {
+    LOG.debug("Finding resources: " + name);
+
+    final List<URL> resources = new ArrayList<URL>();
+    ByteCode entry = byteCodeCache.get(name);
+    if (entry != null) {
+      URL url = urlFactory.getURL(entry.codebase, name);
+      LOG.debug("Adding " + url + " to resources list for " + name);
+      resources.add(url);
+    }
+
+    Iterator<String> jarNameIter = jarNames.iterator();
+    while (jarNameIter.hasNext()) {
+      String resource = jarNameIter.next() + "/" + name;
+      entry = byteCodeCache.get(resource);
+      if (entry != null) {
+        URL url = urlFactory.getURL(entry.codebase, name);
+        LOG.debug("Adding " + url + " to resources list for " + name);
+        resources.add(url);
+      }
+    }
+
+    final Iterator<URL> resIter = resources.iterator();
+    return new Enumeration<URL>() {
+      public boolean hasMoreElements() {
+        return resIter.hasNext();
+      }
+      public URL nextElement() {
+        return resIter.next();
+      }
+    };
+  }
+
+  /**
+   * Return resources from the appropriate codebase.
+   */
+  public InputStream getByteStream(String resource) {
+    InputStream result = null;
+
+    if (!isSystemClass(resource, systemClasses)) {
+      // Make resource canonical (remove ., .., etc).
+      resource = canon(resource);
+
+      ByteCode bytecode = null;
+      String name = resolve(resource);
+      if (name != null) {
+        bytecode = byteCodeCache.get(name);
+      }
+
+      if (bytecode != null) {
+        result = new ByteArrayInputStream(bytecode.bytes);
+      }
+
+      if (result == null) {
+        if (jarNames.contains(resource)) {
+          // resource wanted is an actual jar
+          LOG.debug("Loading resource file directly: " + resource);
+          result = super.getResourceAsStream(resource);
+        }
+      }
+    }
+
+    if (result == null) {
+      // Delegate to parent.
+      result = parent.getResourceAsStream(resource);
+    }
+
+    return result;
+  }
+
+  private boolean isSealed(String name, Manifest man) {
+    String path = name.concat("/");
+    Attributes attr = man.getAttributes(path);
+    String sealed = null;
+    if (attr != null) {
+      sealed = attr.getValue(Name.SEALED);
+    }
+    if (sealed == null) {
+      if ((attr = man.getMainAttributes()) != null) {
+        sealed = attr.getValue(Name.SEALED);
+      }
+    }
+    return "true".equalsIgnoreCase(sealed);
+  }
+
+  /**
+   * Make a path canonical, removing . and ..
+   */
+  private String canon(String path) {
+    path = path.replaceAll("/\\./", "/");
+    String canon = path;
+    String next;
+    do {
+      next = canon;
+      canon = canon.replaceFirst("([^/]*/\\.\\./)", "");
+    } while (!next.equals(canon));
+    return canon;
+  }
+
+  /**
+   * Resolve a resource name.
+   */
+  private String resolve(String name) {
+    if (name.startsWith("/")) {
+      name = name.substring(1);
+    }
+
+    String resource = null;
+    if (byteCodeCache.containsKey(name)) {
+      resource = name;
+    }
+
+    if (resource == null) {
+      Iterator<String> jarNameIter = jarNames.iterator();
+      while (jarNameIter.hasNext()) {
+        String tmp = jarNameIter.next() + "/" + name;
+        if (byteCodeCache.containsKey(tmp)) {
+          resource = tmp;
+          break;
+        }
+      }
+    }
+    return resource;
+  }
+
+  private void load(URL[] urls) throws IOException {
+    for (URL url : urls) {
+      String jarName = url.getPath();
+      JarFile jarFile = null;
+      try {
+        jarFile = new JarFile(jarName);
+        Manifest manifest = jarFile.getManifest();
+
+        Enumeration<JarEntry> entryEnum = jarFile.entries();
+        while (entryEnum.hasMoreElements()) {
+          JarEntry entry = entryEnum.nextElement();
+          if (entry.isDirectory()) {
+            continue;
+          }
+
+          String entryName = entry.getName();
+          InputStream is = jarFile.getInputStream(entry);
+          if (is == null) {
+            throw new IOException("Unable to load resource " + entryName);
+          }
+          try {
+            if (entryName.startsWith(LIB_PREFIX)) {
+              LOG.debug("Caching " + entryName);
+              loadBytesFromJar(is, entryName);
+            } else if (entryName.endsWith(CLASS)) {
+              // A plain vanilla class file rooted at the top of the jar file.
+              loadBytes(entry, is, "/", manifest);
+              LOG.debug("Loaded class: " + jarFile.getName() + "!/" + entry.getName());
+            } else {
+              // A resource
+              loadBytes(entry, is, "/", manifest);
+              LOG.debug("Loaded resource: " + jarFile.getName() + "!/" + entry.getName());
+            }
+          } finally {
+            is.close();
+          }
+        }
+      } finally {
+        if (jarFile != null) {
+          try {
+            jarFile.close();
+          } catch (IOException e) {
+            LOG.debug("Exception closing jarFile: " + jarName, e);
+          }
+        }
+      }
+    }
+  }
+
+  private void loadBytesFromJar(InputStream is, String jar) throws IOException {
+    JarInputStream jis = new JarInputStream(is);
+    Manifest manifest = jis.getManifest();
+    JarEntry entry = null;
+    while ((entry = jis.getNextJarEntry()) != null) {
+      loadBytes(entry, jis, jar, manifest);
+    }
+
+    if (manifest != null) {
+      entry = new JarEntry(MANIFEST);
+      ByteArrayOutputStream baos = new ByteArrayOutputStream();
+      manifest.write(baos);
+      ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+      loadBytes(entry, bais, jar, manifest);
+    }
+  }
+
+  private void loadBytes(JarEntry entry, InputStream is, String jar, Manifest man) throws IOException {
+    String entryName = entry.getName();
+    int index = entryName.lastIndexOf('.');
+
+    // Add package handling to avoid NullPointerException
+    // after calls to getPackage method of this ClassLoader
+    int index2 = entryName.lastIndexOf('/', index - 1);
+    if (entryName.endsWith(CLASS) && index2 > -1) {
+      String packageName = entryName.substring(0, index2).replace('/', '.');
+      if (getPackage(packageName) == null) {
+        if (man != null) {
+          definePackage(packageName, man, urlFactory.getCodeBase(jar));
+        } else {
+          definePackage(packageName, null, null, null, null, null, null, null);
+        }
+      }
+    }
+
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    copy(is, baos);
+
+    // If entry is a class, check to see that it hasn't been defined
+    // already. Class names must be unique within a classloader because
+    // they are cached inside the VM until the classloader is released.
+    if (entryName.endsWith(CLASS)) {
+      if (byteCodeCache.containsKey(entryName)) {
+        return;
+      }
+      byteCodeCache.put(entryName, new ByteCode(baos.toByteArray(), jar, man));
+    } else {
+      // Another kind of resource. Cache this by name and prefixed by the jar name.
+      String localname = jar + "/" + entryName;
+      byte[] bytes = baos.toByteArray();
+      byteCodeCache.put(localname, new ByteCode(bytes, jar, man));
+      // Keep a set of jar names.
+      jarNames.add(jar);
+    }
+  }
+
+  /**
+   * Copying InputStream to OutputStream. Both streams are left open after copy.
+   * @param in Source of bytes to copy.
+   * @param out Destination of bytes to copy.
+   * @throws IOException
+   */
+  private static void copy(InputStream in, OutputStream out) throws IOException {
+    byte[] buf = new byte[1024];
+    while (true) {
+      int len = in.read(buf);
+      if (len < 0) {
+        break;
+      }
+      out.write(buf, 0, len);
+    }
+  }
+
+  /**
+   * Checks if a class should be included as a system class.
+   *
+   * A class is a system class if and only if the longest match pattern is positive.
+   *
+   * @param name the class name to check
+   * @param systemClasses a list of system class configurations.
+   * @return true if the class is a system class
+   */
+  public static boolean isSystemClass(String name, List<String> systemClasses) {
+    boolean result = false;
+    if (systemClasses != null) {
+      String canonicalName = name.replace('/', '.');
+      while (canonicalName.startsWith(".")) {
+        canonicalName=canonicalName.substring(1);
+      }
+      int maxMatchLength = -1;
+      for (String c : systemClasses) {
+        boolean shouldInclude = true;
+        if (c.startsWith("-")) {
+          c = c.substring(1);
+          shouldInclude = false;
+        }
+        if (canonicalName.startsWith(c)) {
+          if (   c.endsWith(".")                                   // package
+              || canonicalName.length() == c.length()              // class
+              ||    canonicalName.length() > c.length()            // nested
+                 && canonicalName.charAt(c.length()) == '$' ) {
+            if (c.length() > maxMatchLength) {
+              maxMatchLength = c.length();
+
+              if (shouldInclude) {
+                result = true;
+              } else {
+                result = false;
+              }
+            } else if (c.length() == maxMatchLength) {
+              if (!shouldInclude) {
+                result = false;
+              }
+            }
+          }
+        }
+      }
+    }
+    return result;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/java/org/apache/sqoop/classloader/ConnectorURLFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/classloader/ConnectorURLFactory.java b/common/src/main/java/org/apache/sqoop/classloader/ConnectorURLFactory.java
new file mode 100644
index 0000000..9ef58f8
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/classloader/ConnectorURLFactory.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.classloader;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.FileNameMap;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.net.URLStreamHandler;
+
+/**
+ * Generates URLs which are efficient, using the in-memory bytecode to access the resources.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings("SIC_INNER_SHOULD_BE_STATIC_ANON")
+public class ConnectorURLFactory implements URLFactory {
+  private URLStreamHandler handler;
+
+  public ConnectorURLFactory(final ConnectorClassLoader loader) {
+    handler = new URLStreamHandler() {
+      @Override
+      protected URLConnection openConnection(final URL u) throws IOException {
+        final String resource = u.getPath();
+        return new URLConnection(u) {
+          public void connect() {
+          }
+
+          public String getContentType() {
+            FileNameMap fileNameMap = java.net.URLConnection.getFileNameMap();
+            String contentType = fileNameMap.getContentTypeFor(resource);
+            if (contentType == null) {
+              contentType = "text/plain";
+            }
+            return contentType;
+          }
+
+          public InputStream getInputStream() throws IOException {
+            InputStream is = loader.getByteStream(resource);
+            if (is == null) {
+              throw new IOException("loader.getByteStream() returned null for " + resource);
+            }
+            return is;
+          }
+        };
+      }
+    };
+  }
+
+  public URL getURL(String codebase, String resource) throws MalformedURLException {
+    String base = resource.endsWith(".class") ? "": codebase + "/";
+    URL url =  new URL(null, PROTOCOL + ":/" + base + resource, handler);
+    return url;
+  }
+
+  public URL getCodeBase(String jar) throws MalformedURLException {
+    return new URL(null, PROTOCOL + ":" + jar, handler);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/java/org/apache/sqoop/classloader/URLFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/classloader/URLFactory.java b/common/src/main/java/org/apache/sqoop/classloader/URLFactory.java
new file mode 100644
index 0000000..b43df45
--- /dev/null
+++ b/common/src/main/java/org/apache/sqoop/classloader/URLFactory.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.classloader;
+
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public interface URLFactory {
+
+  public static final String PROTOCOL = "sqoopconnector";
+
+  public URL getURL(String codebase, String resource) throws MalformedURLException;
+  public URL getCodeBase(String jar) throws MalformedURLException;
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java b/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
deleted file mode 100644
index 4c42a78..0000000
--- a/common/src/main/java/org/apache/sqoop/utils/ConnectorClassLoader.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.utils;
-
-import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.log4j.Logger;
-
-/**
- * A {@link URLClassLoader} for connector isolation. Classes from the
- * connector JARs are loaded in preference to the parent loader.
- */
-public class ConnectorClassLoader extends URLClassLoader {
-  /**
-   * Default value of the system classes if the user did not override them.
-   * JDK classes, sqoop classes and resources, and some select third-party
-   * classes are considered system classes, and are not loaded by the
-   * connector classloader.
-   */
-  public static final String SYSTEM_CLASSES_DEFAULT;
-
-  private static final String PROPERTIES_FILE =
-      "org.apache.sqoop.connector-classloader.properties";
-  private static final String SYSTEM_CLASSES_DEFAULT_KEY =
-      "system.classes.default";
-
-  private static final Logger LOG = Logger.getLogger(ConnectorClassLoader.class);
-
-  private static final FilenameFilter JAR_FILENAME_FILTER =
-    new FilenameFilter() {
-      @Override
-      public boolean accept(File dir, String name) {
-        return name.endsWith(".jar") || name.endsWith(".JAR");
-      }
-  };
-
-  static {
-    try (InputStream is = ConnectorClassLoader.class.getClassLoader()
-        .getResourceAsStream(PROPERTIES_FILE);) {
-      if (is == null) {
-        throw new ExceptionInInitializerError("properties file " +
-            PROPERTIES_FILE + " is not found");
-      }
-      Properties props = new Properties();
-      props.load(is);
-      // get the system classes default
-      String systemClassesDefault =
-          props.getProperty(SYSTEM_CLASSES_DEFAULT_KEY);
-      if (systemClassesDefault == null) {
-        throw new ExceptionInInitializerError("property " +
-            SYSTEM_CLASSES_DEFAULT_KEY + " is not found");
-      }
-      SYSTEM_CLASSES_DEFAULT = systemClassesDefault;
-    } catch (IOException e) {
-      throw new ExceptionInInitializerError(e);
-    }
-  }
-
-  private final ClassLoader parent;
-  private final List<String> systemClasses;
-
-  public ConnectorClassLoader(URL[] urls, ClassLoader parent,
-      List<String> systemClasses, boolean overrideDefaultSystemClasses) {
-    super(urls, parent);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("urls: " + Arrays.toString(urls));
-      LOG.debug("system classes: " + systemClasses);
-    }
-    this.parent = parent;
-    if (parent == null) {
-      throw new IllegalArgumentException("No parent classloader!");
-    }
-    // if the caller-specified system classes are null or empty, use the default
-    this.systemClasses = new ArrayList<String>();
-    if (systemClasses != null && !systemClasses.isEmpty()) {
-      this.systemClasses.addAll(systemClasses);
-    }
-    if (!overrideDefaultSystemClasses || this.systemClasses.isEmpty()) {
-      this.systemClasses.addAll(Arrays.asList(SYSTEM_CLASSES_DEFAULT.split("\\s*,\\s*")));
-    }
-    LOG.info("system classes: " + this.systemClasses);
-  }
-
-  public ConnectorClassLoader(String classpath, ClassLoader parent,
-      List<String> systemClasses) throws MalformedURLException {
-    this(constructUrlsFromClasspath(classpath), parent, systemClasses, true);
-  }
-
-  public ConnectorClassLoader(String classpath, ClassLoader parent,
-      List<String> systemClasses, boolean overrideDefaultSystemClasses) throws MalformedURLException {
-    this(constructUrlsFromClasspath(classpath), parent, systemClasses, overrideDefaultSystemClasses);
-  }
-
-  static URL[] constructUrlsFromClasspath(String classpath)
-      throws MalformedURLException {
-    List<URL> urls = new ArrayList<URL>();
-    for (String element : classpath.split(File.pathSeparator)) {
-      if (element.endsWith("/*")) {
-        String dir = element.substring(0, element.length() - 1);
-        File[] files = new File(dir).listFiles(JAR_FILENAME_FILTER);
-        if (files != null) {
-          for (File file : files) {
-            urls.add(file.toURI().toURL());
-          }
-        }
-      } else {
-        File file = new File(element);
-        if (file.exists()) {
-          urls.add(new File(element).toURI().toURL());
-        }
-      }
-    }
-    return urls.toArray(new URL[urls.size()]);
-  }
-
-  @Override
-  public URL getResource(String name) {
-    URL url = null;
-
-    if (!isSystemClass(name, systemClasses)) {
-      url= findResource(name);
-      if (url == null && name.startsWith("/")) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug("Remove leading / off " + name);
-        }
-        url= findResource(name.substring(1));
-      }
-    }
-
-    if (url == null) {
-      url= parent.getResource(name);
-    }
-
-    if (url != null) {
-      if (LOG.isDebugEnabled()) {
-        LOG.debug("getResource("+name+")=" + url);
-      }
-    }
-
-    return url;
-  }
-
-  @Override
-  public Class<?> loadClass(String name) throws ClassNotFoundException {
-    return this.loadClass(name, false);
-  }
-
-  @Override
-  protected synchronized Class<?> loadClass(String name, boolean resolve)
-      throws ClassNotFoundException {
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Loading class: " + name);
-    }
-
-    Class<?> c = findLoadedClass(name);
-    ClassNotFoundException ex = null;
-
-    if (c == null && !isSystemClass(name, systemClasses)) {
-      // Try to load class from this classloader's URLs. Note that this is like
-      // the servlet spec, not the usual Java 2 behaviour where we ask the
-      // parent to attempt to load first.
-      try {
-        c = findClass(name);
-        if (LOG.isDebugEnabled() && c != null) {
-          LOG.debug("Loaded class: " + name + " ");
-        }
-      } catch (ClassNotFoundException e) {
-        if (LOG.isDebugEnabled()) {
-          LOG.debug(e);
-        }
-        ex = e;
-      }
-    }
-
-    if (c == null) { // try parent
-      c = parent.loadClass(name);
-      if (LOG.isDebugEnabled() && c != null) {
-        LOG.debug("Loaded class from parent: " + name + " ");
-      }
-    }
-
-    if (c == null) {
-      throw ex != null ? ex : new ClassNotFoundException(name);
-    }
-
-    if (resolve) {
-      resolveClass(c);
-    }
-
-    return c;
-  }
-
-  /**
-   * Checks if a class should be included as a system class.
-   *
-   * A class is a system class if and only if it matches one of the positive
-   * patterns and none of the negative ones.
-   *
-   * @param name the class name to check
-   * @param systemClasses a list of system class configurations.
-   * @return true if the class is a system class
-   */
-  public static boolean isSystemClass(String name, List<String> systemClasses) {
-    boolean result = false;
-    if (systemClasses != null) {
-      String canonicalName = name.replace('/', '.');
-      while (canonicalName.startsWith(".")) {
-        canonicalName=canonicalName.substring(1);
-      }
-      for (String c : systemClasses) {
-        boolean shouldInclude = true;
-        if (c.startsWith("-")) {
-          c = c.substring(1);
-          shouldInclude = false;
-        }
-        if (canonicalName.startsWith(c)) {
-          if (   c.endsWith(".")                                   // package
-              || canonicalName.length() == c.length()              // class
-              ||    canonicalName.length() > c.length()            // nested
-                 && canonicalName.charAt(c.length()) == '$' ) {
-            if (shouldInclude) {
-              result = true;
-            } else {
-              return false;
-            }
-          }
-        }
-      }
-    }
-    return result;
-  }
-}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/org.apache.sqoop.connector-classloader.properties b/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
index e2936a9..c0082cc 100644
--- a/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
+++ b/common/src/main/resources/org.apache.sqoop.connector-classloader.properties
@@ -51,5 +51,7 @@ system.classes.default=java.,\
   org.apache.commons.logging.,\
   org.apache.log4j.,\
   org.apache.sqoop.,\
+  -org.apache.sqoop.connector.,\
+  org.xerial.snappy.,\
   sqoop.properties,\
   sqoop_bootstrap.properties

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/java/org/apache/sqoop/classloader/TestConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/classloader/TestConnectorClassLoader.java b/common/src/test/java/org/apache/sqoop/classloader/TestConnectorClassLoader.java
new file mode 100644
index 0000000..87edf3d
--- /dev/null
+++ b/common/src/test/java/org/apache/sqoop/classloader/TestConnectorClassLoader.java
@@ -0,0 +1,311 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.classloader;
+
+import static org.apache.sqoop.classloader.ConnectorClassLoader.constructUrlsFromClasspath;
+import static org.apache.sqoop.classloader.ConnectorClassLoader.isSystemClass;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Enumeration;
+import java.util.List;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.StandardLocation;
+import javax.tools.ToolProvider;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.sqoop.classloader.ConnectorClassLoader;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import com.google.common.base.Splitter;
+import com.google.common.collect.Lists;
+
+public class TestConnectorClassLoader {
+  private static File testDir = new File(System.getProperty("maven.build.directory",
+      System.getProperty("java.io.tmpdir")), "connectorclassloader");
+
+  @BeforeMethod(alwaysRun = true)
+  public void setUp() {
+    FileUtils.deleteQuietly(testDir);
+    testDir.mkdirs();
+  }
+
+  @Test
+  public void testConstructUrlsFromClasspath() throws Exception {
+    File file = new File(testDir, "file");
+    assertTrue(file.createNewFile(), "Create file");
+
+    File dir = new File(testDir, "dir");
+    assertTrue(dir.mkdir(), "Make dir");
+
+    File jarsDir = new File(testDir, "jarsdir");
+    assertTrue(jarsDir.mkdir(), "Make jarsDir");
+    File nonJarFile = new File(jarsDir, "nonjar");
+    assertTrue(nonJarFile.createNewFile(), "Create non-jar file");
+    File jarFile = new File(jarsDir, "a.jar");
+    assertTrue(jarFile.createNewFile(), "Create jar file");
+
+    File nofile = new File(testDir, "nofile");
+    // don't create nofile
+
+    StringBuilder cp = new StringBuilder();
+    cp.append(file.getAbsolutePath()).append(File.pathSeparator)
+      .append(dir.getAbsolutePath()).append(File.pathSeparator)
+      .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator)
+      .append(nofile.getAbsolutePath()).append(File.pathSeparator)
+      .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator);
+
+    URL[] urls = constructUrlsFromClasspath(cp.toString());
+
+    assertEquals(3, urls.length);
+    assertEquals(file.toURI().toURL(), urls[0]);
+    assertEquals(dir.toURI().toURL(), urls[1]);
+    assertEquals(jarFile.toURI().toURL(), urls[2]);
+    // nofile should be ignored
+  }
+
+  @Test
+  public void testIsSystemClass() {
+    testIsSystemClassInternal("");
+  }
+
+  @Test
+  public void testIsSystemNestedClass() {
+    testIsSystemClassInternal("$Klass");
+  }
+
+  private void testIsSystemClassInternal(String nestedClass) {
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass, null));
+    assertTrue(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.Foo")));
+    assertTrue(isSystemClass("/org.example.Foo" + nestedClass,
+        classes("org.example.Foo")));
+    assertTrue(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.")));
+    assertTrue(isSystemClass("net.example.Foo" + nestedClass,
+        classes("org.example.,net.example.")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("-org.example.Foo,org.example.")));
+    assertTrue(isSystemClass("org.example.Bar" + nestedClass,
+        classes("-org.example.Foo.,org.example.")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.,-org.example.Foo")));
+    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
+        classes("org.example.Foo,-org.example.Foo")));
+  }
+
+  private List<String> classes(String classes) {
+    return Lists.newArrayList(Splitter.on(',').split(classes));
+  }
+
+  @Test
+  public void testGetResource() throws IOException {
+    URL testJar = makeTestJar().toURI().toURL();
+
+    ClassLoader currentClassLoader = getClass().getClassLoader();
+    ClassLoader connectorClassloader = new ConnectorClassLoader(
+        new URL[] { testJar }, currentClassLoader, null, false);
+
+    assertNull(currentClassLoader.getResourceAsStream("resource.txt"),
+        "Resource should be null for current classloader");
+    assertNull(currentClassLoader.getResourceAsStream("resource-dep.txt"),
+        "Resource should be null for current classloader");
+
+    InputStream in = connectorClassloader.getResourceAsStream("resource.txt");
+    assertNotNull(in, "Resource should not be null for connector classloader");
+
+    in = connectorClassloader.getResourceAsStream("resource-dep.txt");
+    assertNotNull(in, "Resource should not be null for connector classloader");
+    assertEquals(IOUtils.toString(in), "hello dep");
+  }
+
+  @Test
+  public void testGetResources() throws IOException {
+    URL testJar = makeTestJar().toURI().toURL();
+
+    ClassLoader currentClassLoader = getClass().getClassLoader();
+    ClassLoader connectorClassloader = new ConnectorClassLoader(
+        new URL[] { testJar }, currentClassLoader, null, false);
+
+    List<String> resourceContents = new ArrayList<String>();
+    resourceContents.add("hello A");
+    resourceContents.add("hello B");
+    Enumeration<URL> urlEnum = connectorClassloader.getResources("resource.txt");
+    assertTrue(urlEnum.hasMoreElements());
+    resourceContents.remove(IOUtils.toString(urlEnum.nextElement().openStream()));
+
+    assertTrue(urlEnum.hasMoreElements());
+    resourceContents.remove(IOUtils.toString(urlEnum.nextElement().openStream()));
+
+    assertEquals(resourceContents.size(), 0);
+  }
+
+  @Test
+  public void testLoadClass() throws Exception {
+    URL testJar = makeTestJar().toURI().toURL();
+
+    ClassLoader currentClassLoader = getClass().getClassLoader();
+    ClassLoader connectorClassloader = new ConnectorClassLoader(
+        new URL[] { testJar }, currentClassLoader, null, false);
+
+    try {
+      currentClassLoader.loadClass("A");
+      fail("Should throw ClassNotFoundException when loading class A");
+    } catch (ClassNotFoundException e) {
+      // Expected
+    }
+    try {
+      currentClassLoader.loadClass("B");
+      fail("Should throw ClassNotFoundException when loading class B");
+    } catch (ClassNotFoundException e) {
+      // Expected
+    }
+
+    assertNotNull(connectorClassloader.loadClass("A"));
+    assertNotNull(connectorClassloader.loadClass("B"));
+  }
+
+  private File makeTestJar() throws IOException{
+    File jarFile = new File(testDir, "test.jar");
+
+    Manifest manifest = new Manifest();
+    manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+    manifest.getMainAttributes().put(Attributes.Name.CLASS_PATH, ".");
+
+    // Create test.jar
+    JarOutputStream jos = new JarOutputStream(new FileOutputStream(jarFile), manifest);
+    List<String> classFiles = new ArrayList<String>();
+    classFiles.add("TestConnectorClassLoader/A.java");
+    addFilesToJar(classFiles, jos);
+    JarEntry entry = new JarEntry("resource.txt");
+    addEntry(new ByteArrayInputStream("hello A".getBytes()), jos, entry);
+
+    // Create lib/test-dep.jar
+    ByteArrayOutputStream baos = new ByteArrayOutputStream();
+    JarOutputStream depJos = new JarOutputStream(baos, manifest);
+    classFiles.clear();
+    classFiles.add("TestConnectorClassLoader/lib/B.java");
+    addFilesToJar(classFiles, depJos);
+    entry = new JarEntry("resource.txt");
+    addEntry(new ByteArrayInputStream("hello B".getBytes()), depJos, entry);
+    entry = new JarEntry("resource-dep.txt");
+    addEntry(new ByteArrayInputStream("hello dep".getBytes()), depJos, entry);
+    depJos.close();
+
+    // Add lib/test-dep.jar to test.jar
+    entry = new JarEntry("lib/test-dep.jar");
+    addEntry(new ByteArrayInputStream(baos.toByteArray()), jos, entry);
+
+    jos.close();
+    return jarFile;
+  }
+
+  private void addFilesToJar(List<String> classFiles, JarOutputStream jos) throws IOException {
+    ClassLoader classLoader = getClass().getClassLoader();
+    List<File> sourceFiles = new ArrayList<>();
+    for (String classFile : classFiles) {
+      File sourceFile = new File(classLoader.getResource(classFile).getFile());
+      sourceFiles.add(sourceFile);
+    }
+
+    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+    if (compiler == null) {
+      throw new IllegalStateException(
+        "Cannot find the system Java compiler. "
+          + "Check that your class path includes tools.jar");
+    }
+    StandardJavaFileManager fileManager = compiler.getStandardFileManager
+      (null, null, null);
+
+    fileManager.setLocation(StandardLocation.CLASS_OUTPUT, Arrays.asList(testDir));
+
+    Iterable<? extends JavaFileObject> compilationUnits1 =
+      fileManager.getJavaFileObjectsFromFiles(sourceFiles);
+
+    boolean compiled = compiler.getTask(null, fileManager, null, null, null, compilationUnits1).call();
+    if (!compiled) {
+      throw new RuntimeException("failed to compile");
+    }
+
+    List<String> classesForJar = new ArrayList<String>();
+    // Split the file on dot to get the filename from FILENAME.java
+    for (File source : sourceFiles) {
+      String fileName = source.getName().split("\\.")[0];
+      classesForJar.add(fileName);
+    }
+
+    File[] directoryListing = testDir.listFiles();
+    for (File compiledClass : directoryListing) {
+      String classFileName = compiledClass.getName().split("\\$")[0].split("\\.")[0];
+      if (classesForJar.contains(classFileName)){
+        addFileToJar(compiledClass, jos);
+      }
+    }
+  }
+
+  private void addFileToJar(File source, JarOutputStream jos) throws IOException {
+    JarEntry entry = new JarEntry(source.getName());
+    entry.setTime(source.lastModified());
+
+    BufferedInputStream in = new BufferedInputStream(new FileInputStream(source));
+    addEntry(in, jos, entry);
+
+    if (in != null) {
+      in.close();
+    }
+  }
+
+  private void addEntry(InputStream in, JarOutputStream jos, JarEntry entry) throws IOException {
+    jos.putNextEntry(entry);
+
+    byte[] buffer = new byte[1024];
+    while (true) {
+      int count = in.read(buffer);
+      if (count == -1) {
+        break;
+      }
+      jos.write(buffer, 0, count);
+    }
+
+    jos.closeEntry();
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
index 674ae6a..ec48f82 100644
--- a/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
+++ b/common/src/test/java/org/apache/sqoop/utils/TestClassUtils.java
@@ -32,6 +32,7 @@ import java.util.jar.JarEntry;
 import java.util.jar.JarOutputStream;
 import java.util.jar.Manifest;
 
+import org.apache.sqoop.classloader.ConnectorClassLoader;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java b/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
deleted file mode 100644
index 1ec1bb4..0000000
--- a/common/src/test/java/org/apache/sqoop/utils/TestConnectorClassLoader.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.utils;
-
-import static org.apache.sqoop.utils.ConnectorClassLoader.constructUrlsFromClasspath;
-import static org.apache.sqoop.utils.ConnectorClassLoader.isSystemClass;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URL;
-import java.util.List;
-import java.util.jar.JarOutputStream;
-import java.util.zip.ZipEntry;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import com.google.common.base.Splitter;
-import com.google.common.collect.Lists;
-
-public class TestConnectorClassLoader {
-  private static File testDir = new File(System.getProperty("maven.build.directory",
-      System.getProperty("java.io.tmpdir")), "connectorclassloader");
-
-  @BeforeMethod(alwaysRun = true)
-  public void setUp() {
-    FileUtils.deleteQuietly(testDir);
-    testDir.mkdirs();
-  }
-
-  @Test
-  public void testConstructUrlsFromClasspath() throws Exception {
-    File file = new File(testDir, "file");
-    assertTrue(file.createNewFile(), "Create file");
-
-    File dir = new File(testDir, "dir");
-    assertTrue(dir.mkdir(), "Make dir");
-
-    File jarsDir = new File(testDir, "jarsdir");
-    assertTrue(jarsDir.mkdir(), "Make jarsDir");
-    File nonJarFile = new File(jarsDir, "nonjar");
-    assertTrue(nonJarFile.createNewFile(), "Create non-jar file");
-    File jarFile = new File(jarsDir, "a.jar");
-    assertTrue(jarFile.createNewFile(), "Create jar file");
-
-    File nofile = new File(testDir, "nofile");
-    // don't create nofile
-
-    StringBuilder cp = new StringBuilder();
-    cp.append(file.getAbsolutePath()).append(File.pathSeparator)
-      .append(dir.getAbsolutePath()).append(File.pathSeparator)
-      .append(jarsDir.getAbsolutePath() + "/*").append(File.pathSeparator)
-      .append(nofile.getAbsolutePath()).append(File.pathSeparator)
-      .append(nofile.getAbsolutePath() + "/*").append(File.pathSeparator);
-
-    URL[] urls = constructUrlsFromClasspath(cp.toString());
-
-    assertEquals(3, urls.length);
-    assertEquals(file.toURI().toURL(), urls[0]);
-    assertEquals(dir.toURI().toURL(), urls[1]);
-    assertEquals(jarFile.toURI().toURL(), urls[2]);
-    // nofile should be ignored
-  }
-
-  @Test
-  public void testIsSystemClass() {
-    testIsSystemClassInternal("");
-  }
-
-  @Test
-  public void testIsSystemNestedClass() {
-    testIsSystemClassInternal("$Klass");
-  }
-
-  private void testIsSystemClassInternal(String nestedClass) {
-    assertFalse(isSystemClass("org.example.Foo" + nestedClass, null));
-    assertTrue(isSystemClass("org.example.Foo" + nestedClass,
-        classes("org.example.Foo")));
-    assertTrue(isSystemClass("/org.example.Foo" + nestedClass,
-        classes("org.example.Foo")));
-    assertTrue(isSystemClass("org.example.Foo" + nestedClass,
-        classes("org.example.")));
-    assertTrue(isSystemClass("net.example.Foo" + nestedClass,
-        classes("org.example.,net.example.")));
-    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
-        classes("-org.example.Foo,org.example.")));
-    assertTrue(isSystemClass("org.example.Bar" + nestedClass,
-        classes("-org.example.Foo.,org.example.")));
-    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
-        classes("org.example.,-org.example.Foo")));
-    assertFalse(isSystemClass("org.example.Foo" + nestedClass,
-        classes("org.example.Foo,-org.example.Foo")));
-  }
-
-  private List<String> classes(String classes) {
-    return Lists.newArrayList(Splitter.on(',').split(classes));
-  }
-
-  @Test
-  public void testGetResource() throws IOException {
-    URL testJar = makeTestJar().toURI().toURL();
-
-    ClassLoader currentClassLoader = getClass().getClassLoader();
-    ClassLoader connectorClassloader = new ConnectorClassLoader(
-        new URL[] { testJar }, currentClassLoader, null, false);
-
-    assertNull(currentClassLoader.getResourceAsStream("resource.txt"),
-        "Resource should be null for current classloader");
-
-    InputStream in = connectorClassloader.getResourceAsStream("resource.txt");
-    assertNotNull(in, "Resource should not be null for connector classloader");
-    assertEquals("hello", IOUtils.toString(in));
-  }
-
-  private File makeTestJar() throws IOException {
-    File jarFile = new File(testDir, "test.jar");
-    JarOutputStream out = new JarOutputStream(new FileOutputStream(jarFile));
-    ZipEntry entry = new ZipEntry("resource.txt");
-    out.putNextEntry(entry);
-    out.write("hello".getBytes());
-    out.closeEntry();
-    out.close();
-    return jarFile;
-  }
-}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/resources/TestConnectorClassLoader/A.java
----------------------------------------------------------------------
diff --git a/common/src/test/resources/TestConnectorClassLoader/A.java b/common/src/test/resources/TestConnectorClassLoader/A.java
new file mode 100644
index 0000000..15294be
--- /dev/null
+++ b/common/src/test/resources/TestConnectorClassLoader/A.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class A {
+
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/85743224/common/src/test/resources/TestConnectorClassLoader/lib/B.java
----------------------------------------------------------------------
diff --git a/common/src/test/resources/TestConnectorClassLoader/lib/B.java b/common/src/test/resources/TestConnectorClassLoader/lib/B.java
new file mode 100644
index 0000000..8828670
--- /dev/null
+++ b/common/src/test/resources/TestConnectorClassLoader/lib/B.java
@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+public class B {
+
+}


Mime
View raw message