sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kathl...@apache.org
Subject git commit: SQOOP-943: Sqoop2: Repository wide configuration of immutable disk structures
Date Sun, 23 Jun 2013 01:06:24 GMT
Updated Branches:
  refs/heads/sqoop2 fd4be5f5e -> 681914c85


SQOOP-943: Sqoop2: Repository wide configuration of immutable disk structures

(Jarek Jarcec Cecho via Kate Ting)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/681914c8
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/681914c8
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/681914c8

Branch: refs/heads/sqoop2
Commit: 681914c851cb275f1d803ffe057c2a7a53436330
Parents: fd4be5f
Author: Kate Ting <kathleen@apache.org>
Authored: Sat Jun 22 21:00:42 2013 -0400
Committer: Kate Ting <kathleen@apache.org>
Committed: Sat Jun 22 21:00:42 2013 -0400

----------------------------------------------------------------------
 .../apache/sqoop/repository/JdbcRepository.java | 34 ++++++++++++++++++--
 .../sqoop/repository/JdbcRepositoryContext.java |  9 ------
 .../repository/JdbcRepositoryProvider.java      | 10 +-----
 .../repository/RepoConfigurationConstants.java  | 15 ++++-----
 .../org/apache/sqoop/repository/Repository.java | 20 ++++++++++++
 .../sqoop/repository/RepositoryError.java       |  3 ++
 .../sqoop/repository/RepositoryManager.java     |  9 ++++++
 dist/src/main/server/conf/sqoop.properties      |  4 +--
 .../test/minicluster/SqoopMiniCluster.java      |  2 +-
 9 files changed, 75 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
index 135c2d2..aa1aa8d 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
@@ -31,8 +31,7 @@ import org.apache.sqoop.model.MSubmission;
 
 public class JdbcRepository extends Repository {
 
-  private static final Logger LOG =
-      Logger.getLogger(JdbcRepository.class);
+  private static final Logger LOG = Logger.getLogger(JdbcRepository.class);
 
   private final JdbcRepositoryHandler handler;
   private final JdbcRepositoryContext repoContext;
@@ -124,6 +123,37 @@ public class JdbcRepository extends Repository {
    * {@inheritDoc}
    */
   @Override
+  public void createOrUpdateInternals() {
+    doWithConnection(new DoWithConnection() {
+      @Override
+      public Object doIt(Connection conn) throws Exception {
+        if (!handler.schemaExists()) {
+          LOG.info("Creating repository schema objects");
+          handler.createSchema();
+        }
+
+        return null;
+      }
+    });
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
+  public boolean haveSuitableInternals() {
+    return (Boolean) doWithConnection(new DoWithConnection() {
+      @Override
+      public Object doIt(Connection conn) throws Exception {
+        return handler.schemaExists();
+      }
+    });
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  @Override
   public MConnector registerConnector(final MConnector mConnector) {
 
     return (MConnector) doWithConnection(new DoWithConnection() {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java
index 8989fb6..0a8139a 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryContext.java
@@ -34,7 +34,6 @@ public final class JdbcRepositoryContext {
 
   private final MapContext context;
   private final String handlerClassName;
-  private final boolean createSchema;
   private final String connectionUrl;
   private final String driverClassName;
   private final Properties connectionProperties;
@@ -55,9 +54,6 @@ public final class JdbcRepositoryContext {
           RepoConfigurationConstants.SYSCFG_REPO_JDBC_HANDLER);
     }
 
-    createSchema = context.getBoolean(
-        RepoConfigurationConstants.SYSCFG_REPO_JDBC_CREATE_SCHEMA, false);
-
     connectionUrl = context.getString(
         RepoConfigurationConstants.SYSCFG_REPO_JDBC_URL);
 
@@ -132,7 +128,6 @@ public final class JdbcRepositoryContext {
     if (LOG.isInfoEnabled()) {
       StringBuilder sb = new StringBuilder("[repo-ctx] ");
       sb.append("handler=").append(handlerClassName).append(", ");
-      sb.append("create-schema=").append(createSchema).append(", ");
       sb.append("conn-url=").append(connectionUrl).append(", ");
       sb.append("driver=").append(driverClassName).append(", ");
       sb.append("user=").append(jdbcUserName).append(", ");
@@ -189,10 +184,6 @@ public final class JdbcRepositoryContext {
     return driverClassName;
   }
 
-  public boolean shouldCreateSchema() {
-    return createSchema;
-  }
-
   public JdbcTransactionIsolation getTransactionIsolation() {
     return transactionIsolation;
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java
index 3339c59..1fd092a 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryProvider.java
@@ -40,8 +40,7 @@ import org.apache.sqoop.utils.ClassUtils;
 
 public class JdbcRepositoryProvider implements RepositoryProvider {
 
-  private static final Logger LOG =
-      Logger.getLogger(JdbcRepositoryProvider.class);
+  private static final Logger LOG = Logger.getLogger(JdbcRepositoryProvider.class);
 
   private JdbcRepositoryContext repoContext;
 
@@ -156,13 +155,6 @@ public class JdbcRepositoryProvider implements RepositoryProvider {
 
     handler.initialize(repoContext);
 
-    if (repoContext.shouldCreateSchema()) {
-      if (!handler.schemaExists()) {
-        LOG.info("Creating repository schema objects");
-        handler.createSchema();
-      }
-    }
-
     repository = new JdbcRepository(handler, repoContext);
 
     LOG.info("JdbcRepositoryProvider initialized");

http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
b/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
index 8939f8a..12cb93e 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepoConfigurationConstants.java
@@ -43,6 +43,13 @@ public final class RepoConfigurationConstants {
       + "provider";
 
   /**
+   * Boolean property defining whether repository is allowed to make changes
+   * on disk structures (schema in databases, changing file format, ...).
+   */
+  public static final String SYSCFG_REPO_SCHEMA_IMMUTABLE=
+    PREFIX_REPO_CONFIG + "schema.immutable";
+
+  /**
    * Class name for the JDBC repository handler specified by:
    * <tt>org.apache.sqoop.repository.jdbc.handler</tt>.
    */
@@ -50,14 +57,6 @@ public final class RepoConfigurationConstants {
       + "jdbc.handler";
 
   /**
-   * Indicates if the repository should create the schema objects as necessary,
-   * specified as a boolean value for the key:
-   * <tt>org.apache.sqoop.repository.jdbc.create.schema</tt>
-   */
-  public static final String SYSCFG_REPO_JDBC_CREATE_SCHEMA =
-      PREFIX_REPO_CONFIG + "jdbc.create.schema";
-
-  /**
    * JDBC connection URL specified by:
    * <tt>org.apache.sqoop.repository.jdbc.url</tt>
    */

http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/Repository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java
index 1ba947a..a2911c5 100644
--- a/core/src/main/java/org/apache/sqoop/repository/Repository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java
@@ -56,6 +56,26 @@ public abstract class Repository {
   public abstract RepositoryTransaction getTransaction();
 
   /**
+   * Create or update disk data structures.
+   *
+   * This method will be called only if Sqoop server is enabled with changing
+   * repository on disk structures. Repository should not change its disk structures
+   * outside of this method. This method must be no-op in case that the structures
+   * do not need any maintenance.
+   */
+  public abstract void createOrUpdateInternals();
+
+  /**
+   * Return true if internal repository structures exists and are suitable for use.
+   *
+   * This method should return false in case that the structures do exists, but
+   * are not suitable for use or if they requires upgrade.
+   *
+   * @return Boolean values if internal structures are suitable for use
+   */
+  public abstract boolean haveSuitableInternals();
+
+  /**
    * Registers given connector in the repository and return registered
    * variant. This method might return an exception in case that metadata for
    * given connector are already registered with different structure.

http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
index 4cae7ba..ab5e5a1 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryError.java
@@ -29,6 +29,9 @@ public enum RepositoryError implements ErrorCode {
   /** The system was unable to find or load the repository provider. */
   REPO_0001("Invalid repository provider specified"),
 
+  /** Repository on disk structures are not suitable for use */
+  REPO_0002("Repository structures are not initialized or requires upgrade"),
+
   // JDBC Repository Errors: Prefix JDBCREP
 
   /** An unknown error has occurred. */

http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
index 955306d..a178238 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
@@ -111,6 +111,15 @@ public class RepositoryManager {
 
     provider.initialize(context);
 
+    if(!context.getBoolean(RepoConfigurationConstants.SYSCFG_REPO_SCHEMA_IMMUTABLE, true))
{
+      LOG.info("Creating or upgrading on disk structures if necessary");
+      provider.getRepository().createOrUpdateInternals();
+    }
+
+    if(!provider.getRepository().haveSuitableInternals()) {
+      throw new SqoopException(RepositoryError.REPO_0002);
+    }
+
     LOG.info("Repository initialized: OK");
   }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/dist/src/main/server/conf/sqoop.properties
----------------------------------------------------------------------
diff --git a/dist/src/main/server/conf/sqoop.properties b/dist/src/main/server/conf/sqoop.properties
index 5131aad..07b148e 100755
--- a/dist/src/main/server/conf/sqoop.properties
+++ b/dist/src/main/server/conf/sqoop.properties
@@ -62,20 +62,20 @@ org.apache.sqoop.log4j.category.org.apache.derby=INFO
 # is "org.apache.sqoop.repository.sysprop". Any property that
 # is specified with this prefix is parsed out and set as a
 # system property. For example, if the built in Derby repository
-# is being used, the sysprop prefixed proeprties can be used
+# is being used, the sysprop prefixed properties can be used
 # to affect Derby configuration at startup time by setting
 # the appropriate system properties.
 #
 
 # Repository provider
 org.apache.sqoop.repository.provider=org.apache.sqoop.repository.JdbcRepositoryProvider
+org.apache.sqoop.repository.schema.immutable=false
 
 # JDBC repository provider configuration
 org.apache.sqoop.repository.jdbc.handler=org.apache.sqoop.repository.derby.DerbyRepositoryHandler
 org.apache.sqoop.repository.jdbc.transaction.isolation=READ_COMMITTED
 org.apache.sqoop.repository.jdbc.maximum.connections=10
 org.apache.sqoop.repository.jdbc.url=jdbc:derby:@BASEDIR@/repository/db;create=true
-org.apache.sqoop.repository.jdbc.create.schema=true
 org.apache.sqoop.repository.jdbc.driver=org.apache.derby.jdbc.EmbeddedDriver
 org.apache.sqoop.repository.jdbc.user=sa
 org.apache.sqoop.repository.jdbc.password=

http://git-wip-us.apache.org/repos/asf/sqoop/blob/681914c8/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
index 3620c28..9c71688 100644
--- a/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
+++ b/test/src/main/java/org/apache/sqoop/test/minicluster/SqoopMiniCluster.java
@@ -169,11 +169,11 @@ public abstract class SqoopMiniCluster {
     Map<String, String> properties = new HashMap<String, String>();
 
     properties.put("org.apache.sqoop.repository.provider", "org.apache.sqoop.repository.JdbcRepositoryProvider");
+    properties.put("org.apache.sqoop.repository.schema.immutable", "false");
     properties.put("org.apache.sqoop.repository.jdbc.handler", "org.apache.sqoop.repository.derby.DerbyRepositoryHandler");
     properties.put("org.apache.sqoop.repository.jdbc.transaction.isolation", "READ_COMMITTED");
     properties.put("org.apache.sqoop.repository.jdbc.maximum.connections", "10");
     properties.put("org.apache.sqoop.repository.jdbc.url=jdbc:derby:memory:myDB;create",
"true");
-    properties.put("org.apache.sqoop.repository.jdbc.create.schema", "true");
     properties.put("org.apache.sqoop.repository.jdbc.driver", "org.apache.derby.jdbc.EmbeddedDriver");
     properties.put("org.apache.sqoop.repository.jdbc.user", "sa");
     properties.put("org.apache.sqoop.repository.jdbc.password", "");


Mime
View raw message