sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject [07/13] SQOOP-1498: Sqoop2: Repository Object refactoring (objects prefixed with M)
Date Fri, 03 Oct 2014 06:53:21 GMT
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
index 8d6eb78..847b73d 100644
--- a/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigUpgrader.java
@@ -18,56 +18,54 @@
  */
 package org.apache.sqoop.driver;
 
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import org.apache.log4j.Logger;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.spi.RepositoryUpgrader;
-import org.apache.sqoop.model.MConnectionForms;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MConfig;
 import org.apache.sqoop.model.MInput;
-import org.apache.sqoop.model.MJobForms;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import org.apache.sqoop.model.MLinkConfig;
 
 public class DriverConfigUpgrader extends RepositoryUpgrader{
 
   private static final Logger LOG = Logger.getLogger(DriverConfigUpgrader.class);
 
   @Override
-  public void upgrade(MConnectionForms original,
-    MConnectionForms upgradeTarget) {
-    doUpgrade(original.getForms(), upgradeTarget.getForms());
+  public void upgrade(MLinkConfig original, MLinkConfig upgradeTarget) {
+    // NOTE(VB): There are no link configs anymore for driver, this code remains for previous versions
   }
 
   @Override
-  public void upgrade(MJobForms original, MJobForms upgradeTarget) {
-    doUpgrade(original.getForms(), upgradeTarget.getForms());
-
+  public void upgrade(MConfigList original, MConfigList upgradeTarget) {
+    doUpgrade(original.getConfigs(), upgradeTarget.getConfigs());
   }
 
   @SuppressWarnings("unchecked")
-  private void doUpgrade(List<MForm> original, List<MForm> target) {
-    // Easier to find the form in the original forms list if we use a map.
-    // Since the constructor of MJobForms takes a list,
+  private void doUpgrade(List<MConfig> original, List<MConfig> target) {
+    // Easier to find the config in the original list if we use a map.
+    // Since the constructor takes a list,
     // index is not guaranteed to be the same, so we need to look for
     // equivalence
-    Map<String, MForm> formMap = new HashMap<String, MForm>();
-    for (MForm form : original) {
-      formMap.put(form.getName(), form);
+    Map<String, MConfig> configMap = new HashMap<String, MConfig>();
+    for (MConfig config : original) {
+      configMap.put(config.getName(), config);
     }
-    for (MForm form : target) {
-      List<MInput<?>> inputs = form.getInputs();
-      MForm originalForm = formMap.get(form.getName());
-      if(originalForm == null) {
-        LOG.warn("Form: " + form.getName() + " not present in old " +
+    for (MConfig config : target) {
+      List<MInput<?>> inputs = config.getInputs();
+      MConfig originalConfig = configMap.get(config.getName());
+      if(originalConfig == null) {
+        LOG.warn("Config: " + config.getName() + " not present in old " +
           "driver config. So it will not be transferred by the upgrader.");
         continue;
       }
 
       for (MInput input : inputs) {
         try {
-          MInput originalInput = originalForm.getInput(input.getName());
+          MInput originalInput = originalConfig.getInput(input.getName());
           input.setValue(originalInput.getValue());
         } catch (SqoopException ex) {
           LOG.warn("Input: " + input.getName() + " not present in old " +

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
new file mode 100644
index 0000000..9c3b660
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/DriverConfigValidator.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver;
+
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
+import org.apache.sqoop.driver.configuration.ThrottlingConfig;
+import org.apache.sqoop.validation.Status;
+import org.apache.sqoop.validation.ConfigValidator;
+import org.apache.sqoop.validation.Validator;
+
+public class DriverConfigValidator extends Validator {
+  @Override
+  public ConfigValidator validateConfigForJob(Object jobConfiguration) {
+    ConfigValidator validation = new ConfigValidator(DriverConfiguration.class);
+    DriverConfiguration conf = (DriverConfiguration)jobConfiguration;
+    validateThrottlingConfig(validation,conf.throttlingConfig);
+
+    return validation;
+  };
+
+  private void validateThrottlingConfig(ConfigValidator validation, ThrottlingConfig throttlingConfig) {
+    if(throttlingConfig.numExtractors != null && throttlingConfig.numExtractors < 1) {
+      validation.addMessage(Status.UNACCEPTABLE, "throttlingConfig", "numExtractors", "You need to specify more than one extractor");
+    }
+
+    if(throttlingConfig.numLoaders != null && throttlingConfig.numLoaders < 1) {
+      validation.addMessage(Status.UNACCEPTABLE, "throttlingConfig", "numLoaders", "You need to specify more than one loader");
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java b/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
deleted file mode 100644
index 9cc51dd..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/DriverValidator.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver;
-
-import org.apache.sqoop.driver.configuration.LinkConfiguration;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
-import org.apache.sqoop.driver.configuration.ThrottlingForm;
-import org.apache.sqoop.validation.Status;
-import org.apache.sqoop.validation.Validation;
-import org.apache.sqoop.validation.Validator;
-
-public class DriverValidator extends Validator {
-  @Override
-  public Validation validateLink(Object linkConfiguration) {
-    Validation validation = new Validation(LinkConfiguration.class);
-    // No validation on link object
-    return validation;
-  }
-
-  @Override
-  public Validation validateJob(Object jobConfiguration) {
-    Validation validation = new Validation(JobConfiguration.class);
-    JobConfiguration conf = (JobConfiguration)jobConfiguration;
-    validateThrottlingForm(validation,conf.throttling);
-
-    return validation;
-  };
-
-  private void validateThrottlingForm(Validation validation, ThrottlingForm throttling) {
-    if(throttling.extractors != null && throttling.extractors < 1) {
-      validation.addMessage(Status.UNACCEPTABLE, "throttling", "extractors", "You need to specify more than one extractor");
-    }
-
-    if(throttling.loaders != null && throttling.loaders < 1) {
-      validation.addMessage(Status.UNACCEPTABLE, "throttling", "loaders", "You need to specify more than one loader");
-    }
-  }
-
-}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/JobManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobManager.java b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
index e91c436..df2a5ab 100644
--- a/core/src/main/java/org/apache/sqoop/driver/JobManager.java
+++ b/core/src/main/java/org/apache/sqoop/driver/JobManager.java
@@ -30,13 +30,13 @@ import org.apache.sqoop.connector.spi.SqoopConnector;
 import org.apache.sqoop.core.Reconfigurable;
 import org.apache.sqoop.core.SqoopConfiguration;
 import org.apache.sqoop.core.SqoopConfiguration.CoreConfigurationListener;
-import org.apache.sqoop.driver.configuration.JobConfiguration;
+import org.apache.sqoop.driver.configuration.DriverConfiguration;
 import org.apache.sqoop.job.etl.Destroyer;
 import org.apache.sqoop.job.etl.DestroyerContext;
 import org.apache.sqoop.job.etl.Initializer;
 import org.apache.sqoop.job.etl.InitializerContext;
 import org.apache.sqoop.job.etl.Transferable;
-import org.apache.sqoop.model.FormUtils;
+import org.apache.sqoop.model.ConfigUtils;
 import org.apache.sqoop.model.MJob;
 import org.apache.sqoop.model.MLink;
 import org.apache.sqoop.model.MSubmission;
@@ -311,35 +311,27 @@ public class JobManager implements Reconfigurable {
     SqoopConnector toConnector = getConnector(toConnection.getConnectorId());
     validateSupportedDirection(toConnector, Direction.TO);
 
-    // Transform config to fromConnector specific classes
-    Object fromConnectionConfig = ClassUtils.instantiate(fromConnector
-        .getLinkConfigurationClass());
-    FormUtils.fromForms(fromConnection.getConnectorPart().getForms(), fromConnectionConfig);
+    // link config for the FROM part of the job
+    Object fromLinkConfig = ClassUtils.instantiate(fromConnector.getLinkConfigurationClass());
+    ConfigUtils.fromConfigs(fromConnection.getConnectorLinkConfig().getConfigs(), fromLinkConfig);
 
-    // Transform config to toConnector specific classes
-    Object toConnectorConfig = ClassUtils
-        .instantiate(toConnector.getLinkConfigurationClass());
-    FormUtils.fromForms(toConnection.getConnectorPart().getForms(), toConnectorConfig);
+    // link config for the TO part of the job
+    Object toLinkConfig = ClassUtils.instantiate(toConnector.getLinkConfigurationClass());
+    ConfigUtils.fromConfigs(toConnection.getConnectorLinkConfig().getConfigs(), toLinkConfig);
 
+    // from config for the job
     Object fromJob = ClassUtils.instantiate(fromConnector.getJobConfigurationClass(Direction.FROM));
-    FormUtils.fromForms(job.getConnectorPart(Direction.FROM).getForms(), fromJob);
+    ConfigUtils.fromConfigs(job.getJobConfig(Direction.FROM).getConfigs(), fromJob);
 
+    // to config for the job
     Object toJob = ClassUtils.instantiate(toConnector.getJobConfigurationClass(Direction.TO));
-    FormUtils.fromForms(job.getConnectorPart(Direction.TO).getForms(), toJob);
+    ConfigUtils.fromConfigs(job.getJobConfig(Direction.TO).getConfigs(), toJob);
 
-    // Transform framework specific configs
-    // Q(VB) : Aren't the following 2 exactly the same?
-    Object fromDriverConnection = ClassUtils.instantiate(Driver.getInstance()
-        .getLinkConfigurationClass());
-    FormUtils.fromForms(fromConnection.getFrameworkPart().getForms(), fromDriverConnection);
+    // the only driver config for the job
+    Object driverConfig = ClassUtils
+        .instantiate(Driver.getInstance().getDriverConfigurationGroupClass());
+    ConfigUtils.fromConfigs(job.getDriverConfig().getConfigs(), driverConfig);
 
-    Object toDriverConnection = ClassUtils.instantiate(Driver.getInstance()
-        .getLinkConfigurationClass());
-    FormUtils.fromForms(toConnection.getFrameworkPart().getForms(), toDriverConnection);
-
-    Object frameworkJob = ClassUtils.instantiate(Driver.getInstance()
-        .getJobConfigurationClass());
-    FormUtils.fromForms(job.getFrameworkPart().getForms(), frameworkJob);
 
     // Create a job request for submit/execution
     JobRequest jobRequest = executionEngine.createJobRequest();
@@ -347,14 +339,14 @@ public class JobManager implements Reconfigurable {
     jobRequest.setSummary(submission);
     jobRequest.setConnector(Direction.FROM, fromConnector);
     jobRequest.setConnector(Direction.TO, toConnector);
-    jobRequest.setConnectorLinkConfig(Direction.FROM, fromConnectionConfig);
-    jobRequest.setConnectorLinkConfig(Direction.TO, toConnectorConfig);
-    jobRequest.setConnectorJobConfig(Direction.FROM, fromJob);
-    jobRequest.setConnectorJobConfig(Direction.TO, toJob);
-    // TODO(Abe): Should we actually have 2 different Driver Connection config objects?
-    jobRequest.setFrameworkLinkConfig(Direction.FROM, fromDriverConnection);
-    jobRequest.setFrameworkLinkConfig(Direction.TO, toDriverConnection);
-    jobRequest.setFrameworkJobConfig(frameworkJob);
+
+    jobRequest.setConnectorLinkConfig(Direction.FROM, fromLinkConfig);
+    jobRequest.setConnectorLinkConfig(Direction.TO, toLinkConfig);
+
+    jobRequest.setJobConfig(Direction.FROM, fromJob);
+    jobRequest.setJobConfig(Direction.TO, toJob);
+
+    jobRequest.setDriverConfig(driverConfig);
     jobRequest.setJobName(job.getName());
     jobRequest.setJobId(job.getPersistenceId());
     jobRequest.setNotificationUrl(notificationBaseUrl + jobId);
@@ -453,12 +445,12 @@ public class JobManager implements Reconfigurable {
 
     // Initialize submission from the connector perspective
     initializer.initialize(initializerContext, jobRequest.getConnectorLinkConfig(direction),
-        jobRequest.getConnectorJobConfig(direction));
+        jobRequest.getJobConfig(direction));
 
 
     return initializer.getSchema(initializerContext,
         jobRequest.getConnectorLinkConfig(direction),
-        jobRequest.getConnectorJobConfig(direction));
+        jobRequest.getJobConfig(direction));
   }
 
   private void addConnectorInitializerJars(JobRequest jobRequest, Direction direction) {
@@ -468,7 +460,7 @@ public class JobManager implements Reconfigurable {
     // Add job specific jars to
     jobRequest.addJars(initializer.getJars(initializerContext,
         jobRequest.getConnectorLinkConfig(direction),
-        jobRequest.getConnectorJobConfig(direction)));
+        jobRequest.getJobConfig(direction)));
   }
 
   private Initializer getConnectorInitializer(JobRequest jobRequest, Direction direction) {
@@ -488,12 +480,12 @@ public class JobManager implements Reconfigurable {
   }
 
   void prepareJob(JobRequest request) {
-    JobConfiguration jobConfiguration = (JobConfiguration) request.getFrameworkJobConfig();
+    DriverConfiguration jobConfiguration = (DriverConfiguration) request.getDriverConfig();
     // We're directly moving configured number of extractors and loaders to
     // underlying request object. In the future we might need to throttle this
     // count based on other running jobs to meet our SLAs.
-    request.setExtractors(jobConfiguration.throttling.extractors);
-    request.setLoaders(jobConfiguration.throttling.loaders);
+    request.setExtractors(jobConfiguration.throttlingConfig.numExtractors);
+    request.setLoaders(jobConfiguration.throttlingConfig.numLoaders);
 
     // Delegate rest of the job to execution engine
     executionEngine.prepareJob(request);
@@ -532,9 +524,9 @@ public class JobManager implements Reconfigurable {
 
     // destroy submission from connector perspective
     fromDestroyer.destroy(fromDestroyerContext, request.getConnectorLinkConfig(Direction.FROM),
-        request.getConnectorJobConfig(Direction.FROM));
+        request.getJobConfig(Direction.FROM));
     toDestroyer.destroy(toDestroyerContext, request.getConnectorLinkConfig(Direction.TO),
-        request.getConnectorJobConfig(Direction.TO));
+        request.getJobConfig(Direction.TO));
   }
 
   public MSubmission stop(long jobId, HttpEventContext ctx) {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
index 63e1e49..2666320 100644
--- a/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
+++ b/core/src/main/java/org/apache/sqoop/driver/JobRequest.java
@@ -78,11 +78,11 @@ public class JobRequest {
    */
   Object fromConnectorLinkConfig;
   Object toConnectorLinkConfig;
-  Object fromConnectorJobConfig;
-  Object toConnectorJobConfig;
-  Object fromFrameworkLinkConfig;
-  Object toFrameworkLinkConfig;
-  Object frameworkJobConfig;
+
+  Object fromConfig;
+  Object toConfig;
+
+  Object driverConfig;
 
   /**
    * Connector context (submission specific configuration)
@@ -124,10 +124,9 @@ public class JobRequest {
     this.toConnector = null;
     this.fromConnectorLinkConfig = null;
     this.toConnectorLinkConfig = null;
-    this.fromConnectorJobConfig = null;
-    this.toConnectorJobConfig = null;
-    this.fromFrameworkLinkConfig = null;
-    this.toFrameworkLinkConfig = null;
+    this.fromConfig = null;
+    this.toConfig = null;
+    this.driverConfig = null;
   }
 
   public MSubmission getSummary() {
@@ -244,64 +243,38 @@ public class JobRequest {
     }
   }
 
-  public Object getConnectorJobConfig(Direction type) {
-    switch(type) {
-      case FROM:
-        return fromConnectorJobConfig;
-
-      case TO:
-        return toConnectorJobConfig;
-
-      default:
-        throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
-    }
-  }
-
-  public void setConnectorJobConfig(Direction type, Object config) {
-    switch(type) {
-      case FROM:
-        fromConnectorJobConfig = config;
-        break;
-      case TO:
-        toConnectorJobConfig = config;
-        break;
-      default:
-        throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
-    }
-  }
-
-  public Object getFrameworkLinkConfig(Direction type) {
+  public Object getJobConfig(Direction type) {
     switch(type) {
       case FROM:
-        return fromFrameworkLinkConfig;
+        return fromConfig;
 
       case TO:
-        return toFrameworkLinkConfig;
+        return toConfig;
 
       default:
         throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
     }
   }
 
-  public void setFrameworkLinkConfig(Direction type, Object config) {
+  public void setJobConfig(Direction type, Object config) {
     switch(type) {
       case FROM:
-        fromFrameworkLinkConfig = config;
+        fromConfig = config;
         break;
       case TO:
-        toFrameworkLinkConfig = config;
+        toConfig = config;
         break;
       default:
         throw new SqoopException(DirectionError.DIRECTION_0000, "Direction: " + type);
     }
   }
 
-  public Object getFrameworkJobConfig() {
-    return frameworkJobConfig;
+  public Object getDriverConfig() {
+    return driverConfig;
   }
 
-  public void setFrameworkJobConfig(Object config) {
-    frameworkJobConfig = config;
+  public void setDriverConfig(Object config) {
+    driverConfig = config;
   }
 
   public MutableMapContext getConnectorContext(Direction type) {

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
new file mode 100644
index 0000000..d4e2254
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/DriverConfiguration.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.ConfigurationClass;
+import org.apache.sqoop.model.Config;
+
+/**
+ * Representing the core job configuration
+ */
+@ConfigurationClass
+public class DriverConfiguration {
+  @Config
+  public ThrottlingConfig throttlingConfig;
+
+  public DriverConfiguration() {
+    throttlingConfig = new ThrottlingConfig();
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
deleted file mode 100644
index 908a4eb..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/JobConfiguration.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-import org.apache.sqoop.model.Form;
-
-/**
- * Representing the core job configuration
- */
-@ConfigurationClass
-public class JobConfiguration {
-  @Form
-  public ThrottlingForm throttling;
-
-  public JobConfiguration() {
-    throttling = new ThrottlingForm();
-  }
-}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java b/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
deleted file mode 100644
index 3202844..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/LinkConfiguration.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.ConfigurationClass;
-
-/**
- * Representing the core link configuration
- */
-@ConfigurationClass
-public class LinkConfiguration {
-
-}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
new file mode 100644
index 0000000..357d8e5
--- /dev/null
+++ b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingConfig.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.driver.configuration;
+
+import org.apache.sqoop.model.ConfigClass;
+import org.apache.sqoop.model.Input;
+
+/**
+ * Config to set up number of loaders and extractors
+ */
+@ConfigClass
+public class ThrottlingConfig {
+
+  @Input public Integer numExtractors;
+
+  @Input public Integer numLoaders;
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java b/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
deleted file mode 100644
index e73007e..0000000
--- a/core/src/main/java/org/apache/sqoop/driver/configuration/ThrottlingForm.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.driver.configuration;
-
-import org.apache.sqoop.model.FormClass;
-import org.apache.sqoop.model.Input;
-
-/**
- * Form to set up number of loaders and extractors
- */
-@FormClass
-public class ThrottlingForm {
-
-  @Input public Integer extractors;
-
-  @Input public Integer loaders;
-}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
index 3466116..3ade247 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepository.java
@@ -25,7 +25,7 @@ import org.apache.log4j.Logger;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.model.MLink;
 import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MDriver;
 import org.apache.sqoop.model.MJob;
 import org.apache.sqoop.model.MSubmission;
 
@@ -51,7 +51,7 @@ public class JdbcRepository extends Repository {
     /**
      * Do what is needed to be done with given link object.
      *
-     * @param conn Connection to metadata repository.
+     * @param conn Connection to the repository.
      * @return Arbitrary value
      */
     Object doIt(Connection conn) throws Exception;
@@ -158,8 +158,8 @@ public class JdbcRepository extends Repository {
       public Object doIt(Connection conn) throws Exception {
         String connectorUniqueName = mConnector.getUniqueName();
 
-        MConnector result = handler.findConnector(connectorUniqueName, conn);
-        if (result == null) {
+        MConnector connectorResult = handler.findConnector(connectorUniqueName, conn);
+        if (connectorResult == null) {
           handler.registerConnector(mConnector, conn);
           return mConnector;
         } else {
@@ -167,23 +167,23 @@ public class JdbcRepository extends Repository {
           // For now, use the "string" versions itself - later we should
           // probably include a build number or something that is
           // monotonically increasing.
-          if (result.getUniqueName().equals(mConnector.getUniqueName()) &&
-            mConnector.getVersion().compareTo(result.getVersion()) > 0) {
+          if (connectorResult.getUniqueName().equals(mConnector.getUniqueName()) &&
+            mConnector.getVersion().compareTo(connectorResult.getVersion()) > 0) {
             if (autoUpgrade) {
-              upgradeConnector(result, mConnector);
+              upgradeConnector(connectorResult, mConnector);
               return mConnector;
             } else {
               throw new SqoopException(RepositoryError.JDBCREPO_0026,
                 "Connector: " + mConnector.getUniqueName());
             }
           }
-          if (!result.equals(mConnector)) {
+          if (!connectorResult.equals(mConnector)) {
             throw new SqoopException(RepositoryError.JDBCREPO_0013,
               "Connector: " + mConnector.getUniqueName()
                 + " given: " + mConnector
-                + " found: " + result);
+                + " found: " + connectorResult);
           }
-          return result;
+          return connectorResult;
         }
       }
     });
@@ -220,27 +220,27 @@ public class JdbcRepository extends Repository {
    * {@inheritDoc}
    */
   @Override
-  public MDriverConfig registerDriverConfig(final MDriverConfig mDriverConfig, final boolean autoUpgrade) {
-    return (MDriverConfig) doWithConnection(new DoWithConnection() {
+  public MDriver registerDriver(final MDriver mDriver, final boolean autoUpgrade) {
+    return (MDriver) doWithConnection(new DoWithConnection() {
       @Override
       public Object doIt(Connection conn) {
-        MDriverConfig result = handler.findDriverConfig(conn);
-        if (result == null) {
-          handler.registerDriverConfig(mDriverConfig, conn);
-          return mDriverConfig;
+        MDriver existingDriverConfig = handler.findDriver(conn);
+        if (existingDriverConfig == null) {
+          handler.registerDriver(mDriver, conn);
+          return mDriver;
         } else {
           // We're currently not serializing version into repository
           // so let's just compare the structure to see if we need upgrade.
-          if(!mDriverConfig.equals(result)) {
+          if(!mDriver.equals(existingDriverConfig)) {
             if (autoUpgrade) {
-              upgradeDriverConfig(mDriverConfig);
-              return mDriverConfig;
+              upgradeDriver(mDriver);
+              return mDriver;
             } else {
               throw new SqoopException(RepositoryError.JDBCREPO_0026,
-                "DriverConfig: " + mDriverConfig.getPersistenceId());
+                "DriverConfig: " + mDriver.getPersistenceId());
             }
           }
-          return result;
+          return existingDriverConfig;
         }
       }
     });
@@ -664,11 +664,11 @@ public class JdbcRepository extends Repository {
   }
 
 
-  protected void updateDriverConfig(final MDriverConfig mDriverConfig, RepositoryTransaction tx) {
+  protected void updateDriver(final MDriver mDriver, RepositoryTransaction tx) {
     doWithConnection(new DoWithConnection() {
       @Override
       public Object doIt(Connection conn) throws Exception {
-        handler.updateDriverConfig(mDriverConfig, conn);
+        handler.updateDriver(mDriver, conn);
         return null;
       }
     }, (JdbcRepositoryTransaction) tx);

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
index a743491..97de893 100644
--- a/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
+++ b/core/src/main/java/org/apache/sqoop/repository/JdbcRepositoryHandler.java
@@ -23,7 +23,7 @@ import java.util.List;
 
 import org.apache.sqoop.model.MLink;
 import org.apache.sqoop.model.MConnector;
-import org.apache.sqoop.model.MDriverConfig;
+import org.apache.sqoop.model.MDriver;
 import org.apache.sqoop.model.MJob;
 import org.apache.sqoop.model.MSubmission;
 
@@ -113,11 +113,11 @@ public abstract class JdbcRepositoryHandler {
    * which may not have changed). After this operation the repository is
    * guaranteed to only have the new forms specified in this object.
    *
-   * @param mDriverConfig The new data to be inserted into the repository for
+   * @param mDriver The new data to be inserted into the repository for
    *                     the driverConfig.
    * @param conn JDBC link for querying repository
    */
-  public abstract void updateDriverConfig(MDriverConfig mDriverConfig, Connection conn);
+  public abstract void updateDriver(MDriver mDriver, Connection conn);
 
 
   /**
@@ -127,7 +127,7 @@ public abstract class JdbcRepositoryHandler {
    * @return null if driverConfig are not yet present in repository or
    *  loaded representation.
    */
-  public abstract MDriverConfig findDriverConfig(Connection conn);
+  public abstract MDriver findDriver(Connection conn);
 
   /**
    * Register driver config in repository.
@@ -138,7 +138,7 @@ public abstract class JdbcRepositoryHandler {
    * @param driverConfig Driver config that should be registered.
    * @param conn JDBC link for querying repository.
    */
-  public abstract void registerDriverConfig(MDriverConfig driverConfig, Connection conn);
+  public abstract void registerDriver(MDriver driverConfig, Connection conn);
 
   /**
    * Return true if repository tables exists and are suitable for use.
@@ -243,7 +243,7 @@ public abstract class JdbcRepositoryHandler {
    *
    * @param linkId Link id
    * @param conn Connection to the repository
-   * @return Deserialized form of the link that is saved in repository
+   * @return Deserialized config of the link that is saved in repository
    */
   public abstract MLink findLink(long linkId, Connection conn);
 
@@ -323,7 +323,7 @@ public abstract class JdbcRepositoryHandler {
    *
    * @param jobId Job id
    * @param conn Connection to the repository
-   * @return Deserialized form of the job that is present in the repository
+   * @return Deserialized config of the job that is present in the repository
    */
   public abstract MJob findJob(long jobId, Connection conn);
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/repository/Repository.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/Repository.java b/core/src/main/java/org/apache/sqoop/repository/Repository.java
index ea1608a..95c7a4d 100644
--- a/core/src/main/java/org/apache/sqoop/repository/Repository.java
+++ b/core/src/main/java/org/apache/sqoop/repository/Repository.java
@@ -28,18 +28,21 @@ import org.apache.sqoop.connector.ConnectorManager;
 import org.apache.sqoop.connector.spi.RepositoryUpgrader;
 import org.apache.sqoop.connector.spi.SqoopConnector;
 import org.apache.sqoop.driver.Driver;
-import org.apache.sqoop.model.FormUtils;
-import org.apache.sqoop.model.MConnectionForms;
+import org.apache.sqoop.json.DriverBean;
+import org.apache.sqoop.model.ConfigUtils;
+import org.apache.sqoop.model.MConfig;
 import org.apache.sqoop.model.MConnector;
+import org.apache.sqoop.model.MDriver;
 import org.apache.sqoop.model.MDriverConfig;
-import org.apache.sqoop.model.MForm;
+import org.apache.sqoop.model.MFromConfig;
 import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MJobForms;
 import org.apache.sqoop.model.MLink;
+import org.apache.sqoop.model.MLinkConfig;
 import org.apache.sqoop.model.MPersistableEntity;
 import org.apache.sqoop.model.MSubmission;
+import org.apache.sqoop.model.MToConfig;
 import org.apache.sqoop.utils.ClassUtils;
-import org.apache.sqoop.validation.Validation;
+import org.apache.sqoop.validation.ConfigValidator;
 import org.apache.sqoop.validation.Validator;
 
 
@@ -76,15 +79,26 @@ public abstract class Repository {
 
   /**
    * Registers given connector in the repository and return registered
-   * variant. This method might return an exception in case that 
-   * given connector are already registered with different structure.
+   * variant. This method might return an exception in case that
+   * given connector are already registered with different structure
    *
    * @param mConnector the connector to be registered
-   * autoupgrade whether to upgrade driver config automatically
+   * @param autoUpgrade whether to upgrade driver config automatically
    * @return Registered connector structure
    */
   public abstract MConnector registerConnector(MConnector mConnector, boolean autoUpgrade);
 
+   /**
+   * Registers given driver and its config in the repository and return registered
+   * variant. This method might return an exception in case that the
+   * given driverConfig are already registered with different structure
+   *
+   * @param mDriverConfig driverConfig to be registered
+   * @param autoUpgrade whether to upgrade driverConfig automatically
+   * @return Registered connector structure
+   */
+  public abstract MDriver registerDriver(MDriver mDriverConfig, boolean autoUpgrade);
+
   /**
    * Search for connector with given name in repository.
    *
@@ -103,18 +117,6 @@ public abstract class Repository {
    */
   public abstract List<MConnector> findConnectors();
 
-
-  /**
-   * Registers given driverConfig in the repository and return registered
-   * variant. This method might return an exception in case that the
-   * given driverConfig are already registered with different structure.
-   *
-   * @param mDriverConfig driverConfig to be registered
-   * autoupgrade whether to upgrade driverConfig automatically
-   * @return Registered connector structure
-   */
-  public abstract MDriverConfig registerDriverConfig(MDriverConfig mDriverConfig, boolean autoUpgrade);
-
   /**
    * Save given link to repository. This link must not be already
    * present in the repository otherwise exception will be thrown.
@@ -164,7 +166,7 @@ public abstract class Repository {
    * Find link with given id in repository.
    *
    * @param id Link id
-   * @return Deserialized form of the link that is saved in repository
+   * @return Deserialized config of the link that is saved in repository
    */
   public abstract MLink findLink(long id);
 
@@ -184,7 +186,7 @@ public abstract class Repository {
   public abstract void createJob(MJob job);
 
   /**
-   * Update given job metadata in repository. This object must already be saved
+   * Update given job entity in repository. This object must already be saved
    * in repository otherwise exception will be thrown.
    *
    * @param job Job object that should be updated in the repository
@@ -192,7 +194,7 @@ public abstract class Repository {
   public abstract void updateJob(MJob job);
 
   /**
-   * Update given job metadata in repository. This object must already be saved
+   * Update given job entity in repository. This object must already be saved
    * in repository otherwise exception will be thrown.
    *
    * @param job Job object that should be updated in the repository
@@ -204,7 +206,7 @@ public abstract class Repository {
   public abstract void updateJob(MJob job, RepositoryTransaction tx);
 
   /**
-   * Enable or disable job with given id from metadata repository
+   * Enable or disable job with given id from entity repository
    *
    * @param id Job object that is going to be enabled or disabled
    * @param enabled Enable or disable
@@ -212,7 +214,7 @@ public abstract class Repository {
   public abstract void enableJob(long id, boolean enabled);
 
   /**
-   * Delete job with given id from metadata repository.
+   * Delete job with given id from entity repository.
    *
    * @param id Job id that should be removed
    */
@@ -222,7 +224,7 @@ public abstract class Repository {
    * Find job object with given id.
    *
    * @param id Job id
-   * @return Deserialized form of job loaded from repository
+   * @return Deserialized config of job loaded from repository
    */
   public abstract MJob findJob(long id);
 
@@ -288,8 +290,7 @@ public abstract class Repository {
    * @param connectorID Connector ID whose links should be fetched
    * @return List of MLink that use <code>connectorID</code>.
    */
-  public abstract List<MLink> findLinksForConnector(long
-    connectorID);
+  public abstract List<MLink> findLinksForConnector(long connectorID);
 
   /**
    * Retrieve jobs which use the given link.
@@ -297,17 +298,16 @@ public abstract class Repository {
    * @param connectorID Connector ID whose jobs should be fetched
    * @return List of MJobs that use <code>linkID</code>.
    */
-  public abstract List<MJob> findJobsForConnector(long
-    connectorID);
+  public abstract List<MJob> findJobsForConnector(long connectorID);
 
   /**
    * Update the connector with the new data supplied in the
-   * <tt>newConnector</tt>. Also Update all forms associated with this
-   * connector in the repository with the forms specified in
+   * <tt>newConnector</tt>. Also Update all configs associated with this
+   * connector in the repository with the configs specified in
    * <tt>mConnector</tt>. <tt>mConnector </tt> must
-   * minimally have the connectorID and all required forms (including ones
+   * minimally have the connectorID and all required configs (including ones
    * which may not have changed). After this operation the repository is
-   * guaranteed to only have the new forms specified in this object.
+   * guaranteed to only have the new configs specified in this object.
    *
    * @param newConnector The new data to be inserted into the repository for
    *                     this connector.
@@ -319,22 +319,22 @@ public abstract class Repository {
   protected abstract void updateConnector(MConnector newConnector, RepositoryTransaction tx);
 
   /**
-   * Update the driverConfig with the new data supplied in the
-   * <tt>mDriverConfig</tt>. Also Update all forms associated with the driverConfig
-   * in the repository with the forms specified in
+   * Update the driver with the new data supplied in the
+   * <tt>mDriverConfig</tt>. Also Update all configs associated with the driverConfig
+   * in the repository with the configs specified in
    * <tt>mDriverConfig</tt>. <tt>mDriverConfig </tt> must
-   * minimally have the connectorID and all required forms (including ones
+   * minimally have the connectorID and all required configs (including ones
    * which may not have changed). After this operation the repository is
-   * guaranteed to only have the new forms specified in this object.
+   * guaranteed to only have the new configs specified in this object.
    *
-   * @param mDriverConfig The new data to be inserted into the repository for
+   * @param mDriver The new data to be inserted into the repository for
    *                     the driverConfig.
    * @param tx The repository transaction to use to push the data to the
    *           repository. If this is null, a new transaction will be created.
    *           method will not call begin, commit,
    *           rollback or close on this transaction.
    */
-  protected abstract void updateDriverConfig(MDriverConfig mDriverConfig, RepositoryTransaction tx);
+  protected abstract void updateDriver(MDriver mDriver, RepositoryTransaction tx);
 
   /**
    * Delete all inputs for a job
@@ -365,6 +365,13 @@ public abstract class Repository {
     }
   }
 
+  private void deleteJobs(List<MJob> jobs, RepositoryTransaction tx) {
+    for (MJob job : jobs) {
+      deleteJobInputs(job.getPersistenceId(), tx);
+    }
+  }
+
+
   /**
    * Upgrade the connector with the same {@linkplain MConnector#uniqueName}
    * in the repository with values from <code>newConnector</code>.
@@ -377,7 +384,7 @@ public abstract class Repository {
    *                     upgraded.
    */
   public final void upgradeConnector(MConnector oldConnector, MConnector newConnector) {
-    LOG.info("Upgrading metadata for connector: " + oldConnector.getUniqueName());
+    LOG.info("Upgrading connector: " + oldConnector.getUniqueName());
     long connectorID = oldConnector.getPersistenceId();
     newConnector.setPersistenceId(connectorID);
     /* Algorithms:
@@ -385,8 +392,8 @@ public abstract class Repository {
      * 2. Get all links associated with the connector.
      * 3. Get all jobs associated with the connector.
      * 4. Delete the inputs for all of the jobs and links (in that order)
-     * 5. Remove all inputs and forms associated with the connector, and
-     *    register the new forms and inputs.
+     * 5. Remove all inputs and configs associated with the connector, and
+     *    register the new configs and inputs.
      * 6. Create new links and jobs with connector part being the ones
      *    returned by the upgrader.
      * 7. Validate new links and jobs with connector's validator
@@ -401,85 +408,63 @@ public abstract class Repository {
         ConnectorManager.getInstance().getConnector(newConnector
           .getUniqueName());
 
-      Validator validator = connector.getValidator();
-
+      Validator connectorConfigValidator = connector.getConfigValidator();
       boolean upgradeSuccessful = true;
-
       RepositoryUpgrader upgrader = connector.getRepositoryUpgrader();
-      List<MLink> links = findLinksForConnector(
-        connectorID);
-      List<MJob> jobs = findJobsForConnector(connectorID);
+      List<MLink> linksByConnector = findLinksForConnector(connectorID);
+      List<MJob> jobsByConnector = findJobsForConnector(connectorID);
       // -- BEGIN TXN --
       tx = getTransaction();
       tx.begin();
-      deletelinksAndJobs(links, jobs, tx);
+      deletelinksAndJobs(linksByConnector, jobsByConnector, tx);
       updateConnector(newConnector, tx);
-      for (MLink link : links) {
-        // Make a new copy of the forms from the connector,
-        // else the values will get set in the forms in the connector for
-        // each link.
-        List<MForm> forms = newConnector.getConnectionForms().clone(false).getForms();
-        MConnectionForms newlinkForms = new MConnectionForms(forms);
-        upgrader.upgrade(link.getConnectorPart(), newlinkForms);
-        MLink newlink = new MLink(link, newlinkForms, link.getFrameworkPart());
-
-        // Transform form structures to objects for validations
+      for (MLink oldLink : linksByConnector) {
+        // Make a new copy of the configs
+        List<MConfig> linkConfig = newConnector.getLinkConfig().clone(false).getConfigs();
+        MLinkConfig newLinkConfig = new MLinkConfig(linkConfig);
+        MLinkConfig oldLinkConfig = oldLink.getConnectorLinkConfig();
+        upgrader.upgrade(oldLinkConfig, newLinkConfig);
+
+        MLink newlink = new MLink(oldLink, newLinkConfig);
+
         Object newConfigurationObject = ClassUtils.instantiate(connector.getLinkConfigurationClass());
-        FormUtils.fromForms(newlink.getConnectorPart().getForms(), newConfigurationObject);
+        ConfigUtils.fromConfigs(newlink.getConnectorLinkConfig().getConfigs(), newConfigurationObject);
 
-        Validation validation = validator.validateLink(newConfigurationObject);
-        if (validation.getStatus().canProceed()) {
+        ConfigValidator configValidator = connectorConfigValidator.validateConfigForLink(newConfigurationObject);
+        if (configValidator.getStatus().canProceed()) {
           updateLink(newlink, tx);
         } else {
-          logInvalidModelObject("link", newlink, validation);
+          logInvalidModelObject("link", newlink, configValidator);
           upgradeSuccessful = false;
         }
       }
-      for (MJob job : jobs) {
-        // Make a new copy of the forms from the connector,
-        // else the values will get set in the forms in the connector for
+      for (MJob job : jobsByConnector) {
+        // Make a new copy of the configs
+        // else the values will get set in the configs in the connector for
         // each job.
-        List<MForm> fromForms = newConnector.getJobForms(Direction.FROM).clone(false).getForms();
-        List<MForm> toForms = newConnector.getJobForms(Direction.TO).clone(false).getForms();
+        List<MConfig> fromConfig = newConnector.getConfig(Direction.FROM).clone(false).getConfigs();
+        List<MConfig> toConfig = newConnector.getConfig(Direction.TO).clone(false).getConfigs();
 
-        // New FROM direction forms, old TO direction forms.
+        // New FROM direction configs, old TO direction configs.
         if (job.getConnectorId(Direction.FROM) == newConnector.getPersistenceId()) {
-          MJobForms newFromJobForms = new MJobForms(fromForms);
-          MJobForms oldToJobForms = job.getConnectorPart(Direction.TO);
-          upgrader.upgrade(job.getConnectorPart(Direction.FROM), newFromJobForms);
-          MJob newJob = new MJob(job, newFromJobForms, oldToJobForms, job.getFrameworkPart());
-          updateJob(newJob, tx);
+          MFromConfig newFromConfig = new MFromConfig(fromConfig);
+          MFromConfig oldFromCOnfig = job.getFromJobConfig();
+          upgrader.upgrade(oldFromCOnfig, newFromConfig);
 
-          // Transform form structures to objects for validations
-//          Object newFromConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.FROM));
-//          FormUtils.fromForms(newJob.getConnectorPart(Direction.FROM).getForms(), newFromConfigurationObject);
-//          Validation fromValidation = validator.validateJob(newFromConfigurationObject);
-//          if (fromValidation.getStatus().canProceed()) {
-//            updateJob(newJob, tx);
-//          } else {
-//            logInvalidModelObject("job", newJob, fromValidation);
-//            upgradeSuccessful = false;
-//          }
+          MToConfig oldToConfig = job.getToJobConfig();
+          MJob newJob = new MJob(job, newFromConfig, oldToConfig, job.getDriverConfig());
+          updateJob(newJob, tx);
         }
 
-        // Old FROM direction forms, new TO direction forms.
+        // Old FROM direction configs, new TO direction configs.
         if (job.getConnectorId(Direction.TO) == newConnector.getPersistenceId()) {
-          MJobForms oldFromJobForms = job.getConnectorPart(Direction.FROM);
-          MJobForms newToJobForms = new MJobForms(toForms);
-          upgrader.upgrade(job.getConnectorPart(Direction.TO), newToJobForms);
-          MJob newJob = new MJob(job, oldFromJobForms, newToJobForms, job.getFrameworkPart());
-          updateJob(newJob, tx);
 
-          // Transform form structures to objects for validations
-//          Object newToConfigurationObject = ClassUtils.instantiate(connector.getJobConfigurationClass(Direction.TO));
-//          FormUtils.fromForms(newJob.getConnectorPart(Direction.TO).getForms(), newToConfigurationObject);
-//          Validation toValidation = validator.validateJob(newToConfigurationObject);
-//          if (toValidation.getStatus().canProceed()) {
-//            updateJob(newJob, tx);
-//          } else {
-//            logInvalidModelObject("job", newJob, toValidation);
-//            upgradeSuccessful = false;
-//          }
+          MToConfig oldToConfig = job.getToJobConfig();
+          MToConfig newToConfig = new MToConfig(toConfig);
+          upgrader.upgrade(oldToConfig, newToConfig);
+          MFromConfig oldFromConfig = job.getFromJobConfig();
+          MJob newJob = new MJob(job, oldFromConfig, newToConfig, job.getDriverConfig());
+          updateJob(newJob, tx);
         }
       }
 
@@ -506,60 +491,35 @@ public abstract class Repository {
     }
   }
 
-  public final void upgradeDriverConfig(MDriverConfig driverConfig) {
-    LOG.info("Upgrading driver config");
+  public final void upgradeDriver(MDriver driver) {
+    LOG.info("Upgrading driver");
     RepositoryTransaction tx = null;
     try {
-      RepositoryUpgrader upgrader = Driver.getInstance()
+      RepositoryUpgrader driverConfigUpgrader = Driver.getInstance()
         .getDriverConfigRepositoryUpgrader();
-      List<MLink> links = findLinks();
       List<MJob> jobs = findJobs();
 
       Validator validator = Driver.getInstance().getValidator();
-
       boolean upgradeSuccessful = true;
 
       // -- BEGIN TXN --
       tx = getTransaction();
       tx.begin();
-      deletelinksAndJobs(links, jobs, tx);
-      updateDriverConfig(driverConfig, tx);
-      for (MLink link : links) {
-        // Make a new copy of the forms from the connector,
-        // else the values will get set in the forms in the connector for
-        // each link.
-        // @TODO(Abe): From/To link forms.
-        List<MForm> forms = driverConfig.getConnectionForms().clone(false).getForms();
-        MConnectionForms newlinkForms = new MConnectionForms(forms);
-        upgrader.upgrade(link.getFrameworkPart(), newlinkForms);
-        MLink newlink = new MLink(link, link.getConnectorPart(), newlinkForms);
-
-        // Transform form structures to objects for validations
-        Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getLinkConfigurationClass());
-        FormUtils.fromForms(newlink.getFrameworkPart().getForms(), newConfigurationObject);
-
-        Validation validation = validator.validateLink(newConfigurationObject);
-        if (validation.getStatus().canProceed()) {
-          updateLink(newlink, tx);
-        } else {
-          logInvalidModelObject("link", newlink, validation);
-          upgradeSuccessful = false;
-        }
-      }
+      deleteJobs(jobs, tx);
+      updateDriver(driver, tx);
+
       for (MJob job : jobs) {
-        // Make a new copy of the forms from the framework,
-        // else the values will get set in the forms in the connector for
-        // each link.
-        List<MForm> forms = driverConfig.getJobForms().clone(false).getForms();
-        MJobForms newJobForms = new MJobForms(forms);
-        upgrader.upgrade(job.getFrameworkPart(), newJobForms);
-        MJob newJob = new MJob(job, job.getConnectorPart(Direction.FROM), job.getConnectorPart(Direction.TO), newJobForms);
-
-        // Transform form structures to objects for validations
-        Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getJobConfigurationClass());
-        FormUtils.fromForms(newJob.getFrameworkPart().getForms(), newConfigurationObject);
-
-        Validation validation = validator.validateJob(newConfigurationObject);
+        // Make a new copy of the configs
+        MDriverConfig driverConfig = driver.getDriverConfig().clone(false);
+        MDriver newDriver = new MDriver(driverConfig, DriverBean.CURRENT_DRIVER_VERSION);
+        driverConfigUpgrader.upgrade(job.getDriverConfig(), newDriver.getDriverConfig());
+        MJob newJob = new MJob(job, job.getFromJobConfig(), job.getToJobConfig(), newDriver.getDriverConfig());
+
+        // Transform config structures to objects for validations
+        Object newConfigurationObject = ClassUtils.instantiate(Driver.getInstance().getDriverConfigurationGroupClass());
+        ConfigUtils.fromConfigs(newJob.getDriverConfig().getConfigs(), newConfigurationObject);
+
+        ConfigValidator validation = validator.validateConfigForJob(newConfigurationObject);
         if (validation.getStatus().canProceed()) {
           updateJob(newJob, tx);
         } else {
@@ -587,14 +547,14 @@ public abstract class Repository {
       if(tx != null) {
         tx.close();
       }
-      LOG.info("Driver config upgrade finished");
+      LOG.info("Driver upgrade finished");
     }
   }
 
-  private void logInvalidModelObject(String objectType, MPersistableEntity entity, Validation validation) {
+  private void logInvalidModelObject(String objectType, MPersistableEntity entity, ConfigValidator validation) {
     LOG.error("Upgrader created invalid " + objectType + " with id" + entity.getPersistenceId());
 
-    for(Map.Entry<Validation.FormInput, Validation.Message> entry : validation.getMessages().entrySet()) {
+    for(Map.Entry<ConfigValidator.ConfigInput, ConfigValidator.Message> entry : validation.getMessages().entrySet()) {
       LOG.error("\t" + entry.getKey() + ": " + entry.getValue());
     }
   }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
index ae7be82..c2f8505 100644
--- a/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
+++ b/core/src/main/java/org/apache/sqoop/repository/RepositoryManager.java
@@ -118,7 +118,7 @@ public class RepositoryManager implements Reconfigurable {
     provider.initialize(context);
 
     if(!immutableRepository) {
-      LOG.info("Creating or upgrading on disk structures if necessary");
+      LOG.info("Creating or update respository internals at bootup");
       provider.getRepository().createOrUpdateInternals();
     }
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/main/resources/driver-config.properties
----------------------------------------------------------------------
diff --git a/core/src/main/resources/driver-config.properties b/core/src/main/resources/driver-config.properties
index 9ec5d9e..78c3b38 100644
--- a/core/src/main/resources/driver-config.properties
+++ b/core/src/main/resources/driver-config.properties
@@ -19,12 +19,12 @@
 
 # Throttling From
 #
-throttling.label = Throttling resources
-throttling.help = Set throttling boundaries to not overload your systems
+throttthrottlingConfigling.label = Throttling resources
+throttlingConfig.help = Set throttling boundaries to not overload your systems
 
-throttling.extractors.label = Extractors
-throttling.extractors.help = Number of extractors that Sqoop will use
+throttlingConfig.numExtractors.label = Extractors
+throttlingConfig.numExtractors.help = Number of extractors that Sqoop will use
 
-throttling.loaders.label = Loaders
-throttling.loaders.help = Number of loaders that Sqoop will use
+throttlingConfig.numLoaders.label = Loaders
+throttlingConfig.numLoaders.help = Number of loaders that Sqoop will use
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
index 9c39d23..dc4e8c8 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestDriverConfigUpgrader.java
@@ -18,16 +18,19 @@
  */
 package org.apache.sqoop.driver;
 
-import org.apache.sqoop.driver.DriverConfigUpgrader;
-import org.apache.sqoop.model.*;
-import org.junit.Before;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 import java.util.LinkedList;
 import java.util.List;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
+import org.apache.sqoop.model.MConfig;
+import org.apache.sqoop.model.MConfigList;
+import org.apache.sqoop.model.MInput;
+import org.apache.sqoop.model.MIntegerInput;
+import org.apache.sqoop.model.MStringInput;
+import org.junit.Before;
+import org.junit.Test;
 
 /**
  */
@@ -39,35 +42,14 @@ public class TestDriverConfigUpgrader {
   public void initializeUpgrader() {
     upgrader = new DriverConfigUpgrader();
   }
-
-  /**
-   * We take the same forms on input and output and we
-   * expect that all values will be correctly transferred.
-   */
-  @Test
-  public void testConnectionUpgrade() {
-    MConnectionForms original = connection1();
-    MConnectionForms target = connection1();
-
-    original.getStringInput("f1.s1").setValue("A");
-    original.getStringInput("f1.s2").setValue("B");
-    original.getIntegerInput("f1.i").setValue(3);
-
-    upgrader.upgrade(original, target);
-
-    assertEquals("A", target.getStringInput("f1.s1").getValue());
-    assertEquals("B", target.getStringInput("f1.s2").getValue());
-    assertEquals(3, (long)target.getIntegerInput("f1.i").getValue());
-  }
-
   /**
-   * We take the same forms on input and output and we
+   * We take the same configs on input and output and we
    * expect that all values will be correctly transferred.
    */
   @Test
-  public void testJobUpgrade() {
-    MJobForms original = job1();
-    MJobForms target = job1();
+  public void testJobConfigTyeUpgrade() {
+    MConfigList original = job();
+    MConfigList target = job();
 
     original.getStringInput("f1.s1").setValue("A");
     original.getStringInput("f1.s2").setValue("B");
@@ -85,8 +67,8 @@ public class TestDriverConfigUpgrader {
    */
   @Test
   public void testNonExistingInput() {
-    MConnectionForms original = connection1();
-    MConnectionForms target = connection2();
+    MConfigList original = job1();
+    MConfigList target = job2();
 
     original.getStringInput("f1.s1").setValue("A");
     original.getStringInput("f1.s2").setValue("B");
@@ -104,9 +86,9 @@ public class TestDriverConfigUpgrader {
    * therefore is missing in the original.
    */
   @Test
-  public void testNonExistingForm() {
-    MConnectionForms original = connection1();
-    MConnectionForms target = connection3();
+  public void testNonExistingConfig() {
+    MConfigList original = job1();
+    MConfigList target = job3();
 
     original.getStringInput("f1.s1").setValue("A");
     original.getStringInput("f1.s2").setValue("B");
@@ -119,25 +101,25 @@ public class TestDriverConfigUpgrader {
     assertNull(target.getIntegerInput("f2.i").getValue());
   }
 
-  MJobForms job1() {
-    return new MJobForms(forms1());
+  MConfigList job() {
+    return new MConfigList(configs1());
   }
 
-  MConnectionForms connection1() {
-    return new MConnectionForms(forms1());
+  MConfigList job1() {
+    return new MConfigList(configs1());
   }
 
-  MConnectionForms connection2() {
-    return new MConnectionForms(forms2());
+  MConfigList job2() {
+    return new MConfigList(configs2());
   }
 
-  MConnectionForms connection3() {
-    return new MConnectionForms(forms3());
+  MConfigList job3() {
+    return new MConfigList(configs3());
   }
 
-  List<MForm> forms1() {
-    List<MForm> list = new LinkedList<MForm>();
-    list.add(new MForm("f1", inputs1("f1")));
+  List<MConfig> configs1() {
+    List<MConfig> list = new LinkedList<MConfig>();
+    list.add(new MConfig("f1", inputs1("f1")));
     return list;
   }
 
@@ -149,9 +131,9 @@ public class TestDriverConfigUpgrader {
     return list;
   }
 
-  List<MForm> forms2() {
-    List<MForm> list = new LinkedList<MForm>();
-    list.add(new MForm("f1", inputs2("f1")));
+  List<MConfig> configs2() {
+    List<MConfig> list = new LinkedList<MConfig>();
+    list.add(new MConfig("f1", inputs2("f1")));
     return list;
   }
 
@@ -163,9 +145,9 @@ public class TestDriverConfigUpgrader {
     return list;
   }
 
-  List<MForm> forms3() {
-    List<MForm> list = new LinkedList<MForm>();
-    list.add(new MForm("f2", inputs1("f2")));
+  List<MConfig> configs3() {
+    List<MConfig> list = new LinkedList<MConfig>();
+    list.add(new MConfig("f2", inputs1("f2")));
     return list;
   }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f63c080d/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
index aa3af89..3b475c6 100644
--- a/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
+++ b/core/src/test/java/org/apache/sqoop/driver/TestJobManager.java
@@ -20,6 +20,13 @@ package org.apache.sqoop.driver;
 import java.util.Arrays;
 import java.util.List;
 
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
 import org.apache.sqoop.common.Direction;
 import org.apache.sqoop.common.SqoopException;
 import org.apache.sqoop.connector.ConnectorManager;
@@ -34,13 +41,6 @@ import org.apache.sqoop.request.HttpEventContext;
 import org.junit.Before;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
 public class TestJobManager {
   private JobManager jobManager;
   private SqoopConnector sqoopConnectorMock;
@@ -99,7 +99,7 @@ public class TestJobManager {
 
   @Test
   public void testGetLink() {
-    MLink testLink = new MLink(123l, null, null);
+    MLink testLink = new MLink(123l, null);
     testLink.setEnabled(true);
     MLink mConnectionSpy = org.mockito.Mockito.spy(testLink);
     when(repositoryManagerMock.getRepository()).thenReturn(jdbcRepoMock);
@@ -111,7 +111,7 @@ public class TestJobManager {
 
   @Test
   public void testDisabledLink() {
-    MLink testConnection = new MLink(123l, null, null);
+    MLink testConnection = new MLink(123l, null);
     testConnection.setPersistenceId(1234);
     testConnection.setEnabled(false);
     SqoopException exception = new SqoopException(DriverError.DRIVER_0010, "Connection id: "


Mime
View raw message