sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject sqoop git commit: SQOOP-1577: Sqoop2: Refactor repository dump and load tools
Date Fri, 20 Feb 2015 16:45:36 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 98d0db1f5 -> 70a08a029


SQOOP-1577: Sqoop2: Refactor repository dump and load tools

(Abraham Elmahrek via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/70a08a02
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/70a08a02
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/70a08a02

Branch: refs/heads/sqoop2
Commit: 70a08a029a2c48cb9b23ad5e06654c7325872575
Parents: 98d0db1
Author: Jarek Jarcec Cecho <jarcec@apache.org>
Authored: Fri Feb 20 08:44:57 2015 -0800
Committer: Jarek Jarcec Cecho <jarcec@apache.org>
Committed: Fri Feb 20 08:44:57 2015 -0800

----------------------------------------------------------------------
 .../apache/sqoop/tools/tool/JSONConstants.java  |  3 +-
 .../sqoop/tools/tool/RepositoryDumpTool.java    | 38 ++++++----
 .../sqoop/tools/tool/RepositoryLoadTool.java    | 78 +++++++++++---------
 3 files changed, 69 insertions(+), 50 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
index 9bec0d0..9301707 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
@@ -25,8 +25,9 @@ public final class JSONConstants {
   }
 
   public static final String CONNECTOR_ID = "connector-id";
+  public static final String FROM_CONNECTOR_ID = "from-connector-id";
+  public static final String TO_CONNECTOR_ID = "from-connector-id";
   public static final String CONNECTOR_NAME = "connector-name";
-  public static final String ALL = "all";
   public static final String NAME = "name";
   public static final String LINK_ID = "link-id";
   public static final String JOB_ID = "job-id";

http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
index 53dac1b..1027baf 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
@@ -21,6 +21,7 @@ import java.io.BufferedWriter;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.util.Iterator;
+import java.util.List;
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
@@ -31,10 +32,10 @@ import org.apache.commons.cli.ParseException;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.common.VersionInfo;
 import org.apache.sqoop.connector.ConnectorManager;
-import org.apache.sqoop.json.JobBean;
-import org.apache.sqoop.json.JsonBean;
-import org.apache.sqoop.json.LinkBean;
+import org.apache.sqoop.json.JobsBean;
+import org.apache.sqoop.json.LinksBean;
 import org.apache.sqoop.json.SubmissionsBean;
+import org.apache.sqoop.model.MLink;
 import org.apache.sqoop.repository.Repository;
 import org.apache.sqoop.repository.RepositoryManager;
 import org.apache.sqoop.tools.ConfiguredTool;
@@ -101,17 +102,26 @@ public class RepositoryDumpTool extends ConfiguredTool {
 
     JSONObject result = new JSONObject();
 
-    LOG.info("Dumping Connections with skipSensitive=" + String.valueOf(skipSensitive));
-    LinkBean links = new LinkBean(repository.findLinks());
-    result.put(JSONConstants.LINKS, addConnectorName(links.extract(skipSensitive)));
+    LOG.info("Dumping Links with skipSensitive=" + String.valueOf(skipSensitive));
+    List<MLink> links = repository.findLinks();
+    LinksBean linkBeans = new LinksBean(links);
+    JSONObject linksJsonObject = linkBeans.extract(skipSensitive);
+    JSONArray linksJsonArray = (JSONArray)linksJsonObject.get(JSONConstants.LINKS);
+    addConnectorName(linksJsonArray, JSONConstants.CONNECTOR_ID);
+    result.put(JSONConstants.LINKS, linksJsonObject);
 
     LOG.info("Dumping Jobs with skipSensitive=" + String.valueOf(skipSensitive));
-    JobBean jobs = new JobBean(repository.findJobs());
-    result.put(JSONConstants.JOBS, addConnectorName(jobs.extract(skipSensitive)));
+    JobsBean jobs = new JobsBean(repository.findJobs());
+    JSONObject jobsJsonObject = jobs.extract(skipSensitive);
+    JSONArray jobsJsonArray = (JSONArray)jobsJsonObject.get(JSONConstants.JOBS);
+    addConnectorName(jobsJsonArray, JSONConstants.FROM_CONNECTOR_ID);
+    addConnectorName(jobsJsonArray, JSONConstants.TO_CONNECTOR_ID);
+    result.put(JSONConstants.JOBS, jobsJsonObject);
 
     LOG.info("Dumping Submissions with skipSensitive=" + String.valueOf(skipSensitive));
     SubmissionsBean submissions = new SubmissionsBean(repository.findSubmissions());
-    result.put(JSONConstants.SUBMISSIONS, submissions.extract(skipSensitive));
+    JSONObject submissionsJsonObject = submissions.extract(skipSensitive);
+    result.put(JSONConstants.SUBMISSIONS, submissionsJsonObject);
 
     result.put(JSONConstants.METADATA, repoMetadata(skipSensitive));
 
@@ -129,19 +139,17 @@ public class RepositoryDumpTool extends ConfiguredTool {
     return metadata;
   }
 
-  private JSONObject addConnectorName(JSONObject json) {
+  private JSONArray addConnectorName(JSONArray jsonArray, String connectorKey) {
     ConnectorManager connectorManager = ConnectorManager.getInstance();
 
-    JSONArray results = (JSONArray) json.get(JsonBean.ALL);
-
-    Iterator<JSONObject> iterator = results.iterator();
+    Iterator<JSONObject> iterator = jsonArray.iterator();
 
     while (iterator.hasNext()) {
       JSONObject result = iterator.next();
-      Long connectorId = (Long) result.get(JSONConstants.CONNECTOR_ID);
+      Long connectorId = (Long) result.get(connectorKey);
       result.put(JSONConstants.CONNECTOR_NAME,  connectorManager.getConnectorConfigurable(connectorId).getUniqueName());
     }
 
-    return json;
+    return jsonArray;
   }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/70a08a02/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
index d0ebaed..e13cf1e 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
@@ -43,9 +43,9 @@ import org.apache.sqoop.connector.spi.SqoopConnector;
 import org.apache.sqoop.driver.Driver;
 import org.apache.sqoop.driver.DriverUpgrader;
 import org.apache.sqoop.json.JSONUtils;
-import org.apache.sqoop.json.JobBean;
-import org.apache.sqoop.json.LinkBean;
-import org.apache.sqoop.json.SubmissionBean;
+import org.apache.sqoop.json.JobsBean;
+import org.apache.sqoop.json.LinksBean;
+import org.apache.sqoop.json.SubmissionsBean;
 import org.apache.sqoop.model.ConfigUtils;
 import org.apache.sqoop.model.MConfig;
 import org.apache.sqoop.model.MConnector;
@@ -136,19 +136,23 @@ public class RepositoryLoadTool extends ConfiguredTool {
     ConnectorManager.getInstance().initialize();
     LOG.info("Loading Connections");
 
-    JSONObject jsonConns = (JSONObject) repo.get(JSONConstants.LINKS);
+    JSONObject jsonLinks = (JSONObject) repo.get(JSONConstants.LINKS);
 
-    if (jsonConns == null) {
+    if (jsonLinks == null) {
       LOG.error("Malformed JSON file. Key " + JSONConstants.LINKS + " not found.");
       return false;
     }
 
-    LinkBean linkBean = new LinkBean();
-    linkBean.restore(updateConnectorIDUsingName(jsonConns));
+    updateConnectorIDUsingName(
+        (JSONArray)jsonLinks.get(JSONConstants.LINKS),
+        JSONConstants.CONNECTOR_ID);
 
-    HashMap<Long, Long> connectionIds = new HashMap<Long, Long>();
+    LinksBean linksBean = new LinksBean();
+    linksBean.restore(jsonLinks);
 
-    for (MLink link : linkBean.getLinks()) {
+    HashMap<Long, Long> linkIds = new HashMap<Long, Long>();
+
+    for (MLink link : linksBean.getLinks()) {
       long oldId = link.getPersistenceId();
       long newId = loadLink(link);
       if (newId == link.PERSISTANCE_ID_DEFAULT) {
@@ -156,9 +160,9 @@ public class RepositoryLoadTool extends ConfiguredTool {
             + " failed. Aborting repository load. Check log for details.");
         return false;
       }
-      connectionIds.put(oldId, newId);
+      linkIds.put(oldId, newId);
     }
-    LOG.info("Loaded " + connectionIds.size() + " connections");
+    LOG.info("Loaded " + linkIds.size() + " links");
 
     LOG.info("Loading Jobs");
     JSONObject jsonJobs = (JSONObject) repo.get(JSONConstants.JOBS);
@@ -168,12 +172,20 @@ public class RepositoryLoadTool extends ConfiguredTool {
       return false;
     }
 
-    JobBean jobBean = new JobBean();
-    jobBean.restore(updateIdUsingMap(updateConnectorIDUsingName(jsonJobs), connectionIds,
-        JSONConstants.LINK_ID));
+    updateConnectorIDUsingName(
+        (JSONArray)jsonJobs.get(JSONConstants.JOBS),
+        JSONConstants.FROM_CONNECTOR_ID);
+    updateConnectorIDUsingName(
+        (JSONArray)jsonJobs.get(JSONConstants.JOBS),
+        JSONConstants.TO_CONNECTOR_ID);
+    updateIdUsingMap((JSONArray)jsonJobs.get(JSONConstants.JOBS),
+        linkIds, JSONConstants.LINK_ID);
+
+    JobsBean jobsBean = new JobsBean();
+    jobsBean.restore(jsonJobs);
 
     HashMap<Long, Long> jobIds = new HashMap<Long, Long>();
-    for (MJob job : jobBean.getJobs()) {
+    for (MJob job : jobsBean.getJobs()) {
       long oldId = job.getPersistenceId();
       long newId = loadJob(job);
 
@@ -195,10 +207,12 @@ public class RepositoryLoadTool extends ConfiguredTool {
       return false;
     }
 
-    SubmissionBean submissionBean = new SubmissionBean();
-    submissionBean.restore(updateIdUsingMap(jsonSubmissions, jobIds, JSONConstants.JOB_ID));
+    updateIdUsingMap((JSONArray)jsonSubmissions.get(JSONConstants.SUBMISSIONS), jobIds, JSONConstants.JOB_ID);
+
+    SubmissionsBean submissionsBean = new SubmissionsBean();
+    submissionsBean.restore(jsonSubmissions);
     int submissionCount = 0;
-    for (MSubmission submission : submissionBean.getSubmissions()) {
+    for (MSubmission submission : submissionsBean.getSubmissions()) {
       resetPersistenceId(submission);
       repository.createSubmission(submission);
       submissionCount++;
@@ -238,7 +252,7 @@ public class RepositoryLoadTool extends ConfiguredTool {
 
   private long loadLink(MLink link) {
 
-    // starting by pretending we have a brand new connection
+    // starting by pretending we have a brand new link
     resetPersistenceId(link);
 
     Repository repository = RepositoryManager.getInstance().getRepository();
@@ -345,9 +359,7 @@ public class RepositoryLoadTool extends ConfiguredTool {
 
   }
 
-  private JSONObject updateConnectorIDUsingName(JSONObject json) {
-    JSONArray array = (JSONArray) json.get(JSONConstants.ALL);
-
+  private JSONArray updateConnectorIDUsingName(JSONArray jsonArray, String connectorIdKey)
{
     Repository repository = RepositoryManager.getInstance().getRepository();
 
     List<MConnector> connectors = repository.findConnectors();
@@ -357,34 +369,32 @@ public class RepositoryLoadTool extends ConfiguredTool {
       connectorMap.put(connector.getUniqueName(), connector.getPersistenceId());
     }
 
-    for (Object obj : array) {
+    for (Object obj : jsonArray) {
       JSONObject object = (JSONObject) obj;
-      long connectorId = (Long) object.get(JSONConstants.CONNECTOR_ID);
+      long connectorId = (Long) object.get(connectorIdKey);
       String connectorName = (String) object.get(JSONConstants.CONNECTOR_NAME);
       long currentConnectorId = connectorMap.get(connectorName);
-      String connectionName = (String) object.get(JSONConstants.NAME);
+      String linkName = (String) object.get(JSONConstants.NAME);
 
       // If a given connector now has a different ID, we need to update the ID
       if (connectorId != currentConnectorId) {
-        LOG.warn("Connection " + connectionName + " uses connector " + connectorName + ".
"
+        LOG.warn("Link " + linkName + " uses connector " + connectorName + ". "
             + "Replacing previous ID " + connectorId + " with new ID " + currentConnectorId);
 
-        object.put(JSONConstants.CONNECTOR_ID, currentConnectorId);
+        object.put(connectorIdKey, currentConnectorId);
       }
     }
-    return json;
-  }
 
-  private JSONObject updateIdUsingMap(JSONObject json, HashMap<Long, Long> idMap, String
fieldName) {
-    JSONArray array = (JSONArray) json.get(JSONConstants.ALL);
+    return jsonArray;
+  }
 
-    for (Object obj : array) {
+  private JSONArray updateIdUsingMap(JSONArray jsonArray, HashMap<Long, Long> idMap,
String fieldName) {
+    for (Object obj : jsonArray) {
       JSONObject object = (JSONObject) obj;
 
       object.put(fieldName, idMap.get(object.get(fieldName)));
     }
 
-    return json;
+    return jsonArray;
   }
-
 }


Mime
View raw message