sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From a..@apache.org
Subject sqoop git commit: SQOOP-1256: Sqoop2: Tool: Load repository dump into a different repository
Date Thu, 30 Jul 2015 06:33:34 GMT
Repository: sqoop
Updated Branches:
  refs/heads/sqoop2 3c88f553d -> 97c65879f


SQOOP-1256: Sqoop2: Tool: Load repository dump into a different repository

(Dian Fu via Abraham Elmahrek)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/97c65879
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/97c65879
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/97c65879

Branch: refs/heads/sqoop2
Commit: 97c65879f21bc1ef8ed491ff0f13f8c2370fca49
Parents: 3c88f55
Author: Abraham Elmahrek <abe@apache.org>
Authored: Wed Jul 29 23:32:58 2015 -0700
Committer: Abraham Elmahrek <abe@apache.org>
Committed: Wed Jul 29 23:32:58 2015 -0700

----------------------------------------------------------------------
 .../org/apache/sqoop/json/SubmissionBean.java   |   9 +-
 .../apache/sqoop/tools/tool/JSONConstants.java  |  10 +-
 .../sqoop/tools/tool/RepositoryDumpTool.java    |  40 ++++++-
 .../sqoop/tools/tool/RepositoryLoadTool.java    | 104 ++++++++++++++++---
 4 files changed, 142 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/97c65879/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java b/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
index ad05057..c7d6958 100644
--- a/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
+++ b/common/src/main/java/org/apache/sqoop/json/SubmissionBean.java
@@ -44,7 +44,9 @@ import org.json.simple.JSONObject;
 public class SubmissionBean implements JsonBean {
 
   private static final String SUBMISSION = "submission";
+  @Deprecated
   private static final String JOB = "job";
+  private static final String JOB_ID = "job-id";
   private static final String CREATION_USER = "creation-user";
   private static final String CREATION_DATE = "creation-date";
   private static final String LAST_UPDATE_USER = "last-udpate-user";
@@ -103,6 +105,7 @@ public class SubmissionBean implements JsonBean {
     JSONObject object = new JSONObject();
 
     object.put(JOB, submission.getJobId());
+    object.put(JOB_ID, submission.getJobId());
     object.put(STATUS, submission.getStatus().name());
     object.put(PROGRESS, submission.getProgress());
 
@@ -173,7 +176,11 @@ public class SubmissionBean implements JsonBean {
   private MSubmission restoreSubmission(Object obj) {
     JSONObject object = (JSONObject) obj;
     MSubmission submission = new MSubmission();
-    submission.setJobId((Long) object.get(JOB));
+    Long jobId = (Long) object.get(JOB_ID);
+    if (jobId == null) {
+      jobId = (Long) object.get(JOB);
+    }
+    submission.setJobId(jobId);
     submission.setStatus(SubmissionStatus.valueOf((String) object.get(STATUS)));
     submission.setProgress((Double) object.get(PROGRESS));
 

http://git-wip-us.apache.org/repos/asf/sqoop/blob/97c65879/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
index 9301707..7df7f3b 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/JSONConstants.java
@@ -26,11 +26,17 @@ public final class JSONConstants {
 
   public static final String CONNECTOR_ID = "connector-id";
   public static final String FROM_CONNECTOR_ID = "from-connector-id";
-  public static final String TO_CONNECTOR_ID = "from-connector-id";
+  public static final String TO_CONNECTOR_ID = "to-connector-id";
   public static final String CONNECTOR_NAME = "connector-name";
+  public static final String FROM_CONNECTOR_NAME = "from-connector-name";
+  public static final String TO_CONNECTOR_NAME = "to-connector-name";
   public static final String NAME = "name";
-  public static final String LINK_ID = "link-id";
+  public static final String FROM_LINK_ID = "from-link-id";
+  public static final String TO_LINK_ID = "to-link-id";
+  public static final String FROM_LINK_NAME = "from-link-name";
+  public static final String TO_LINK_NAME = "to-link-name";
   public static final String JOB_ID = "job-id";
+  public static final String JOB_NAME = "job-name";
   public static final String LINKS = "links";
   public static final String JOBS = "jobs";
   public static final String SUBMISSIONS = "submissions";

http://git-wip-us.apache.org/repos/asf/sqoop/blob/97c65879/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
index 1027baf..6cb6210 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryDumpTool.java
@@ -107,20 +107,24 @@ public class RepositoryDumpTool extends ConfiguredTool {
     LinksBean linkBeans = new LinksBean(links);
     JSONObject linksJsonObject = linkBeans.extract(skipSensitive);
     JSONArray linksJsonArray = (JSONArray)linksJsonObject.get(JSONConstants.LINKS);
-    addConnectorName(linksJsonArray, JSONConstants.CONNECTOR_ID);
+    addConnectorName(linksJsonArray, JSONConstants.CONNECTOR_ID, JSONConstants.CONNECTOR_NAME);
     result.put(JSONConstants.LINKS, linksJsonObject);
 
     LOG.info("Dumping Jobs with skipSensitive=" + String.valueOf(skipSensitive));
     JobsBean jobs = new JobsBean(repository.findJobs());
     JSONObject jobsJsonObject = jobs.extract(skipSensitive);
     JSONArray jobsJsonArray = (JSONArray)jobsJsonObject.get(JSONConstants.JOBS);
-    addConnectorName(jobsJsonArray, JSONConstants.FROM_CONNECTOR_ID);
-    addConnectorName(jobsJsonArray, JSONConstants.TO_CONNECTOR_ID);
+    addConnectorName(jobsJsonArray, JSONConstants.FROM_CONNECTOR_ID, JSONConstants.FROM_CONNECTOR_NAME);
+    addConnectorName(jobsJsonArray, JSONConstants.TO_CONNECTOR_ID, JSONConstants.TO_CONNECTOR_NAME);
+    addLinkName(jobsJsonArray, JSONConstants.FROM_LINK_ID, JSONConstants.FROM_LINK_NAME);
+    addLinkName(jobsJsonArray, JSONConstants.TO_LINK_ID, JSONConstants.TO_LINK_NAME);
     result.put(JSONConstants.JOBS, jobsJsonObject);
 
     LOG.info("Dumping Submissions with skipSensitive=" + String.valueOf(skipSensitive));
     SubmissionsBean submissions = new SubmissionsBean(repository.findSubmissions());
     JSONObject submissionsJsonObject = submissions.extract(skipSensitive);
+    JSONArray submissionsJsonArray = (JSONArray)submissionsJsonObject.get(JSONConstants.SUBMISSIONS);
+    addJobName(submissionsJsonArray, JSONConstants.JOB_ID);
     result.put(JSONConstants.SUBMISSIONS, submissionsJsonObject);
 
     result.put(JSONConstants.METADATA, repoMetadata(skipSensitive));
@@ -139,7 +143,7 @@ public class RepositoryDumpTool extends ConfiguredTool {
     return metadata;
   }
 
-  private JSONArray addConnectorName(JSONArray jsonArray, String connectorKey) {
+  private JSONArray addConnectorName(JSONArray jsonArray, String connectorKey, String connectorName)
{
     ConnectorManager connectorManager = ConnectorManager.getInstance();
 
     Iterator<JSONObject> iterator = jsonArray.iterator();
@@ -147,7 +151,33 @@ public class RepositoryDumpTool extends ConfiguredTool {
     while (iterator.hasNext()) {
       JSONObject result = iterator.next();
       Long connectorId = (Long) result.get(connectorKey);
-      result.put(JSONConstants.CONNECTOR_NAME,  connectorManager.getConnectorConfigurable(connectorId).getUniqueName());
+      result.put(connectorName,  connectorManager.getConnectorConfigurable(connectorId).getUniqueName());
+    }
+
+    return jsonArray;
+  }
+
+  private JSONArray addLinkName(JSONArray jsonArray, String linkKey, String linkName) {
+    Repository repository = RepositoryManager.getInstance().getRepository();
+    Iterator<JSONObject> iterator = jsonArray.iterator();
+
+    while (iterator.hasNext()) {
+      JSONObject jobObject = iterator.next();
+      Long linkId = (Long) jobObject.get(linkKey);
+      jobObject.put(linkName, repository.findLink(linkId).getName());
+    }
+
+    return jsonArray;
+  }
+
+  private JSONArray addJobName(JSONArray jsonArray, String jobKey) {
+    Repository repository = RepositoryManager.getInstance().getRepository();
+    Iterator<JSONObject> iterator = jsonArray.iterator();
+
+    while (iterator.hasNext()) {
+      JSONObject submissionObject = iterator.next();
+      Long jobId = (Long) submissionObject.get(jobKey);
+      submissionObject.put(JSONConstants.JOB_NAME,  repository.findJob(jobId).getName());
     }
 
     return jsonArray;

http://git-wip-us.apache.org/repos/asf/sqoop/blob/97c65879/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
index a25ebe8..3cbfcf0 100644
--- a/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
+++ b/tools/src/main/java/org/apache/sqoop/tools/tool/RepositoryLoadTool.java
@@ -23,6 +23,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -134,6 +135,7 @@ public class RepositoryLoadTool extends ConfiguredTool {
     Repository repository = RepositoryManager.getInstance().getRepository();
 
     ConnectorManager.getInstance().initialize();
+    Driver.getInstance().initialize();
     LOG.info("Loading Connections");
 
     JSONObject jsonLinks = (JSONObject) repo.get(JSONConstants.LINKS);
@@ -145,7 +147,9 @@ public class RepositoryLoadTool extends ConfiguredTool {
 
     updateConnectorIDUsingName(
         (JSONArray)jsonLinks.get(JSONConstants.LINKS),
-        JSONConstants.CONNECTOR_ID);
+        JSONConstants.CONNECTOR_ID,
+        JSONConstants.CONNECTOR_NAME,
+        true);
 
     LinksBean linksBean = new LinksBean();
     linksBean.restore(jsonLinks);
@@ -174,12 +178,21 @@ public class RepositoryLoadTool extends ConfiguredTool {
 
     updateConnectorIDUsingName(
         (JSONArray)jsonJobs.get(JSONConstants.JOBS),
-        JSONConstants.FROM_CONNECTOR_ID);
+        JSONConstants.FROM_CONNECTOR_ID,
+        JSONConstants.FROM_CONNECTOR_NAME,
+        false);
     updateConnectorIDUsingName(
         (JSONArray)jsonJobs.get(JSONConstants.JOBS),
-        JSONConstants.TO_CONNECTOR_ID);
-    updateIdUsingMap((JSONArray)jsonJobs.get(JSONConstants.JOBS),
-        linkIds, JSONConstants.LINK_ID);
+        JSONConstants.TO_CONNECTOR_ID,
+        JSONConstants.TO_CONNECTOR_NAME,
+        false);
+
+    updateLinkIdUsingName((JSONArray)jsonJobs.get(JSONConstants.JOBS),
+        JSONConstants.FROM_LINK_ID,
+        JSONConstants.FROM_LINK_NAME);
+    updateLinkIdUsingName((JSONArray)jsonJobs.get(JSONConstants.JOBS),
+        JSONConstants.TO_LINK_ID,
+        JSONConstants.TO_LINK_NAME);
 
     JobsBean jobsBean = new JobsBean();
     jobsBean.restore(jsonJobs);
@@ -207,7 +220,7 @@ public class RepositoryLoadTool extends ConfiguredTool {
       return false;
     }
 
-    updateIdUsingMap((JSONArray)jsonSubmissions.get(JSONConstants.SUBMISSIONS), jobIds, JSONConstants.JOB_ID);
+    updateJobIdUsingName((JSONArray)jsonSubmissions.get(JSONConstants.SUBMISSIONS));
 
     SubmissionsBean submissionsBean = new SubmissionsBean();
     submissionsBean.restore(jsonSubmissions);
@@ -359,7 +372,7 @@ public class RepositoryLoadTool extends ConfiguredTool {
 
   }
 
-  private JSONArray updateConnectorIDUsingName(JSONArray jsonArray, String connectorIdKey)
{
+  private JSONArray updateConnectorIDUsingName(JSONArray jsonArray, String connectorIdKey,
String connectorNameKey, boolean isLink) {
     Repository repository = RepositoryManager.getInstance().getRepository();
 
     List<MConnector> connectors = repository.findConnectors();
@@ -369,16 +382,22 @@ public class RepositoryLoadTool extends ConfiguredTool {
       connectorMap.put(connector.getUniqueName(), connector.getPersistenceId());
     }
 
-    for (Object obj : jsonArray) {
-      JSONObject object = (JSONObject) obj;
+    for (Iterator iterator = jsonArray.iterator(); iterator.hasNext(); ) {
+      JSONObject object = (JSONObject) iterator.next();
       long connectorId = (Long) object.get(connectorIdKey);
-      String connectorName = (String) object.get(JSONConstants.CONNECTOR_NAME);
-      long currentConnectorId = connectorMap.get(connectorName);
-      String linkName = (String) object.get(JSONConstants.NAME);
+      String connectorName = (String) object.get(connectorNameKey);
+      Long currentConnectorId = connectorMap.get(connectorName);
+      String name = (String) object.get(JSONConstants.NAME);
+
+      if (currentConnectorId == null) {
+        // If a connector doesn't exist, remove the links and jobs relating to it
+        iterator.remove();
+        continue;
+      }
 
       // If a given connector now has a different ID, we need to update the ID
       if (connectorId != currentConnectorId) {
-        LOG.warn("Link " + linkName + " uses connector " + connectorName + ". "
+        LOG.warn((isLink ? "Link " : "Job ") + name + " uses connector " + connectorName
+ ". "
             + "Replacing previous ID " + connectorId + " with new ID " + currentConnectorId);
 
         object.put(connectorIdKey, currentConnectorId);
@@ -388,6 +407,65 @@ public class RepositoryLoadTool extends ConfiguredTool {
     return jsonArray;
   }
 
+  private JSONArray updateLinkIdUsingName(JSONArray jobsJsonArray, String linkIdKey, String
linkNameKey) {
+    Repository repository = RepositoryManager.getInstance().getRepository();
+
+    List<MLink> links = repository.findLinks();
+    Map<String, Long> linkMap = new HashMap<String, Long>();
+    for (MLink link : links) {
+      linkMap.put(link.getName(), link.getPersistenceId());
+    }
+
+    for(Iterator iterator = jobsJsonArray.iterator(); iterator.hasNext(); ) {
+      JSONObject jobObject = (JSONObject) iterator.next();
+      long linkId = (Long) jobObject.get(linkIdKey);
+      String linkName = (String) jobObject.get(linkNameKey);
+      Long currentLinkId = linkMap.get(linkName);
+      String jobName = (String) jobObject.get(JSONConstants.NAME);
+
+      if (currentLinkId == null) {
+        // If a link doesn't exist, remove the jobs relating to it
+        iterator.remove();
+        continue;
+      }
+
+      if (linkId != currentLinkId) {
+        LOG.warn("Job " + jobName + " uses link " + linkName + "."
+            + "Replacing previous ID " + linkId + " with new ID " + currentLinkId);
+
+        jobObject.put(linkIdKey, currentLinkId);
+      }
+    }
+
+    return jobsJsonArray;
+  }
+
+  private JSONArray updateJobIdUsingName(JSONArray submissionsJsonArray) {
+    Repository repository = RepositoryManager.getInstance().getRepository();
+
+    List<MJob> jobs = repository.findJobs();
+    Map<String, Long> jobMap = new HashMap<String, Long>();
+    for (MJob job : jobs) {
+      jobMap.put(job.getName(), job.getPersistenceId());
+    }
+
+    for(Iterator iterator = submissionsJsonArray.iterator(); iterator.hasNext(); ) {
+      JSONObject submissionObject = (JSONObject) iterator.next();
+      String jobName = (String) submissionObject.get(JSONConstants.JOB_NAME);
+      Long currentJobId = jobMap.get(jobName);
+
+      if (currentJobId == null) {
+        // If a job doesn't exist, remove the submissions relating to it
+        iterator.remove();
+        continue;
+      }
+
+      submissionObject.put(JSONConstants.JOB_ID, currentJobId);
+    }
+
+    return submissionsJsonArray;
+  }
+
   private JSONArray updateIdUsingMap(JSONArray jsonArray, HashMap<Long, Long> idMap,
String fieldName) {
     for (Object obj : jsonArray) {
       JSONObject object = (JSONObject) obj;


Mime
View raw message