sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kathl...@apache.org
Subject git commit: SQOOP-946: Sqoop2: Integration: Enhance the POC test case (Jarek Jarcec Cecho via Kate Ting)
Date Tue, 26 Mar 2013 08:15:53 GMT
Updated Branches:
  refs/heads/sqoop2 27271af13 -> 3fd8bdffc


SQOOP-946: Sqoop2: Integration: Enhance the POC test case
(Jarek Jarcec Cecho via Kate Ting)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3fd8bdff
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3fd8bdff
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3fd8bdff

Branch: refs/heads/sqoop2
Commit: 3fd8bdffc9510cc9c5bbd159e9036891171496a8
Parents: 27271af
Author: Kate Ting <kathleen@apache.org>
Authored: Tue Mar 26 04:15:05 2013 -0400
Committer: Kate Ting <kathleen@apache.org>
Committed: Tue Mar 26 04:15:05 2013 -0400

----------------------------------------------------------------------
 .../apache/sqoop/integration/TomcatTestCase.java   |   31 ++++-
 .../integration/connector/ConnectorTestCase.java   |   92 +++++++++++++-
 .../connector/jdbc/generic/TableImportTest.java    |   99 ++++++++-------
 3 files changed, 164 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/3fd8bdff/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
index 5631144..6eb3184 100644
--- a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
+++ b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
@@ -23,6 +23,8 @@ import org.apache.sqoop.client.SqoopClient;
 import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -45,17 +47,29 @@ abstract public class TomcatTestCase {
 
   private static final Logger LOG = Logger.getLogger(TomcatTestCase.class);
 
+  @Rule public TestName name = new TestName();
+
   /**
-   * Temporary path that will be used for this test.
+   * Temporary base path that will be used for tests.
    *
-   * By default we will take look for sqoop.integration.tmpdir property that is
+   * By default we will take a look for sqoop.integration.tmpdir property that is
    * filled up by maven. If the test is not started from maven (IDE) we will
    * pick up configured java.io.tmpdir value. The last results is /tmp/ directory
    * in case that no property is set.
    */
-  private final String TMP_PATH =
-    System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp"))
-      + "/sqoop-cargo-tests/" + getClass().getName() + "/";
+  private static final String TMP_PATH_BASE =
+    System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp"))
+ "/sqoop-cargo-tests/";
+
+  /**
+   * Temporary directory that will be used by the test.
+   *
+   * We will take TMP_PATH_BASE and append two subdirectories. First will be named
+   * after fully qualified class name of current test class, second directory will
+   * be named after current test method name. For example:
+   *
+   * TMP_PATH_BASE/org.apache.sqoop.TestClass/testMethod/
+   */
+  private String tmpPath;
 
   /**
    * Tomcat based Sqoop mini cluster
@@ -69,9 +83,14 @@ abstract public class TomcatTestCase {
 
   @Before
   public void startServer() throws Exception {
+    // Set up the temporary path
+    tmpPath = TMP_PATH_BASE + getClass().getName() + "/" + name.getMethodName() + "/";
+
+    // Set up and start server
     cluster = new TomcatSqoopMiniCluster(getTemporaryPath());
     cluster.start();
 
+    // Initialize Sqoop Client API
     client = new SqoopClient(getServerUrl());
   }
 
@@ -90,7 +109,7 @@ abstract public class TomcatTestCase {
   }
 
   public String getTemporaryPath() {
-    return TMP_PATH;
+    return tmpPath;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3fd8bdff/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
index 14f7dc4..837a1aa 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
@@ -18,11 +18,21 @@
 package org.apache.sqoop.integration.connector;
 
 import org.apache.log4j.Logger;
+import org.apache.sqoop.framework.configuration.OutputFormat;
+import org.apache.sqoop.framework.configuration.StorageType;
 import org.apache.sqoop.integration.TomcatTestCase;
+import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MPersistableEntity;
 import org.apache.sqoop.test.db.DatabaseProvider;
 import org.apache.sqoop.test.db.DatabaseProviderFactory;
-import org.junit.After;
-import org.junit.Before;
+import org.apache.sqoop.validation.Status;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
 
 /**
  * Base test case for connector testing.
@@ -33,17 +43,18 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
 
   private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class);
 
-  protected DatabaseProvider provider;
+  protected static DatabaseProvider provider;
 
-  @Before
-  public void startProvider() throws Exception {
+  @BeforeClass
+  public static void startProvider() throws Exception {
     provider = DatabaseProviderFactory.getProvider(System.getProperties());
     LOG.info("Starting database provider: " + provider.getClass().getName());
     provider.start();
   }
 
-  @After
-  public void stopProvider() {
+  @AfterClass
+  public static void stopProvider() {
+    LOG.info("Stopping database provider: " + provider.getClass().getName());
     provider.stop();
   }
 
@@ -62,4 +73,71 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
   protected void insertRow(Object ...values) {
     provider.insertRow(getTableName(), values);
   }
+
+  /**
+   * Fill connection form based on currently active provider.
+   *
+   * @param connection MConnection object to fill
+   */
+  protected void fillConnectionForm(MConnection connection) {
+    MFormList forms = connection.getConnectorPart();
+    forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver());
+    forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl());
+    forms.getStringInput("connection.username").setValue(provider.getConnectionUsername());
+    forms.getStringInput("connection.password").setValue(provider.getConnectionPassword());
+  }
+
+  /**
+   * Fill output form with specific storage and output type. Mapreduce output directory
+   * will be set to default test value.
+   *
+   * @param job MJOb object to fill
+   * @param storage Storage type that should be set
+   * @param output Output type that should be set
+   */
+  protected void fillOutputForm(MJob job, StorageType storage, OutputFormat output) {
+    MFormList forms = job.getFrameworkPart();
+    forms.getEnumInput("output.storageType").setValue(storage);
+    forms.getEnumInput("output.outputFormat").setValue(output);
+    forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory());
+  }
+
+  /**
+   * Create table cities.
+   */
+  protected void createAndLoadTableCities() {
+    createTable("id",
+      "id", "int",
+      "country", "varchar(50)",
+      "city", "varchar(50)"
+    );
+    insertRow(1, "USA", "San Francisco");
+    insertRow(2, "USA", "Sunnyvale");
+    insertRow(3, "Czech Republic", "Brno");
+    insertRow(4, "USA", "Palo Alto");
+  }
+
+  /**
+   * Create connection.
+   *
+   * With asserts to make sure that it was created correctly.
+   *
+   * @param connection
+   */
+  protected void createConnection(MConnection connection) {
+    assertEquals(Status.FINE, getClient().createConnection(connection));
+    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId());
+  }
+
+ /**
+   * Create job.
+   *
+   * With asserts to make sure that it was created correctly.
+   *
+   * @param job
+   */
+ protected void createJob(MJob job) {
+    assertEquals(Status.FINE, getClient().createJob(job));
+    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
+  }
 }

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3fd8bdff/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
index 05a799c..adcfbaf 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
@@ -24,78 +24,87 @@ import org.apache.sqoop.integration.connector.ConnectorTestCase;
 import org.apache.sqoop.model.MConnection;
 import org.apache.sqoop.model.MFormList;
 import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MPersistableEntity;
 import org.apache.sqoop.model.MSubmission;
-import org.apache.sqoop.validation.Status;
 import org.junit.Test;
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
 import static org.junit.Assert.assertTrue;
 
 /**
- * Proof of concept implementation of first "real" integration test.
- *
- * Will be improved when client API will be created.
+ * Import simple table with various configurations.
  */
 public class TableImportTest extends ConnectorTestCase {
 
   private static final Logger LOG = Logger.getLogger(TableImportTest.class);
 
-  /**
-   * This test is proof of concept.
-   *
-   * It will be refactored once we will create reasonable client interface.
-   */
   @Test
-  public void testBasicTableImport() throws Exception {
-    createTable("id",
-      "id", "int",
-      "txt", "varchar(50)"
-    );
-    insertRow(1, "San Francisco");
-    insertRow(2, "Sunnyvale");
-    insertRow(3, "Brno");
-
-    // Connection creation and job submission will be refactored once
-    // the client API for embedding Sqoop client will be ready.
+  public void testBasicImport() throws Exception {
+    createAndLoadTableCities();
 
     // Connection creation
     MConnection connection = getClient().newConnection(1L);
+    fillConnectionForm(connection);
+    createConnection(connection);
+
+    // Job creation
+    MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT);
 
-    MFormList forms;
     // Connector values
-    forms = connection.getConnectorPart();
-    forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver());
-    forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl());
-    forms.getStringInput("connection.username").setValue(provider.getConnectionUsername());
-    forms.getStringInput("connection.password").setValue(provider.getConnectionPassword());
+    MFormList forms = job.getConnectorPart();
+    forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName()));
+    forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName("id"));
     // Framework values
-    // No need to set anything
+    fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE);
+    createJob(job);
 
-    assertEquals(Status.FINE, getClient().createConnection(connection));
-    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId());
+    MSubmission submission = getClient().startSubmission(job.getPersistenceId());
+    assertTrue(submission.getStatus().isRunning());
+
+    // Wait until the job finish - this active waiting will be removed once
+    // Sqoop client API will get blocking support.
+    do {
+      Thread.sleep(5000);
+      submission = getClient().getSubmissionStatus(job.getPersistenceId());
+    } while(submission.getStatus().isRunning());
+
+    // Assert correct output
+    assertMapreduceOutput(
+      "1,'USA','San Francisco'",
+      "2,'USA','Sunnyvale'",
+      "3,'Czech Republic','Brno'",
+      "4,'USA','Palo Alto'"
+    );
+
+    // Clean up testing table
+    dropTable();
+  }
+
+  @Test
+  public void testColumns() throws Exception {
+    createAndLoadTableCities();
+
+    // Connection creation
+    MConnection connection = getClient().newConnection(1L);
+    fillConnectionForm(connection);
+
+    createConnection(connection);
 
     // Job creation
     MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT);
 
     // Connector values
-    forms = job.getConnectorPart();
+    MFormList forms = job.getConnectorPart();
     forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName()));
     forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName("id"));
+    forms.getStringInput("table.columns").setValue(provider.escapeColumnName("id") + ","
+ provider.escapeColumnName("country"));
     // Framework values
-    forms = job.getFrameworkPart();
-    forms.getEnumInput("output.storageType").setValue(StorageType.HDFS);
-    forms.getEnumInput("output.outputFormat").setValue(OutputFormat.TEXT_FILE);
-    forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory());
-
-    assertEquals(Status.FINE, getClient().createJob(job));
-    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
+    fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE);
+    createJob(job);
 
     MSubmission submission = getClient().startSubmission(job.getPersistenceId());
     assertTrue(submission.getStatus().isRunning());
 
-    // Wait until the job finish
+    // Wait until the job finish - this active waiting will be removed once
+    // Sqoop client API will get blocking support.
     do {
       Thread.sleep(5000);
       submission = getClient().getSubmissionStatus(job.getPersistenceId());
@@ -103,13 +112,13 @@ public class TableImportTest extends ConnectorTestCase {
 
     // Assert correct output
     assertMapreduceOutput(
-      "1,'San Francisco'",
-      "2,'Sunnyvale'",
-      "3,'Brno'"
+      "1,'USA'",
+      "2,'USA'",
+      "3,'Czech Republic'",
+      "4,'USA'"
     );
 
     // Clean up testing table
     dropTable();
   }
-
 }


Mime
View raw message