sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From kathl...@apache.org
Subject git commit: SQOOP-1087: Sqoop2: Integration: Abstract common functionality into src module
Date Sun, 23 Jun 2013 22:45:18 GMT
Updated Branches:
  refs/heads/sqoop2 fe54d473a -> f980e90fc


SQOOP-1087: Sqoop2: Integration: Abstract common functionality into src module

(Jarek Jarcec Cecho via Kate Ting)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f980e90f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f980e90f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f980e90f

Branch: refs/heads/sqoop2
Commit: f980e90fc9570b01305166b19d7381c2051bc49e
Parents: fe54d47
Author: Kate Ting <kathleen@apache.org>
Authored: Sun Jun 23 18:43:50 2013 -0400
Committer: Kate Ting <kathleen@apache.org>
Committed: Sun Jun 23 18:43:50 2013 -0400

----------------------------------------------------------------------
 test/pom.xml                                    |   1 -
 .../apache/sqoop/test/asserts/HdfsAsserts.java  |  83 ++++++++
 .../sqoop/test/asserts/ProviderAsserts.java     |  74 +++++++
 .../java/org/apache/sqoop/test/data/Cities.java |  53 +++++
 .../org/apache/sqoop/test/data/DataSet.java     |  66 ++++++
 .../sqoop/test/testcases/ConnectorTestCase.java | 177 ++++++++++++++++
 .../sqoop/test/testcases/TomcatTestCase.java    | 146 +++++++++++++
 .../org/apache/sqoop/test/utils/HdfsUtils.java  |  69 ++++++
 .../sqoop/integration/TomcatTestCase.java       | 197 -----------------
 .../connector/ConnectorTestCase.java            | 209 -------------------
 .../connector/jdbc/generic/TableExportTest.java |  10 +-
 .../connector/jdbc/generic/TableImportTest.java |   2 +-
 .../sqoop/integration/server/VersionTest.java   |   2 +-
 13 files changed, 675 insertions(+), 414 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index b7ea1ed..8001fce 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -33,7 +33,6 @@ limitations under the License.
     <dependency>
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
-      <scope>test</scope>
     </dependency>
 
     <dependency>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
new file mode 100644
index 0000000..056e612
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.asserts;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.test.utils.HdfsUtils;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Assert methods suitable for checking HDFS files and directories.
+ *
+ * TODO: This module will require clean up to work on MiniCluster/Real cluster.
+ */
+public class HdfsAsserts {
+
+  private static final Logger LOG = Logger.getLogger(HdfsAsserts.class);
+
+  /**
+   * Verify that mapreduce output (across all files) is as expected.
+   *
+   * @param directory Mapreduce output directory
+   * @param lines Expected lines
+   * @throws IOException
+   */
+  public static void assertMapreduceOutput(String directory, String... lines) throws IOException
{
+    Set<String> setLines = new HashSet<String>(Arrays.asList(lines));
+    List<String> notFound = new LinkedList<String>();
+
+    String []files = HdfsUtils.getOutputMapreduceFiles(directory);
+
+    for(String file : files) {
+      String filePath = directory + "/" + file;
+      BufferedReader br = new BufferedReader(new FileReader((filePath)));
+
+      String line;
+      while ((line = br.readLine()) != null) {
+        if (!setLines.remove(line)) {
+          notFound.add(line);
+        }
+      }
+      br.close();
+    }
+
+    if(!setLines.isEmpty() || !notFound.isEmpty()) {
+      LOG.error("Expected lines that weren't present in the files:");
+      LOG.error("\t" + StringUtils.join(setLines, "\n\t"));
+      LOG.error("Extra lines in files that weren't expected:");
+      LOG.error("\t" + StringUtils.join(notFound, "\n\t"));
+      fail("Output do not match expectations.");
+    }
+  }
+
+
+  private HdfsAsserts() {
+    // Instantiation is prohibited
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java
new file mode 100644
index 0000000..364f86d
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.asserts;
+
+import org.apache.sqoop.test.db.DatabaseProvider;
+import org.apache.log4j.Logger;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Database provider related asserts.
+ */
+public class ProviderAsserts {
+
+  private static final Logger LOG = Logger.getLogger(ProviderAsserts.class);
+
+  /**
+   * Assert row in the table.
+   *
+   * @param provider Provider that should be used to query the database
+   * @param tableName Table name
+   * @param conditions Conditions for identifying the row
+   * @param values Values that should be present in the table
+   */
+  public static void assertRow(DatabaseProvider provider, String tableName,  Object []conditions,
Object ...values) {
+    ResultSet rs = null;
+    try {
+      rs = provider.getRows(tableName, conditions);
+
+      if(! rs.next()) {
+        fail("No rows found.");
+      }
+
+      int i = 1;
+      for(Object expectedValue : values) {
+        Object actualValue = rs.getObject(i);
+        assertEquals("Columns do not match on position: " + i, expectedValue, actualValue);
+        i++;
+      }
+
+      if(rs.next()) {
+        fail("Found more than one row.");
+      }
+    } catch (SQLException e) {
+      LOG.error("Unexpected SQLException", e);
+      fail("Unexpected SQLException: " + e);
+    } finally {
+      provider.closeResultSetWithStatement(rs);
+    }
+  }
+
+  private ProviderAsserts() {
+    // Instantiation is prohibited
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/data/Cities.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/data/Cities.java b/test/src/main/java/org/apache/sqoop/test/data/Cities.java
new file mode 100644
index 0000000..fb9be13
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/data/Cities.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.data;
+
+import org.apache.sqoop.test.db.DatabaseProvider;
+
+/**
+ * Simple listing of few world's cities to do basic sanity tests.
+ */
+public class Cities extends DataSet {
+
+  public Cities(DatabaseProvider provider, String tableBaseName) {
+    super(provider, tableBaseName);
+  }
+
+  @Override
+  public DataSet createTables() {
+    provider.createTable(
+      tableBaseName,
+      "id",
+      "id", "int",
+      "country", "varchar(50)",
+      "city", "varchar(50)"
+    );
+
+    return this;
+  }
+
+  @Override
+  public DataSet loadBasicData() {
+    provider.insertRow(tableBaseName, 1, "USA", "San Francisco");
+    provider.insertRow(tableBaseName, 2, "USA", "Sunnyvale");
+    provider.insertRow(tableBaseName, 3, "Czech Republic", "Brno");
+    provider.insertRow(tableBaseName, 4, "USA", "Palo Alto");
+
+    return this;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/data/DataSet.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/data/DataSet.java b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java
new file mode 100644
index 0000000..6999408
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.data;
+
+import org.apache.sqoop.test.db.DatabaseProvider;
+
+/**
+ * Abstract class for basic testing data sets.
+ *
+ * Each data set provides couple of generic methods that can be used to set up
+ * the tables and load example data.
+ */
+public abstract class DataSet {
+
+  /**
+   * Database provider that will be used to populate the data.
+   */
+  protected DatabaseProvider provider;
+
+  /**
+   * Base name for created tables.
+   */
+  protected String tableBaseName;
+
+  public DataSet(DatabaseProvider provider, String tableBaseName) {
+    setProvider(provider);
+    setTableBaseName(tableBaseName);
+  }
+
+  public DataSet setProvider(DatabaseProvider provider) {
+    this.provider = provider;
+    return this;
+  }
+
+  public DataSet setTableBaseName(String tableBaseName) {
+    this.tableBaseName = tableBaseName;
+    return this;
+  }
+
+  /**
+   * Crate all tables that this testing data set might need.
+   */
+  public abstract DataSet createTables();
+
+  /**
+   * Load basic data.
+   *
+   * Basic data set should be small (around 10 rows) without any specialities.
+   */
+  public abstract DataSet loadBasicData();
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
new file mode 100644
index 0000000..6aeadd4
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.testcases;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.framework.configuration.OutputFormat;
+import org.apache.sqoop.framework.configuration.StorageType;
+import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MPersistableEntity;
+import org.apache.sqoop.test.asserts.ProviderAsserts;
+import org.apache.sqoop.test.data.Cities;
+import org.apache.sqoop.test.db.DatabaseProvider;
+import org.apache.sqoop.test.db.DatabaseProviderFactory;
+import org.apache.sqoop.validation.Status;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+
+/**
+ * Base test case suitable for connector testing.
+ *
+ * In addition to pure Tomcat based test case it will also create and initialize
+ * the database provider prior every test execution.
+ */
+abstract public class ConnectorTestCase extends TomcatTestCase {
+
+  private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class);
+
+  protected static DatabaseProvider provider;
+
+  @BeforeClass
+  public static void startProvider() throws Exception {
+    provider = DatabaseProviderFactory.getProvider(System.getProperties());
+    LOG.info("Starting database provider: " + provider.getClass().getName());
+    provider.start();
+  }
+
+  @AfterClass
+  public static void stopProvider() {
+    LOG.info("Stopping database provider: " + provider.getClass().getName());
+    provider.stop();
+  }
+
+  public String getTableName() {
+    return getClass().getSimpleName();
+  }
+
+  protected void createTable(String primaryKey, String ...columns) {
+    provider.createTable(getTableName(), primaryKey, columns);
+  }
+
+  protected void dropTable() {
+    provider.dropTable(getTableName());
+  }
+
+  protected void insertRow(Object ...values) {
+    provider.insertRow(getTableName(), values);
+  }
+
+  protected long rowCount() {
+    return provider.rowCount(getTableName());
+  }
+
+  /**
+   * Fill connection form based on currently active provider.
+   *
+   * @param connection MConnection object to fill
+   */
+  protected void fillConnectionForm(MConnection connection) {
+    MFormList forms = connection.getConnectorPart();
+    forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver());
+    forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl());
+    forms.getStringInput("connection.username").setValue(provider.getConnectionUsername());
+    forms.getStringInput("connection.password").setValue(provider.getConnectionPassword());
+  }
+
+  /**
+   * Fill output form with specific storage and output type. Mapreduce output directory
+   * will be set to default test value.
+   *
+   * @param job MJOb object to fill
+   * @param storage Storage type that should be set
+   * @param output Output type that should be set
+   */
+  protected void fillOutputForm(MJob job, StorageType storage, OutputFormat output) {
+    MFormList forms = job.getFrameworkPart();
+    forms.getEnumInput("output.storageType").setValue(storage);
+    forms.getEnumInput("output.outputFormat").setValue(output);
+    forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory());
+  }
+
+  /**
+   * Fill input form. Mapreduce input directory will be set to default test value.
+   *
+   * @param job MJOb object to fill
+   */
+  protected void fillInputForm(MJob job) {
+    MFormList forms = job.getFrameworkPart();
+    forms.getStringInput("input.inputDirectory").setValue(getMapreduceDirectory());
+  }
+
+  /**
+   * Create table cities.
+   */
+  protected void createTableCities() {
+    new Cities(provider, getTableName()).createTables();
+  }
+
+  /**
+   * Create table cities and load few rows.
+   */
+  protected void createAndLoadTableCities() {
+    new Cities(provider, getTableName()).createTables().loadBasicData();
+  }
+
+  /**
+   * Assert row in testing table.
+   *
+   * @param conditions Conditions in form that are expected by the database provider
+   * @param values Values that are expected in the table (with corresponding types)
+   */
+  protected void assertRow(Object []conditions, Object ...values) {
+    ProviderAsserts.assertRow(provider, getTableName(), conditions, values);
+  }
+
+  /**
+   * Assert row in table "cities".
+   *
+   * @param values Values that are expected
+   */
+  protected void assertRowInCities(Object... values) {
+    assertRow(new Object[]{"id", values[0]}, values);
+  }
+
+  /**
+   * Create connection.
+   *
+   * With asserts to make sure that it was created correctly.
+   *
+   * @param connection
+   */
+  protected void createConnection(MConnection connection) {
+    assertEquals(Status.FINE, getClient().createConnection(connection));
+    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId());
+  }
+
+ /**
+   * Create job.
+   *
+   * With asserts to make sure that it was created correctly.
+   *
+   * @param job
+   */
+ protected void createJob(MJob job) {
+    assertEquals(Status.FINE, getClient().createJob(job));
+    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
new file mode 100644
index 0000000..ca77e64
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.testcases;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.test.asserts.HdfsAsserts;
+import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import java.io.IOException;
+
+/**
+ * Basic test case that will bootstrap Sqoop server running in external Tomcat
+ * process.
+ */
+abstract public class TomcatTestCase {
+
+  private static final Logger LOG = Logger.getLogger(TomcatTestCase.class);
+
+  @Rule public TestName name = new TestName();
+
+  /**
+   * Temporary base path that will be used for tests.
+   *
+   * By default we will take a look for sqoop.integration.tmpdir property that is
+   * filled up by maven. If the test is not started from maven (IDE) we will
+   * pick up configured java.io.tmpdir value. The last results is /tmp/ directory
+   * in case that no property is set.
+   */
+  private static final String TMP_PATH_BASE =
+    System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp"))
+ "/sqoop-cargo-tests/";
+
+  /**
+   * Temporary directory that will be used by the test.
+   *
+   * We will take TMP_PATH_BASE and append two subdirectories. First will be named
+   * after fully qualified class name of current test class, second directory will
+   * be named after current test method name. For example:
+   *
+   * TMP_PATH_BASE/org.apache.sqoop.TestClass/testMethod/
+   */
+  private String tmpPath;
+
+  /**
+   * Tomcat based Sqoop mini cluster
+   */
+  private TomcatSqoopMiniCluster cluster;
+
+  /**
+   * Sqoop client API.
+   */
+  private SqoopClient client;
+
+  @Before
+  public void startServer() throws Exception {
+    // Set up the temporary path
+    tmpPath = TMP_PATH_BASE + getClass().getName() + "/" + name.getMethodName() + "/";
+
+    // Set up and start server
+    cluster = new TomcatSqoopMiniCluster(getTemporaryPath());
+    cluster.start();
+
+    // Initialize Sqoop Client API
+    client = new SqoopClient(getServerUrl());
+  }
+
+  @After
+  public void stopServer() throws Exception {
+    cluster.stop();
+  }
+
+  /**
+   * Return SqoopClient configured to talk to testing server.
+   *
+   * @return
+   */
+  public SqoopClient getClient() {
+    return client;
+  }
+
+  public String getTemporaryPath() {
+    return tmpPath;
+  }
+
+  /**
+   * Return testing server URL
+   *
+   * @return
+   */
+  public String getServerUrl() {
+    return cluster.getServerUrl();
+  }
+
+  /**
+   * Get input/output directory for mapreduce job.
+   *
+   * @return
+   */
+  public String getMapreduceDirectory() {
+    return getTemporaryPath() + "/mapreduce-job-io";
+  }
+
+  /**
+   * Assert that mapreduce has generated following lines.
+   *
+   * As the lines can be spread between multiple files the ordering do not make
+   * a difference.
+   *
+   * @param lines
+   * @throws IOException
+   */
+  protected void assertMapreduceOutput(String... lines) throws IOException {
+    HdfsAsserts.assertMapreduceOutput(getMapreduceDirectory(), lines);
+  }
+
+  /**
+   * Create mapreduce input file with specified content.
+   *
+   * @param filename Input file name
+   * @param lines Individual lines that should be written into the file
+   * @throws IOException
+   */
+  protected void createInputMapreduceFile(String filename, String...lines) throws IOException
{
+    HdfsUtils.createFile(getMapreduceDirectory(), filename, lines);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
new file mode 100644
index 0000000..95dd177
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.utils;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Handy utilities to work with HDFS
+ *
+ * TODO: This module will require clean up to work on MiniCluster/Real cluster.
+ */
+public class HdfsUtils {
+
+  private static final Logger LOG = Logger.getLogger(HdfsUtils.class);
+
+  /**
+   * Get list of mapreduce output files from given directory.
+   *
+   * @param directory Directory to be searched for files generated by MR
+   * @return
+   */
+  public static String [] getOutputMapreduceFiles(String directory) {
+    File dir = new File(directory);
+    return dir.list(new FilenameFilter() {
+      @Override
+      public boolean accept(File dir, String name) {
+        return name.startsWith("part-");
+      }
+    });
+  }
+
+  /**
+   * Create HDFS file with given content.
+   *
+   * @param directory Directory where the file should be created
+   * @param filename File name
+   * @param lines Individual lines that should be written into the file
+   * @throws IOException
+   */
+  public static void createFile(String directory, String filename, String ...lines) throws
IOException {
+    File outputFile = new File(directory, filename);
+    FileUtils.writeLines(outputFile, Arrays.asList(lines));
+  }
+
+  private HdfsUtils() {
+    // Instantiation is not allowed
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
deleted file mode 100644
index fa2c2b4..0000000
--- a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.integration;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.log4j.Logger;
-import org.apache.commons.lang.StringUtils;
-import org.apache.sqoop.client.SqoopClient;
-import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import static org.junit.Assert.fail;
-
-/**
- * Basic test case that will bootstrap Sqoop server running in external Tomcat
- * process.
- */
-abstract public class TomcatTestCase {
-
-  private static final Logger LOG = Logger.getLogger(TomcatTestCase.class);
-
-  @Rule public TestName name = new TestName();
-
-  /**
-   * Temporary base path that will be used for tests.
-   *
-   * By default we will take a look for sqoop.integration.tmpdir property that is
-   * filled up by maven. If the test is not started from maven (IDE) we will
-   * pick up configured java.io.tmpdir value. The last results is /tmp/ directory
-   * in case that no property is set.
-   */
-  private static final String TMP_PATH_BASE =
-    System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp"))
+ "/sqoop-cargo-tests/";
-
-  /**
-   * Temporary directory that will be used by the test.
-   *
-   * We will take TMP_PATH_BASE and append two subdirectories. First will be named
-   * after fully qualified class name of current test class, second directory will
-   * be named after current test method name. For example:
-   *
-   * TMP_PATH_BASE/org.apache.sqoop.TestClass/testMethod/
-   */
-  private String tmpPath;
-
-  /**
-   * Tomcat based Sqoop mini cluster
-   */
-  private TomcatSqoopMiniCluster cluster;
-
-  /**
-   * Sqoop client API.
-   */
-  private SqoopClient client;
-
-  @Before
-  public void startServer() throws Exception {
-    // Set up the temporary path
-    tmpPath = TMP_PATH_BASE + getClass().getName() + "/" + name.getMethodName() + "/";
-
-    // Set up and start server
-    cluster = new TomcatSqoopMiniCluster(getTemporaryPath());
-    cluster.start();
-
-    // Initialize Sqoop Client API
-    client = new SqoopClient(getServerUrl());
-  }
-
-  @After
-  public void stopServer() throws Exception {
-    cluster.stop();
-  }
-
-  /**
-   * Return SqoopClient configured to talk to testing server.
-   *
-   * @return
-   */
-  public SqoopClient getClient() {
-    return client;
-  }
-
-  public String getTemporaryPath() {
-    return tmpPath;
-  }
-
-  /**
-   * Return testing server URL
-   *
-   * @return
-   */
-  public String getServerUrl() {
-    return cluster.getServerUrl();
-  }
-
-  /**
-   * Get input/output directory for mapreduce job.
-   *
-   * @return
-   */
-  public String getMapreduceDirectory() {
-    return getTemporaryPath() + "/mapreduce-job-io";
-  }
-
-  /**
-   * Return list of file names that are outputs of mapreduce job.
-   *
-   * @return
-   */
-  public String[] getOutputFilesMapreduce() {
-    File dir = new File(getMapreduceDirectory());
-    return dir.list(new FilenameFilter() {
-      @Override
-      public boolean accept(File dir, String name) {
-        return name.startsWith("part-");
-      }
-    });
-  }
-
-  /**
-   * Assert that mapreduce has generated following lines.
-   *
-   * As the lines can be spread between multiple files the ordering do not make
-   * a difference.
-   *
-   * @param lines
-   * @throws IOException
-   */
-  protected void assertMapreduceOutput(String... lines) throws IOException {
-    Set<String> setLines = new HashSet<String>(Arrays.asList(lines));
-    List<String> notFound = new LinkedList<String>();
-
-    String []files = getOutputFilesMapreduce();
-
-    for(String file : files) {
-      String filePath = getMapreduceDirectory() + "/" + file;
-      BufferedReader br = new BufferedReader(new FileReader((filePath)));
-
-      String line;
-      while ((line = br.readLine()) != null) {
-        if (!setLines.remove(line)) {
-          notFound.add(line);
-        }
-      }
-      br.close();
-    }
-
-    if(!setLines.isEmpty() || !notFound.isEmpty()) {
-      LOG.error("Expected lines that weren't present in the files:");
-      LOG.error("\t" + StringUtils.join(setLines, "\n\t"));
-      LOG.error("Extra lines in files that weren't expected:");
-      LOG.error("\t" + StringUtils.join(notFound, "\n\t"));
-      fail("Output do not match expectations.");
-    }
-  }
-
-  /**
-   * Create mapreduce input file with specified content.
-   *
-   * @param filename Input file name
-   * @param lines Individual lines that should be written into the file
-   * @throws IOException
-   */
-  protected void createInputMapreduceFile(String filename, String...lines) throws IOException
{
-    File outputFile = new File(getMapreduceDirectory(), filename);
-    FileUtils.writeLines(outputFile, Arrays.asList(lines));
-  }
-}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
deleted file mode 100644
index d4e432d..0000000
--- a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.integration.connector;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.framework.configuration.OutputFormat;
-import org.apache.sqoop.framework.configuration.StorageType;
-import org.apache.sqoop.integration.TomcatTestCase;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MFormList;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MPersistableEntity;
-import org.apache.sqoop.test.db.DatabaseProvider;
-import org.apache.sqoop.test.db.DatabaseProviderFactory;
-import org.apache.sqoop.validation.Status;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.fail;
-
-/**
- * Base test case for connector testing.
- *
- * It will create and initialize database provider prior every test execution.
- */
-abstract public class ConnectorTestCase extends TomcatTestCase {
-
-  private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class);
-
-  protected static DatabaseProvider provider;
-
-  @BeforeClass
-  public static void startProvider() throws Exception {
-    provider = DatabaseProviderFactory.getProvider(System.getProperties());
-    LOG.info("Starting database provider: " + provider.getClass().getName());
-    provider.start();
-  }
-
-  @AfterClass
-  public static void stopProvider() {
-    LOG.info("Stopping database provider: " + provider.getClass().getName());
-    provider.stop();
-  }
-
-  public String getTableName() {
-    return getClass().getSimpleName();
-  }
-
-  protected void createTable(String primaryKey, String ...columns) {
-    provider.createTable(getTableName(), primaryKey, columns);
-  }
-
-  protected void dropTable() {
-    provider.dropTable(getTableName());
-  }
-
-  protected void insertRow(Object ...values) {
-    provider.insertRow(getTableName(), values);
-  }
-
-  protected long rowCount() {
-    return provider.rowCount(getTableName());
-  }
-
-  /**
-   * Fill connection form based on currently active provider.
-   *
-   * @param connection MConnection object to fill
-   */
-  protected void fillConnectionForm(MConnection connection) {
-    MFormList forms = connection.getConnectorPart();
-    forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver());
-    forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl());
-    forms.getStringInput("connection.username").setValue(provider.getConnectionUsername());
-    forms.getStringInput("connection.password").setValue(provider.getConnectionPassword());
-  }
-
-  /**
-   * Fill output form with specific storage and output type. Mapreduce output directory
-   * will be set to default test value.
-   *
-   * @param job MJOb object to fill
-   * @param storage Storage type that should be set
-   * @param output Output type that should be set
-   */
-  protected void fillOutputForm(MJob job, StorageType storage, OutputFormat output) {
-    MFormList forms = job.getFrameworkPart();
-    forms.getEnumInput("output.storageType").setValue(storage);
-    forms.getEnumInput("output.outputFormat").setValue(output);
-    forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory());
-  }
-
-  /**
-   * Fill input form. Mapreduce input directory will be set to default test value.
-   *
-   * @param job MJOb object to fill
-   */
-  protected void fillInputForm(MJob job) {
-    MFormList forms = job.getFrameworkPart();
-    forms.getStringInput("input.inputDirectory").setValue(getMapreduceDirectory());
-  }
-
-  /**
-   * Create table cities.
-   */
-  protected void createTableCities() {
-     createTable("id",
-       "id", "int",
-       "country", "varchar(50)",
-       "city", "varchar(50)"
-     );
-  }
-
-  /**
-   * Create table cities and load few rows.
-   */
-  protected void createAndLoadTableCities() {
-    createTableCities();
-    insertRow(1, "USA", "San Francisco");
-    insertRow(2, "USA", "Sunnyvale");
-    insertRow(3, "Czech Republic", "Brno");
-    insertRow(4, "USA", "Palo Alto");
-  }
-
-  /**
-   * Assert row in testing table.
-   *
-   * @param conditions Conditions in form that are expected by the database provider
-   * @param values Values that are expected in the table (with corresponding types)
-   */
-  protected void assertRow(Object []conditions, Object ...values) {
-    ResultSet rs = provider.getRows(getTableName(), conditions);
-
-    try {
-      if(! rs.next()) {
-        fail("No rows found.");
-      }
-
-      int i = 1;
-      for(Object expectedValue : values) {
-        Object actualValue = rs.getObject(i);
-        assertEquals("Columns do not match on position: " + i, expectedValue, actualValue);
-        i++;
-      }
-
-      if(rs.next()) {
-        fail("Found more than one row.");
-      }
-    } catch (SQLException e) {
-      LOG.error("Unexpected SQLException", e);
-      fail("Unexpected SQLException: " + e);
-    } finally {
-      provider.closeResultSetWithStatement(rs);
-    }
-  }
-
-  /**
-   * Assert row in table "cities".
-   *
-   * @param values Values that are expected
-   */
-  protected void assertRowInCitiesTable(Object ... values) {
-    assertRow(new Object[]{"id", values[0]}, values);
-  }
-
-  /**
-   * Create connection.
-   *
-   * With asserts to make sure that it was created correctly.
-   *
-   * @param connection
-   */
-  protected void createConnection(MConnection connection) {
-    assertEquals(Status.FINE, getClient().createConnection(connection));
-    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId());
-  }
-
- /**
-   * Create job.
-   *
-   * With asserts to make sure that it was created correctly.
-   *
-   * @param job
-   */
- protected void createJob(MJob job) {
-    assertEquals(Status.FINE, getClient().createJob(job));
-    assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
-  }
-}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
index b920655..fbf46e9 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
@@ -18,7 +18,7 @@
 package org.apache.sqoop.integration.connector.jdbc.generic;
 
 import org.apache.log4j.Logger;
-import org.apache.sqoop.integration.connector.ConnectorTestCase;
+import org.apache.sqoop.test.testcases.ConnectorTestCase;
 import org.apache.sqoop.model.MConnection;
 import org.apache.sqoop.model.MFormList;
 import org.apache.sqoop.model.MJob;
@@ -70,10 +70,10 @@ public class TableExportTest extends ConnectorTestCase {
     } while(submission.getStatus().isRunning());
 
     assertEquals(4L, rowCount());
-    assertRowInCitiesTable(1, "USA", "San Francisco");
-    assertRowInCitiesTable(2, "USA", "Sunnyvale");
-    assertRowInCitiesTable(3, "Czech Republic", "Brno");
-    assertRowInCitiesTable(4, "USA", "Palo Alto");
+    assertRowInCities(1, "USA", "San Francisco");
+    assertRowInCities(2, "USA", "Sunnyvale");
+    assertRowInCities(3, "Czech Republic", "Brno");
+    assertRowInCities(4, "USA", "Palo Alto");
 
     // Clean up testing table
     dropTable();

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
index adcfbaf..639d9ad 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
@@ -20,7 +20,7 @@ package org.apache.sqoop.integration.connector.jdbc.generic;
 import org.apache.log4j.Logger;
 import org.apache.sqoop.framework.configuration.OutputFormat;
 import org.apache.sqoop.framework.configuration.StorageType;
-import org.apache.sqoop.integration.connector.ConnectorTestCase;
+import org.apache.sqoop.test.testcases.ConnectorTestCase;
 import org.apache.sqoop.model.MConnection;
 import org.apache.sqoop.model.MFormList;
 import org.apache.sqoop.model.MJob;

http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
index 0f48a8b..cea24b9 100644
--- a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.integration.server;
 
 import org.apache.sqoop.client.request.VersionRequest;
 import org.apache.sqoop.common.VersionInfo;
-import org.apache.sqoop.integration.TomcatTestCase;
+import org.apache.sqoop.test.testcases.TomcatTestCase;
 import org.apache.sqoop.json.VersionBean;
 import org.junit.Test;
 


Mime
View raw message