sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From hshreedha...@apache.org
Subject git commit: SQOOP-1114. Sqoop2: Integration: Add tests for various Generic JDBC Connector splitters
Date Wed, 31 Jul 2013 06:29:47 GMT
Updated Branches:
  refs/heads/sqoop2 9c7adb4e1 -> 3631b29e0


SQOOP-1114. Sqoop2: Integration: Add tests for various Generic JDBC Connector splitters

(Jarek Jarcec Cecho via Hari Shreedharan)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/3631b29e
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/3631b29e
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/3631b29e

Branch: refs/heads/sqoop2
Commit: 3631b29e06c5e78d666752a65b0fe01601dc3259
Parents: 9c7adb4
Author: Hari Shreedharan <hshreedharan@apache.org>
Authored: Tue Jul 30 23:28:50 2013 -0700
Committer: Hari Shreedharan <hshreedharan@apache.org>
Committed: Tue Jul 30 23:28:50 2013 -0700

----------------------------------------------------------------------
 pom.xml                                         |   2 +-
 .../apache/sqoop/test/asserts/HdfsAsserts.java  |  11 ++
 .../apache/sqoop/test/data/UbuntuReleases.java  |  72 +++++++++++
 .../sqoop/test/testcases/ConnectorTestCase.java |  15 +++
 .../sqoop/test/testcases/TomcatTestCase.java    |   9 ++
 .../sqoop/test/utils/ParametrizedUtils.java     |  91 ++++++++++++++
 .../jdbc/generic/imports/PartitionerTest.java   | 125 +++++++++++++++++++
 7 files changed, 324 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 04cf479..c1ed306 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,7 +99,7 @@ limitations under the License.
     <hadoop.2.version>2.0.2-alpha</hadoop.2.version>
     <guava.version>11.0.2</guava.version>
     <json-simple.version>1.1</json-simple.version>
-    <junit.version>4.9</junit.version>
+    <junit.version>4.11</junit.version>
     <mockito.version>1.9.5</mockito.version>
     <log4j.version>1.2.16</log4j.version>
     <servlet.version>2.5</servlet.version>

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
index 056e612..c175272 100644
--- a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
+++ b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
@@ -30,6 +30,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 
+import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;
 
 /**
@@ -76,6 +77,16 @@ public class HdfsAsserts {
     }
   }
 
+  /**
+   * Verify number of output mapreduce files.
+   *
+   * @param directory Mapreduce output directory
+   * @param expectedFiles Expected number of files
+   */
+  public static void assertMapreduceOutputFiles(String directory, int expectedFiles) {
+    String []files = HdfsUtils.getOutputMapreduceFiles(directory);
+    assertEquals(expectedFiles, files.length);
+  }
 
   private HdfsAsserts() {
     // Instantiation is prohibited

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java b/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java
new file mode 100644
index 0000000..a4730c2
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/data/UbuntuReleases.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.data;
+
+import org.apache.sqoop.test.db.DatabaseProvider;
+
+/**
+ * Releases of Ubuntu Linux.
+ *
+ * Purpose of this set is to cover most common data types (varchar, int, numeric, date, boolean).
+ */
+public class UbuntuReleases extends DataSet {
+
+  public UbuntuReleases(DatabaseProvider provider, String tableBaseName) {
+    super(provider, tableBaseName);
+  }
+
+  @Override
+  public DataSet createTables() {
+    provider.createTable(
+      tableBaseName,
+      "id",
+      "id", "int",
+      "code_name", "varchar(64)",
+      "version", "numeric(4,2)",
+      "release_date", "date",
+      "lts", "boolean"
+    );
+
+    return this;
+  }
+
+  @Override
+  public DataSet loadBasicData() {
+    provider.insertRow(tableBaseName,  1, "Warty Warthog",    4.10,  "2004-10-20", false);
+    provider.insertRow(tableBaseName,  2, "Hoary Hedgehog",   5.04,  "2005-04-08", false);
+    provider.insertRow(tableBaseName,  3, "Breezy Badger",    5.10,  "2005-10-13", false);
+    provider.insertRow(tableBaseName,  4, "Dapper Drake",     6.06,  "2006-06-01", true);
+    provider.insertRow(tableBaseName,  5, "Edgy Eft",         6.10,  "2006-10-26", false);
+    provider.insertRow(tableBaseName,  6, "Feisty Fawn",      7.04,  "2007-04-19", false);
+    provider.insertRow(tableBaseName,  7, "Gutsy Gibbon",     7.10,  "2007-10-18", false);
+    provider.insertRow(tableBaseName,  8, "Hardy Heron",      8.04,  "2008-04-24", true);
+    provider.insertRow(tableBaseName,  9, "Intrepid Ibex",    8.10,  "2008-10-18", false);
+    provider.insertRow(tableBaseName, 10, "Jaunty Jackalope", 9.04,  "2009-04-23", false);
+    provider.insertRow(tableBaseName, 11, "Karmic Koala",     9.10,  "2009-10-29", false);
+    provider.insertRow(tableBaseName, 12, "Lucid Lynx",      10.04,  "2010-04-29", true);
+    provider.insertRow(tableBaseName, 13, "Maverick Meerkat",10.10,  "2010-10-10", false);
+    provider.insertRow(tableBaseName, 14, "Natty Narwhal",   11.04,  "2011-04-28", false);
+    provider.insertRow(tableBaseName, 15, "Oneiric Ocelot",  11.10,  "2011-10-10", false);
+    provider.insertRow(tableBaseName, 16, "Precise Pangolin",12.04,  "2012-04-26", true);
+    provider.insertRow(tableBaseName, 17, "Quantal Quetzal", 12.10,  "2012-10-18", false);
+    provider.insertRow(tableBaseName, 18, "Raring Ringtail", 13.04,  "2013-04-25", false);
+    provider.insertRow(tableBaseName, 19, "Saucy Salamander",13.10,  "2013-10-17", false);
+
+    return this;
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
index bf1c91f..d10b942 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
@@ -28,6 +28,7 @@ import org.apache.sqoop.model.MPersistableEntity;
 import org.apache.sqoop.model.MSubmission;
 import org.apache.sqoop.test.asserts.ProviderAsserts;
 import org.apache.sqoop.test.data.Cities;
+import org.apache.sqoop.test.data.UbuntuReleases;
 import org.apache.sqoop.test.db.DatabaseProvider;
 import org.apache.sqoop.test.db.DatabaseProviderFactory;
 import org.apache.sqoop.validation.Status;
@@ -155,6 +156,20 @@ abstract public class ConnectorTestCase extends TomcatTestCase {
   }
 
   /**
+   * Create table for ubuntu releases.
+   */
+  protected void createTableUbuntuReleases() {
+    new UbuntuReleases(provider, getTableName()).createTables();
+  }
+
+  /**
+   * Create table for ubuntu releases.
+   */
+  protected void createAndLoadTableUbuntuReleases() {
+    new UbuntuReleases(provider, getTableName()).createTables().loadBasicData();
+  }
+
+  /**
    * Assert row in testing table.
    *
    * @param conditions Conditions in form that are expected by the database provider

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
index ca77e64..efdfed4 100644
--- a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
@@ -134,6 +134,15 @@ abstract public class TomcatTestCase {
   }
 
   /**
+   * Verify number of output mapreduce files.
+   *
+   * @param expectedFiles Expected number of files
+   */
+  protected void assertMapreduceOutputFiles(int expectedFiles) {
+    HdfsAsserts.assertMapreduceOutputFiles(getMapreduceDirectory(), expectedFiles);
+  }
+
+  /**
    * Create mapreduce input file with specified content.
    *
    * @param filename Input file name

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java
new file mode 100644
index 0000000..656bc34
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/utils/ParametrizedUtils.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.utils;
+
+import org.apache.commons.lang.ArrayUtils;
+
+import java.util.LinkedList;
+
+/**
+ * Various utils to help build expected structure for JUNIT Parametrized runner.
+ */
+public class ParametrizedUtils {
+
+  /**
+   * Join two arrays of parameters by providing their cross product to make
+   * testing matrix for Parameterized runner easier.
+   *
+   * This method accepts both arguments as array of arrays. In such case
+   * the inner array will be flattened during the join to meet the Parametrized
+   * runner requirements.
+   *
+   * @param v1 First array of test values
+   * @param v2 Second array of test values
+   * @return Cross product of v1 and v2
+   */
+  public static Iterable<Object []> crossProduct(Object[] v1, Object[] v2) {
+    LinkedList<Object []> ret = new LinkedList<Object []>();
+
+    for(Object o1 : v1) {
+      for(Object o2 : v2) {
+        ret.add(mergeObjects(o1, o2));
+      }
+    }
+
+    return ret;
+  }
+
+  /**
+   * Merge two objects into array.
+   *
+   * This method will flatten any argument that is already an array, e.g.:
+   *
+   * mergeObjects(1, 2) = [1, 2]
+   * mergeObjects([1], 2) = [1, 2]
+   * mergeObjects(1, [2]) = [1, 2]
+   * mergeObjects([1], [2]) = [1, 2]
+   *
+   * @param o1
+   * @param o2
+   * @return
+   */
+  public static Object [] mergeObjects(Object o1, Object o2) {
+    if(!o1.getClass().isArray() && !o2.getClass().isArray()) {
+      return new Object[] { o1, o2 };
+    }
+
+    Object []a1 = toArray(o1);
+    Object []a2 = toArray(o2);
+
+    return ArrayUtils.addAll(a1, a2);
+  }
+
+  /**
+   * Convert object into one element array if it's not an array already.
+   *
+   * @param o
+   * @return
+   */
+  public static Object[] toArray(Object o) {
+    if(o.getClass().isArray()) {
+      return (Object [])o;
+    }
+
+    return new Object[] { o };
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/3631b29e/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java
b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java
new file mode 100644
index 0000000..50fce74
--- /dev/null
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/imports/PartitionerTest.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.integration.connector.jdbc.generic.imports;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.framework.configuration.OutputFormat;
+import org.apache.sqoop.framework.configuration.StorageType;
+import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.test.testcases.ConnectorTestCase;
+import org.apache.sqoop.test.utils.ParametrizedUtils;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+/**
+ *
+ */
+@RunWith(Parameterized.class)
+public class PartitionerTest extends ConnectorTestCase {
+
+  private static final Logger LOG = Logger.getLogger(PartitionerTest.class);
+
+  /**
+   * Columns that we will use as partition column with maximal number of
+   * partitions that can be created for such column.
+   */
+  public static Object[] COLUMNS = new Object [][] {
+    {"id",           13},
+    {"code_name",    13},
+    {"version",      13},
+    {"release_date", 13},
+    {"lts",           2},
+  };
+
+  /**
+   * Number of extractors that we will use to transfer the table.
+   */
+  public static Object [] EXTRACTORS = new Object[] {
+    3, 5, 10, 13,
+  };
+
+  @Parameterized.Parameters(name = "{0}-{1}-{2}")
+  public static Iterable<Object[]> data() {
+    return ParametrizedUtils.crossProduct(COLUMNS, EXTRACTORS);
+  }
+
+  private String partitionColumn;
+  private int extractors;
+  private int maxOutputFiles;
+
+  public PartitionerTest(String partitionColumn, int expectedOutputFiles, int extractors)
{
+    this.partitionColumn = partitionColumn;
+    this.maxOutputFiles = expectedOutputFiles;
+    this.extractors = extractors;
+  }
+
+  @Test
+  public void testSplitter() throws Exception {
+    createAndLoadTableUbuntuReleases();
+
+    // Connection creation
+    MConnection connection = getClient().newConnection(1L);
+    fillConnectionForm(connection);
+    createConnection(connection);
+
+    // Job creation
+    MJob job = getClient().newJob(connection.getPersistenceId(), MJob.Type.IMPORT);
+
+    // Connector values
+    MFormList forms = job.getConnectorPart();
+    forms.getStringInput("table.tableName").setValue(provider.escapeTableName(getTableName()));
+    forms.getStringInput("table.partitionColumn").setValue(provider.escapeColumnName(partitionColumn));
+    // Framework values
+    fillOutputForm(job, StorageType.HDFS, OutputFormat.TEXT_FILE);
+    forms = job.getFrameworkPart();
+    forms.getIntegerInput("throttling.extractors").setValue(extractors);
+    createJob(job);
+
+    runJob(job);
+
+    // Assert correct output
+    assertMapreduceOutputFiles((extractors > maxOutputFiles) ? maxOutputFiles : extractors);
+    assertMapreduceOutput(
+      "1,'Warty Warthog',4.10,2004-10-20,false",
+      "2,'Hoary Hedgehog',5.04,2005-04-08,false",
+      "3,'Breezy Badger',5.10,2005-10-13,false",
+      "4,'Dapper Drake',6.06,2006-06-01,true",
+      "5,'Edgy Eft',6.10,2006-10-26,false",
+      "6,'Feisty Fawn',7.04,2007-04-19,false",
+      "7,'Gutsy Gibbon',7.10,2007-10-18,false",
+      "8,'Hardy Heron',8.04,2008-04-24,true",
+      "9,'Intrepid Ibex',8.10,2008-10-18,false",
+      "10,'Jaunty Jackalope',9.04,2009-04-23,false",
+      "11,'Karmic Koala',9.10,2009-10-29,false",
+      "12,'Lucid Lynx',10.04,2010-04-29,true",
+      "13,'Maverick Meerkat',10.10,2010-10-10,false",
+      "14,'Natty Narwhal',11.04,2011-04-28,false",
+      "15,'Oneiric Ocelot',11.10,2011-10-10,false",
+      "16,'Precise Pangolin',12.04,2012-04-26,true",
+      "17,'Quantal Quetzal',12.10,2012-10-18,false",
+      "18,'Raring Ringtail',13.04,2013-04-25,false",
+      "19,'Saucy Salamander',13.10,2013-10-17,false"
+    );
+
+    // Clean up testing table
+    dropTable();
+  }
+}


Mime
View raw message