sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From va...@apache.org
Subject [1/2] sqoop git commit: SQOOP-3396: Add parquet numeric support for Parquet in Hive import
Date Fri, 07 Dec 2018 12:34:23 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 302674d96 -> a50394977


http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/importjob/numerictypes/parquet/OracleNumericTypesParquetImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/importjob/numerictypes/parquet/OracleNumericTypesParquetImportTest.java
b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/OracleNumericTypesParquetImportTest.java
new file mode 100644
index 0000000..65ed0eb
--- /dev/null
+++ b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/OracleNumericTypesParquetImportTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.importjob.numerictypes.parquet;
+
+import org.apache.sqoop.importjob.configuration.AvroTestConfiguration;
+import org.apache.sqoop.importjob.configuration.OracleImportJobTestConfiguration;
+import org.apache.sqoop.importjob.configuration.OracleImportJobTestConfigurationForNumber;
+import org.apache.sqoop.importjob.configuration.ParquetTestConfiguration;
+import org.apache.sqoop.importjob.numerictypes.NumericTypesAvroImportTestBase;
+import org.apache.sqoop.testcategories.thirdpartytest.OracleTest;
+import org.apache.sqoop.testutil.adapter.DatabaseAdapter;
+import org.apache.sqoop.testutil.adapter.OracleDatabaseAdapter;
+import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITHOUT_EXTRA_ARGS;
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITH_PADDING_ONLY;
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITH_PADDING_ONLY;
+
+@Category(OracleTest.class)
+@RunWith(Parameterized.class)
+@Parameterized.UseParametersRunnerFactory(BlockJUnit4ClassRunnerWithParametersFactory.class)
+public class OracleNumericTypesParquetImportTest<T extends AvroTestConfiguration &
ParquetTestConfiguration> extends NumericTypesAvroImportTestBase<T> {
+
+  @Override
+  public DatabaseAdapter createAdapter() {
+    return new OracleDatabaseAdapter();
+  }
+
+  @Parameterized.Parameters(name = "Config: {0}| failWithoutExtraArgs: {1}| failWithPadding:
{2}")
+  public static Iterable<? extends Object> testConfigurations() {
+    return Arrays.asList(
+        new Object[]{new OracleImportJobTestConfigurationForNumber(), FAIL_WITHOUT_EXTRA_ARGS,
FAIL_WITH_PADDING_ONLY},
+        new Object[]{new OracleImportJobTestConfiguration(), FAIL_WITHOUT_EXTRA_ARGS, SUCCEED_WITH_PADDING_ONLY}
+    );
+  }
+
+  public OracleNumericTypesParquetImportTest(T configuration, boolean failWithoutExtraArgs,
boolean failWithPaddingOnly) {
+    super(configuration, failWithoutExtraArgs, failWithPaddingOnly);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/importjob/numerictypes/parquet/PostgresNumericTypesParquetImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/importjob/numerictypes/parquet/PostgresNumericTypesParquetImportTest.java
b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/PostgresNumericTypesParquetImportTest.java
new file mode 100644
index 0000000..6e84c83
--- /dev/null
+++ b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/PostgresNumericTypesParquetImportTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.importjob.numerictypes.parquet;
+
+import org.apache.sqoop.importjob.configuration.ParquetTestConfiguration;
+import org.apache.sqoop.importjob.configuration.PostgresqlImportJobTestConfigurationForNumeric;
+import org.apache.sqoop.importjob.configuration.PostgresqlImportJobTestConfigurationPaddingShouldSucceed;
+import org.apache.sqoop.importjob.numerictypes.NumericTypesParquetImportTestBase;
+import org.apache.sqoop.testcategories.thirdpartytest.PostgresqlTest;
+import org.apache.sqoop.testutil.adapter.DatabaseAdapter;
+import org.apache.sqoop.testutil.adapter.PostgresDatabaseAdapter;
+import org.apache.sqoop.util.BlockJUnit4ClassRunnerWithParametersFactory;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.util.Arrays;
+
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITHOUT_EXTRA_ARGS;
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.FAIL_WITH_PADDING_ONLY;
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITHOUT_EXTRA_ARGS;
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITH_PADDING_ONLY;
+
+@Category(PostgresqlTest.class)
+@RunWith(Parameterized.class)
+@Parameterized.UseParametersRunnerFactory(BlockJUnit4ClassRunnerWithParametersFactory.class)
+public class PostgresNumericTypesParquetImportTest<T extends ParquetTestConfiguration>
extends NumericTypesParquetImportTestBase<T> {
+
+  @Override
+  public DatabaseAdapter createAdapter() {
+    return new PostgresDatabaseAdapter();
+  }
+
+  @Parameterized.Parameters(name = "Config: {0}| failWithoutExtraArgs: {1}| failWithPadding:
{2}")
+  public static Iterable<? extends Object> testConfigurations() {
+    return Arrays.asList(
+        new Object[]{new PostgresqlImportJobTestConfigurationForNumeric(), FAIL_WITHOUT_EXTRA_ARGS,
FAIL_WITH_PADDING_ONLY},
+        new Object[]{new PostgresqlImportJobTestConfigurationPaddingShouldSucceed(), SUCCEED_WITHOUT_EXTRA_ARGS,
SUCCEED_WITH_PADDING_ONLY}
+    );
+  }
+
+  public PostgresNumericTypesParquetImportTest(T configuration, boolean failWithoutExtraArgs,
boolean failWithPaddingOnly) {
+    super(configuration, failWithoutExtraArgs, failWithPaddingOnly);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/importjob/numerictypes/parquet/SqlServerNumericTypesParquetImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/importjob/numerictypes/parquet/SqlServerNumericTypesParquetImportTest.java
b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/SqlServerNumericTypesParquetImportTest.java
new file mode 100644
index 0000000..7f5eb35
--- /dev/null
+++ b/src/test/org/apache/sqoop/importjob/numerictypes/parquet/SqlServerNumericTypesParquetImportTest.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.importjob.numerictypes.parquet;
+
+import org.apache.sqoop.importjob.configuration.SqlServerImportJobTestConfiguration;
+import org.apache.sqoop.importjob.numerictypes.NumericTypesParquetImportTestBase;
+import org.apache.sqoop.testcategories.thirdpartytest.SqlServerTest;
+import org.apache.sqoop.testutil.adapter.DatabaseAdapter;
+import org.apache.sqoop.testutil.adapter.SqlServerDatabaseAdapter;
+import org.junit.experimental.categories.Category;
+
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITHOUT_EXTRA_ARGS;
+import static org.apache.sqoop.testutil.NumericTypesTestUtils.SUCCEED_WITH_PADDING_ONLY;
+
+@Category(SqlServerTest.class)
+public class SqlServerNumericTypesParquetImportTest extends NumericTypesParquetImportTestBase
{
+
+  @Override
+  public DatabaseAdapter createAdapter() {
+    return new SqlServerDatabaseAdapter();
+  }
+
+  public SqlServerNumericTypesParquetImportTest() {
+    super(new SqlServerImportJobTestConfiguration(), SUCCEED_WITHOUT_EXTRA_ARGS, SUCCEED_WITH_PADDING_ONLY);
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/testutil/NumericTypesTestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/testutil/NumericTypesTestUtils.java b/src/test/org/apache/sqoop/testutil/NumericTypesTestUtils.java
new file mode 100644
index 0000000..60371b9
--- /dev/null
+++ b/src/test/org/apache/sqoop/testutil/NumericTypesTestUtils.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.testutil;
+
+import org.junit.rules.ExpectedException;
+
+import java.io.IOException;
+
+public class NumericTypesTestUtils {
+  // Constants for the basic test case, that doesn't use extra arguments
+  // that are required to avoid errors, i.e. padding and default precision and scale.
+  public final static boolean SUCCEED_WITHOUT_EXTRA_ARGS = false;
+  public final static boolean FAIL_WITHOUT_EXTRA_ARGS = true;
+  // Constants for the test case that has padding specified but not default precision and
scale.
+  public final static boolean SUCCEED_WITH_PADDING_ONLY = false;
+  public final static boolean FAIL_WITH_PADDING_ONLY = true;
+
+  /**
+   * Adds properties to the given arg builder for decimal precision and scale.
+   * @param builder
+   */
+  public static void addPrecisionAndScale(ArgumentArrayBuilder builder) {
+    builder.withProperty("sqoop.avro.logical_types.decimal.default.precision", "38");
+    builder.withProperty("sqoop.avro.logical_types.decimal.default.scale", "3");
+  }
+
+  /**
+   * Enables padding for decimals in avro and parquet import.
+   * @param builder
+   */
+  public static void addPadding(ArgumentArrayBuilder builder) {
+    builder.withProperty("sqoop.avro.decimal_padding.enable", "true");
+  }
+
+  public static void addEnableAvroDecimal(ArgumentArrayBuilder builder) {
+    builder.withProperty("sqoop.avro.logical_types.decimal.enable", "true");
+  }
+
+  public static void addEnableParquetDecimal(ArgumentArrayBuilder builder) {
+    builder.withProperty("sqoop.parquet.logical_types.decimal.enable", "true");
+  }
+
+  public static void configureJunitToExpectFailure(ExpectedException thrown) {
+    thrown.expect(IOException.class);
+    thrown.expectMessage("Failure during job; return status 1");
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/a5039497/src/test/org/apache/sqoop/testutil/ThirdPartyTestBase.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/testutil/ThirdPartyTestBase.java b/src/test/org/apache/sqoop/testutil/ThirdPartyTestBase.java
new file mode 100644
index 0000000..a2087e3
--- /dev/null
+++ b/src/test/org/apache/sqoop/testutil/ThirdPartyTestBase.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.testutil;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.importjob.DatabaseAdapterFactory;
+import org.apache.sqoop.importjob.configuration.ImportJobTestConfiguration;
+import org.apache.sqoop.testutil.adapter.DatabaseAdapter;
+import org.junit.After;
+import org.junit.Before;
+
+import java.sql.SQLException;
+import java.util.List;
+
+public abstract class ThirdPartyTestBase<T extends ImportJobTestConfiguration> extends
ImportJobTestCase implements DatabaseAdapterFactory {
+
+  private final DatabaseAdapter adapter;
+  protected final T configuration;
+
+  private Configuration conf = new Configuration();
+
+  public DatabaseAdapter getAdapter() {
+    return adapter;
+  }
+
+  public T getConfiguration() {
+    return configuration;
+  }
+
+  protected ThirdPartyTestBase(T configuration) {
+    this.adapter = createAdapter();
+    this.configuration = configuration;
+  }
+
+  @Override
+  protected String getConnectString() {
+    return adapter.getConnectionString();
+  }
+
+  @Override
+  protected SqoopOptions getSqoopOptions(Configuration conf) {
+    SqoopOptions opts = new SqoopOptions(conf);
+    adapter.injectConnectionParameters(opts);
+    return opts;
+  }
+
+  @Override
+  protected void dropTableIfExists(String table) throws SQLException {
+    adapter.dropTableIfExists(table, getManager());
+  }
+
+  @Override
+  protected Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  protected boolean useHsqldbTestServer() {
+    return false;
+  }
+
+  @Before
+  public void setUp() {
+    super.setUp();
+    String[] names = configuration.getNames();
+    String[] types = configuration.getTypes();
+    createTableWithColTypesAndNames(names, types, new String[0]);
+    List<String[]> inputData = configuration.getSampleData();
+    for (String[] input  : inputData) {
+      insertIntoTable(names, types, input);
+    }
+  }
+
+  @After
+  public void tearDown() {
+    try {
+      dropTableIfExists(getTableName());
+    } catch (SQLException e) {
+      LOG.warn("Error trying to drop table on tearDown: " + e);
+    }
+    super.tearDown();
+  }
+
+}


Mime
View raw message