sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject [1/2] SQOOP-846: Provide a connector for Netezza appliances
Date Tue, 05 Mar 2013 06:23:19 GMT
http://git-wip-us.apache.org/repos/asf/sqoop/blob/0d5f73ad/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java b/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java
new file mode 100644
index 0000000..97399fe
--- /dev/null
+++ b/src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.cloudera.sqoop.manager;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.ArrayList;
+import java.util.Arrays;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.sqoop.manager.NetezzaManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.cloudera.sqoop.SqoopOptions;
+import com.cloudera.sqoop.testutil.CommonArgs;
+import com.cloudera.sqoop.testutil.ImportJobTestCase;
+import com.cloudera.sqoop.util.FileListing;
+
+/**
+ * Test the Netezza  implementation.
+ *
+ * This uses JDBC to import data from an Netezza database into HDFS.
+ *
+ * Since this requires an Netezza SErver installation, this class is named
+ * in such a way that Sqoop's default QA process does not run it. You need to
+ * run this manually with -Dtestcase=NetezzaManagerImportManualTest.
+ *
+ */
+public class NetezzaImportManualTest extends ImportJobTestCase {
+
+  public static final Log LOG = LogFactory
+      .getLog(NetezzaImportManualTest.class.getName());
+
+
+
+  // instance variables populated during setUp, used during tests
+  private NetezzaManager manager;
+
+
+  @Override
+  protected boolean useHsqldbTestServer() {
+    return false;
+  }
+
+  @Override
+  protected String getTableName() {
+    return NetezzaTestUtils.TABLE_NAME;
+  }
+
+  @Before
+  public void setUp() {
+    super.setUp();
+
+    SqoopOptions options = new SqoopOptions(
+        NetezzaTestUtils.getNZConnectString(), getTableName());
+    options.setUsername(NetezzaTestUtils.getNZUser());
+    options.setPassword(NetezzaTestUtils.getNZPassword());
+
+    manager = new NetezzaManager(options);
+
+    // Drop the existing table, if there is one.
+    Connection conn = null;
+    Statement stmt = null;
+    try {
+      conn = manager.getConnection();
+      stmt = conn.createStatement();
+      stmt.execute("DROP TABLE " + getTableName());
+    } catch (SQLException sqlE) {
+      LOG.info("Table was not dropped: " + sqlE.getMessage());
+    } finally {
+      try {
+        if (null != stmt) {
+          stmt.close();
+        }
+      } catch (Exception ex) {
+        LOG.warn("Exception while closing stmt", ex);
+      }
+    }
+
+    // Create and populate table
+    try {
+      conn = manager.getConnection();
+      conn.setAutoCommit(false);
+      stmt = conn.createStatement();
+
+      // create the database table and populate it with data.
+      stmt.executeUpdate("CREATE TABLE " + getTableName() + " ("
+          + "id INT NOT NULL, " + "name VARCHAR(24) NOT NULL, "
+          + "salary FLOAT, " + "dept VARCHAR(32), " + "PRIMARY KEY (id))");
+
+      stmt.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
+          + "1,'Aaron', " + "1000000.00,'engineering')");
+      stmt.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
+          + "2,'Bob', " + "400.00,'sales')");
+      stmt.executeUpdate("INSERT INTO " + getTableName() + " VALUES("
+          + "3,'Fred', 15.00," + "'marketing')");
+      conn.commit();
+    } catch (SQLException sqlE) {
+      LOG.error("Encountered SQL Exception: ", sqlE);
+      sqlE.printStackTrace();
+      fail("SQLException when running test setUp(): " + sqlE);
+    } finally {
+      try {
+        if (null != stmt) {
+          stmt.close();
+        }
+      } catch (Exception ex) {
+        LOG.warn("Exception while closing connection/stmt", ex);
+      }
+    }
+  }
+
+  @After
+  public void tearDown() {
+    super.tearDown();
+    try {
+      manager.close();
+    } catch (SQLException sqlE) {
+      LOG.error("Got SQLException: " + sqlE.toString());
+      fail("Got SQLException: " + sqlE.toString());
+    }
+  }
+
+  @Test
+  public void testNetezzaImport() throws IOException {
+
+    runNetezzaTest(getExpectedResults());
+  }
+
+  private String[] getExpectedResults() {
+    return new String[] { "1,Aaron,1000000.0,engineering", "2,Bob,400.0,sales",
+        "3,Fred,15.0,marketing", };
+  }
+
+  private String[] getArgv() {
+    ArrayList<String> args = new ArrayList<String>();
+
+    CommonArgs.addHadoopFlags(args);
+
+    args.add("--table");
+    args.add(getTableName());
+    args.add("--warehouse-dir");
+    args.add(getWarehouseDir());
+    args.add("--connect");
+    args.add(NetezzaTestUtils.getNZConnectString());
+    args.add("--username");
+    args.add(NetezzaTestUtils.getNZUser());
+    args.add("--password");
+    args.add(NetezzaTestUtils.getNZPassword());
+    args.add("--num-mappers");
+    args.add("1");
+    return args.toArray(new String[args.size()]);
+  }
+
+  private void runNetezzaTest(String[] expectedResults) throws IOException {
+
+    Path warehousePath = new Path(this.getWarehouseDir());
+    Path tablePath = new Path(warehousePath, getTableName());
+    Path filePath = new Path(tablePath, "part-m-00000");
+
+    File tableFile = new File(tablePath.toString());
+    if (tableFile.exists() && tableFile.isDirectory()) {
+      // remove the directory before running the import.
+      FileListing.recursiveDeleteDir(tableFile);
+    }
+
+    String[] argv = getArgv();
+    try {
+      runImport(argv);
+    } catch (IOException ioe) {
+      LOG.error("Got IOException during import: " + ioe.toString());
+      ioe.printStackTrace();
+      fail(ioe.toString());
+    }
+
+    File f = new File(filePath.toString());
+    assertTrue("Could not find imported data file", f.exists());
+    BufferedReader r = null;
+    try {
+      // Read through the file and make sure it's all there.
+      r = new BufferedReader(new InputStreamReader(new FileInputStream(f)));
+      String[] s = new String[3];
+      for (int i = 0; i < s.length; ++i) {
+        s[i] = r.readLine();
+      }
+      Arrays.sort(s);
+      for (int i = 0; i < expectedResults.length; ++i) {
+        assertEquals(expectedResults[i], s[i]);
+      }
+    } catch (IOException ioe) {
+      LOG.error("Got IOException verifying results: " + ioe.toString());
+      ioe.printStackTrace();
+      fail(ioe.toString());
+    } finally {
+      IOUtils.closeStream(r);
+    }
+  }
+
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/0d5f73ad/src/test/com/cloudera/sqoop/manager/NetezzaTestUtils.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/NetezzaTestUtils.java b/src/test/com/cloudera/sqoop/manager/NetezzaTestUtils.java
new file mode 100644
index 0000000..9f4c07c
--- /dev/null
+++ b/src/test/com/cloudera/sqoop/manager/NetezzaTestUtils.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.cloudera.sqoop.manager;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+/**
+ * Utilities for Netezza tests.
+ */
+public final class NetezzaTestUtils {
+
+  public static final Log LOG = LogFactory.getLog(
+      NetezzaTestUtils.class.getName());
+
+  public static final String NZ_HOST = System.getProperty(
+      "sqoop.test.netezza.host", "nz-host");
+  public static final String NZ_PORT = System.getProperty(
+      "sqoop.test.netezza.port", "5480");
+  public static final String NZ_JDBC_URL_PREFIX = "jdbc:netezza:";
+
+  public static final String NZ_DB_USER = System.getProperty(
+      "sqoop.test.netezza.username", "ADMIN");
+
+  public static final String NZ_DB_PASSWORD = System.getProperty(
+      "sqoop.test.netezza.password", "password");
+
+  public static final String NZ_DB_NAME = System.getProperty(
+      "sqoop.test.netezza.db.name", "SQOOP");
+  public static final String TABLE_NAME = System.getProperty(
+      "sqoop.test.netezza.table.name", "EMP");
+
+
+  private NetezzaTestUtils() { }
+
+  /** @return the current username. */
+  public static String getNZUser() {
+    // First, check the $NZ_USER environment variable.
+    String nzUser = System.getenv("NZ_USER");
+    if (nzUser == null) {
+      // Else return what is in the NZ Properties
+      nzUser = NZ_DB_USER;
+    }
+    return nzUser;
+  }
+
+  public static String getNZPassword() {
+    String nzPass = System.getenv("NZ_PASSWORD");
+    if (nzPass == null) {
+      nzPass = NZ_DB_PASSWORD;
+    }
+    return nzPass;
+  }
+  public static String getNZConnectString() {
+    String nzHost = System.getenv("NZ_HOST");
+    if (nzHost == null) {
+      nzHost = NZ_HOST;
+    }
+
+    String nzPort = System.getenv("NZ_PORT");
+    if (nzPort == null) {
+      nzPort = NZ_PORT;
+    }
+    String nzDB = System.getenv("NZ_DB_NAME");
+    if (nzDB == null) {
+      nzDB = NZ_DB_NAME;
+    }
+
+    StringBuilder url = new StringBuilder(NZ_JDBC_URL_PREFIX);
+    url.append("//").append(nzHost).append(':').append(nzPort);
+    url.append('/').append(nzDB);
+
+    LOG.info("NZ Connect string generated : " + url.toString());
+    return url.toString();
+  }
+}


Mime
View raw message