sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jar...@apache.org
Subject git commit: SQOOP-916: Add an abort validation handler
Date Fri, 05 Apr 2013 18:08:30 GMT
Updated Branches:
  refs/heads/trunk 75bc21b16 -> ef093399b


SQOOP-916: Add an abort validation handler

(Venkatesh Seetharam via Jarek Jarcec Cecho)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/ef093399
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/ef093399
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/ef093399

Branch: refs/heads/trunk
Commit: ef093399bf01fed4a7c17caebd05676f1e566b8a
Parents: 75bc21b
Author: Jarek Jarcec Cecho <jarcec@cloudera.com>
Authored: Fri Apr 5 11:07:50 2013 -0700
Committer: Jarek Jarcec Cecho <jarcec@cloudera.com>
Committed: Fri Apr 5 11:07:50 2013 -0700

----------------------------------------------------------------------
 src/docs/user/validation.txt                       |    4 +-
 src/java/org/apache/sqoop/SqoopOptions.java        |    4 +-
 .../sqoop/validation/AbortOnFailureHandler.java    |   55 +++++++
 .../sqoop/validation/LogOnFailureHandler.java      |    5 +-
 .../apache/sqoop/validation/RowCountValidator.java |    2 +-
 .../sqoop/validation/ValidationException.java      |    4 +-
 .../validation/AbortOnFailureHandlerTest.java      |   62 ++++++++
 .../AbsoluteValidationThresholdTest.java           |   38 +++++
 .../validation/RowCountValidatorImportTest.java    |  123 +++++++++++++--
 9 files changed, 275 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/docs/user/validation.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/validation.txt b/src/docs/user/validation.txt
index f28b420..282cfd6 100644
--- a/src/docs/user/validation.txt
+++ b/src/docs/user/validation.txt
@@ -90,7 +90,7 @@ described below.
  Description:      Responsible for handling failures, must implement
                    org.apache.sqoop.validation.ValidationFailureHandler
  Supported values: The value has to be a fully qualified class name.
- Default value:    org.apache.sqoop.validation.LogOnFailureHandler
+ Default value:    org.apache.sqoop.validation.AbortOnFailureHandler
 
 
 Limitations
@@ -132,5 +132,5 @@ $ sqoop import --connect jdbc:mysql://db.foo.com/corp --table EMPLOYEES
\
     --validation-threshold \
           org.apache.sqoop.validation.AbsoluteValidationThreshold \
     --validation-failurehandler \
-          org.apache.sqoop.validation.LogOnFailureHandler
+          org.apache.sqoop.validation.AbortOnFailureHandler
 ----

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/SqoopOptions.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/SqoopOptions.java b/src/java/org/apache/sqoop/SqoopOptions.java
index 906542b..f423c0f 100644
--- a/src/java/org/apache/sqoop/SqoopOptions.java
+++ b/src/java/org/apache/sqoop/SqoopOptions.java
@@ -41,8 +41,8 @@ import com.cloudera.sqoop.util.RandomHash;
 import com.cloudera.sqoop.util.StoredAsProperty;
 import org.apache.sqoop.util.CredentialsUtil;
 import org.apache.sqoop.util.LoggingUtils;
+import org.apache.sqoop.validation.AbortOnFailureHandler;
 import org.apache.sqoop.validation.AbsoluteValidationThreshold;
-import org.apache.sqoop.validation.LogOnFailureHandler;
 import org.apache.sqoop.validation.RowCountValidator;
 
 /**
@@ -870,7 +870,7 @@ public class SqoopOptions implements Cloneable {
     this.isValidationEnabled = false; // validation is disabled by default
     this.validatorClass = RowCountValidator.class;
     this.validationThresholdClass = AbsoluteValidationThreshold.class;
-    this.validationFailureHandlerClass = LogOnFailureHandler.class;
+    this.validationFailureHandlerClass = AbortOnFailureHandler.class;
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java b/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java
new file mode 100644
index 0000000..32f4fd9
--- /dev/null
+++ b/src/java/org/apache/sqoop/validation/AbortOnFailureHandler.java
@@ -0,0 +1,55 @@
+/**
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+/**
+ * A specific implementation of ValidationFailureHandler that aborts the
+ * processing by throwing an exception with failure message and the reason.
+ *
+ * This is used as the default handler unless overridden in configuration.
+ */
+public class AbortOnFailureHandler implements ValidationFailureHandler {
+
+  static final ValidationFailureHandler INSTANCE = new AbortOnFailureHandler();
+
+  /**
+   * Method that handles the validation failure.
+   *
+   * @param validationContext validation context
+   * @return if failure was handled or not
+   * @throws ValidationException
+   */
+  @Override
+  public boolean handle(ValidationContext validationContext)
+    throws ValidationException {
+
+    StringBuilder messageBuffer = new StringBuilder();
+    messageBuffer.append("Validation failed by ");
+    messageBuffer.append(validationContext.getMessage());
+    messageBuffer.append(". Reason: ").append(validationContext.getReason());
+    messageBuffer.append(", Row Count at Source: ");
+    messageBuffer.append(validationContext.getSourceRowCount());
+    messageBuffer.append(", Row Count at Target: ");
+    messageBuffer.append(validationContext.getTargetRowCount());
+
+    throw new ValidationException(messageBuffer.toString());
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java b/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java
index 3ded6ad..3c20136 100644
--- a/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java
+++ b/src/java/org/apache/sqoop/validation/LogOnFailureHandler.java
@@ -23,9 +23,10 @@ import org.apache.commons.logging.LogFactory;
 
 /**
  * A specific implementation of ValidationFailureHandler that logs the failure
- * message and the reason with the configured logger.
+ * message and the reason with the configured logger. A note of caution in
+ * using this since it fails silently by logging the failure to a log file.
  *
- * This is used as the default handler unless overridden in configuration.
+ * This is mostly used for testing purposes since this fails silently.
  */
 public class LogOnFailureHandler implements ValidationFailureHandler {
   private static final Log LOG =

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/validation/RowCountValidator.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/validation/RowCountValidator.java b/src/java/org/apache/sqoop/validation/RowCountValidator.java
index 2896192..ca21232 100644
--- a/src/java/org/apache/sqoop/validation/RowCountValidator.java
+++ b/src/java/org/apache/sqoop/validation/RowCountValidator.java
@@ -37,7 +37,7 @@ public class RowCountValidator implements Validator {
   public boolean validate(ValidationContext context)
     throws ValidationException {
     return validate(context,
-      AbsoluteValidationThreshold.INSTANCE, LogOnFailureHandler.INSTANCE);
+      AbsoluteValidationThreshold.INSTANCE, AbortOnFailureHandler.INSTANCE);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/java/org/apache/sqoop/validation/ValidationException.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/validation/ValidationException.java b/src/java/org/apache/sqoop/validation/ValidationException.java
index 0c10241..118f1ca 100644
--- a/src/java/org/apache/sqoop/validation/ValidationException.java
+++ b/src/java/org/apache/sqoop/validation/ValidationException.java
@@ -24,8 +24,8 @@ package org.apache.sqoop.validation;
  */
 public class ValidationException extends Exception {
 
-  public ValidationException(String s, Throwable throwable) {
-    super(s, throwable);
+  public ValidationException(String message) {
+    super(message);
   }
 
   @Override

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java b/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java
new file mode 100644
index 0000000..f38164c
--- /dev/null
+++ b/src/test/org/apache/sqoop/validation/AbortOnFailureHandlerTest.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+import com.cloudera.sqoop.SqoopOptions;
+import junit.framework.TestCase;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Tests for AbortOnFailureHandler.
+ */
+public class AbortOnFailureHandlerTest extends TestCase {
+
+  public void testAbortOnFailureHandlerIsDefaultOption() {
+    assertEquals(AbortOnFailureHandler.class,
+      new SqoopOptions(new Configuration()).getValidationFailureHandlerClass());
+  }
+
+  /**
+   * Positive case.
+   */
+  public void testAbortOnFailureHandlerAborting() {
+    try {
+      Validator validator = new RowCountValidator();
+      validator.validate(new ValidationContext(100, 90));
+      fail("AbortOnFailureHandler should have thrown an exception");
+    } catch (ValidationException e) {
+      assertEquals("Validation failed by RowCountValidator. "
+        + "Reason: The expected counter value was 100 but the actual value "
+        + "was 90, Row Count at Source: 100, Row Count at Target: 90",
+        e.getMessage());
+    }
+  }
+
+  /**
+   * Negative case.
+   */
+  public void testAbortOnFailureHandlerNotAborting() {
+    try {
+      Validator validator = new RowCountValidator();
+      validator.validate(new ValidationContext(100, 100));
+    } catch (ValidationException e) {
+      fail("AbortOnFailureHandler should NOT have thrown an exception");
+    }
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java b/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java
new file mode 100644
index 0000000..9ac5074
--- /dev/null
+++ b/src/test/org/apache/sqoop/validation/AbsoluteValidationThresholdTest.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.validation;
+
+import junit.framework.TestCase;
+
+/**
+ * Tests for AbsoluteValidationThreshold.
+ */
+public class AbsoluteValidationThresholdTest extends TestCase {
+
+  /**
+   * Test the implementation for AbsoluteValidationThreshold.
+   * Both arguments should be same else fail.
+   */
+  public void testAbsoluteValidationThreshold() {
+    ValidationThreshold validationThreshold = new AbsoluteValidationThreshold();
+    assertTrue(validationThreshold.compare(100, 100));
+    assertFalse(validationThreshold.compare(100, 90));
+    assertFalse(validationThreshold.compare(90, 100));
+  }
+}

http://git-wip-us.apache.org/repos/asf/sqoop/blob/ef093399/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java b/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java
index a53e281..035d3b1 100644
--- a/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java
+++ b/src/test/org/apache/sqoop/validation/RowCountValidatorImportTest.java
@@ -18,8 +18,10 @@
 
 package org.apache.sqoop.validation;
 
+import com.cloudera.sqoop.SqoopOptions;
 import com.cloudera.sqoop.testutil.ImportJobTestCase;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.sqoop.tool.ImportTool;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -37,22 +39,11 @@ public class RowCountValidatorImportTest extends ImportJobTestCase {
   }
 
   /**
-   * Test the implementation for AbsoluteValidationThreshold.
-   * Both arguments should be same else fail.
-   */
-  public void testAbsoluteValidationThreshold() {
-    ValidationThreshold validationThreshold = new AbsoluteValidationThreshold();
-    assertTrue(validationThreshold.compare(100, 100));
-    assertFalse(validationThreshold.compare(100, 90));
-    assertFalse(validationThreshold.compare(90, 100));
-  }
-
-  /**
-   * Test if teh --validate flag actually made it through the options.
+   * Test if the --validate flag actually made it through the options.
    *
    * @throws Exception
    */
-  public void testValidateOptionIsEnabled() throws Exception {
+  public void testValidateOptionIsEnabledInCLI() throws Exception {
     String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
     String[] insertVals = {"1", "'Bob'", "'sales'"};
 
@@ -68,6 +59,112 @@ public class RowCountValidatorImportTest extends ImportJobTestCase {
     }
   }
 
+  public void testValidationOptionsParsedCorrectly() throws Exception {
+    String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
+    String[] insertVals = {"1", "'Bob'", "'sales'"};
+
+    try {
+      createTableWithColTypes(types, insertVals);
+
+      String[] args = getArgv(true, null, getConf());
+      ArrayList<String> argsList = new ArrayList<String>();
+      argsList.add("--validator");
+      argsList.add("org.apache.sqoop.validation.RowCountValidator");
+      argsList.add("--validation-threshold");
+      argsList.add("org.apache.sqoop.validation.AbsoluteValidationThreshold");
+      argsList.add("--validation-failurehandler");
+      argsList.add("org.apache.sqoop.validation.AbortOnFailureHandler");
+      Collections.addAll(argsList, args);
+
+      assertTrue("Validate option missing.", argsList.contains("--validate"));
+      assertTrue("Validator option missing.", argsList.contains("--validator"));
+
+      String[] optionArgs = toStringArray(argsList);
+
+      SqoopOptions validationOptions = new ImportTool().parseArguments(
+        optionArgs, getConf(), getSqoopOptions(getConf()), true);
+      assertEquals(RowCountValidator.class,
+        validationOptions.getValidatorClass());
+      assertEquals(AbsoluteValidationThreshold.class,
+        validationOptions.getValidationThresholdClass());
+      assertEquals(AbortOnFailureHandler.class,
+        validationOptions.getValidationFailureHandlerClass());
+    } catch (Exception e) {
+      fail("The validation options are passed correctly: " + e.getMessage());
+    } finally {
+      dropTableIfExists(getTableName());
+    }
+  }
+
+  public void testInvalidValidationOptions() throws Exception {
+    String[] types = {"INT NOT NULL PRIMARY KEY", "VARCHAR(32)", "VARCHAR(32)"};
+    String[] insertVals = {"1", "'Bob'", "'sales'"};
+
+    try {
+      createTableWithColTypes(types, insertVals);
+
+      String[] args = getArgv(true, null, getConf());
+      ArrayList<String> argsList = new ArrayList<String>();
+      argsList.add("--validator");
+      argsList.add("org.apache.sqoop.validation.NullValidator");
+      argsList.add("--validation-threshold");
+      argsList.add("org.apache.sqoop.validation.NullValidationThreshold");
+      argsList.add("--validation-failurehandler");
+      argsList.add("org.apache.sqoop.validation.NullFailureHandler");
+      Collections.addAll(argsList, args);
+
+      String[] optionArgs = toStringArray(argsList);
+
+      new ImportTool().parseArguments(optionArgs, getConf(),
+        getSqoopOptions(getConf()), true);
+      fail("The validation options are incorrect and must throw an exception");
+    } catch (Exception e) {
+      System.out.println("e.getMessage() = " + e.getMessage());
+      System.out.println("e.getClass() = " + e.getClass());
+      assertEquals(
+        com.cloudera.sqoop.SqoopOptions.InvalidOptionsException.class,
+        e.getClass());
+    } finally {
+      dropTableIfExists(getTableName());
+    }
+  }
+
+  private String[] toStringArray(ArrayList<String> argsList) {
+    String[] optionArgs = new String[argsList.size()];
+    for (int i = 0; i < argsList.size(); i++) {
+      optionArgs[i] = argsList.get(i);
+    }
+    return optionArgs;
+  }
+
+  /**
+   * Negative case where the row counts do NOT match.
+   */
+  public void testValidatorWithDifferentRowCounts() {
+    try {
+      Validator validator = new RowCountValidator();
+      validator.validate(new ValidationContext(100, 90));
+      fail("FailureHandler should have thrown an exception");
+    } catch (ValidationException e) {
+      assertEquals("Validation failed by RowCountValidator. "
+        + "Reason: The expected counter value was 100 but the actual value "
+        + "was 90, Row Count at Source: 100, Row Count at Target: 90",
+        e.getMessage());
+    }
+  }
+
+  /**
+   * Positive case where the row counts match.
+   */
+  public void testValidatorWithMatchingRowCounts() {
+    try {
+      Validator validator = new RowCountValidator();
+      validator.validate(new ValidationContext(100, 100));
+    } catch (ValidationException e) {
+      fail("FailureHandler should NOT have thrown an exception");
+    }
+  }
+
   /**
    * Test the validation for a sample import, positive case.
    *


Mime
View raw message