sqoop-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From f...@apache.org
Subject sqoop git commit: SQOOP-3408 Introduce a Gradle build parameter to set the default forkEvery value for the tests
Date Thu, 22 Nov 2018 15:23:35 GMT
Repository: sqoop
Updated Branches:
  refs/heads/trunk 0a7407613 -> 74252163f


SQOOP-3408 Introduce a Gradle build parameter to set the default forkEvery value for the tests

(Szabolcs Vasas via Fero Szabo)


Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/74252163
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/74252163
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/74252163

Branch: refs/heads/trunk
Commit: 74252163f0455747ed2a99eb6c2c9d0e48f3dedc
Parents: 0a74076
Author: Fero Szabo <fero@apache.org>
Authored: Thu Nov 22 16:20:30 2018 +0100
Committer: Fero Szabo <fero@apache.org>
Committed: Thu Nov 22 16:20:30 2018 +0100

----------------------------------------------------------------------
 COMPILING.txt | 12 ++++++++++++
 build.gradle  | 36 ++++++++++++++++++++----------------
 2 files changed, 32 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/sqoop/blob/74252163/COMPILING.txt
----------------------------------------------------------------------
diff --git a/COMPILING.txt b/COMPILING.txt
index 0383707..b399ba8 100644
--- a/COMPILING.txt
+++ b/COMPILING.txt
@@ -149,6 +149,18 @@ This the same as running unitTest, integrationTest and kerberizedTest.
 * +allTest+: Runs all Sqoop tests.
 
 
+The https://docs.gradle.org/current/dsl/org.gradle.api.tasks.testing.Test.html#org.gradle.api.tasks.testing.Test:forkEvery[forkEvery]
+parameter of most of the Gradle test tasks is set to +0+ which means that all of the tests
run in a single JVM.
+The only exception is the +kerberizedTest+ task which requires a new JVM for every test class.
+The benefit of this setup is that the test tasks finish much faster since the JVM creation
is a slow operation however
+the Sqoop test framework seems to consume/leak too much memory which can lead to an +OutOfMemoryError+
during the build.
+To prevent the JVM running out of memory you can use the +-DforkEvery.default+ property to
set the forkEvery
+parameter for all the test tasks except +kerberizedTest+:
+
+----
+./gradlew -DforkEvery.default=30 test
+----
+
 === Third party tests
 
 ==== Installing the necessary databases

http://git-wip-us.apache.org/repos/asf/sqoop/blob/74252163/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 954935d..efe980d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -87,6 +87,7 @@ configurations.all {
 }
 
 def sqoopThirdPartyLib = System.getProperty("sqoop.thirdparty.lib.dir")
+def forkEveryDefault = Integer.valueOf(System.getProperty("forkEvery.default", "0"))
 
 dependencies {
     if (sqoopThirdPartyLib != null) runtime fileTree(dir: sqoopThirdPartyLib, include: '*.jar')
@@ -171,20 +172,6 @@ task integrationTest(type: Test) {
     }
 }
 
-task kerberizedTest (type: Test){
-    description 'Run Kerberized Test'
-    // A Gradle test task with forkEvery 1 and includeCategories performs poorly because
it starts a new JVM even for the filtered tests.
-    // To work around this performance problem we need to add every kerberized test in a
separate include field here.
-    include '**/TestKerberosAuthenticator*'
-    include '**/HBaseKerberizedConnectivityTest*'
-    include '**/TestHiveMiniCluster*'
-    include '**/TestHiveServer2TextImport*'
-    useJUnit {
-        includeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
-    }
-    forkEvery 1
-}
-
 task thirdPartyTest (type: Test) {
     description 'Run Third-party Tests - you need to specify -Dsqoop.thirdparty.lib.dir where
the Third party driver ' +
             'jars reside (relative to the project directory)'
@@ -210,7 +197,6 @@ test {
         excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
     }
 }
-test.finalizedBy(kerberizedTest)
 
 task s3Test(type: Test) {
     description 'Run S3 tests'
@@ -229,7 +215,6 @@ task allTest (type: Test){
         excludeCategories 'org.apache.sqoop.testcategories.sqooptest.ManualTest'
     }
 }
-allTest.finalizedBy(kerberizedTest)
 
 def testBuildDir = "$buildDir/test/"
 def testBuildDirData ="$testBuildDir/data/"
@@ -265,8 +250,27 @@ tasks.withType(Test) {
     jacoco{
         excludes = ["**/SqoopVersion*"]
     }
+
+    forkEvery forkEveryDefault
+}
+
+task kerberizedTest (type: Test){
+    description 'Run Kerberized Test'
+    // A Gradle test task with forkEvery 1 and includeCategories performs poorly because
it starts a new JVM even for the filtered tests.
+    // To work around this performance problem we need to add every kerberized test in a
separate include field here.
+    include '**/TestKerberosAuthenticator*'
+    include '**/HBaseKerberizedConnectivityTest*'
+    include '**/TestHiveMiniCluster*'
+    include '**/TestHiveServer2TextImport*'
+    useJUnit {
+        includeCategories 'org.apache.sqoop.testcategories.KerberizedTest'
+    }
+    forkEvery 1
 }
 
+test.finalizedBy(kerberizedTest)
+allTest.finalizedBy(kerberizedTest)
+
 tasks.withType(Checkstyle) {
     reports {
         xml.enabled false


Mime
View raw message