kafka-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From nehanarkh...@apache.org
Subject svn commit: r1396687 [3/4] - in /incubator/kafka/branches/0.8/system_test: ./ mirror_maker_testsuite/ mirror_maker_testsuite/config/ mirror_maker_testsuite/testcase_5001/ replication_testsuite/ replication_testsuite/config/ replication_testsuite/testca...
Date Wed, 10 Oct 2012 16:57:01 GMT
Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0114/testcase_0114_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0114/testcase_0114_properties.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0114/testcase_0114_properties.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0114/testcase_0114_properties.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,79 @@
+{
+  "description": {"01":"Multi Leader Failures in Replication : 1. mode => async; 2. acks => 1",
+                  "02":"Produce and consume messages to a single topic - three partition.",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:async, acks:1, comp:0",
+                  "10":"Log segment size    : 102400"
+  },
+  "testcase_args": {
+    "bounce_leader": "true",
+    "replica_factor": "3",
+    "num_partition": "3",
+    "num_iteration": "3",
+    "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15",
+    "num_messages_to_produce_per_producer_call": "50"
+  },
+  "entities": [
+    {
+      "entity_id": "0",
+      "clientPort": "2188",
+      "dataDir": "/tmp/zookeeper_0",
+      "log_filename": "zookeeper_2188.log",
+      "config_filename": "zookeeper_2188.properties"
+    },
+    {
+      "entity_id": "1",
+      "port": "9091",
+      "brokerid": "1",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_1_logs",
+      "log_filename": "kafka_server_9091.log",
+      "config_filename": "kafka_server_9091.properties"
+    },
+    {
+      "entity_id": "2",
+      "port": "9092",
+      "brokerid": "2",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_2_logs",
+      "log_filename": "kafka_server_9092.log",
+      "config_filename": "kafka_server_9092.properties"
+    },
+    {
+      "entity_id": "3",
+      "port": "9093",
+      "brokerid": "3",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_3_logs",
+      "log_filename": "kafka_server_9093.log",
+      "config_filename": "kafka_server_9093.properties"
+    },
+    {
+      "entity_id": "4",
+      "topic": "test_1",
+      "threads": "5",
+      "compression-codec": "0",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "1",
+      "async":"true",
+      "log_filename": "producer_performance.log",
+      "config_filename": "producer_performance.properties"
+    },
+    {
+      "entity_id": "5",
+      "topic": "test_1",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "zookeeper": "localhost:2188",
+      "log_filename": "console_consumer.log",
+      "config_filename": "console_consumer.properties"
+    }
+  ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0115/testcase_0115_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0115/testcase_0115_properties.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0115/testcase_0115_properties.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0115/testcase_0115_properties.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,79 @@
+{
+  "description": {"01":"Multi Leader Failures in Replication : 1. comp => 1",
+                  "02":"Produce and consume messages to a single topic - three partition.",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:sync, acks:-1, comp:1",
+                  "10":"Log segment size    : 102400"
+  },
+  "testcase_args": {
+    "bounce_leader": "true",
+    "replica_factor": "3",
+    "num_partition": "3",
+    "num_iteration": "3",
+    "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15",
+    "num_messages_to_produce_per_producer_call": "50"
+  },
+  "entities": [
+    {
+      "entity_id": "0",
+      "clientPort": "2188",
+      "dataDir": "/tmp/zookeeper_0",
+      "log_filename": "zookeeper_2188.log",
+      "config_filename": "zookeeper_2188.properties"
+    },
+    {
+      "entity_id": "1",
+      "port": "9091",
+      "brokerid": "1",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_1_logs",
+      "log_filename": "kafka_server_9091.log",
+      "config_filename": "kafka_server_9091.properties"
+    },
+    {
+      "entity_id": "2",
+      "port": "9092",
+      "brokerid": "2",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_2_logs",
+      "log_filename": "kafka_server_9092.log",
+      "config_filename": "kafka_server_9092.properties"
+    },
+    {
+      "entity_id": "3",
+      "port": "9093",
+      "brokerid": "3",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_3_logs",
+      "log_filename": "kafka_server_9093.log",
+      "config_filename": "kafka_server_9093.properties"
+    },
+    {
+      "entity_id": "4",
+      "topic": "test_1",
+      "threads": "5",
+      "compression-codec": "1",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "-1",
+      "async":"false",
+      "log_filename": "producer_performance.log",
+      "config_filename": "producer_performance.properties"
+    },
+    {
+      "entity_id": "5",
+      "topic": "test_1",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "zookeeper": "localhost:2188",
+      "log_filename": "console_consumer.log",
+      "config_filename": "console_consumer.properties"
+    }
+  ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0116/testcase_0116_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0116/testcase_0116_properties.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0116/testcase_0116_properties.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0116/testcase_0116_properties.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,79 @@
+{
+  "description": {"01":"Multi Leader Failures in Replication : 1. mode => async; 2. comp => 1",
+                  "02":"Produce and consume messages to a single topic - three partition.",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:async, acks:-1, comp:1",
+                  "10":"Log segment size    : 102400"
+  },
+  "testcase_args": {
+    "bounce_leader": "true",
+    "replica_factor": "3",
+    "num_partition": "3",
+    "num_iteration": "3",
+    "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15",
+    "num_messages_to_produce_per_producer_call": "50"
+  },
+  "entities": [
+    {
+      "entity_id": "0",
+      "clientPort": "2188",
+      "dataDir": "/tmp/zookeeper_0",
+      "log_filename": "zookeeper_2188.log",
+      "config_filename": "zookeeper_2188.properties"
+    },
+    {
+      "entity_id": "1",
+      "port": "9091",
+      "brokerid": "1",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_1_logs",
+      "log_filename": "kafka_server_9091.log",
+      "config_filename": "kafka_server_9091.properties"
+    },
+    {
+      "entity_id": "2",
+      "port": "9092",
+      "brokerid": "2",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_2_logs",
+      "log_filename": "kafka_server_9092.log",
+      "config_filename": "kafka_server_9092.properties"
+    },
+    {
+      "entity_id": "3",
+      "port": "9093",
+      "brokerid": "3",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_3_logs",
+      "log_filename": "kafka_server_9093.log",
+      "config_filename": "kafka_server_9093.properties"
+    },
+    {
+      "entity_id": "4",
+      "topic": "test_1",
+      "threads": "5",
+      "compression-codec": "1",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "-1",
+      "async":"true",
+      "log_filename": "producer_performance.log",
+      "config_filename": "producer_performance.properties"
+    },
+    {
+      "entity_id": "5",
+      "topic": "test_1",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "zookeeper": "localhost:2188",
+      "log_filename": "console_consumer.log",
+      "config_filename": "console_consumer.properties"
+    }
+  ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0117/testcase_0117_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0117/testcase_0117_properties.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0117/testcase_0117_properties.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0117/testcase_0117_properties.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,79 @@
+{
+  "description": {"01":"Multi Leader Failures in Replication : 1. acks => 1; 2. comp => 1",
+                  "02":"Produce and consume messages to a single topic - three partition.",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:sync, acks:1, comp:1",
+                  "10":"Log segment size    : 102400"
+  },
+  "testcase_args": {
+    "bounce_leader": "true",
+    "replica_factor": "3",
+    "num_partition": "3",
+    "num_iteration": "3",
+    "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15",
+    "num_messages_to_produce_per_producer_call": "50"
+  },
+  "entities": [
+    {
+      "entity_id": "0",
+      "clientPort": "2188",
+      "dataDir": "/tmp/zookeeper_0",
+      "log_filename": "zookeeper_2188.log",
+      "config_filename": "zookeeper_2188.properties"
+    },
+    {
+      "entity_id": "1",
+      "port": "9091",
+      "brokerid": "1",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_1_logs",
+      "log_filename": "kafka_server_9091.log",
+      "config_filename": "kafka_server_9091.properties"
+    },
+    {
+      "entity_id": "2",
+      "port": "9092",
+      "brokerid": "2",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_2_logs",
+      "log_filename": "kafka_server_9092.log",
+      "config_filename": "kafka_server_9092.properties"
+    },
+    {
+      "entity_id": "3",
+      "port": "9093",
+      "brokerid": "3",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_3_logs",
+      "log_filename": "kafka_server_9093.log",
+      "config_filename": "kafka_server_9093.properties"
+    },
+    {
+      "entity_id": "4",
+      "topic": "test_1",
+      "threads": "5",
+      "compression-codec": "1",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "1",
+      "async":"false",
+      "log_filename": "producer_performance.log",
+      "config_filename": "producer_performance.properties"
+    },
+    {
+      "entity_id": "5",
+      "topic": "test_1",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "zookeeper": "localhost:2188",
+      "log_filename": "console_consumer.log",
+      "config_filename": "console_consumer.properties"
+    }
+  ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0118/testcase_0118_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0118/testcase_0118_properties.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0118/testcase_0118_properties.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0118/testcase_0118_properties.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,79 @@
+{
+  "description": {"01":"Multi Leader Failures in Replication : 1. mode => async; 2. acks => 1; 3. comp => 1",
+                  "02":"Produce and consume messages to a single topic - three partition.",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:async, acks:1, comp:1",
+                  "10":"Log segment size    : 102400"
+  },
+  "testcase_args": {
+    "bounce_leader": "true",
+    "replica_factor": "3",
+    "num_partition": "3",
+    "num_iteration": "3",
+    "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15",
+    "num_messages_to_produce_per_producer_call": "50"
+  },
+  "entities": [
+    {
+      "entity_id": "0",
+      "clientPort": "2188",
+      "dataDir": "/tmp/zookeeper_0",
+      "log_filename": "zookeeper_2188.log",
+      "config_filename": "zookeeper_2188.properties"
+    },
+    {
+      "entity_id": "1",
+      "port": "9091",
+      "brokerid": "1",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_1_logs",
+      "log_filename": "kafka_server_9091.log",
+      "config_filename": "kafka_server_9091.properties"
+    },
+    {
+      "entity_id": "2",
+      "port": "9092",
+      "brokerid": "2",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_2_logs",
+      "log_filename": "kafka_server_9092.log",
+      "config_filename": "kafka_server_9092.properties"
+    },
+    {
+      "entity_id": "3",
+      "port": "9093",
+      "brokerid": "3",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_3_logs",
+      "log_filename": "kafka_server_9093.log",
+      "config_filename": "kafka_server_9093.properties"
+    },
+    {
+      "entity_id": "4",
+      "topic": "test_1",
+      "threads": "5",
+      "compression-codec": "1",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "1",
+      "async":"true",
+      "log_filename": "producer_performance.log",
+      "config_filename": "producer_performance.properties"
+    },
+    {
+      "entity_id": "5",
+      "topic": "test_1",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "zookeeper": "localhost:2188",
+      "log_filename": "console_consumer.log",
+      "config_filename": "console_consumer.properties"
+    }
+  ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0121/cluster_config.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0121/cluster_config.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0121/cluster_config.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0121/cluster_config.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,76 @@
+{
+    "cluster_config": [
+        {
+            "entity_id": "0",
+            "hostname": "localhost",
+            "role": "zookeeper",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9990"
+        },
+        {
+            "entity_id": "1",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9991"
+        },
+        {
+            "entity_id": "2",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9992"
+        },
+        {
+            "entity_id": "3",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9993"
+        },
+        {
+            "entity_id": "4",
+            "hostname": "localhost",
+            "role": "producer_performance",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9997"
+        },
+        {
+            "entity_id": "5",
+            "hostname": "localhost",
+            "role": "producer_performance",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9998"
+        },
+        {
+            "entity_id": "6",
+            "hostname": "localhost",
+            "role": "console_consumer",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9999"
+        },
+        {
+            "entity_id": "7",
+            "hostname": "localhost",
+            "role": "console_consumer",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9099"
+        }
+    ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0121/testcase_0121_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0121/testcase_0121_properties.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0121/testcase_0121_properties.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0121/testcase_0121_properties.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,97 @@
+{
+  "description": {"01":"Leader Failure in Replication with multi topics & partitions : Base Test",
+                  "02":"Produce and consume messages to 2 topics - 3 partitions",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:sync, acks:-1, comp:0",
+                  "10":"Log segment size    : 102400"
+  },
+  "testcase_args": {
+    "bounce_leader": "true",
+    "replica_factor": "3",
+    "num_partition": "3",
+    "num_iteration": "3",
+    "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15"
+  },
+  "entities": [
+    {
+      "entity_id": "0",
+      "clientPort": "2188",
+      "dataDir": "/tmp/zookeeper_0",
+      "log_filename": "zookeeper_2188.log",
+      "config_filename": "zookeeper_2188.properties"
+    },
+    {
+      "entity_id": "1",
+      "port": "9091",
+      "brokerid": "1",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_1_logs",
+      "log_filename": "kafka_server_9091.log",
+      "config_filename": "kafka_server_9091.properties"
+    },
+    {
+      "entity_id": "2",
+      "port": "9092",
+      "brokerid": "2",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_2_logs",
+      "log_filename": "kafka_server_9092.log",
+      "config_filename": "kafka_server_9092.properties"
+    },
+    {
+      "entity_id": "3",
+      "port": "9093",
+      "brokerid": "3",
+      "log.file.size": "102400",
+      "log.dir": "/tmp/kafka_server_3_logs",
+      "log_filename": "kafka_server_9093.log",
+      "config_filename": "kafka_server_9093.properties"
+    },
+    {
+      "entity_id": "4",
+      "topic": "test_1",
+      "threads": "5",
+      "compression-codec": "0",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "-1",
+      "async":"false",
+      "log_filename": "producer_performance_4.log",
+      "config_filename": "producer_performance_4.properties"
+    },
+    {
+      "entity_id": "5",
+      "topic": "test_2",
+      "threads": "5",
+      "compression-codec": "0",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "-1",
+      "async":"false",
+      "log_filename": "producer_performance_5.log",
+      "config_filename": "producer_performance_5.properties"
+    },
+    {
+      "entity_id": "6",
+      "topic": "test_1",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "log_filename": "console_consumer_6.log",
+      "config_filename": "console_consumer_6.properties"
+    },
+    {
+      "entity_id": "7",
+      "topic": "test_2",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "log_filename": "console_consumer_7.log",
+      "config_filename": "console_consumer_7.properties"
+    }
+  ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0122/cluster_config.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0122/cluster_config.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0122/cluster_config.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0122/cluster_config.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,76 @@
+{
+    "cluster_config": [
+        {
+            "entity_id": "0",
+            "hostname": "localhost",
+            "role": "zookeeper",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9990"
+        },
+        {
+            "entity_id": "1",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9991"
+        },
+        {
+            "entity_id": "2",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9992"
+        },
+        {
+            "entity_id": "3",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9993"
+        },
+        {
+            "entity_id": "4",
+            "hostname": "localhost",
+            "role": "producer_performance",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9997"
+        },
+        {
+            "entity_id": "5",
+            "hostname": "localhost",
+            "role": "producer_performance",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9998"
+        },
+        {
+            "entity_id": "6",
+            "hostname": "localhost",
+            "role": "console_consumer",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9999"
+        },
+        {
+            "entity_id": "7",
+            "hostname": "localhost",
+            "role": "console_consumer",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9099"
+        }
+    ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0122/testcase_0122_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0122/testcase_0122_properties.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0122/testcase_0122_properties.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0122/testcase_0122_properties.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,97 @@
+{
+  "description": {"01":"Leader Failure in Replication with multi topics & partitions : 1. acks => 1",
+                  "02":"Produce and consume messages to 2 topics - 3 partitions",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:sync, acks:1, comp:0",
+                  "10":"Log segment size    : 512000"
+  },
+  "testcase_args": {
+    "bounce_leader": "true",
+    "replica_factor": "3",
+    "num_partition": "3",
+    "num_iteration": "3",
+    "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15"
+  },
+  "entities": [
+    {
+      "entity_id": "0",
+      "clientPort": "2188",
+      "dataDir": "/tmp/zookeeper_0",
+      "log_filename": "zookeeper_2188.log",
+      "config_filename": "zookeeper_2188.properties"
+    },
+    {
+      "entity_id": "1",
+      "port": "9091",
+      "brokerid": "1",
+      "log.file.size": "512000",
+      "log.dir": "/tmp/kafka_server_1_logs",
+      "log_filename": "kafka_server_9091.log",
+      "config_filename": "kafka_server_9091.properties"
+    },
+    {
+      "entity_id": "2",
+      "port": "9092",
+      "brokerid": "2",
+      "log.file.size": "512000",
+      "log.dir": "/tmp/kafka_server_2_logs",
+      "log_filename": "kafka_server_9092.log",
+      "config_filename": "kafka_server_9092.properties"
+    },
+    {
+      "entity_id": "3",
+      "port": "9093",
+      "brokerid": "3",
+      "log.file.size": "512000",
+      "log.dir": "/tmp/kafka_server_3_logs",
+      "log_filename": "kafka_server_9093.log",
+      "config_filename": "kafka_server_9093.properties"
+    },
+    {
+      "entity_id": "4",
+      "topic": "test_1",
+      "threads": "5",
+      "compression-codec": "0",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "1",
+      "async":"false",
+      "log_filename": "producer_performance_4.log",
+      "config_filename": "producer_performance_4.properties"
+    },
+    {
+      "entity_id": "5",
+      "topic": "test_2",
+      "threads": "5",
+      "compression-codec": "0",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "1",
+      "async":"false",
+      "log_filename": "producer_performance_5.log",
+      "config_filename": "producer_performance_5.properties"
+    },
+    {
+      "entity_id": "6",
+      "topic": "test_1",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "log_filename": "console_consumer_6.log",
+      "config_filename": "console_consumer_6.properties"
+    },
+    {
+      "entity_id": "7",
+      "topic": "test_2",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "log_filename": "console_consumer_7.log",
+      "config_filename": "console_consumer_7.properties"
+    }
+  ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0123/cluster_config.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0123/cluster_config.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0123/cluster_config.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0123/cluster_config.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,76 @@
+{
+    "cluster_config": [
+        {
+            "entity_id": "0",
+            "hostname": "localhost",
+            "role": "zookeeper",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9990"
+        },
+        {
+            "entity_id": "1",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9991"
+        },
+        {
+            "entity_id": "2",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9992"
+        },
+        {
+            "entity_id": "3",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9993"
+        },
+        {
+            "entity_id": "4",
+            "hostname": "localhost",
+            "role": "producer_performance",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9997"
+        },
+        {
+            "entity_id": "5",
+            "hostname": "localhost",
+            "role": "producer_performance",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9998"
+        },
+        {
+            "entity_id": "6",
+            "hostname": "localhost",
+            "role": "console_consumer",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9999"
+        },
+        {
+            "entity_id": "7",
+            "hostname": "localhost",
+            "role": "console_consumer",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9099"
+        }
+    ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0123/testcase_0123_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0123/testcase_0123_properties.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0123/testcase_0123_properties.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_0123/testcase_0123_properties.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,97 @@
+{
+  "description": {"01":"Leader Failure in Replication with multi topics & partitions : 1. mode => async; 2. comp => 0",
+                  "02":"Produce and consume messages to 2 topics - 3 partitions",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:async, acks:1, comp:1",
+                  "10":"Log segment size    : 512000"
+  },
+  "testcase_args": {
+    "bounce_leader": "true",
+    "replica_factor": "3",
+    "num_partition": "3",
+    "num_iteration": "3",
+    "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15"
+  },
+  "entities": [
+    {
+      "entity_id": "0",
+      "clientPort": "2188",
+      "dataDir": "/tmp/zookeeper_0",
+      "log_filename": "zookeeper_2188.log",
+      "config_filename": "zookeeper_2188.properties"
+    },
+    {
+      "entity_id": "1",
+      "port": "9091",
+      "brokerid": "1",
+      "log.file.size": "512000",
+      "log.dir": "/tmp/kafka_server_1_logs",
+      "log_filename": "kafka_server_9091.log",
+      "config_filename": "kafka_server_9091.properties"
+    },
+    {
+      "entity_id": "2",
+      "port": "9092",
+      "brokerid": "2",
+      "log.file.size": "512000",
+      "log.dir": "/tmp/kafka_server_2_logs",
+      "log_filename": "kafka_server_9092.log",
+      "config_filename": "kafka_server_9092.properties"
+    },
+    {
+      "entity_id": "3",
+      "port": "9093",
+      "brokerid": "3",
+      "log.file.size": "512000",
+      "log.dir": "/tmp/kafka_server_3_logs",
+      "log_filename": "kafka_server_9093.log",
+      "config_filename": "kafka_server_9093.properties"
+    },
+    {
+      "entity_id": "4",
+      "topic": "test_1",
+      "threads": "5",
+      "compression-codec": "1",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "1",
+      "async":"true",
+      "log_filename": "producer_performance_4.log",
+      "config_filename": "producer_performance_4.properties"
+    },
+    {
+      "entity_id": "5",
+      "topic": "test_2",
+      "threads": "5",
+      "compression-codec": "1",
+      "message-size": "500",
+      "message": "100",
+      "request-num-acks": "1",
+      "async":"true",
+      "log_filename": "producer_performance_5.log",
+      "config_filename": "producer_performance_5.properties"
+    },
+    {
+      "entity_id": "6",
+      "topic": "test_1",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "log_filename": "console_consumer_6.log",
+      "config_filename": "console_consumer_6.properties"
+    },
+    {
+      "entity_id": "7",
+      "topic": "test_2",
+      "groupid": "mytestgroup",
+      "consumer-timeout-ms": "10000",
+      "log_filename": "console_consumer_7.log",
+      "config_filename": "console_consumer_7.properties"
+    }
+  ]
+}

Added: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_1/cluster_config.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_1/cluster_config.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_1/cluster_config.json (added)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_1/cluster_config.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,58 @@
+{
+    "cluster_config": [
+        {
+            "entity_id": "0",
+            "hostname": "localhost",
+            "role": "zookeeper",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9990"
+        },
+        {
+            "entity_id": "1",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9991"
+        },
+        {
+            "entity_id": "2",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9992"
+        },
+        {
+            "entity_id": "3",
+            "hostname": "localhost",
+            "role": "broker",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9993"
+        },
+        {
+            "entity_id": "4",
+            "hostname": "localhost",
+            "role": "producer_performance",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9994"
+        },
+        {
+            "entity_id": "5",
+            "hostname": "localhost",
+            "role": "console_consumer",
+            "cluster_name": "source",
+            "kafka_home": "default",
+            "java_home": "default",
+            "jmx_port": "9995"
+        }
+    ]
+}

Modified: incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_1/testcase_1_properties.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_1/testcase_1_properties.json?rev=1396687&r1=1396686&r2=1396687&view=diff
==============================================================================
--- incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_1/testcase_1_properties.json (original)
+++ incubator/kafka/branches/0.8/system_test/replication_testsuite/testcase_1/testcase_1_properties.json Wed Oct 10 16:56:57 2012
@@ -1,11 +1,22 @@
 {
-  "description": "Basic test to produce and consume messages to a single topic partition. This test sends messages to n replicas and at the end verifies the log size and contents as well as using a consumer to verify no message loss. Optionally, the test bounces the leader periodically to introduce failures during the message replication.",
+  "description": {"01":"To Test : 'Leader Failure in Replication'",
+                  "02":"Produce and consume messages to a single topic - single partition.",
+                  "03":"This test sends messages to 3 replicas",
+                  "04":"To trigger leader election: find the leader and terminate by controlled failure (kill -15)",
+                  "05":"Restart the terminated broker",
+                  "06":"Lookup brokers' log4j messages and verify that leader is re-elected successfully",
+                  "07":"At the end it verifies the log size and contents",
+                  "08":"Use a consumer to verify no message loss.",
+                  "09":"Producer dimensions : mode:sync, acks:-1, comp:0",
+                  "10":"Log segment size    : 10240"
+  },
   "testcase_args": {
     "bounce_leader": "true",
     "replica_factor": "3",
     "num_partition": "2",
     "num_iteration": "2",
     "sleep_seconds_between_producer_calls": "1",
+    "message_producing_free_time_sec": "15",
     "num_messages_to_produce_per_producer_call": "50"
   },
   "entities": [
@@ -47,9 +58,11 @@
       "entity_id": "4",
       "topic": "test_1",
       "threads": "5",
-      "compression-codec": "1",
+      "compression-codec": "0",
       "message-size": "500",
       "message": "500",
+      "request-num-acks": "-1",
+      "async":"false",
       "log_filename": "producer_performance.log",
       "config_filename": "producer_performance.properties"
     },
@@ -58,6 +71,7 @@
       "topic": "test_1",
       "groupid": "mytestgroup",
       "consumer-timeout-ms": "10000",
+      "zookeeper": "localhost:2188",
       "log_filename": "console_consumer.log",
       "config_filename": "console_consumer.properties"
     }

Added: incubator/kafka/branches/0.8/system_test/run_sanity.sh
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/run_sanity.sh?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/run_sanity.sh (added)
+++ incubator/kafka/branches/0.8/system_test/run_sanity.sh Wed Oct 10 16:56:57 2012
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+my_ts=`date +"%s"`
+
+cp testcase_to_run.json testcase_to_run.json_${my_ts}
+cp testcase_to_run_sanity.json testcase_to_run.json
+
+python -B system_test_runner.py
+
+

Modified: incubator/kafka/branches/0.8/system_test/system_test_env.py
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/system_test_env.py?rev=1396687&r1=1396686&r2=1396687&view=diff
==============================================================================
--- incubator/kafka/branches/0.8/system_test/system_test_env.py (original)
+++ incubator/kafka/branches/0.8/system_test/system_test_env.py Wed Oct 10 16:56:57 2012
@@ -20,10 +20,13 @@
 # system_test_env.py
 # ===================================
 
+import copy
 import json
 import os
 import sys
 
+from   utils import system_test_utils
+
 class SystemTestEnv():
 
     # private:
@@ -41,21 +44,82 @@ class SystemTestEnv():
     CLUSTER_CONFIG_PATHNAME   = os.path.abspath(SYSTEM_TEST_BASE_DIR + "/" + CLUSTER_CONFIG_FILENAME)
     METRICS_FILENAME          = "metrics.json"
     METRICS_PATHNAME          = os.path.abspath(SYSTEM_TEST_BASE_DIR + "/" + METRICS_FILENAME)
+    TESTCASE_TO_RUN_FILENAME  = "testcase_to_run.json"
+    TESTCASE_TO_RUN_PATHNAME  = os.path.abspath(SYSTEM_TEST_BASE_DIR + "/" + TESTCASE_TO_RUN_FILENAME)
+    TESTCASE_TO_SKIP_FILENAME = "testcase_to_skip.json"
+    TESTCASE_TO_SKIP_PATHNAME = os.path.abspath(SYSTEM_TEST_BASE_DIR + "/" + TESTCASE_TO_SKIP_FILENAME)
+
+    clusterEntityConfigDictList                      = []   # cluster entity config for current level
+    clusterEntityConfigDictListInSystemTestLevel     = []   # cluster entity config defined in system level
+    clusterEntityConfigDictListLastFoundInTestSuite  = []   # cluster entity config last found in testsuite level
+    clusterEntityConfigDictListLastFoundInTestCase   = []   # cluster entity config last found in testcase level
 
-    clusterEntityConfigDictList  = []
     systemTestResultsList        = []
+    testCaseToRunListDict        = {}
+    testCaseToSkipListDict       = {}
+
+    printTestDescriptionsOnly    = False
+    doNotValidateRemoteHost      = False
 
     def __init__(self):
         "Create an object with this system test session environment"
 
-        # retrieve each entity's data from cluster config json file
-        # as "dict" and enter them into a "list"
-        jsonFileContent = open(self.CLUSTER_CONFIG_PATHNAME, "r").read()
-        jsonData        = json.loads(jsonFileContent)
-        for key, cfgList in jsonData.items():
-            if key == "cluster_config":
-                for cfg in cfgList:
-                    self.clusterEntityConfigDictList.append(cfg)
+        # load the system level cluster config
+        system_test_utils.load_cluster_config(self.CLUSTER_CONFIG_PATHNAME, self.clusterEntityConfigDictList)
+
+        # save the system level cluster config
+        self.clusterEntityConfigDictListInSystemTestLevel = copy.deepcopy(self.clusterEntityConfigDictList)
+
+        # retrieve testcases to run from testcase_to_run.json
+        try:
+            testcaseToRunFileContent  = open(self.TESTCASE_TO_RUN_PATHNAME, "r").read()
+            testcaseToRunData        = json.loads(testcaseToRunFileContent)
+            for testClassName, caseList in testcaseToRunData.items():
+                self.testCaseToRunListDict[testClassName] = caseList
+        except:
+            pass
+
+        # retrieve testcases to skip from testcase_to_skip.json
+        try:
+            testcaseToSkipFileContent = open(self.TESTCASE_TO_SKIP_PATHNAME, "r").read()
+            testcaseToSkipData        = json.loads(testcaseToSkipFileContent)
+            for testClassName, caseList in testcaseToSkipData.items():
+                self.testCaseToSkipListDict[testClassName] = caseList
+        except:
+            pass
+
+    def isTestCaseToSkip(self, testClassName, testcaseDirName):
+        testCaseToRunList  = {}
+        testCaseToSkipList = {}
+
+        try:
+            testCaseToRunList  = self.testCaseToRunListDict[testClassName]
+        except:
+            # no 'testClassName' found => no need to run any cases for this test class
+            return True
+
+        try:
+            testCaseToSkipList = self.testCaseToSkipListDict[testClassName]
+        except:
+            pass
+
+        # if testCaseToRunList has elements, it takes precedence:
+        if len(testCaseToRunList) > 0:
+            #print "#### testClassName => ", testClassName
+            #print "#### testCaseToRunList => ", testCaseToRunList
+            #print "#### testcaseDirName => ", testcaseDirName
+            if not testcaseDirName in testCaseToRunList:
+                #self.log_message("Skipping : " + testcaseDirName)
+                return True
+        elif len(testCaseToSkipList) > 0:
+            #print "#### testClassName => ", testClassName
+            #print "#### testCaseToSkipList => ", testCaseToSkipList
+            #print "#### testcaseDirName => ", testcaseDirName
+            if testcaseDirName in testCaseToSkipList:
+                #self.log_message("Skipping : " + testcaseDirName)
+                return True
+
+        return False
 
 
     def getSystemTestEnvDict(self):

Modified: incubator/kafka/branches/0.8/system_test/system_test_runner.py
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/system_test_runner.py?rev=1396687&r1=1396686&r2=1396687&view=diff
==============================================================================
--- incubator/kafka/branches/0.8/system_test/system_test_runner.py (original)
+++ incubator/kafka/branches/0.8/system_test/system_test_runner.py Wed Oct 10 16:56:57 2012
@@ -16,63 +16,66 @@
 # under the License.
 #!/usr/bin/evn python
 
-# ===================================
+# =================================================================
 # system_test_runner.py
-# ===================================
+#
+# - This script is the test driver for a distributed environment
+#   system testing framework. It is located at the top level of the
+#   framework hierachy (in this case - system_test/).
+#
+# - This test driver servers as an entry point to launch a series
+#   of test suites (module) with multiple functionally similar test
+#   cases which can be grouped together.
+#
+# - Please refer to system_test/README.txt for more details on
+#   how to add test suite and test case.
+#
+# - In most cases, it is not necessary to make any changes to this
+#   script.
+# =================================================================
 
+from optparse        import OptionParser
 from system_test_env import SystemTestEnv
-from utils import system_test_utils
+from utils           import system_test_utils
 
-import logging
+import logging.config
 import os
+import pprint
 import sys
 
 
-# ====================================================================
-# Two logging formats are defined in system_test/system_test_runner.py
-# ====================================================================
-
-# 1. "namedLogger" is defined to log message in this format:
-#    "%(asctime)s - %(levelname)s - %(message)s %(name_of_class)s"
-# 
-# usage: to log message and showing the class name of the message
-
-namedLogger = logging.getLogger("namedLogger")
-namedLogger.setLevel(logging.INFO)
-#namedLogger.setLevel(logging.DEBUG)
-namedConsoleHandler = logging.StreamHandler()
-namedConsoleHandler.setLevel(logging.DEBUG)
-namedFormatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s %(name_of_class)s")
-namedConsoleHandler.setFormatter(namedFormatter)
-namedLogger.addHandler(namedConsoleHandler)
-
-# 2. "anonymousLogger" is defined to log message in this format:
-#    "%(asctime)s - %(levelname)s - %(message)s"
-# 
-# usage: to log message without showing class name and it's appropriate
-#        for logging generic message such as "sleeping for 5 seconds"
-
-anonymousLogger = logging.getLogger("anonymousLogger")
-anonymousLogger.setLevel(logging.INFO)
-#anonymousLogger.setLevel(logging.DEBUG)
-anonymousConsoleHandler = logging.StreamHandler()
-anonymousConsoleHandler.setLevel(logging.DEBUG)
-anonymousFormatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
-anonymousConsoleHandler.setFormatter(anonymousFormatter)
-anonymousLogger.addHandler(anonymousConsoleHandler)
+# load the config file for logging
+logging.config.fileConfig('logging.conf')
 
+# 'd' is an argument to be merged into the log message (see Python doc for logging).
+# In this case, corresponding class name can be appended to the end of the logging
+# message to facilitate debugging.
 d = {'name_of_class': '(system_test_runner)'}
 
 def main():
+    nLogger = logging.getLogger('namedLogger')
+    aLogger = logging.getLogger('anonymousLogger')
 
-    print
-    print
-    print
-    anonymousLogger.info("=================================================")
-    anonymousLogger.info("        System Regression Test Framework")
-    anonymousLogger.info("=================================================")
-    print
-    print
+    optionParser = OptionParser()
+    optionParser.add_option("-p", "--print-test-descriptions-only",
+                            dest="printTestDescriptionsOnly",
+                            default=False,
+                            action="store_true",
+                            help="print test descriptions only - don't run the test")
+
+    optionParser.add_option("-n", "--do-not-validate-remote-host",
+                            dest="doNotValidateRemoteHost",
+                            default=False,
+                            action="store_true",
+                            help="do not validate remote host (due to different kafka versions are installed)")
+
+    (options, args) = optionParser.parse_args()
+
+    print "\n"
+    aLogger.info("=================================================")
+    aLogger.info("        System Regression Test Framework")
+    aLogger.info("=================================================")
+    print "\n"
 
     testSuiteClassDictList = []
 
@@ -80,47 +83,53 @@ def main():
     # such as the SYSTEM_TEST_BASE_DIR, SYSTEM_TEST_UTIL_DIR, ...
     systemTestEnv = SystemTestEnv()
 
-    # sanity check on remote hosts to make sure:
-    # - all directories (eg. java_home) specified in cluster_config.json exists in all hosts
-    # - no conflicting running processes in remote hosts
-    anonymousLogger.info("=================================================")
-    anonymousLogger.info("setting up remote hosts ...")
-    anonymousLogger.info("=================================================")
-    if not system_test_utils.setup_remote_hosts(systemTestEnv):
-        namedLogger.error("Remote hosts sanity check failed. Aborting test ...", extra=d)
+    if options.printTestDescriptionsOnly:
+        systemTestEnv.printTestDescriptionsOnly = True
+    if options.doNotValidateRemoteHost:
+        systemTestEnv.doNotValidateRemoteHost = True
+
+    if not systemTestEnv.printTestDescriptionsOnly:
+        if not systemTestEnv.doNotValidateRemoteHost:
+            if not system_test_utils.setup_remote_hosts(systemTestEnv):
+                nLogger.error("Remote hosts sanity check failed. Aborting test ...", extra=d)
+                print
+                sys.exit(1)
+        else:
+            nLogger.info("SKIPPING : checking remote machines", extra=d)
         print
-        sys.exit(1)
-    print
 
     # get all defined names within a module: 
     definedItemList = dir(SystemTestEnv)
-    anonymousLogger.debug("=================================================")
-    anonymousLogger.debug("SystemTestEnv keys:")
+    aLogger.debug("=================================================")
+    aLogger.debug("SystemTestEnv keys:")
     for item in definedItemList:
-        anonymousLogger.debug("    " + item)
-    anonymousLogger.debug("=================================================")
+        aLogger.debug("    " + item)
+    aLogger.debug("=================================================")
 
-    anonymousLogger.info("=================================================")
-    anonymousLogger.info("looking up test suites ...")
-    anonymousLogger.info("=================================================")
+    aLogger.info("=================================================")
+    aLogger.info("looking up test suites ...")
+    aLogger.info("=================================================")
     # find all test suites in SYSTEM_TEST_BASE_DIR
     for dirName in os.listdir(systemTestEnv.SYSTEM_TEST_BASE_DIR):
 
         # make sure this is a valid testsuite directory
         if os.path.isdir(dirName) and dirName.endswith(systemTestEnv.SYSTEM_TEST_SUITE_SUFFIX):
-            
-            namedLogger.info("found a testsuite : " + dirName, extra=d)
+            print
+            nLogger.info("found a testsuite : " + dirName, extra=d)
             testModulePathName = os.path.abspath(systemTestEnv.SYSTEM_TEST_BASE_DIR + "/" + dirName)
 
+            if not systemTestEnv.printTestDescriptionsOnly:
+                system_test_utils.setup_remote_hosts_with_testsuite_level_cluster_config(systemTestEnv, testModulePathName)
+
             # go through all test modules file in this testsuite
             for moduleFileName in os.listdir(testModulePathName):
 
                 # make sure it is a valid test module
                 if moduleFileName.endswith(systemTestEnv.SYSTEM_TEST_MODULE_EXT) \
-                   and not moduleFileName.startswith("__"):
+                    and not moduleFileName.startswith("__"):
 
                     # found a test module file
-                    namedLogger.info("found a test module file : " + moduleFileName, extra=d) 
+                    nLogger.info("found a test module file : " + moduleFileName, extra=d) 
 
                     testModuleClassName = system_test_utils.sys_call("grep ^class " + testModulePathName + "/" + \
                                           moduleFileName + " | sed 's/^class //g' | sed 's/(.*):.*//g'")
@@ -134,42 +143,44 @@ def main():
                     testSuiteClassDict["class"]  = testModuleClassName
                     testSuiteClassDictList.append(testSuiteClassDict)
 
-    # loop through testSuiteClassDictList and start the test class one by one
-    for testSuiteClassDict in testSuiteClassDictList:
+                    suiteName  = testSuiteClassDict["suite"]
+                    moduleName = testSuiteClassDict["module"]
+                    className  = testSuiteClassDict["class"]
+
+                    # add testsuite directory to sys.path such that the module can be loaded
+                    sys.path.append(systemTestEnv.SYSTEM_TEST_BASE_DIR + "/" + suiteName)
+ 
+                    if not systemTestEnv.printTestDescriptionsOnly:
+                        aLogger.info("=================================================")
+                        aLogger.info("Running Test for : ")
+                        aLogger.info("    suite  : " + suiteName)
+                        aLogger.info("    module : " + moduleName)
+                        aLogger.info("    class  : " + className)
+                        aLogger.info("=================================================")
+
+                    # dynamically loading a module and starting the test class
+                    mod      = __import__(moduleName)
+                    theClass = getattr(mod, className)
+                    instance = theClass(systemTestEnv)
+                    instance.runTest()
+                print
 
-        suiteName  = testSuiteClassDict["suite"]
-        moduleName = testSuiteClassDict["module"]
-        className  = testSuiteClassDict["class"]
-
-        # add testsuite directory to sys.path such that the module can be loaded
-        sys.path.append(systemTestEnv.SYSTEM_TEST_BASE_DIR + "/" + suiteName)
-
-        anonymousLogger.info("=================================================")
-        anonymousLogger.info("Running Test for : ")
-        anonymousLogger.info("    suite  : " + suiteName)
-        anonymousLogger.info("    module : " + moduleName)
-        anonymousLogger.info("    class  : " + className)
-        anonymousLogger.info("=================================================")
-
-        # dynamically loading a module and starting the test class
-        mod      = __import__(moduleName)
-        theClass = getattr(mod, className)
-        instance = theClass(systemTestEnv)
-        instance.runTest()
-
-    print
-    anonymousLogger.info("=================================================")
-    anonymousLogger.info("                 TEST REPORTS")
-    anonymousLogger.info("=================================================")
-    for systemTestResult in systemTestEnv.systemTestResultsList:
-        for key,val in systemTestResult.items():
-            if key == "validation_status":
-                anonymousLogger.info(key + " : ")
-                for validation, status in val.items():
-                     anonymousLogger.info("    " + validation + " : " + status)
-            else:
-                anonymousLogger.info(key + " : " + val)
+    if not systemTestEnv.printTestDescriptionsOnly:
         print
+        print "========================================================"
+        print "                 TEST REPORTS"
+        print "========================================================"
+        for systemTestResult in systemTestEnv.systemTestResultsList:
+            for key in sorted(systemTestResult.iterkeys()):
+                if key == "validation_status":
+                    print key, " : "
+                    for validatedItem in sorted(systemTestResult[key].iterkeys()):
+                         print "    ", validatedItem, " : ", systemTestResult[key][validatedItem]
+                else:
+                    print key, " : ", systemTestResult[key]
+            print
+            print "========================================================"
+            print
 
 # =========================
 # main entry point

Added: incubator/kafka/branches/0.8/system_test/testcase_to_run.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/testcase_to_run.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/testcase_to_run.json (added)
+++ incubator/kafka/branches/0.8/system_test/testcase_to_run.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,6 @@
+{
+    "ReplicaBasicTest"  : [
+        "testcase_0001",
+        "testcase_1"
+    ]
+}

Added: incubator/kafka/branches/0.8/system_test/testcase_to_run_sanity.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/testcase_to_run_sanity.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/testcase_to_run_sanity.json (added)
+++ incubator/kafka/branches/0.8/system_test/testcase_to_run_sanity.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,5 @@
+{
+    "ReplicaBasicTest"   : [
+        "testcase_1"
+    ]
+}

Added: incubator/kafka/branches/0.8/system_test/testcase_to_skip.json
URL: http://svn.apache.org/viewvc/incubator/kafka/branches/0.8/system_test/testcase_to_skip.json?rev=1396687&view=auto
==============================================================================
--- incubator/kafka/branches/0.8/system_test/testcase_to_skip.json (added)
+++ incubator/kafka/branches/0.8/system_test/testcase_to_skip.json Wed Oct 10 16:56:57 2012
@@ -0,0 +1,3 @@
+{
+    "ReplicaBasicTest": [ "testcase_1" ]
+}



Mime
View raw message