kafka-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From guozh...@apache.org
Subject kafka git commit: KAFKA-2820: systest log level
Date Wed, 18 Nov 2015 23:11:21 GMT
Repository: kafka
Updated Branches:
  refs/heads/trunk 0f00ec97a -> 2e91806db


KAFKA-2820: systest log level

Restores control over log level in system test service class KafkaService.

Author: Geoff Anderson <geoff@confluent.io>

Reviewers: Ismael Juma, Ewen Cheslack-Postava

Closes #538 from granders/KAFKA-2820-systest-log-level


Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/2e91806d
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/2e91806d
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/2e91806d

Branch: refs/heads/trunk
Commit: 2e91806db45b3854bd2fadb69cee4ecae9fd43ac
Parents: 0f00ec9
Author: Geoff Anderson <geoff@confluent.io>
Authored: Wed Nov 18 15:11:17 2015 -0800
Committer: Confluent <confluent@Confluents-MacBook-Pro.local>
Committed: Wed Nov 18 15:11:17 2015 -0800

----------------------------------------------------------------------
 tests/kafkatest/services/kafka/kafka.py         | 29 +++++++-----
 .../services/kafka/templates/log4j.properties   | 46 ++++++++++----------
 tests/kafkatest/services/verifiable_producer.py |  1 -
 3 files changed, 40 insertions(+), 36 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kafka/blob/2e91806d/tests/kafkatest/services/kafka/kafka.py
----------------------------------------------------------------------
diff --git a/tests/kafkatest/services/kafka/kafka.py b/tests/kafkatest/services/kafka/kafka.py
index a7a1581..15f541d 100644
--- a/tests/kafkatest/services/kafka/kafka.py
+++ b/tests/kafkatest/services/kafka/kafka.py
@@ -34,17 +34,22 @@ import os.path
 class KafkaService(JmxMixin, Service):
 
     PERSISTENT_ROOT = "/mnt"
-    LOG4J_CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "kafka-log4j.properties")
+    STDOUT_CAPTURE = os.path.join(PERSISTENT_ROOT, "kafka.log")
+    STDERR_CAPTURE = os.path.join(PERSISTENT_ROOT, "kafka.log")
+    LOG4J_CONFIG = os.path.join(PERSISTENT_ROOT, "kafka-log4j.properties")
+    # Logs such as controller.log, server.log, etc all go here
+    OPERATIONAL_LOG_DIR = os.path.join(PERSISTENT_ROOT, "kafka-operational-logs")
+    # Kafka log segments etc go here
+    DATA_LOG_DIR = os.path.join(PERSISTENT_ROOT, "kafka-data-logs")
+    CONFIG_FILE = os.path.join(PERSISTENT_ROOT, "kafka.properties")
+
 
     logs = {
-        "kafka_log": {
-            "path": "/mnt/kafka.log",
-            "collect_default": True},
         "kafka_operational_logs": {
-            "path": "/mnt/kafka-operational-logs",
+            "path": OPERATIONAL_LOG_DIR,
             "collect_default": True},
         "kafka_data": {
-            "path": "/mnt/kafka-data-logs",
+            "path": DATA_LOG_DIR,
             "collect_default": False}
     }
 
@@ -107,24 +112,24 @@ class KafkaService(JmxMixin, Service):
 
     def start_cmd(self, node):
         cmd = "export JMX_PORT=%d; " % self.jmx_port
-        cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG_FILE
-        cmd += "export LOG_DIR=/mnt/kafka-operational-logs/; "
+        cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG
+        cmd += "export LOG_DIR=%s; " % KafkaService.OPERATIONAL_LOG_DIR
         cmd += "export KAFKA_OPTS=%s; " % self.security_config.kafka_opts
-        cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-server-start.sh /mnt/kafka.properties
1>> /mnt/kafka.log 2>> /mnt/kafka.log &"
+        cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-server-start.sh %s 1>> %s 2>>
%s &" % (KafkaService.CONFIG_FILE, KafkaService.STDOUT_CAPTURE, KafkaService.STDERR_CAPTURE)
         return cmd
 
     def start_node(self, node):
         prop_file = self.prop_file(node)
         self.logger.info("kafka.properties:")
         self.logger.info(prop_file)
-        node.account.create_file("/mnt/kafka.properties", prop_file)
-        node.account.create_file(self.LOG4J_CONFIG_FILE, self.render('log4j.properties'))
+        node.account.create_file(KafkaService.CONFIG_FILE, prop_file)
+        node.account.create_file(self.LOG4J_CONFIG, self.render('log4j.properties', log_dir=KafkaService.OPERATIONAL_LOG_DIR))
 
         self.security_config.setup_node(node)
 
         cmd = self.start_cmd(node)
         self.logger.debug("Attempting to start KafkaService on %s with command: %s" % (str(node.account),
cmd))
-        with node.account.monitor_log("/mnt/kafka.log") as monitor:
+        with node.account.monitor_log(KafkaService.STDOUT_CAPTURE) as monitor:
             node.account.ssh(cmd)
             monitor.wait_until("Kafka Server.*started", timeout_sec=30, err_msg="Kafka server
didn't finish startup")
 

http://git-wip-us.apache.org/repos/asf/kafka/blob/2e91806d/tests/kafkatest/services/kafka/templates/log4j.properties
----------------------------------------------------------------------
diff --git a/tests/kafkatest/services/kafka/templates/log4j.properties b/tests/kafkatest/services/kafka/templates/log4j.properties
index bf816e7..fb38e1f 100644
--- a/tests/kafkatest/services/kafka/templates/log4j.properties
+++ b/tests/kafkatest/services/kafka/templates/log4j.properties
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-log4j.rootLogger=INFO, stdout 
+log4j.rootLogger={{ log_level }}, stdout
 
 log4j.appender.stdout=org.apache.log4j.ConsoleAppender
 log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
@@ -21,67 +21,67 @@ log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
 
 log4j.appender.kafkaAppender=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.kafkaAppender.DatePattern='.'yyyy-MM-dd-HH
-log4j.appender.kafkaAppender.File=${kafka.logs.dir}/server.log
+log4j.appender.kafkaAppender.File={{ log_dir }}/server.log
 log4j.appender.kafkaAppender.layout=org.apache.log4j.PatternLayout
 log4j.appender.kafkaAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
 
 log4j.appender.stateChangeAppender=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.stateChangeAppender.DatePattern='.'yyyy-MM-dd-HH
-log4j.appender.stateChangeAppender.File=${kafka.logs.dir}/state-change.log
+log4j.appender.stateChangeAppender.File={{ log_dir }}/state-change.log
 log4j.appender.stateChangeAppender.layout=org.apache.log4j.PatternLayout
 log4j.appender.stateChangeAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
 
 log4j.appender.requestAppender=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.requestAppender.DatePattern='.'yyyy-MM-dd-HH
-log4j.appender.requestAppender.File=${kafka.logs.dir}/kafka-request.log
+log4j.appender.requestAppender.File={{ log_dir }}/kafka-request.log
 log4j.appender.requestAppender.layout=org.apache.log4j.PatternLayout
 log4j.appender.requestAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
 
 log4j.appender.cleanerAppender=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.cleanerAppender.DatePattern='.'yyyy-MM-dd-HH
-log4j.appender.cleanerAppender.File=${kafka.logs.dir}/log-cleaner.log
+log4j.appender.cleanerAppender.File={{ log_dir }}/log-cleaner.log
 log4j.appender.cleanerAppender.layout=org.apache.log4j.PatternLayout
 log4j.appender.cleanerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
 
 log4j.appender.controllerAppender=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.controllerAppender.DatePattern='.'yyyy-MM-dd-HH
-log4j.appender.controllerAppender.File=${kafka.logs.dir}/controller.log
+log4j.appender.controllerAppender.File={{ log_dir }}/controller.log
 log4j.appender.controllerAppender.layout=org.apache.log4j.PatternLayout
 log4j.appender.controllerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
 
 log4j.appender.authorizerAppender=org.apache.log4j.DailyRollingFileAppender
 log4j.appender.authorizerAppender.DatePattern='.'yyyy-MM-dd-HH
-log4j.appender.authorizerAppender.File=${kafka.logs.dir}/kafka-authorizer.log
+log4j.appender.authorizerAppender.File={{ log_dir }}/kafka-authorizer.log
 log4j.appender.authorizerAppender.layout=org.apache.log4j.PatternLayout
 log4j.appender.authorizerAppender.layout.ConversionPattern=[%d] %p %m (%c)%n
 
 # Turn on all our debugging info
-#log4j.logger.kafka.producer.async.DefaultEventHandler=DEBUG, kafkaAppender
-#log4j.logger.kafka.client.ClientUtils=DEBUG, kafkaAppender
-#log4j.logger.kafka.perf=DEBUG, kafkaAppender
-#log4j.logger.kafka.perf.ProducerPerformance$ProducerThread=DEBUG, kafkaAppender
-#log4j.logger.org.I0Itec.zkclient.ZkClient=DEBUG
-log4j.logger.kafka=INFO, kafkaAppender
-
-log4j.logger.kafka.network.RequestChannel$=WARN, requestAppender
+log4j.logger.kafka.producer.async.DefaultEventHandler={{ log_level }}, kafkaAppender
+log4j.logger.kafka.client.ClientUtils={{ log_level }}, kafkaAppender
+log4j.logger.kafka.perf={{ log_level }}, kafkaAppender
+log4j.logger.kafka.perf.ProducerPerformance$ProducerThread={{ log_level }}, kafkaAppender
+log4j.logger.org.I0Itec.zkclient.ZkClient={{ log_level }}
+log4j.logger.kafka={{ log_level }}, kafkaAppender
+
+log4j.logger.kafka.network.RequestChannel$={{ log_level }}, requestAppender
 log4j.additivity.kafka.network.RequestChannel$=false
 
-#log4j.logger.kafka.network.Processor=TRACE, requestAppender
-#log4j.logger.kafka.server.KafkaApis=TRACE, requestAppender
-#log4j.additivity.kafka.server.KafkaApis=false
-log4j.logger.kafka.request.logger=WARN, requestAppender
+log4j.logger.kafka.network.Processor={{ log_level }}, requestAppender
+log4j.logger.kafka.server.KafkaApis={{ log_level }}, requestAppender
+log4j.additivity.kafka.server.KafkaApis=false
+log4j.logger.kafka.request.logger={{ log_level }}, requestAppender
 log4j.additivity.kafka.request.logger=false
 
-log4j.logger.kafka.controller=TRACE, controllerAppender
+log4j.logger.kafka.controller={{ log_level }}, controllerAppender
 log4j.additivity.kafka.controller=false
 
-log4j.logger.kafka.log.LogCleaner=INFO, cleanerAppender
+log4j.logger.kafka.log.LogCleaner={{ log_level }}, cleanerAppender
 log4j.additivity.kafka.log.LogCleaner=false
 
-log4j.logger.state.change.logger=TRACE, stateChangeAppender
+log4j.logger.state.change.logger={{ log_level }}, stateChangeAppender
 log4j.additivity.state.change.logger=false
 
 #Change this to debug to get the actual audit log for authorizer.
-log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender
+log4j.logger.kafka.authorizer.logger={{ log_level }}, authorizerAppender
 log4j.additivity.kafka.authorizer.logger=false
 

http://git-wip-us.apache.org/repos/asf/kafka/blob/2e91806d/tests/kafkatest/services/verifiable_producer.py
----------------------------------------------------------------------
diff --git a/tests/kafkatest/services/verifiable_producer.py b/tests/kafkatest/services/verifiable_producer.py
index 9985bbd..b2d2b97 100644
--- a/tests/kafkatest/services/verifiable_producer.py
+++ b/tests/kafkatest/services/verifiable_producer.py
@@ -17,7 +17,6 @@ from ducktape.services.background_thread import BackgroundThreadService
 
 from kafkatest.services.kafka.directory import kafka_dir, KAFKA_TRUNK
 from kafkatest.services.kafka.version import TRUNK, LATEST_0_8_2
-from kafkatest.services.security.security_config import SecurityConfig
 
 import json
 import os


Mime
View raw message