kafka-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From guozh...@apache.org
Subject kafka git commit: MINOR: increase RocksDb parallelism
Date Fri, 10 Mar 2017 21:45:23 GMT
Repository: kafka
Updated Branches:
  refs/heads/0.10.2 beb0d5700 -> f9660e16d


MINOR: increase RocksDb parallelism

Author: Eno Thereska <eno@confluent.io>
Author: Eno Thereska <eno.thereska@gmail.com>

Reviewers: Damian Guy, Guozhang Wang

Closes #2663 from enothereska/minor-rocksdb-parallel

(cherry picked from commit 0011a3dc337a41ceb21b1567db7bdc3219a07552)
Signed-off-by: Guozhang Wang <wangguoz@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/kafka/repo
Commit: http://git-wip-us.apache.org/repos/asf/kafka/commit/f9660e16
Tree: http://git-wip-us.apache.org/repos/asf/kafka/tree/f9660e16
Diff: http://git-wip-us.apache.org/repos/asf/kafka/diff/f9660e16

Branch: refs/heads/0.10.2
Commit: f9660e16dafb9e1109704bdecb978b7046f0cc6b
Parents: beb0d57
Author: Eno Thereska <eno@confluent.io>
Authored: Fri Mar 10 13:43:04 2017 -0800
Committer: Guozhang Wang <wangguoz@gmail.com>
Committed: Fri Mar 10 13:45:09 2017 -0800

----------------------------------------------------------------------
 .../org/apache/kafka/streams/state/internals/RocksDBStore.java | 6 ++++++
 1 file changed, 6 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/kafka/blob/f9660e16/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
----------------------------------------------------------------------
diff --git a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
index 5c83ddf..6bb14dd 100644
--- a/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
+++ b/streams/src/main/java/org/apache/kafka/streams/state/internals/RocksDBStore.java
@@ -35,6 +35,7 @@ import org.rocksdb.BlockBasedTableConfig;
 import org.rocksdb.CompactionStyle;
 import org.rocksdb.CompressionType;
 import org.rocksdb.FlushOptions;
+import org.rocksdb.InfoLogLevel;
 import org.rocksdb.Options;
 import org.rocksdb.RocksDB;
 import org.rocksdb.RocksDBException;
@@ -120,6 +121,11 @@ public class RocksDBStore<K, V> implements KeyValueStore<K,
V> {
         options.setMaxWriteBufferNumber(MAX_WRITE_BUFFERS);
         options.setCreateIfMissing(true);
         options.setErrorIfExists(false);
+        options.setInfoLogLevel(InfoLogLevel.ERROR_LEVEL);
+        // this is the recommended way to increase parallelism in RocksDb
+        // note that the current implementation increases the number of compaction threads
+        // but not flush threads.
+        options.setIncreaseParallelism(Runtime.getRuntime().availableProcessors());
 
         wOptions = new WriteOptions();
         wOptions.setDisableWAL(true);


Mime
View raw message