kafka-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jun...@apache.org
Subject svn commit: r1476886 - in /kafka/site: 07/includes/header.html 08/ 08/api.html 08/configuration.html 08/design.html 08/includes/ 08/includes/footer.html 08/includes/header.html 08/quickstart.html 08/tools.html includes/header.html
Date Mon, 29 Apr 2013 03:50:25 GMT
Author: junrao
Date: Mon Apr 29 03:50:25 2013
New Revision: 1476886

URL: http://svn.apache.org/r1476886
Log:
add 0.8 docs

Added:
    kafka/site/08/
    kafka/site/08/api.html
    kafka/site/08/configuration.html
    kafka/site/08/design.html
    kafka/site/08/includes/
    kafka/site/08/includes/footer.html
    kafka/site/08/includes/header.html
    kafka/site/08/quickstart.html
    kafka/site/08/tools.html
Modified:
    kafka/site/07/includes/header.html
    kafka/site/includes/header.html

Modified: kafka/site/07/includes/header.html
URL: http://svn.apache.org/viewvc/kafka/site/07/includes/header.html?rev=1476886&r1=1476885&r2=1476886&view=diff
==============================================================================
--- kafka/site/07/includes/header.html (original)
+++ kafka/site/07/includes/header.html Mon Apr 29 03:50:25 2013
@@ -25,7 +25,7 @@
 			</div>
 			<div class="lsidebar">
 				<ul>
-                                        <li>0.7</li>
+                                        <li>0.7
                                                 <ul>
 							<li><a href="quickstart.html">quickstart</a></li>
 							<li><a href="design.html">design</a></li>
@@ -34,6 +34,7 @@
 							<li><a href="configuration.html">configuration</a></li>
 							<li><a href="performance.html">performance</a></li>
 						</ul>
+					</li>
 				</ul>
 			</div>
 		<div class='content'>

Added: kafka/site/08/api.html
URL: http://svn.apache.org/viewvc/kafka/site/08/api.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/api.html (added)
+++ kafka/site/08/api.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,151 @@
+<!--#include virtual="includes/header.html" -->
+
+<h3>Producer Api</h3>
+<pre>
+/**
+ *  V: type of the message
+ *  K: type of the optional key assciated with the message
+ */
+class kafka.javaapi.producer.Producer&lt;K,V&gt;
+{
+  public Producer(ProducerConfig config);
+
+  /**
+   * Sends the data to a single topic, partitioned by key, using either the
+   * synchronous or the asynchronous producer
+   * @param message the producer data object that encapsulates the topic, key and message
data
+   */
+  public void send(KeyedMessage&lt;K,V&gt; message);
+
+  /**
+   * Use this API to send data to multiple topics
+   * @param messages list of producer data objects that encapsulate the topic, key and message
data
+   */
+  public void send(List&lt;KeyedMessage&lt;K,V&gt;&gt; messages);
+
+  /**
+   * Close API to close the producer pool connections to all Kafka brokers.
+   */
+  public void close();
+}
+
+</pre>
+You can follow 
+<a href="https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+Producer+Example" title="Kafka
0.8 producer example">this</a> to learn how to use the producer api.
+
+<h3>High level Consumer Api</h3>
+<pre>
+class Consumer {
+  /**
+   *  Create a ConsumerConnector
+   *
+   *  @param config  at the minimum, need to specify the groupid of the consumer and the
zookeeper
+   *                 connection string zookeeper.connect.
+   */
+  public static kafka.javaapi.consumer.ConsumerConnector createJavaConsumerConnector(config:
ConsumerConfig);
+}
+
+/**
+ *  V: type of the message
+ *  K: type of the optional key assciated with the message
+ */
+public interface kafka.javaapi.consumer.ConsumerConnector {
+  /**
+   *  Create a list of message streams of type T for each topic.
+   *
+   *  @param topicCountMap  a map of (topic, #streams) pair
+   *  @param decoder a decoder that converts from Message to T
+   *  @return a map of (topic, list of  KafkaStream) pairs.
+   *          The number of items in the list is #streams. Each stream supports
+   *          an iterator over message/metadata pairs.
+   */
+  public &lt;K,V&gt; Map&lt;String, List&lt;KafkaStream&lt;K,V&gt;&gt;&gt;

+    createMessageStreams(Map&lt;String, Integer&gt; topicCountMap, Decoder&lt;K&gt;
keyDecoder, Decoder&lt;V&gt; valueDecoder);
+  
+  /**
+   *  Create a list of message streams of type T for each topic, using the default decoder.
+   */
+  public Map&lt;String, List&lt;KafkaStream&lt;byte[], byte[]&gt;&gt;&gt;
createMessageStreams(Map&lt;String, Integer&gt; topicCountMap);
+
+  /**
+   *  Create a list of message streams for topics matching a wildcard.
+   *
+   *  @param topicFilter a TopicFilter that specifies which topics to
+   *                    subscribe to (encapsulates a whitelist or a blacklist).
+   *  @param numStreams the number of message streams to return.
+   *  @param keyDecoder a decoder that decodes the message key
+   *  @param valueDecoder a decoder that decodes the message itself
+   *  @return a list of KafkaStream. Each stream supports an
+   *          iterator over its MessageAndMetadata elements.
+   */
+  public &lt;K,V> List&lt;KafkaStream&lt;K,V&gt;&gt; 
+    createMessageStreamsByFilter(TopicFilter topicFilter, int numStreams, Decoder&lt;K&gt;
keyDecoder, Decoder&lt;V&gt; valueDecoder);
+  
+  /**
+   *  Create a list of message streams for topics matching a wildcard, using the default
decoder.
+   */
+  public List&lt;KafkaStream&lt;byte[], byte[]&gt;&gt; createMessageStreamsByFilter(TopicFilter
topicFilter, int numStreams);
+  
+  /**
+   *  Commit the offsets of all topic/partitions connected by this connector.
+   */
+  public void commitOffsets();
+
+  /**
+   *  Shut down the connector
+   */
+  public void shutdown();
+}
+
+</pre>
+You can follow 
+<a href="" title="Kafka 0.8 consumer example">this</a> to learn how to use the
high level consumer api.
+
+<h3>SimpleConsumer Api</h3>
+<pre>
+class kafka.javaapi.consumer.SimpleConsumer {
+  /**
+   *  Fetch a set of messages from a topic.
+   *
+   *  @param request specifies the topic name, topic partition, starting byte offset, maximum
bytes to be fetched.
+   *  @return a set of fetched messages
+   */
+  public FetchResponse fetch(request: kafka.javaapi.FetchRequest);
+
+  /**
+   *  Fetch metadata for a sequence of topics.
+   *  
+   *  @param request specifies the versionId, clientId, sequence of topics.
+   *  @return metadata for each topic in the request.
+   */
+  public kafka.javaapi.TopicMetadataResponse send(request: kafka.javaapi.TopicMetadataRequest);
+
+  /**
+   *  Get a list of valid offsets (up to maxSize) before the given time.
+   *
+   *  @param request a [[kafka.javaapi.OffsetRequest]] object.
+   *  @return a [[kafka.javaapi.OffsetResponse]] object.
+   */
+  public kafka.javaapi.OffsetResponse getOffsetsBefore(request: OffsetRequest);
+
+  /**
+   * Close the SimpleConsumer.
+   */
+  public void close();
+}
+</pre>
+For most applications, the high level consumer Api is good enough. Some applications want
features not exposed to the high level consumer yet (e.g., set initial offset when restarting
the consumer). They can instead use our low level SimpleConsumer Api. The logic will be a
bit more complicated and you can follow the examples in
+<a href="https://cwiki.apache.org/confluence/display/KAFKA/0.8.0+SimpleConsumer+Example"
title="Kafka 0.8 SimpleConsumer example">here</a>.
+
+<h3>Kafka Hadoop Consumer</h3>
+
+<p>
+Providing a horizontally scalable solution for aggregating and loading data into Hadoop was
one of our basic use cases. To support this use case, we provide a Hadoop-based consumer which
spawns off many map tasks to pull data from the Kafka cluster in parallel. This provides extremely
fast pull-based Hadoop data load capabilities (we were able to fully saturate the network
with only a handful of Kafka servers).
+</p>
+
+<p>
+Usage information on the hadoop consumer can be found <a href="https://github.com/linkedin/camus/tree/camus-kafka-0.8/">here</a>.
+</p>
+
+<!--#include virtual="includes/footer.html" -->
+

Added: kafka/site/08/configuration.html
URL: http://svn.apache.org/viewvc/kafka/site/08/configuration.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/configuration.html (added)
+++ kafka/site/08/configuration.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,512 @@
+<!--#include virtual="includes/header.html" -->
+
+<h2> Configuration </h2>
+
+<h3> Important configuration properties for Kafka broker are: </h3>
+<ul>
+	<li><code>broker.id</code>
+	<li><code>log.dirs</code>
+	<li><code>zookeeper.connect</code>
+</ul>
+
+<p>More details about server configuration can be found in the scala class <code>kafka.server.KafkaConfig</code>.</p>

+
+<table class="data-table">
+<tbody><tr>
+      <th>Property</th>
+      <th>Default</th>
+      <th>Description</th>
+    </tr>
+    <tr>
+      <td>broker.id</td>
+      <td></td>
+      <td>Each broker is uniquely identified by a non-negative integer id. This id
serves as the brokers "name", and allows the broker to be moved to a different host/port without
confusing consumers.</td>
+    </tr>
+    <tr>
+      <td>log.dirs</td>
+      <td>"/tmp/kafka-logs"</td>
+      <td>The directories in which the log data is kept</td>
+    </tr>
+    <tr>
+      <td>zookeeper.connect</td>
+      <td>null</td>
+      <td>Specifies the zookeeper connection string in the form hostname:port/chroot.
Here the chroot is a base directory which is prepended to all path operations (this effectively
namespaces all kafka znodes to allow sharing with other applications on the same zookeeper
cluster).</td>
+    </tr>
+    <tr>
+      <td>message.max.bytes</td>
+      <td>
+        <span>1000000</span>
+     </td>
+      <td>The maximum size of a message that the server can receive</td>
+    </tr>
+    <tr>
+      <td>num.network.threads</td>
+      <td>3</td>
+      <td>The number of network threads that the server uses for handling network requests</td>
+    </tr>
+    <tr>
+      <td>num.io.threads</td>
+      <td>8</td>
+      <td>The number of io threads that the server uses for carrying out network requests</td>
+    </tr>
+    <tr>
+      <td>queued.max.requests</td>
+      <td>500</td>
+      <td>The number of queued requests allowed before blocking the network threads</td>
+    </tr>
+    <tr>
+      <td>port</td>
+      <td>6667</td>
+      <td>The port to listen and accept connections on</td>
+    </tr>
+    <tr>
+      <td>host.name</td>
+      <td>null</td>
+      <td>
+        <p>Hostname of broker. If this is set, it will only bind to this address. If
this is not set, it will bind to all interfaces, and publish one to ZK</p>
+     </td>
+    </tr>
+    <tr>
+      <td>socket.send.buffer.bytes</td>
+      <td>100 * 1024</td>
+      <td>The SO_SNDBUFF buffer of the socket sever sockets</td>
+    </tr>
+    <tr>
+      <td>socket.receive.buffer.bytes</td>
+      <td>100 * 1024</td>
+      <td>The SO_RCVBUFF buffer of the socket sever sockets</td>
+    </tr>
+    <tr>
+      <td>socket.request.max.bytes</td>
+      <td>100 * 1024 * 1024</td>
+      <td>The maximum number of bytes in a socket request</td>
+    </tr>
+    <tr>
+      <td>num.partitions</td>
+      <td>1</td>
+      <td>The default number of log partitions per topic</td>
+    </tr>
+    <tr>
+      <td>log.segment.bytes</td>
+      <td>1024 * 1024 * 1024</td>
+      <td>The maximum size of a single log file</td>
+    </tr>
+    <tr>
+      <td>log.segment.bytes.per.topic</td>
+      <td>""</td>
+      <td>The maximum size of a single log file for some specific topic</td>
+    </tr>
+    <tr>
+      <td>log.roll.hours</td>
+      <td>24 * 7</td>
+      <td>The maximum time before a new log segment is rolled out</td>
+    </tr>
+    <tr>
+      <td>log.roll.hours.per.topic</td>
+      <td>""</td>
+      <td>The number of hours before rolling out a new log segment for some specific
topic</td>
+    </tr>
+    <tr>
+      <td>log.retention.hours</td>
+      <td>24 * 7</td>
+      <td>The number of hours to keep a log file before deleting it</td>
+    </tr>
+    <tr>
+      <td>log.retention.hours.per.topic</td>
+      <td>""</td>
+      <td>The number of hours to keep a log file before deleting it for some specific
topic</td>
+    </tr>
+    <tr>
+      <td>log.retention.bytes</td>
+      <td>-1</td>
+      <td>The maximum size of the log before deleting it</td>
+    </tr>
+    <tr>
+      <td>log.retention.bytes.per.topic</td>
+      <td>""</td>
+      <td>The maximum size of the log for some specific topic before deleting it</td>
+    </tr>
+    <tr>
+      <td>log.cleanup.interval.mins</td>
+      <td>10</td>
+      <td>The frequency in minutes that the log cleaner checks whether any log is eligible
for deletion</td>
+    </tr>
+    <tr>
+      <td>log.index.size.max.bytes</td>
+      <td>10 * 1024 * 1024</td>
+      <td>The maximum size in bytes of the offset index</td>
+    </tr>
+    <tr>
+      <td>log.index.interval.bytes</td>
+      <td>4096</td>
+      <td>The interval with which we add an entry to the offset index</td>
+    </tr>
+    <tr>
+      <td>log.flush.interval.messages</td>
+      <td>10000</td>
+      <td>The number of messages accumulated on a log partition before messages are
flushed to disk</td>
+    </tr>
+    <tr>
+      <td>log.flush.interval.ms.per.topic</td>
+      <td>""</td>
+      <td>The maximum time in ms that a message in selected topics is kept in memory
before flushed to disk, e.g., topic1:3000,topic2:6000</td>
+    </tr>
+    <tr>
+      <td>log.flush.scheduler.interval.ms</td>
+      <td>3000</td>
+      <td>The frequency in ms that the log flusher checks whether any log needs to
be flushed to disk</td>
+    </tr>
+    <tr>
+      <td>log.flush.interval.ms</td>
+      <td>${<span>log.flush.scheduler.interval.ms}</span>
+     </td>
+      <td>The maximum time in ms that a message in any topic is kept in memory before
flushed to disk</td>
+    </tr>
+    <tr>
+      <td>auto.create.topics.enable</td>
+      <td>true</td>
+      <td>Enable auto creation of topic on the server</td>
+    </tr>
+    <tr>
+      <td>controller.socket.timeout.ms</td>
+      <td>30000</td>
+      <td>The socket timeout for controller-to-broker channels</td>
+    </tr>
+    <tr>
+      <td>controller.message.queue.size</td>
+      <td>10</td>
+      <td>The buffer size for controller-to-broker-channels</td>
+    </tr>
+    <tr>
+      <td>default.replication.factor</td>
+      <td>1</td>
+      <td>Default replication factors for automatically created topics</td>
+    </tr>
+    <tr>
+      <td>replica.lag.time.max.ms</td>
+      <td>10000</td>
+      <td>If a follower hasn't sent any fetch requests during this time, the leader
will remove the follower from isr</td>
+    </tr>
+    <tr>
+      <td>replica.lag.max.messages</td>
+      <td>4000</td>
+      <td>If the lag in messages between a leader and a follower exceeds this number,
the leader will remove the follower from isr</td>
+    </tr>
+    <tr>
+      <td>replica.socket.timeout.ms</td>
+      <td>30 * 1000</td>
+      <td>The socket timeout for network requests</td>
+    </tr>
+    <tr>
+      <td>replica.socket.receive.buffer.bytes</td>
+      <td>64 * 1024</td>
+      <td>The socket receive buffer for network requests</td>
+    </tr>
+    <tr>
+      <td>replica.fetch.max.bytes</td>
+      <td>1024 * 1024</td>
+      <td>The number of byes of messages to attempt to fetch</td>
+    </tr>
+    <tr>
+      <td>replica.fetch.wait.max.ms</td>
+      <td>500</td>
+      <td>Max wait time for each fetcher request issued by follower replicas</td>
+    </tr>
+    <tr>
+      <td>replica.fetch.min.bytes</td>
+      <td>1</td>
+      <td>Minimum bytes expected for each fetch response. If not enough bytes, wait
up to replicaMaxWaitTimeMs</td>
+    </tr>
+    <tr>
+      <td>num.replica.fetchers</td>
+      <td>1</td>
+      <td>
+        <p>Number of fetcher threads used to replicate messages from a source broker.
Increasing this value can increase the degree of I/O parallelism in the follower broker.</p>
+     </td>
+    </tr>
+    <tr>
+      <td>replica.high.watermark.checkpoint.interval.ms</td>
+      <td>5000</td>
+      <td>The frequency with which the high watermark is saved out to disk</td>
+    </tr>
+    <tr>
+      <td>fetch.purgatory.purge.interval.requests</td>
+      <td>10000</td>
+      <td>The purge interval (in number of requests) of the fetch request purgatory</td>
+    </tr>
+    <tr>
+      <td>producer.purgatory.purge.interval.requests</td>
+      <td>10000</td>
+      <td>The purge interval (in number of requests) of the producer request purgatory</td>
+    </tr>
+    <tr>
+      <td>zookeeper.session.timeout.ms</td>
+      <td>6000</td>
+      <td>Zookeeper session timeout</td>
+    </tr>
+    <tr>
+      <td>zookeeper.connection.timeout.ms</td>
+      <td>${zookeeper.session.timeout.ms}</td>
+      <td>The max time that the client waits to establish a connection to zookeeper</td>
+    </tr>
+    <tr>
+      <td>zookeeper.sync.time.ms</td>
+      <td>2000</td>
+      <td>How far a ZK follower can be behind a ZK leader</td>
+    </tr>
+</tbody></table>
+
+
+<h3> Important configuration properties for the high-level consumer are: </h3>
+<ul>
+        <li><code>group.id</code>
+        <li><code>zookeeper.connect</code>
+</ul>
+
+<p>More details about consumer configuration can be found in the scala class <code>kafka.consumer.ConsumerConfig</code>.</p>

+
+<table class="data-table">
+<tbody><tr>
+        <th>Property</th>
+        <th>Default</th>
+        <th>Description</th>
+</tr>
+    <tr>
+      <td>group.id</td>
+      <td colspan="1"></td>
+      <td>A string that uniquely identifies a set of consumers within the same consumer
group</td>
+    </tr>
+    <tr>
+      <td>zookeeper.connect</td>
+      <td colspan="1">null</td>
+      <td>Specifies the zookeeper connection string in the form hostname:port/chroot.
Here the chroot is a base directory which is prepended to all path operations (this effectively
namespaces all kafka znodes to allow sharing with other applications on the same zookeeper
cluster.</td>
+    </tr>
+    <tr>
+      <td>consumer.id</td>
+      <td colspan="1">null</td>
+      <td>
+        <p>Generated automatically if not set.</p>
+     </td>
+    </tr>
+    <tr>
+      <td>socket.timeout.ms</td>
+      <td colspan="1">30 * 1000</td>
+      <td>The socket timeout for network requests. The actual timeout set will be max.fetch.wait
+ socket.timeout.ms.</td>
+    </tr>
+    <tr>
+      <td>socket.receive.buffer.bytes</td>
+      <td colspan="1">64 * 1024</td>
+      <td>The socket receive buffer for network requests</td>
+    </tr>
+    <tr>
+      <td>fetch.message.max.bytes</td>
+      <td colspan="1">1024 * 1024</td>
+      <td>The number of byes of messages to attempt to fetch</td>
+    </tr>
+    <tr>
+      <td>auto.commit.enable</td>
+      <td colspan="1">true</td>
+      <td>If true, periodically commit to zookeeper the offset of messages already
fetched by the consumer</td>
+    </tr>
+    <tr>
+      <td>auto.commit.interval.ms</td>
+      <td colspan="1">60 * 1000</td>
+      <td>The frequency in ms that the consumer offsets are committed to zookeeper</td>
+    </tr>
+    <tr>
+      <td>queued.max.messages</td>
+      <td colspan="1">10</td>
+      <td>Max number of messages buffered for consumption</td>
+    </tr>
+    <tr>
+      <td>rebalance.max.retries</td>
+      <td colspan="1">4</td>
+      <td>Max number of retries during rebalance</td>
+    </tr>
+    <tr>
+      <td>fetch.min.bytes</td>
+      <td colspan="1">1</td>
+      <td>The minimum amount of data the server should return for a fetch request.
If insufficient data is available the request will block</td>
+    </tr>
+    <tr>
+      <td>fetch.wait.max.ms</td>
+      <td colspan="1">100</td>
+      <td>The maximum amount of time the server will block before answering the fetch
request if there isn't sufficient data to immediately satisfy fetch.min.bytes</td>
+    </tr>
+    <tr>
+      <td>rebalance.backoff.ms</td>
+      <td>${zookeeper.sync.time.ms}</td>
+      <td>Backoff time between retries during rebalance</td>
+    </tr>
+    <tr>
+      <td>refresh.leader.backoff.ms</td>
+      <td colspan="1">200</td>
+      <td>Backoff time to refresh the leader of a partition after it loses the current
leader</td>
+    </tr>
+    <tr>
+      <td>auto.offset.reset</td>
+      <td colspan="1">"smallest"</td>
+      <td>
+        <p>What to do if an offset is out of range:<br/>* smallest : automatically
reset the offset to the smallest offset<br/>* largest : automatically reset the offset
to the largest offset<br/>* anything else: throw exception to the consumer</p>
+     </td>
+    </tr>
+    <tr>
+      <td>consumer.timeout.ms</td>
+      <td colspan="1">-1</td>
+      <td>Throw a timeout exception to the consumer if no message is available for
consumption after the specified interval</td>
+    </tr>
+    <tr>
+      <td>client.id</td>
+      <td colspan="1">${group.id}</td>
+      <td>Client id is specified by the kafka consumer client, used to distinguish
different clients</td>
+    </tr>
+    <tr>
+      <td>zookeeper.session.timeout.ms </td>
+      <td colspan="1">6000</td>
+      <td>Zookeeper session timeout</td>
+    </tr>
+    <tr>
+      <td>zookeeper.connection.timeout.ms</td>
+      <td colspan="1">${<span>zookeeper.session.timeout.ms}</span>
+     </td>
+      <td>The max time that the client waits to establish a connection to zookeeper</td>
+    </tr>
+    <tr>
+      <td>zookeeper.sync.time.ms </td>
+      <td colspan="1">2000</td>
+      <td>How far a ZK follower can be behind a ZK leader</td>
+    </tr>
+</tbody></table>
+
+
+<h3> Important configuration properties for the producer are: </h3>
+<ul>
+        <li><code>metadata.broker.list</code>
+        <li><code>request.required.acks</code>
+        <li><code>producer.type</code>
+        <li><code>serializer.class</code>
+</ul>
+
+<p>More details about producer configuration can be found in the scala class <code>kafka.producer.ProducerConfig</code>.</p>

+
+<table class="data-table">
+<tbody><tr>
+        <th>Property</th>
+        <th>Default</th>
+        <th>Description</th>
+      </tr>
+    <tr>
+      <td>metadata.broker.list</td>
+      <td colspan="1"></td>
+      <td>
+        <p>This is for bootstrapping and the producer will only use it for getting
metadata (topics, partitions and replicas). The socket connections for sending the actual
data will be established based on the broker information returned in the metadata. The format
is host1:port1,host2:port2, and the list can be a subset of brokers or a VIP pointing to a
subset of brokers.</p>
+     </td>
+    </tr>
+    <tr>
+      <td>request.required.acks</td>
+      <td colspan="1">0</td>
+      <td>
+        <p>This value controls when the producer receives an acknowledgement from the
broker. Typical values are (1) 0, which means that the producer never waits for an acknowledgement
from the broker (the same behavior as 0.7); (2) 1, which means that the producer gets an acknowledgement
after the leader replica has received the data; (3) -1, which means that the producer gets
an acknowledgement after all in-sync replicas have received the data. The first option provides
the lowest latency (no network delay), but the worst durabilty (some data loss when the leader
replica fails). The second option provides lower latency (one network round trip) and better
durability (few data loss when the leader replica fails). The last option provides low latency
(two network round trips) and the best durability (no data loss as long as the number of failed
brokers is less the replication factor of the topic).</p>
+     </td>
+    </tr>
+    <tr>
+      <td>producer.type</td>
+      <td colspan="1">"sync"</td>
+      <td>
+        <p>This parameter specifies whether the messages are sent asynchronously or
not. Valid values are (1) async for asynchronous send and (2) sync for synchronous send.</p>
+     </td>
+    <tr>
+      <td>serializer.class</td>
+      <td colspan="1">"kafka.serializer.DefaultEncoder"</td>
+      <td>The serializer class for values</td>
+    </tr>
+    <tr>
+      <td>key.serializer.class</td>
+      <td colspan="1">${serializer.class}</td>
+      <td>The serializer class for keys (defaults to the same as for values)</td>
+    </tr>
+    <tr>
+      <td>partitioner.class</td>
+      <td colspan="1">"kafka.producer.DefaultPartitioner"</td>
+      <td>The partitioner class for partitioning events amongst sub-topics</td>
+    </tr>
+    </tr>
+    <tr>
+      <td>compression.codec</td>
+      <td colspan="1">"none"</td>
+      <td>
+        <p>This parameter allows you to specify the compression codec for all data
generated by this producer. The default is NoCompressionCodec</p>
+     </td>
+    </tr>
+    <tr>
+      <td>compressed.topics</td>
+      <td colspan="1">null</td>
+      <td>
+        <p>This parameter allows you to set whether compression should be turned on
for particular topics. If the compression codec is anything other than NoCompressionCodec,
enable compression only for specified topics if any. If the list of compressed topics is empty,
then enable the specified compression codec for all topics. If the compression codec is NoCompressionCodec,
compression is disabled for all topics</p>
+     </td>
+    </tr>
+    <tr>
+      <td>message.send.max.retries</td>
+      <td colspan="1">3</td>
+      <td>
+        <p>The leader may be unavailable transiently, which can fail the sending of
a message. This property specifies the number of retries when such failures occur.</p>
+     </td>
+    </tr>
+    <tr>
+      <td>retry.backoff.ms</td>
+      <td colspan="1">100</td>
+      <td>
+        <p>Before each retry, the producer refreshes the metadata of relevant topics.
Since leader election takes a bit of time, this property specifies the amount of time that
the producer waits before refreshing the metadata.</p>
+     </td>
+    </tr>
+    <tr>
+      <td>topic.metadata.refresh.interval.ms</td>
+      <td colspan="1">600* 1000</td>
+      <td>
+        <p>The producer generally refreshes the topic metadata from brokers when there
is a failure (partition missing, leader not available...). It will also poll regularly (default:
every 10min so 600000ms). If you set this to a negative value, metadata will only get refreshed
on failure. If you set this to zero, the metadata will get refreshed after each message sent
(not recommended). Important note: the refresh happen only AFTER the message is sent, so if
the producer never sends a message the metadata is never refreshed</p>
+     </td>
+    </tr>
+    <tr>
+      <td>queue.buffering.max.ms</td>
+      <td colspan="1">5000</td>
+      <td>Maximum time, in milliseconds, for buffering data on the producer queue</td>
+    </tr>
+    <tr>
+      <td>queue.buffering.max.messages</td>
+      <td colspan="1">10000</td>
+      <td>The maximum size of the blocking queue for buffering on the producer</td>
+    </tr>
+    <tr>
+      <td>queue.enqueue.timeout.ms</td>
+      <td colspan="1">-1</td>
+      <td>
+        <p>Timeout for event enqueue:<br/> * 0: events will be enqueued immediately
or dropped if the queue is full<br/> * -ve: enqueue will block indefinitely if the queue
is full<br/> * +ve: enqueue will block up to this many milliseconds if the queue is
full</p>
+     </td>
+    </tr>
+    <tr>
+      <td>batch.num.messages</td>
+      <td colspan="1">200</td>
+      <td>The number of messages batched at the producer</td>
+    </tr>
+    <tr>
+      <td>send.buffer.bytes</td>
+      <td colspan="1">100 * 1024</td>
+      <td>Socket write buffer size</td>
+    </tr>
+    <tr>
+      <td>client.id</td>
+      <td colspan="1">""</td>
+      <td>The client application sending the producer requests</td>
+    </tr>
+    <tr>
+      <td>request.timeout.ms</td>
+      <td colspan="1">1500</td>
+      <td>The ack timeout of the producer requests. Value must be non-negative and
non-zero</td>
+    </tr>
+</tbody></table>
+
+
+		</div>
+	
+
+</div></body></html>

Added: kafka/site/08/design.html
URL: http://svn.apache.org/viewvc/kafka/site/08/design.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/design.html (added)
+++ kafka/site/08/design.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,10 @@
+<!--#include virtual="includes/header.html" -->
+
+<h3> General design</h3>
+The motivation of Kafka and the general design can be found in the <a href="http://kafka.apache.org/07/design.html">design
doc of 0.7</a>.
+
+<h3> Replication design</h3>
+The design of intra-cluster replication can be found in the <a href="http://www.slideshare.net/junrao/kafka-replication-apachecon2013">presentation</a>
at ApacheCon 2013.
+
+<!--#include virtual="includes/footer.html" -->
+

Added: kafka/site/08/includes/footer.html
URL: http://svn.apache.org/viewvc/kafka/site/08/includes/footer.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/includes/footer.html (added)
+++ kafka/site/08/includes/footer.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,3 @@
+		</div>
+	</body>
+</html>
\ No newline at end of file

Added: kafka/site/08/includes/header.html
URL: http://svn.apache.org/viewvc/kafka/site/08/includes/header.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/includes/header.html (added)
+++ kafka/site/08/includes/header.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,39 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd">
+<html xmlns:og="http://ogp.me/ns#">
+	<head>
+		<title>Apache Kafka</title>
+		<link rel='stylesheet' href='../styles.css' type='text/css'>
+		<link rel="icon" type="image/png" href="http://path/to/favicon.png"><meta name="keywords"
content="keywords!">
+		<meta name="description" content="A high-throughput, distributed, publish-subscribe
messaging system.">
+		<meta http-equiv='Content-Type' content='text/html;charset=utf-8' />
+		<meta property="og:title" value="Kafka" />
+		<meta property="og:image" value="" />
+		<meta property="og:description" value="A high-throughput, distributed, publish-subscribe
messaging system." />
+		<meta property="og:site_name" value="sna-projects" />
+		<meta property="og:type" value="website" />
+	</head>
+	<body>
+		<div>
+			<div id="header" style="overflow: hidden">
+				<div>
+					<a href="http://www.apache.org">
+						<img style="float: right; margin: 5px" src="http://www.apache.org/images/feather-small.png"/>
+					</a>
+					<div class="title"><a href="../index.html">Apache Kafka</a></div>
+					<div class="subtitle">A high-throughput distributed messaging system.</div>
+				</div>
+			</div>
+			<div class="lsidebar">
+				<ul>
+                                        <li>0.8
+                                                <ul>
+							<li><a href="quickstart.html">quickstart</a></li>
+							<li><a href="design.html">design</a></li>
+                                        		<li><a href="api.html">api docs</a></li>
+                                        		<li><a href="configuration.html">configuration</a></li>
+							<li><a href="tools.html">tools</a></li>
+						</ul>
+					</li>
+				</ul>
+			</div>
+		<div class='content'>

Added: kafka/site/08/quickstart.html
URL: http://svn.apache.org/viewvc/kafka/site/08/quickstart.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/quickstart.html (added)
+++ kafka/site/08/quickstart.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,78 @@
+<!--#include virtual="includes/header.html" -->
+
+<h2>Quick Start</h3>
+
+<h3> Step 0: Clean up 0.7 data, if exists </h3>
+Since 0.8 is not backward compatible with 0.7.x, if you had an 0.7 installation already,
you will need to delete all existing Zookeeper data and Kafka log data (unless, you use a
different Zookeeper namespace and a new Kafka log directory).
+	
+<h3> Step 1: Download the code </h3>
+
+<a href="../downloads.html" title="Kafka downloads">Download</a> a recent stable
release.
+
+<pre>
+<b>&gt; tar xzf kafka-&lt;VERSION&gt;.tgz</b>
+<b>&gt; cd kafka-&lt;VERSION&gt;</b>
+<b>&gt; ./sbt update</b>
+<b>&gt; ./sbt package</b>
+<b>&gt; ./sbt assembly-package-dependency</b>
+</pre>
+
+<h3>Step 2: Start the server</h3>
+
+<p>
+First start the zookeeper server. You can use the convenience script packaged with kafka
to get a quick-and-dirty single-node zookeeper instance.
+
+<pre>
+<b>&gt; bin/zookeeper-server-start.sh config/zookeeper.properties</b>
+[2013-04-22 15:01:37,495] INFO Reading configuration from: config/zookeeper.properties (org.apache.zookeeper.server.quorum.QuorumPeerConfig)
+...
+</pre>
+
+Now start the Kafka server:
+<pre>
+<b>&gt; bin/kafka-server-start.sh config/server.properties</b>
+jkreps-mn-2:kafka-trunk jkreps$ bin/kafka-server-start.sh config/server.properties 
+[2013-04-22 15:01:47,028] INFO Verifying properties (kafka.utils.VerifiableProperties)
+[2013-04-22 15:01:47,051] INFO Property socket.send.buffer.bytes is overridden to 1048576
(kafka.utils.VerifiableProperties)
+...
+</pre>
+
+<h3>Step 3: Send some messages</h3>
+
+Kafka comes with a command line client that will take input from standard in and send it
out as messages to the Kafka cluster. By default each line will be sent as a separate message.
The topic <i>test</i> is created automatically when messages are sent to it. Omitting
logging you should see something like this:
+
+<pre>
+&gt; <b>bin/kafka-console-producer.sh --broker-list localhost:9092 --topic test</b>

+This is a message
+This is another message
+</pre>
+
+<h3>Step 4: Start a consumer</h3>
+
+Kafka also has a command line consumer that will dump out messages to standard out.
+
+<pre>
+<b>&gt; bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic test --from-beginning</b>
+This is a message
+This is another message
+</pre>
+<p>
+If you have each of the above commands running in a different terminal then you should now
be able to type messages into the producer terminal and see them appear in the consumer terminal.
+</p>
+<p>
+Both of these command line tools have additional options. Running the command with no arguments
will display usage information documenting them in more detail.	
+</p>
+
+
+<h3>Setting up replication</h3>
+If you want to set up replication on more than one broker, you can follow
+<a href="https://cwiki.apache.org/confluence/display/KAFKA/Kafka+0.8+Quick+Start" title="Kafka
0.8 replication quickstart">this</a>.
+
+<h3>Build a release zip or tar file</h3>
+<pre>
+<b>&gt; ./sbt release-zip</b>
+<b>&gt; ./sbt release-tar</b>
+</pre>
+
+<!--#include virtual="includes/footer.html" -->
+

Added: kafka/site/08/tools.html
URL: http://svn.apache.org/viewvc/kafka/site/08/tools.html?rev=1476886&view=auto
==============================================================================
--- kafka/site/08/tools.html (added)
+++ kafka/site/08/tools.html Mon Apr 29 03:50:25 2013
@@ -0,0 +1,7 @@
+<!--#include virtual="includes/header.html" -->
+
+<h3> Migrating data from 0.7 to 0.8</h3>
+Since 0.8 is not backward compatible with 0.7.x, we provide a tool for migrating data in
an 0.7 cluster to an 0.8 cluster. Details of the tool can be found <a href="https://cwiki.apache.org/confluence/display/KAFKA/Migrating+from+0.7+to+0.8">here</a>.
+
+<!--#include virtual="includes/footer.html" -->
+

Modified: kafka/site/includes/header.html
URL: http://svn.apache.org/viewvc/kafka/site/includes/header.html?rev=1476886&r1=1476885&r2=1476886&view=diff
==============================================================================
--- kafka/site/includes/header.html (original)
+++ kafka/site/includes/header.html Mon Apr 29 03:50:25 2013
@@ -26,10 +26,12 @@
 			<div class="lsidebar">
 				<ul>
 					<li><a href="downloads.html">download</a></li>
-					<li>releases</li>
+					<li>releases
 						<ul>
 							<li><a href="07/quickstart.html">0.7</a></li>
+							<li><a href="08/quickstart.html">0.8</a></li>
 						</ul>
+                                        </li>
 					<li><a href="https://cwiki.apache.org/confluence/display/KAFKA/Operations">operations</a></li>
 					<li><a href="faq.html">faq</a></li>
 					<li><a href="https://cwiki.apache.org/confluence/display/KAFKA">wiki</li>



Mime
View raw message