html/server/7.2/kafka-output.html

<!DOCTYPE html>
<html lang="en-us">
  <head>
    
<meta charset="UTF-8">
<title>Configure the Kafka output | APM Server Reference [7.2] | Elastic</title>
<meta class="elastic" name="content" content="Configure the Kafka output | APM Server Reference [7.2]">

<link rel="home" href="index.html" title="APM Server Reference [7.2]"/>
<link rel="up" href="configuring-output.html" title="Configure the output"/>
<link rel="prev" href="logstash-output.html" title="Configure the Logstash output"/>
<link rel="next" href="file-output.html" title="Configure the File output"/>
<meta class="elastic" name="product_version" content="7.2"/>
<meta class="elastic" name="product_name" content="APM"/>
<meta class="elastic" name="website_area" content="documentation"/>
<meta name="DC.type" content="Learn/Docs/APM Server/Reference/7.2"/>
<meta name="DC.subject" content="APM"/>
<meta name="DC.identifier" content="7.2"/>
<meta name="robots" content="noindex,nofollow"/>

    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd" />
    <meta name="yandex-verification" content="d8a47e95d0972434" />
    <meta name="localized" content="true" />
    <meta name="st:robots" content="follow,index" />
    <meta property="og:image" content="https://static-www.elastic.co/v3/assets/bltefdd0b53724fa2ce/blt280217a63b82a734/6202d3378b1f312528798412/elastic-logo.svg" />
    <meta property="og:image:width" content="500" />
    <meta property="og:image:height" content="172" />
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles-v1.css" />
  </head>

  <!--© 2015-2025 Elasticsearch B.V. -->
  <!-- All Elastic documentation is licensed under a Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License. -->
  <!-- http://creativecommons.org/licenses/by-nc-nd/4.0/ -->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!-- Google Tag Manager for GA4 -->
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-KNJMG2M');</script>
    <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KNJMG2M" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <!-- End Google Tag Manager for GA4-->

    <div id='elastic-nav' style="display:none;"></div>
    <script src='https://www.elastic.co/elastic-nav.js'></script>

    <div class="main-container">
      <section id="content" >
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container-fluid">
              <div class="row pb-3">
                <div class="col-12 order-2 col-md-4 order-md-1 col-lg-3 h-almost-full-md sticky-top-md" id="left_col">
                  <!-- The TOC is appended here -->
                </div>

                <div class="col-12 order-1 col-md-8 order-md-2 col-lg-7 order-lg-2 guide-section" id="middle_col">
                  <!-- start body -->
                  
<div class="page_header">
A newer version is available. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div class="navheader">
<span class="prev">
<a href="logstash-output.html">« Configure the Logstash output</a>
</span>
<span class="next">
<a href="file-output.html">Configure the File output »</a>
</span>
</div>
<div class="book" lang="en">
<div class="titlepage">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="/guide/">Elastic Docs</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link">
  <div id="related-products" class="dropdown">
  <div class="related-products-title">APM:</div>
  <div class="dropdown-anchor" tabindex="0">Server Reference<span class="dropdown-icon"></span></div>
  <div class="dropdown-content">
  <ul>
  <li class="dropdown-category">APM</li>
  <ul>
  <li><a href="/guide/en/observability/current/apm.html">Observability › APM</a></li>
  </ul>
  <li class="dropdown-category">APM agents</li>
  <ul>
  <li><a href="/guide/en/apm/agent/android/current/intro.html">Android Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/go/current/introduction.html">Go Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/swift/current/intro.html">iOS Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/java/current/intro.html">Java Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/dotnet/current/intro.html">.NET Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/nodejs/current/intro.html">Node.js Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/php/current/intro.html">PHP Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/python/current/getting-started.html">Python Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/ruby/current/introduction.html">Ruby Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/rum-js/current/intro.html">Real User Monitoring JavaScript Agent Reference</a></li>
  </ul>
  <li class="dropdown-category">APM extensions</li>
  <ul>
  <li><a href="/guide/en/apm/lambda/current/aws-lambda-arch.html">AWS Lambda extension</a></li>
  <li><a href="/guide/en/apm/attacher/current/apm-attacher.html">Attacher</a></li>
  </ul>
  </ul>
  </div>
  </div>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="configuring-howto-apm-server.html">Configuring APM Server</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="configuring-output.html">Configure the output</a></span>
</div>
<div>
<div><h1 class="title"><a id="id-1"></a>Configure the Kafka output</h1><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div>
<!--EXTRA-->
</div>
<div id="content">
<div id="url-to-v3" class="version-warning">
    <strong>IMPORTANT</strong>: This documentation is no longer updated. Refer to <a href="https://www.elastic.co/support/eol">Elastic's version policy</a> and the <a href="https://www.elastic.co/docs">latest documentation</a>.
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h2 class="title"><a id="kafka-output"></a>Configure the Kafka output</h2><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>

<p>The Kafka output sends the events to Apache Kafka.</p>
<p>Example configuration:</p>
<div class="pre_wrapper lang-yaml">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-yaml">output.kafka:
  # initial brokers for reading cluster metadata
  hosts: ["kafka1:9092", "kafka2:9092", "kafka3:9092"]

  # message topic selection + partitioning
  topic: '%{[fields.log_topic]}'
  partition.round_robin:
    reachable_only: false

  required_acks: 1
  compression: gzip
  max_message_bytes: 1000000</pre>
</div>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Events bigger than <a class="xref" href="kafka-output.html#kafka-max_message_bytes" title="max_message_bytes"><code class="literal">max_message_bytes</code></a> will be dropped. To avoid this problem, make sure APM Server does not generate events bigger than <a class="xref" href="kafka-output.html#kafka-max_message_bytes" title="max_message_bytes"><code class="literal">max_message_bytes</code></a>.</p>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h3 class="title"><a id="kafka-compatibility"></a>Compatibility</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>This output works with all Kafka versions in between 0.11 and 2.1.0. Older versions
might work as well, but are not supported.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h3 class="title"><a id="_configuration_options_3"></a>Configuration options</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>You can specify the following options in the <code class="literal">kafka</code> section of the <code class="literal">apm-server.yml</code> config file:</p>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_enabled_3"></a><code class="literal">enabled</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The <code class="literal">enabled</code> config is a boolean setting to enable or disable the output. If set
to false, the output is disabled.</p>
<p>The default value is true.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_hosts"></a><code class="literal">hosts</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The list of Kafka broker addresses from where to fetch the cluster metadata.
The cluster metadata contain the actual Kafka brokers events are published to.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_version"></a><code class="literal">version</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka version apm-server is assumed to run against. Defaults to 1.0.0.</p>
<p>Event timestamps will be added, if version 0.10.0.0+ is enabled.</p>
<p>Valid values are all kafka releases in between <code class="literal">0.8.2.0</code> and <code class="literal">2.0.0</code>.</p>
<p>See <a class="xref" href="kafka-output.html#kafka-compatibility" title="Compatibility">Compatibility</a> for information on supported versions.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_username_2"></a><code class="literal">username</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The username for connecting to Kafka. If username is configured, the password
must be configured as well. Only SASL/PLAIN is supported.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_password_2"></a><code class="literal">password</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The password for connecting to Kafka.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="topic-option-kafka"></a><code class="literal">topic</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The Kafka topic used for produced events.</p>
<p>You can set the topic dynamically by using a format string to access any
event field. For example, this configuration uses a custom field,
<code class="literal">fields.log_topic</code>, to set the topic for each event:</p>
<div class="pre_wrapper lang-yaml">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-yaml">topic: '%{[fields.log_topic]}'</pre>
</div>
<div class="tip admon">
<div class="icon"></div>
<div class="admon_content">
<p>To learn how to add custom fields to events, see the
<a class="xref" href="configuring-output.html#libbeat-configuration-fields" title="fields"><code class="literal">fields</code></a> option.</p>
</div>
</div>
<p>See the <a class="xref" href="kafka-output.html#topics-option-kafka" title="topics"><code class="literal">topics</code></a> setting for other ways to set the
topic dynamically.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="topics-option-kafka"></a><code class="literal">topics</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>An array of topic selector rules. Each rule specifies the <code class="literal">topic</code> to use for
events that match the rule. During publishing, APM Server sets the <code class="literal">topic</code>
for each event based on the first matching rule in the array. Rules
can contain conditionals, format string-based fields, and name mappings. If the
<code class="literal">topics</code> setting is missing or no rule matches, the
<a class="xref" href="kafka-output.html#topic-option-kafka" title="topic"><code class="literal">topic</code></a> field is used.</p>
<p>Rule settings:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">topic</code></strong></span>
</span>
</dt>
<dd>
The topic format string to use.  If this string contains field
references, such as <code class="literal">%{[fields.name]}</code>, the fields must exist, or the rule
fails.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">mappings</code></strong></span>
</span>
</dt>
<dd>
A dictionary that takes the value returned by <code class="literal">topic</code> and maps it
to a new name.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">default</code></strong></span>
</span>
</dt>
<dd>
The default string value to use if <code class="literal">mappings</code> does not find a
match.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">when</code></strong></span>
</span>
</dt>
<dd>
A condition that must succeed in order to execute the current rule.
</dd>
</dl>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_key"></a><code class="literal">key</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Optional Kafka event key. If configured, the event key must be unique and can be extracted from the event using a format string.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_partition"></a><code class="literal">partition</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka output broker event partitioning strategy. Must be one of <code class="literal">random</code>,
<code class="literal">round_robin</code>, or <code class="literal">hash</code>. By default the <code class="literal">hash</code> partitioner is used.</p>
<p><span class="strong strong"><strong><code class="literal">random.group_events</code></strong></span>: Sets the number of events to be published to the same
 partition, before the partitioner selects a new partition by random. The
 default value is 1 meaning after each event a new partition is picked randomly.</p>
<p><span class="strong strong"><strong><code class="literal">round_robin.group_events</code></strong></span>: Sets the number of events to be published to the
 same partition, before the partitioner selects the next partition. The default
 value is 1 meaning after each event the next partition will be selected.</p>
<p><span class="strong strong"><strong><code class="literal">hash.hash</code></strong></span>: List of fields used to compute the partitioning hash value from.
 If no field is configured, the events <code class="literal">key</code> value will be used.</p>
<p><span class="strong strong"><strong><code class="literal">hash.random</code></strong></span>: Randomly distribute events if no hash or key value can be computed.</p>
<p>All partitioners will try to publish events to all partitions by default. If a
partition&#8217;s leader becomes unreachable for the beat, the output might block. All
partitioners support setting <code class="literal">reachable_only</code> to overwrite this
behavior. If <code class="literal">reachable_only</code> is set to <code class="literal">true</code>, events will be published to
available partitions only.</p>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Publishing to a subset of available partitions potentially increases resource usage because events may become unevenly distributed.</p>
</div>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_client_id"></a><code class="literal">client_id</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The configurable ClientID used for logging, debugging, and auditing purposes. The default is "beats".</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_worker_3"></a><code class="literal">worker</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The number of concurrent load-balanced Kafka output workers.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_codec"></a><code class="literal">codec</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Output codec configuration. If the <code class="literal">codec</code> section is missing, events will be json encoded.</p>
<p>See <a class="xref" href="configuration-output-codec.html" title="Change the output codec">Change the output codec</a> for more information.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_metadata"></a><code class="literal">metadata</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka metadata update settings. The metadata do contain information about
brokers, topics, partition, and active leaders to use for publishing.</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">refresh_frequency</code></strong></span>
</span>
</dt>
<dd>
Metadata refresh interval. Defaults to 10 minutes.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">full</code></strong></span>
</span>
</dt>
<dd>
Strategy to use when fetching metadata, when this option is <code class="literal">true</code>, the client will maintain
a full set of metadata for all the available topics, if the this option is set to <code class="literal">false</code> it will only refresh the
metadata for the configured topics. The default is true.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">retry.max</code></strong></span>
</span>
</dt>
<dd>
Total number of metadata update retries when cluster is in middle of leader election. The default is 3.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">retry.backoff</code></strong></span>
</span>
</dt>
<dd>
Waiting time between retries during leader elections. Default is 250ms.
</dd>
</dl>
</div>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_max_retries_3"></a><code class="literal">max_retries</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The number of times to retry publishing an event after a publishing failure.
After the specified number of retries, the events are typically dropped.</p>
<p>Set <code class="literal">max_retries</code> to a value less than 0 to retry until all events are published.</p>
<p>The default is 3.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_bulk_max_size_3"></a><code class="literal">bulk_max_size</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum number of events to bulk in a single Kafka request. The default is 2048.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_timeout_3"></a><code class="literal">timeout</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The number of seconds to wait for responses from the Kafka brokers before timing
out. The default is 30 (seconds).</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_broker_timeout"></a><code class="literal">broker_timeout</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum duration a broker will wait for number of required ACKs. The default is 10s.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_channel_buffer_size"></a><code class="literal">channel_buffer_size</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Per Kafka broker number of messages buffered in output pipeline. The default is 256.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_keep_alive"></a><code class="literal">keep_alive</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The keep-alive period for an active network connection. If 0s, keep-alives are disabled. The default is 0 seconds.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_compression"></a><code class="literal">compression</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Sets the output compression codec. Must be one of <code class="literal">none</code>, <code class="literal">snappy</code>, <code class="literal">lz4</code> and <code class="literal">gzip</code>. The default is <code class="literal">gzip</code>.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_compression_level_3"></a><code class="literal">compression_level</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Sets the compression level used by gzip. Setting this value to 0 disables compression.
The compression level must be in the range of 1 (best speed) to 9 (best compression).</p>
<p>Increasing the compression level will reduce the network usage but will increase the cpu usage.</p>
<p>The default value is 4.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="kafka-max_message_bytes"></a><code class="literal">max_message_bytes</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum permitted size of JSON-encoded messages. Bigger messages will be dropped. The default value is 1000000 (bytes). This value should be equal to or less than the broker&#8217;s <code class="literal">message.max.bytes</code>.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_required_acks"></a><code class="literal">required_acks</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The ACK reliability level required from broker. 0=no response, 1=wait for local commit, -1=wait for all replicas to commit. The default is 1.</p>
<p>Note: If set to 0, no ACKs are returned by Kafka. Messages might be lost silently on error.</p>
</div>

<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_ssl_3"></a><code class="literal">ssl</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.2/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Configuration options for SSL parameters like the root CA for Kafka connections. See
<a class="xref" href="configuration-ssl.html" title="SSL output settings">SSL output settings</a> for more information.</p>
</div>

</div>

</div>
</div>
</div><div class="navfooter">
<span class="prev">
<a href="logstash-output.html">« Configure the Logstash output</a>
</span>
<span class="next">
<a href="file-output.html">Configure the File output »</a>
</span>
</div>

                  <!-- end body -->
                </div>

                <div class="col-12 order-3 col-lg-2 order-lg-3 h-almost-full-lg sticky-top-lg" id="right_col">
                  <div id="sticky_content">
                    <!-- The OTP is appended here -->
                    <div class="row">
                      <div class="col-0 col-md-4 col-lg-0" id="bottom_left_col"></div>
                      <div class="col-12 col-md-8 col-lg-12">
                        <div id="rtpcontainer">
                          <div class="mktg-promo" id="most-popular">
                            <p class="aside-heading">Most Popular</p>
                            <div class="pb-2">
                              <p class="media-type">Video</p>
                              <a href="https://www.elastic.co/webinars/getting-started-elasticsearch?page=docs&placement=top-video">
                                <p class="mb-0">Get Started with Elasticsearch</p>
                              </a>
                            </div>
                            <div class="pb-2">
                              <p class="media-type">Video</p>
                              <a href="https://www.elastic.co/webinars/getting-started-kibana?page=docs&placement=top-video">
                                <p class="mb-0">Intro to Kibana</p>
                              </a>
                            </div>
                            <div class="pb-2">
                              <p class="media-type">Video</p>
                              <a href="https://www.elastic.co/webinars/introduction-elk-stack?page=docs&placement=top-video">
                                <p class="mb-0">ELK for Logs & Metrics</p>
                              </a>
                            </div>
                          </div>
                        </div>

                        <!-- Feedback widget -->
                        <div id="feedbackWidgetContainer"></div>
                      </div>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


        <div id='elastic-footer'></div>
        <script src='https://www.elastic.co/elastic-footer.js'></script>
        <!-- Footer Section end-->

      </section>
    </div>

    <!-- Feedback modal -->
    <div id="feedbackModalContainer"></div>

    <script src="/guide/static/jquery.js"></script>
    <script type="text/javascript" src="/guide/static/docs-v1.js"></script>
    <script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>