-
Notifications
You must be signed in to change notification settings - Fork 70
/
Copy pathkafka-output.html
530 lines (480 loc) · 31.3 KB
/
kafka-output.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
<!DOCTYPE html>
<html lang="en-us">
<head>
<meta charset="UTF-8">
<title>Configure the Kafka output | APM Server Reference [6.7] | Elastic</title>
<meta class="elastic" name="content" content="Configure the Kafka output | APM Server Reference [6.7]">
<link rel="home" href="index.html" title="APM Server Reference [6.7]"/>
<link rel="up" href="configuring-output.html" title="Configure the output"/>
<link rel="prev" href="logstash-output.html" title="Configure the Logstash output"/>
<link rel="next" href="file-output.html" title="Configure the File output"/>
<meta class="elastic" name="product_version" content="6.7"/>
<meta class="elastic" name="product_name" content="APM"/>
<meta class="elastic" name="website_area" content="documentation"/>
<meta name="DC.type" content="Learn/Docs/APM Server/Reference/6.7"/>
<meta name="DC.subject" content="APM"/>
<meta name="DC.identifier" content="6.7"/>
<meta name="robots" content="noindex,nofollow"/>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdn.optimizely.com/js/18132920325.js"></script>
<link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
<link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
<link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
<link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
<link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
<link rel="manifest" href="/manifest.json">
<meta name="apple-mobile-web-app-title" content="Elastic">
<meta name="application-name" content="Elastic">
<meta name="msapplication-TileColor" content="#ffffff">
<meta name="msapplication-TileImage" content="/mstile-144x144.png">
<meta name="theme-color" content="#ffffff">
<meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd" />
<meta name="yandex-verification" content="d8a47e95d0972434" />
<meta name="localized" content="true" />
<meta name="st:robots" content="follow,index" />
<meta property="og:image" content="https://static-www.elastic.co/v3/assets/bltefdd0b53724fa2ce/blt280217a63b82a734/6202d3378b1f312528798412/elastic-logo.svg" />
<meta property="og:image:width" content="500" />
<meta property="og:image:height" content="172" />
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
<link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
<link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
<!-- Give IE8 a fighting chance -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->
<link rel="stylesheet" type="text/css" href="/guide/static/styles-v1.css" />
</head>
<!--© 2015-2025 Elasticsearch B.V. -->
<!-- All Elastic documentation is licensed under a Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License. -->
<!-- http://creativecommons.org/licenses/by-nc-nd/4.0/ -->
<body>
<!-- Google Tag Manager -->
<script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
<!-- End Google Tag Manager -->
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-12395217-16');
</script>
<!-- Google Tag Manager for GA4 -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-KNJMG2M');</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KNJMG2M" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager for GA4-->
<div id='elastic-nav' style="display:none;"></div>
<script src='https://www.elastic.co/elastic-nav.js'></script>
<div class="main-container">
<section id="content" >
<div class="content-wrapper">
<section id="guide" lang="en">
<div class="container-fluid">
<div class="row pb-3">
<div class="col-12 order-2 col-md-4 order-md-1 col-lg-3 h-almost-full-md sticky-top-md" id="left_col">
<!-- The TOC is appended here -->
</div>
<div class="col-12 order-1 col-md-8 order-md-2 col-lg-7 order-lg-2 guide-section" id="middle_col">
<!-- start body -->
<div class="page_header">
A newer version is available. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div class="navheader">
<span class="prev">
<a href="logstash-output.html">« Configure the Logstash output</a>
</span>
<span class="next">
<a href="file-output.html">Configure the File output »</a>
</span>
</div>
<div class="book" lang="en">
<div class="titlepage">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="/guide/">Elastic Docs</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link">
<div id="related-products" class="dropdown">
<div class="related-products-title">APM:</div>
<div class="dropdown-anchor" tabindex="0">Server Reference<span class="dropdown-icon"></span></div>
<div class="dropdown-content">
<ul>
<li class="dropdown-category">APM</li>
<ul>
<li><a href="/guide/en/observability/current/apm.html">Observability › APM</a></li>
</ul>
<li class="dropdown-category">APM agents</li>
<ul>
<li><a href="/guide/en/apm/agent/android/current/intro.html">Android Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/go/current/introduction.html">Go Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/swift/current/intro.html">iOS Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/java/current/intro.html">Java Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/dotnet/current/intro.html">.NET Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/nodejs/current/intro.html">Node.js Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/php/current/intro.html">PHP Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/python/current/getting-started.html">Python Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/ruby/current/introduction.html">Ruby Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/rum-js/current/intro.html">Real User Monitoring JavaScript Agent Reference</a></li>
</ul>
<li class="dropdown-category">APM extensions</li>
<ul>
<li><a href="/guide/en/apm/lambda/current/aws-lambda-arch.html">AWS Lambda extension</a></li>
<li><a href="/guide/en/apm/attacher/current/apm-attacher.html">Attacher</a></li>
</ul>
</ul>
</div>
</div>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="configuring-howto-apm-server.html">Configuring APM Server</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="configuring-output.html">Configure the output</a></span>
</div>
<div>
<div><h1 class="title"><a id="id-1"></a>Configure the Kafka output</h1><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div>
<!--EXTRA-->
</div>
<div id="content">
<div id="url-to-v3" class="version-warning">
<strong>IMPORTANT</strong>: This documentation is no longer updated. Refer to <a href="https://www.elastic.co/support/eol">Elastic's version policy</a> and the <a href="https://www.elastic.co/docs">latest documentation</a>.
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h2 class="title"><a id="kafka-output"></a>Configure the Kafka output</h2><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The Kafka output sends the events to Apache Kafka.</p>
<p>Example configuration:</p>
<div class="pre_wrapper lang-yaml">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-yaml">output.kafka:
# initial brokers for reading cluster metadata
hosts: ["kafka1:9092", "kafka2:9092", "kafka3:9092"]
# message topic selection + partitioning
topic: '%{[fields.log_topic]}'
partition.round_robin:
reachable_only: false
required_acks: 1
compression: gzip
max_message_bytes: 1000000</pre>
</div>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Events bigger than <a class="xref" href="kafka-output.html#kafka-max_message_bytes" title="max_message_bytes"><code class="literal">max_message_bytes</code></a> will be dropped. To avoid this problem, make sure APM Server does not generate events bigger than <a class="xref" href="kafka-output.html#kafka-max_message_bytes" title="max_message_bytes"><code class="literal">max_message_bytes</code></a>.</p>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h3 class="title"><a id="_compatibility_3"></a>Compatibility</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>This output works with all Kafka in between 0.11 and 1.1.1. Older versions
might work as well, but are not supported.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h3 class="title"><a id="_configuration_options_3"></a>Configuration options</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>You can specify the following options in the <code class="literal">kafka</code> section of the <code class="literal">apm-server.yml</code> config file:</p>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_enabled_3"></a><code class="literal">enabled</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The <code class="literal">enabled</code> config is a boolean setting to enable or disable the output. If set
to false, the output is disabled.</p>
<p>The default value is true.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_hosts"></a><code class="literal">hosts</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The list of Kafka broker addresses from where to fetch the cluster metadata.
The cluster metadata contain the actual Kafka brokers events are published to.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_version"></a><code class="literal">version</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka version $apm-server is assumed to run against. Defaults to 1.0.0.</p>
<p>Event timestamps will be added, if version 0.10.0.0+ is enabled.</p>
<p>Valid values are all kafka releases in between <code class="literal">0.8.2.0</code> and <code class="literal">1.1.1</code>.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_username_2"></a><code class="literal">username</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The username for connecting to Kafka. If username is configured, the password
must be configured as well. Only SASL/PLAIN is supported.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_password_2"></a><code class="literal">password</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The password for connecting to Kafka.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_topic"></a><code class="literal">topic</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The Kafka topic used for produced events. The setting can be a format string
using any event field. For example, you can use the
<a class="xref" href="configuring-output.html#libbeat-configuration-fields" title="fields"><code class="literal">fields</code></a> configuration option to add a custom
field called <code class="literal">log_topic</code> to the event, and then set <code class="literal">topic</code> to the value of the
custom field:</p>
<div class="pre_wrapper lang-yaml">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-yaml">topic: '%{[fields.log_topic]}'</pre>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_topics"></a><code class="literal">topics</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Array of topic selector rules supporting conditionals, format string
based field access and name mappings. The first rule matching will be used to
set the <code class="literal">topic</code> for the event to be published. If <code class="literal">topics</code> is missing or no
rule matches, the <code class="literal">topic</code> field will be used.</p>
<p>Rule settings:</p>
<p><span class="strong strong"><strong><code class="literal">topic</code></strong></span>: The topic format string to use. If the fields used are missing, the
rule fails.</p>
<p><span class="strong strong"><strong><code class="literal">mapping</code></strong></span>: Dictionary mapping index names to new names</p>
<p><span class="strong strong"><strong><code class="literal">default</code></strong></span>: Default string value if <code class="literal">mapping</code> does not find a match.</p>
<p><span class="strong strong"><strong><code class="literal">when</code></strong></span>: Condition which must succeed in order to execute the current rule.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_key"></a><code class="literal">key</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Optional Kafka event key. If configured, the event key must be unique and can be extracted from the event using a format string.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_partition"></a><code class="literal">partition</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka output broker event partitioning strategy. Must be one of <code class="literal">random</code>,
<code class="literal">round_robin</code>, or <code class="literal">hash</code>. By default the <code class="literal">hash</code> partitioner is used.</p>
<p><span class="strong strong"><strong><code class="literal">random.group_events</code></strong></span>: Sets the number of events to be published to the same
partition, before the partitioner selects a new partition by random. The
default value is 1 meaning after each event a new partition is picked randomly.</p>
<p><span class="strong strong"><strong><code class="literal">round_robin.group_events</code></strong></span>: Sets the number of events to be published to the
same partition, before the partitioner selects the next partition. The default
value is 1 meaning after each event the next partition will be selected.</p>
<p><span class="strong strong"><strong><code class="literal">hash.hash</code></strong></span>: List of fields used to compute the partitioning hash value from.
If no field is configured, the events <code class="literal">key</code> value will be used.</p>
<p><span class="strong strong"><strong><code class="literal">hash.random</code></strong></span>: Randomly distribute events if no hash or key value can be computed.</p>
<p>All partitioners will try to publish events to all partitions by default. If a
partition’s leader becomes unreachable for the beat, the output might block. All
partitioners support setting <code class="literal">reachable_only</code> to overwrite this
behavior. If <code class="literal">reachable_only</code> is set to <code class="literal">true</code>, events will be published to
available partitions only.</p>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Publishing to a subset of available partitions potentially increases resource usage because events may become unevenly distributed.</p>
</div>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_client_id"></a><code class="literal">client_id</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The configurable ClientID used for logging, debugging, and auditing purposes. The default is "beats".</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_worker_3"></a><code class="literal">worker</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The number of concurrent load-balanced Kafka output workers.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_codec"></a><code class="literal">codec</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Output codec configuration. If the <code class="literal">codec</code> section is missing, events will be json encoded.</p>
<p>See <a class="xref" href="configuration-output-codec.html" title="Change the output codec">Change the output codec</a> for more information.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_metadata"></a><code class="literal">metadata</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka metadata update settings. The metadata do contain information about
brokers, topics, partition, and active leaders to use for publishing.</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">refresh_frequency</code></strong></span>
</span>
</dt>
<dd>
Metadata refresh interval. Defaults to 10 minutes.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">retry.max</code></strong></span>
</span>
</dt>
<dd>
Total number of metadata update retries when cluster is in middle of leader election. The default is 3.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">retry.backoff</code></strong></span>
</span>
</dt>
<dd>
Waiting time between retries during leader elections. Default is 250ms.
</dd>
</dl>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_max_retries_3"></a><code class="literal">max_retries</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The number of times to retry publishing an event after a publishing failure.
After the specified number of retries, the events are typically dropped.</p>
<p>Set <code class="literal">max_retries</code> to a value less than 0 to retry until all events are published.</p>
<p>The default is 3.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_bulk_max_size_3"></a><code class="literal">bulk_max_size</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum number of events to bulk in a single Kafka request. The default is 2048.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_timeout_3"></a><code class="literal">timeout</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The number of seconds to wait for responses from the Kafka brokers before timing
out. The default is 30 (seconds).</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_broker_timeout"></a><code class="literal">broker_timeout</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum duration a broker will wait for number of required ACKs. The default is 10s.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_channel_buffer_size"></a><code class="literal">channel_buffer_size</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Per Kafka broker number of messages buffered in output pipeline. The default is 256.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_keep_alive"></a><code class="literal">keep_alive</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The keep-alive period for an active network connection. If 0s, keep-alives are disabled. The default is 0 seconds.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_compression"></a><code class="literal">compression</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Sets the output compression codec. Must be one of <code class="literal">none</code>, <code class="literal">snappy</code>, <code class="literal">lz4</code> and <code class="literal">gzip</code>. The default is <code class="literal">gzip</code>.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_compression_level_3"></a><code class="literal">compression_level</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Sets the compression level used by gzip. Setting this value to 0 disables compression.
The compression level must be in the range of 1 (best speed) to 9 (best compression).</p>
<p>Increasing the compression level will reduce the network usage but will increase the cpu usage.</p>
<p>The default value is 4.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="kafka-max_message_bytes"></a><code class="literal">max_message_bytes</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum permitted size of JSON-encoded messages. Bigger messages will be dropped. The default value is 1000000 (bytes). This value should be equal to or less than the broker’s <code class="literal">message.max.bytes</code>.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_required_acks"></a><code class="literal">required_acks</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>The ACK reliability level required from broker. 0=no response, 1=wait for local commit, -1=wait for all replicas to commit. The default is 1.</p>
<p>Note: If set to 0, no ACKs are returned by Kafka. Messages might be lost silently on error.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_ssl_3"></a><code class="literal">ssl</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/6.7/docs/copied-from-beats/outputconfig.asciidoc">edit</a></div>
</div></div></div>
<p>Configuration options for SSL parameters like the root CA for Kafka connections. See
<a class="xref" href="configuration-ssl.html" title="Specify SSL settings"><em>Specify SSL settings</em></a> for more information.</p>
</div>
</div>
</div>
</div>
</div><div class="navfooter">
<span class="prev">
<a href="logstash-output.html">« Configure the Logstash output</a>
</span>
<span class="next">
<a href="file-output.html">Configure the File output »</a>
</span>
</div>
<!-- end body -->
</div>
<div class="col-12 order-3 col-lg-2 order-lg-3 h-almost-full-lg sticky-top-lg" id="right_col">
<div id="sticky_content">
<!-- The OTP is appended here -->
<div class="row">
<div class="col-0 col-md-4 col-lg-0" id="bottom_left_col"></div>
<div class="col-12 col-md-8 col-lg-12">
<div id="rtpcontainer">
<div class="mktg-promo" id="most-popular">
<p class="aside-heading">Most Popular</p>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/getting-started-elasticsearch?page=docs&placement=top-video">
<p class="mb-0">Get Started with Elasticsearch</p>
</a>
</div>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/getting-started-kibana?page=docs&placement=top-video">
<p class="mb-0">Intro to Kibana</p>
</a>
</div>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/introduction-elk-stack?page=docs&placement=top-video">
<p class="mb-0">ELK for Logs & Metrics</p>
</a>
</div>
</div>
</div>
<!-- Feedback widget -->
<div id="feedbackWidgetContainer"></div>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
</div>
<div id='elastic-footer'></div>
<script src='https://www.elastic.co/elastic-footer.js'></script>
<!-- Footer Section end-->
</section>
</div>
<!-- Feedback modal -->
<div id="feedbackModalContainer"></div>
<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs-v1.js"></script>
<script type="text/javascript">
window.initial_state = {}</script>
</body>
</html>