-
Notifications
You must be signed in to change notification settings - Fork 70
/
Copy pathkafka-output.html
628 lines (577 loc) · 35.3 KB
/
kafka-output.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
<!DOCTYPE html>
<html lang="en-us">
<head>
<meta charset="UTF-8">
<title>Configure the Kafka output | APM Server Reference [7.9] | Elastic</title>
<meta class="elastic" name="content" content="Configure the Kafka output | APM Server Reference [7.9]">
<link rel="home" href="index.html" title="APM Server Reference [7.9]"/>
<link rel="up" href="configuring-output.html" title="Configure the output"/>
<link rel="prev" href="logstash-output.html" title="Configure the Logstash output"/>
<link rel="next" href="redis-output.html" title="Configure the Redis output"/>
<meta class="elastic" name="product_version" content="7.9"/>
<meta class="elastic" name="product_name" content="APM"/>
<meta class="elastic" name="website_area" content="documentation"/>
<meta name="DC.type" content="Learn/Docs/APM Server/Reference/7.9"/>
<meta name="DC.subject" content="APM"/>
<meta name="DC.identifier" content="7.9"/>
<meta name="robots" content="noindex,nofollow"/>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdn.optimizely.com/js/18132920325.js"></script>
<link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
<link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
<link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
<link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
<link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
<link rel="manifest" href="/manifest.json">
<meta name="apple-mobile-web-app-title" content="Elastic">
<meta name="application-name" content="Elastic">
<meta name="msapplication-TileColor" content="#ffffff">
<meta name="msapplication-TileImage" content="/mstile-144x144.png">
<meta name="theme-color" content="#ffffff">
<meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd" />
<meta name="yandex-verification" content="d8a47e95d0972434" />
<meta name="localized" content="true" />
<meta name="st:robots" content="follow,index" />
<meta property="og:image" content="https://static-www.elastic.co/v3/assets/bltefdd0b53724fa2ce/blt280217a63b82a734/6202d3378b1f312528798412/elastic-logo.svg" />
<meta property="og:image:width" content="500" />
<meta property="og:image:height" content="172" />
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
<link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
<link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
<!-- Give IE8 a fighting chance -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->
<link rel="stylesheet" type="text/css" href="/guide/static/styles-v1.css" />
</head>
<!--© 2015-2025 Elasticsearch B.V. -->
<!-- All Elastic documentation is licensed under a Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License. -->
<!-- http://creativecommons.org/licenses/by-nc-nd/4.0/ -->
<body>
<!-- Google Tag Manager -->
<script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
<!-- End Google Tag Manager -->
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-12395217-16');
</script>
<!-- Google Tag Manager for GA4 -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-KNJMG2M');</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KNJMG2M" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager for GA4-->
<div id='elastic-nav' style="display:none;"></div>
<script src='https://www.elastic.co/elastic-nav.js'></script>
<div class="main-container">
<section id="content" >
<div class="content-wrapper">
<section id="guide" lang="en">
<div class="container-fluid">
<div class="row pb-3">
<div class="col-12 order-2 col-md-4 order-md-1 col-lg-3 h-almost-full-md sticky-top-md" id="left_col">
<!-- The TOC is appended here -->
</div>
<div class="col-12 order-1 col-md-8 order-md-2 col-lg-7 order-lg-2 guide-section" id="middle_col">
<!-- start body -->
<div class="page_header">
A newer version is available. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div class="navheader">
<span class="prev">
<a href="logstash-output.html">« Configure the Logstash output</a>
</span>
<span class="next">
<a href="redis-output.html">Configure the Redis output »</a>
</span>
</div>
<div class="book" lang="en">
<div class="titlepage">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="/guide/">Elastic Docs</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link">
<div id="related-products" class="dropdown">
<div class="related-products-title">APM:</div>
<div class="dropdown-anchor" tabindex="0">Server Reference<span class="dropdown-icon"></span></div>
<div class="dropdown-content">
<ul>
<li class="dropdown-category">APM</li>
<ul>
<li><a href="/guide/en/observability/current/apm.html">Observability › APM</a></li>
</ul>
<li class="dropdown-category">APM agents</li>
<ul>
<li><a href="/guide/en/apm/agent/android/current/intro.html">Android Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/go/current/introduction.html">Go Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/swift/current/intro.html">iOS Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/java/current/intro.html">Java Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/dotnet/current/intro.html">.NET Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/nodejs/current/intro.html">Node.js Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/php/current/intro.html">PHP Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/python/current/getting-started.html">Python Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/ruby/current/introduction.html">Ruby Agent Reference</a></li>
<li><a href="/guide/en/apm/agent/rum-js/current/intro.html">Real User Monitoring JavaScript Agent Reference</a></li>
</ul>
<li class="dropdown-category">APM extensions</li>
<ul>
<li><a href="/guide/en/apm/lambda/current/aws-lambda-arch.html">AWS Lambda extension</a></li>
<li><a href="/guide/en/apm/attacher/current/apm-attacher.html">Attacher</a></li>
</ul>
</ul>
</div>
</div>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="configuring-howto-apm-server.html">Configure APM Server</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="configuring-output.html">Configure the output</a></span>
</div>
<div>
<div><h1 class="title"><a id="id-1"></a>Configure the Kafka output</h1><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div>
<!--EXTRA-->
</div>
<div id="content">
<div id="url-to-v3" class="version-warning">
<strong>IMPORTANT</strong>: This documentation is no longer updated. Refer to <a href="https://www.elastic.co/support/eol">Elastic's version policy</a> and the <a href="https://www.elastic.co/docs">latest documentation</a>.
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h2 class="title"><a id="kafka-output"></a>Configure the Kafka output</h2><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<div class="important admon">
<div class="icon"></div>
<div class="admon_content">
<p class="admon_title">Known issue in version 7.8.0</p>
<p>The Kafka output fails to connect when using multiple TLS brokers. We advise
not to upgrade to APM Server 7.8.0 if you’re using the Kafka output in this
configuration.</p>
</div>
</div>
<p>The Kafka output sends events to Apache Kafka.</p>
<p>To use this output, edit the APM Server configuration file to disable the Elasticsearch
output by commenting it out, and enable the Kafka output by uncommenting the
Kafka section.</p>
<p>Example configuration:</p>
<div class="pre_wrapper lang-yaml">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-yaml">output.kafka:
# initial brokers for reading cluster metadata
hosts: ["kafka1:9092", "kafka2:9092", "kafka3:9092"]
# message topic selection + partitioning
topic: '%{[fields.log_topic]}'
partition.round_robin:
reachable_only: false
required_acks: 1
compression: gzip
max_message_bytes: 1000000</pre>
</div>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Events bigger than <a class="xref" href="kafka-output.html#kafka-max_message_bytes" title="max_message_bytes"><code class="literal">max_message_bytes</code></a> will be dropped. To avoid this problem, make sure APM Server does not generate events bigger than <a class="xref" href="kafka-output.html#kafka-max_message_bytes" title="max_message_bytes"><code class="literal">max_message_bytes</code></a>.</p>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h3 class="title"><a id="kafka-compatibility"></a>Compatibility</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>This output works with all Kafka versions in between 0.11 and 2.2.2. Older versions
might work as well, but are not supported.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h3 class="title"><a id="_configuration_options_3"></a>Configuration options</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>You can specify the following options in the <code class="literal">kafka</code> section of the <code class="literal">apm-server.yml</code> config file:</p>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_enabled_3"></a><code class="literal">enabled</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The <code class="literal">enabled</code> config is a boolean setting to enable or disable the output. If set
to false, the output is disabled.</p>
<p>The default value is <code class="literal">false</code>.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_hosts"></a><code class="literal">hosts</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The list of Kafka broker addresses from where to fetch the cluster metadata.
The cluster metadata contain the actual Kafka brokers events are published to.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_version"></a><code class="literal">version</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka version apm-server is assumed to run against. Defaults to 1.0.0.</p>
<p>Event timestamps will be added, if version 0.10.0.0+ is enabled.</p>
<p>Valid values are all kafka releases in between <code class="literal">0.8.2.0</code> and <code class="literal">2.0.0</code>.</p>
<p>See <a class="xref" href="kafka-output.html#kafka-compatibility" title="Compatibility">Compatibility</a> for information on supported versions.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_username_2"></a><code class="literal">username</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The username for connecting to Kafka. If username is configured, the password
must be configured as well. Only SASL/PLAIN is supported.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_password_2"></a><code class="literal">password</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The password for connecting to Kafka.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="topic-option-kafka"></a><code class="literal">topic</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The Kafka topic used for produced events.</p>
<p>You can set the topic dynamically by using a format string to access any
event field. For example, this configuration uses a custom field,
<code class="literal">fields.log_topic</code>, to set the topic for each event:</p>
<div class="pre_wrapper lang-yaml">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-yaml">topic: '%{[fields.log_topic]}'</pre>
</div>
<div class="tip admon">
<div class="icon"></div>
<div class="admon_content">
<p>To learn how to add custom fields to events, see the
<a class="xref" href="configuring-output.html#libbeat-configuration-fields" title="fields"><code class="literal">fields</code></a> option.</p>
</div>
</div>
<p>See the <a class="xref" href="kafka-output.html#topics-option-kafka" title="topics"><code class="literal">topics</code></a> setting for other ways to set the
topic dynamically.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="topics-option-kafka"></a><code class="literal">topics</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>An array of topic selector rules. Each rule specifies the <code class="literal">topic</code> to use for
events that match the rule. During publishing, APM Server sets the <code class="literal">topic</code>
for each event based on the first matching rule in the array. Rules
can contain conditionals, format string-based fields, and name mappings. If the
<code class="literal">topics</code> setting is missing or no rule matches, the
<a class="xref" href="kafka-output.html#topic-option-kafka" title="topic"><code class="literal">topic</code></a> field is used.</p>
<p>Rule settings:</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">topic</code></strong></span>
</span>
</dt>
<dd>
The topic format string to use. If this string contains field
references, such as <code class="literal">%{[fields.name]}</code>, the fields must exist, or the rule
fails.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">mappings</code></strong></span>
</span>
</dt>
<dd>
A dictionary that takes the value returned by <code class="literal">topic</code> and maps it
to a new name.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">default</code></strong></span>
</span>
</dt>
<dd>
The default string value to use if <code class="literal">mappings</code> does not find a
match.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">when</code></strong></span>
</span>
</dt>
<dd>
A condition that must succeed in order to execute the current rule.
</dd>
</dl>
</div>
<p>The following example sets the topic based on whether the message field contains
the specified string:</p>
<div class="pre_wrapper lang-yaml">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-yaml">output.kafka:
hosts: ["localhost:9092"]
topic: "logs-%{[agent.version]}"
topics:
- topic: "critical-%{[agent.version]}"
when.contains:
message: "CRITICAL"
- topic: "error-%{[agent.version]}"
when.contains:
message: "ERR"</pre>
</div>
<p>This configuration results in topics named <code class="literal">critical-7.9.3</code>,
<code class="literal">error-7.9.3</code>, and <code class="literal">logs-7.9.3</code>.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_key"></a><code class="literal">key</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Optional formatted string specifying the Kafka event key. If configured, the
event key can be extracted from the event using a format string.</p>
<p>See the Kafka documentation for the implications of a particular choice of key;
by default, the key is chosen by the Kafka cluster.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_partition"></a><code class="literal">partition</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka output broker event partitioning strategy. Must be one of <code class="literal">random</code>,
<code class="literal">round_robin</code>, or <code class="literal">hash</code>. By default the <code class="literal">hash</code> partitioner is used.</p>
<p><span class="strong strong"><strong><code class="literal">random.group_events</code></strong></span>: Sets the number of events to be published to the same
partition, before the partitioner selects a new partition by random. The
default value is 1 meaning after each event a new partition is picked randomly.</p>
<p><span class="strong strong"><strong><code class="literal">round_robin.group_events</code></strong></span>: Sets the number of events to be published to the
same partition, before the partitioner selects the next partition. The default
value is 1 meaning after each event the next partition will be selected.</p>
<p><span class="strong strong"><strong><code class="literal">hash.hash</code></strong></span>: List of fields used to compute the partitioning hash value from.
If no field is configured, the events <code class="literal">key</code> value will be used.</p>
<p><span class="strong strong"><strong><code class="literal">hash.random</code></strong></span>: Randomly distribute events if no hash or key value can be computed.</p>
<p>All partitioners will try to publish events to all partitions by default. If a
partition’s leader becomes unreachable for the beat, the output might block. All
partitioners support setting <code class="literal">reachable_only</code> to overwrite this
behavior. If <code class="literal">reachable_only</code> is set to <code class="literal">true</code>, events will be published to
available partitions only.</p>
<div class="note admon">
<div class="icon"></div>
<div class="admon_content">
<p>Publishing to a subset of available partitions potentially increases resource usage because events may become unevenly distributed.</p>
</div>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_client_id"></a><code class="literal">client_id</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The configurable ClientID used for logging, debugging, and auditing purposes. The default is "beats".</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_worker_3"></a><code class="literal">worker</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The number of concurrent load-balanced Kafka output workers.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_codec"></a><code class="literal">codec</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Output codec configuration. If the <code class="literal">codec</code> section is missing, events will be json encoded.</p>
<p>See <a class="xref" href="configuration-output-codec.html" title="Change the output codec">Change the output codec</a> for more information.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_metadata"></a><code class="literal">metadata</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Kafka metadata update settings. The metadata do contain information about
brokers, topics, partition, and active leaders to use for publishing.</p>
<div class="variablelist">
<dl class="variablelist">
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">refresh_frequency</code></strong></span>
</span>
</dt>
<dd>
Metadata refresh interval. Defaults to 10 minutes.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">full</code></strong></span>
</span>
</dt>
<dd>
Strategy to use when fetching metadata, when this option is <code class="literal">true</code>, the client will maintain
a full set of metadata for all the available topics, if the this option is set to <code class="literal">false</code> it will only refresh the
metadata for the configured topics. The default is false.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">retry.max</code></strong></span>
</span>
</dt>
<dd>
Total number of metadata update retries when cluster is in middle of leader election. The default is 3.
</dd>
<dt>
<span class="term">
<span class="strong strong"><strong><code class="literal">retry.backoff</code></strong></span>
</span>
</dt>
<dd>
Waiting time between retries during leader elections. Default is 250ms.
</dd>
</dl>
</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_max_retries_3"></a><code class="literal">max_retries</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The number of times to retry publishing an event after a publishing failure.
After the specified number of retries, the events are typically dropped.</p>
<p>Set <code class="literal">max_retries</code> to a value less than 0 to retry until all events are published.</p>
<p>The default is 3.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_bulk_max_size_3"></a><code class="literal">bulk_max_size</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum number of events to bulk in a single Kafka request. The default is 2048.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_bulk_flush_frequency"></a><code class="literal">bulk_flush_frequency</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Duration to wait before sending bulk Kafka request. 0 is no delay. The default is 0.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_timeout_3"></a><code class="literal">timeout</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The number of seconds to wait for responses from the Kafka brokers before timing
out. The default is 30 (seconds).</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_broker_timeout"></a><code class="literal">broker_timeout</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum duration a broker will wait for number of required ACKs. The default is 10s.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_channel_buffer_size"></a><code class="literal">channel_buffer_size</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Per Kafka broker number of messages buffered in output pipeline. The default is 256.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_keep_alive"></a><code class="literal">keep_alive</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The keep-alive period for an active network connection. If 0s, keep-alives are disabled. The default is 0 seconds.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_compression"></a><code class="literal">compression</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Sets the output compression codec. Must be one of <code class="literal">none</code>, <code class="literal">snappy</code>, <code class="literal">lz4</code> and <code class="literal">gzip</code>. The default is <code class="literal">gzip</code>.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_compression_level_3"></a><code class="literal">compression_level</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Sets the compression level used by gzip. Setting this value to 0 disables compression.
The compression level must be in the range of 1 (best speed) to 9 (best compression).</p>
<p>Increasing the compression level will reduce the network usage but will increase the cpu usage.</p>
<p>The default value is 4.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="kafka-max_message_bytes"></a><code class="literal">max_message_bytes</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The maximum permitted size of JSON-encoded messages. Bigger messages will be dropped. The default value is 1000000 (bytes). This value should be equal to or less than the broker’s <code class="literal">message.max.bytes</code>.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_required_acks"></a><code class="literal">required_acks</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>The ACK reliability level required from broker. 0=no response, 1=wait for local commit, -1=wait for all replicas to commit. The default is 1.</p>
<p>Note: If set to 0, no ACKs are returned by Kafka. Messages might be lost silently on error.</p>
</div>
<div class="section">
<div class="titlepage"><div><div>
<div class="position-relative"><h4 class="title"><a id="_ssl_3"></a><code class="literal">ssl</code></h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.9/docs/copied-from-beats/outputs/kafka/docs/kafka.asciidoc">edit</a></div>
</div></div></div>
<p>Configuration options for SSL parameters like the root CA for Kafka connections.
The Kafka host keystore should be created with the
<code class="literal">-keyalg RSA</code> argument to ensure it uses a cipher supported by
<a href="https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions#why-cant-sarama-connect-to-my-kafka-cluster-using-ssl" class="ulink" target="_top">Filebeat’s Kafka library</a>.
See <a class="xref" href="configuration-ssl.html" title="SSL output settings">SSL output settings</a> for more information.</p>
</div>
</div>
</div>
</div>
</div><div class="navfooter">
<span class="prev">
<a href="logstash-output.html">« Configure the Logstash output</a>
</span>
<span class="next">
<a href="redis-output.html">Configure the Redis output »</a>
</span>
</div>
<!-- end body -->
</div>
<div class="col-12 order-3 col-lg-2 order-lg-3 h-almost-full-lg sticky-top-lg" id="right_col">
<div id="sticky_content">
<!-- The OTP is appended here -->
<div class="row">
<div class="col-0 col-md-4 col-lg-0" id="bottom_left_col"></div>
<div class="col-12 col-md-8 col-lg-12">
<div id="rtpcontainer">
<div class="mktg-promo" id="most-popular">
<p class="aside-heading">Most Popular</p>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/getting-started-elasticsearch?page=docs&placement=top-video">
<p class="mb-0">Get Started with Elasticsearch</p>
</a>
</div>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/getting-started-kibana?page=docs&placement=top-video">
<p class="mb-0">Intro to Kibana</p>
</a>
</div>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/introduction-elk-stack?page=docs&placement=top-video">
<p class="mb-0">ELK for Logs & Metrics</p>
</a>
</div>
</div>
</div>
<!-- Feedback widget -->
<div id="feedbackWidgetContainer"></div>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
</div>
<div id='elastic-footer'></div>
<script src='https://www.elastic.co/elastic-footer.js'></script>
<!-- Footer Section end-->
</section>
</div>
<!-- Feedback modal -->
<div id="feedbackModalContainer"></div>
<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs-v1.js"></script>
<script type="text/javascript">
window.initial_state = {}</script>
</body>
</html>