forked from elastic/built-docs
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathpipeline.html
369 lines (351 loc) · 21.9 KB
/
pipeline.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
<!DOCTYPE html>
<html lang="en-us">
<head>
<meta charset="UTF-8">
<title>Logstash Processing Pipeline | Logstash Reference [2.0] | Elastic</title>
<meta class="elastic" name="content" content="Logstash Processing Pipeline | Logstash Reference [2.0]">
<link rel="home" href="index.html" title="Logstash Reference [2.0]"/>
<link rel="up" href="getting-started-with-logstash.html" title="Getting Started with Logstash"/>
<link rel="prev" href="advanced-pipeline.html" title="Setting Up an Advanced Logstash Pipeline"/>
<link rel="next" href="package-repositories.html" title="Package Repositories"/>
<meta class="elastic" name="product_version" content="2.0"/>
<meta class="elastic" name="product_name" content="Logstash"/>
<meta class="elastic" name="website_area" content="documentation"/>
<meta name="DC.type" content="Learn/Docs/Logstash/Reference/2.0"/>
<meta name="DC.subject" content="Logstash"/>
<meta name="DC.identifier" content="2.0"/>
<meta name="robots" content="noindex,nofollow"/>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdn.optimizely.com/js/18132920325.js"></script>
<link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
<link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
<link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
<link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
<link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
<link rel="manifest" href="/manifest.json">
<meta name="apple-mobile-web-app-title" content="Elastic">
<meta name="application-name" content="Elastic">
<meta name="msapplication-TileColor" content="#ffffff">
<meta name="msapplication-TileImage" content="/mstile-144x144.png">
<meta name="theme-color" content="#ffffff">
<meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd" />
<meta name="yandex-verification" content="d8a47e95d0972434" />
<meta name="localized" content="true" />
<meta name="st:robots" content="follow,index" />
<meta property="og:image" content="https://static-www.elastic.co/v3/assets/bltefdd0b53724fa2ce/blt280217a63b82a734/6202d3378b1f312528798412/elastic-logo.svg" />
<meta property="og:image:width" content="500" />
<meta property="og:image:height" content="172" />
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
<link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
<link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
<!-- Give IE8 a fighting chance -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->
<link rel="stylesheet" type="text/css" href="/guide/static/styles.css" />
</head>
<!--© 2015-2022 Elasticsearch B.V. -->
<!-- All Elastic documentation is licensed under a Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License. -->
<!-- http://creativecommons.org/licenses/by-nc-nd/4.0/ -->
<body>
<!-- Google Tag Manager -->
<script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
<!-- End Google Tag Manager -->
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-12395217-16');
</script>
<!-- Google Tag Manager for GA4 -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-KNJMG2M');</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KNJMG2M" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager for GA4-->
<div id='elastic-nav' style="display:none;"></div>
<script src='https://www.elastic.co/elastic-nav.js'></script>
<div class="main-container">
<section id="content" >
<div class="content-wrapper">
<section id="guide" lang="en">
<div class="container-fluid">
<div class="row pb-3">
<div class="col-12 order-2 col-md-4 order-md-1 col-lg-3 h-almost-full-md sticky-top-md" id="left_col">
<!-- The TOC is appended here -->
</div>
<div class="col-12 order-1 col-md-8 order-md-2 col-lg-7 order-lg-2 guide-section" id="middle_col">
<!-- start body -->
<div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="/guide/">Elastic Docs</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="index.html">Logstash Reference [2.0]</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="logstash-reference.html">Logstash Reference</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="getting-started-with-logstash.html">Getting Started with Logstash</a></span>
</div>
<div class="navheader">
<span class="prev">
<a href="advanced-pipeline.html">« Setting Up an Advanced Logstash Pipeline</a>
</span>
<span class="next">
<a href="package-repositories.html">Package Repositories »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title"><a id="pipeline"></a>Logstash Processing Pipeline<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/master/docs/asciidoc/static/life-of-an-event.asciidoc">edit</a></h2>
</div></div></div>
<p>The Logstash event processing pipeline has three stages: inputs → filters →
outputs. Inputs generate events, filters modify them, and outputs ship them
elsewhere. Inputs and outputs support codecs that enable you to encode or decode
the data as it enters or exits the pipeline without having to use a separate
filter.</p>
<h4><a id="_inputs"></a>Inputs<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/master/docs/asciidoc/static/life-of-an-event.asciidoc">edit</a></h4>
<p>You use inputs to get data into Logstash. Some of the more commonly-used inputs
are:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<span class="strong strong"><strong>file</strong></span>: reads from a file on the filesystem, much like the UNIX command
<code class="literal">tail -0F</code>
</li>
<li class="listitem">
<span class="strong strong"><strong>syslog</strong></span>: listens on the well-known port 514 for syslog messages and parses
according to the RFC3164 format
</li>
<li class="listitem">
<span class="strong strong"><strong>redis</strong></span>: reads from a redis server, using both redis channels and redis lists.
Redis is often used as a "broker" in a centralized Logstash installation, which
queues Logstash events from remote Logstash "shippers".
</li>
<li class="listitem">
<span class="strong strong"><strong>beats</strong></span>: processes events sent by <a href="/downloads/beats/filebeat" class="ulink" target="_top">Filebeat</a>.
</li>
</ul>
</div>
<p>For more information about the available inputs, see
<a class="xref" href="input-plugins.html" title="Input plugins">Input Plugins</a>.</p>
<h4><a id="_filters"></a>Filters<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/master/docs/asciidoc/static/life-of-an-event.asciidoc">edit</a></h4>
<p>Filters are intermediary processing devices in the Logstash pipeline. You can
combine filters with conditionals to perform an action on an event if it meets
certain criteria. Some useful filters include:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<span class="strong strong"><strong>grok</strong></span>: parse and structure arbitrary text. Grok is currently the best way in
Logstash to parse unstructured log data into something structured and queryable.
With 120 patterns built-in to Logstash, it’s more than likely you’ll find one
that meets your needs!
</li>
<li class="listitem">
<span class="strong strong"><strong>mutate</strong></span>: perform general transformations on event fields. You can rename,
remove, replace, and modify fields in your events.
</li>
<li class="listitem">
<span class="strong strong"><strong>drop</strong></span>: drop an event completely, for example, <em>debug</em> events.
</li>
<li class="listitem">
<span class="strong strong"><strong>clone</strong></span>: make a copy of an event, possibly adding or removing fields.
</li>
<li class="listitem">
<span class="strong strong"><strong>geoip</strong></span>: add information about geographical location of IP addresses (also
displays amazing charts in Kibana!)
</li>
</ul>
</div>
<p>For more information about the available filters, see
<a class="xref" href="filter-plugins.html" title="Filter plugins">Filter Plugins</a>.</p>
<h4><a id="_outputs"></a>Outputs<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/master/docs/asciidoc/static/life-of-an-event.asciidoc">edit</a></h4>
<p>Outputs are the final phase of the Logstash pipeline. An event can pass through
multiple outputs, but once all output processing is complete, the event has
finished its execution. Some commonly used outputs include:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<span class="strong strong"><strong>elasticsearch</strong></span>: send event data to Elasticsearch. If you’re planning to save
your data in an efficient, convenient, and easily queryable format…​
Elasticsearch is the way to go. Period. Yes, we’re biased :)
</li>
<li class="listitem">
<span class="strong strong"><strong>file</strong></span>: write event data to a file on disk.
</li>
<li class="listitem">
<span class="strong strong"><strong>graphite</strong></span>: send event data to graphite, a popular open source tool for
storing and graphing metrics. <a href="http://graphite.wikidot.com/" class="ulink" target="_top">http://graphite.wikidot.com/</a>
</li>
<li class="listitem">
<span class="strong strong"><strong>statsd</strong></span>: send event data to statsd, a service that "listens for statistics,
like counters and timers, sent over UDP and sends aggregates to one or more
pluggable backend services". If you’re already using statsd, this could be
useful for you!
</li>
</ul>
</div>
<p>For more information about the available outputs, see
<a class="xref" href="output-plugins.html" title="Output plugins">Output Plugins</a>.</p>
<h4><a id="_codecs"></a>Codecs<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/master/docs/asciidoc/static/life-of-an-event.asciidoc">edit</a></h4>
<p>Codecs are basically stream filters that can operate as part of an input or
output. Codecs enable you to easily separate the transport of your messages from
the serialization process. Popular codecs include <code class="literal">json</code>, <code class="literal">msgpack</code>, and <code class="literal">plain</code>
(text).</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<span class="strong strong"><strong>json</strong></span>: encode or decode data in the JSON format.
</li>
<li class="listitem">
<span class="strong strong"><strong>multiline</strong></span>: merge multiple-line text events such as java exception and
stacktrace messages into a single event.
</li>
</ul>
</div>
<p>For more information about the available codecs, see
<a class="xref" href="codec-plugins.html" title="Codec plugins">Codec Plugins</a>.</p>
<h3><a id="_fault_tolerance"></a>Fault Tolerance<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/master/docs/asciidoc/static/life-of-an-event.asciidoc">edit</a></h3>
<p>Events are passed from stage to stage using internal queues implemented with a
Ruby <code class="literal">SizedQueue</code>. A <code class="literal">SizedQueue</code> has a maximum number of items it can contain.
When the queue is at maximum capacity, all writes to the queue are blocked.</p>
<p>Logstash sets the size of each queue to 20. This means a maximum of 20 events
can be pending for the next stage, which helps prevent data loss and keeps
Logstash from acting as a data storage system. These internal queues are not
intended for storing messages long-term.</p>
<p>The small queue sizes mean that Logstash simply blocks and stalls safely when
there’s a heavy load or temporary pipeline problems. The alternatives would be
to either have an unlimited queue or drop messages when there’s a problem. An
unlimited queue can grow unbounded and eventually exceed memory, causing a crash
that loses all of the queued messages. In most cases, dropping messages outright
is equally undesirable.</p>
<p>An output can fail or have problems due to downstream issues, such as a full
disk, permissions problems, temporary network failures, or service outages. Most
outputs keep retrying to ship events affected by the failure.</p>
<p>If an output is failing, the output thread waits until the output is able to
successfully send the message. The output stops reading from the output queue,
which means the queue can fill up with events.</p>
<p>When the output queue is full, filters are blocked because they cannot write new
events to the output queue. While they are blocked from writing to the output
queue, filters stop reading from the filter queue. Eventually, this can cause
the filter queue (input → filter) to fill up.</p>
<p>A full filter queue blocks inputs from writing to the filters. This causes all
inputs to stop processing data from wherever they’re getting new events.</p>
<p>In ideal circumstances, this behaves similarly to when the tcp window closes to
0. No new data is sent because the receiver hasn’t finished processing the
current queue of data, but as soon as the downstream (output) problem is
resolved, messages start flowing again.</p>
<h3><a id="_thread_model"></a>Thread Model<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/master/docs/asciidoc/static/life-of-an-event.asciidoc">edit</a></h3>
<p>The thread model in Logstash is currently:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">input threads | filter worker threads | output worker</pre>
</div>
<p>Filters are optional, so if you have no filters defined it is simply:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">input threads | output worker</pre>
</div>
<p>Each input runs in a thread by itself. This prevents busier inputs from being
blocked by slower ones. It also allows for easier containment of scope because
each input has a thread.</p>
<p>The filter thread model is a <em>worker</em> model where each worker receives an event
and applies all filters, in order, before sending it on to the output queue.
This allows scalability across CPUs because many filters are CPU intensive
(permitting that we have thread safety).</p>
<p>The default number of filter workers is 1, but you can increase this number by
specifying the <em>-w</em> flag when you run the Logstash agent.</p>
<p>The output worker model is currently a single thread. Outputs receive events in
the order the outputs are defined in the config file.</p>
<p>Outputs might decide to temporarily buffer events before publishing them. One
example of this is the <code class="literal">elasticsearch</code> output, which buffers events and flushes
them all at once using a separate thread. This mechanism (buffering many events
and writing in a separate thread) can improve performance because it prevents
the Logstash pipeline from being stalled waiting for a response from
elasticsearch.</p>
<h3><a id="_resource_usage"></a>Resource Usage<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/master/docs/asciidoc/static/life-of-an-event.asciidoc">edit</a></h3>
<p>Logstash typically has at least 3 threads (2 if you have no filters). One input
thread, one filter worker thread, and one output thread. If you see Logstash
using multiple CPUs, this is likely why. If you want to know more about what
each thread is doing, you should read this article:
<a href="http://www.semicomplete.com/blog/geekery/debugging-java-performance.html" class="ulink" target="_top">Debugging Java Performance</a>.
Threads in Java have names and you can use <code class="literal">jstack</code> and <code class="literal">top</code> to figure out who
is using what resources.</p>
<p>On Linux platforms, Logstash labels all the threads it can with something
descriptive. For example, inputs show up as <code class="literal"><inputname</code>, filter workers show up
as <code class="literal">|worker</code>, and outputs show up as <code class="literal">>outputworker</code>. Where possible, other
threads are also labeled to help you identify their purpose should you wonder
why they are consuming resources!</p>
</div>
<div class="navfooter">
<span class="prev">
<a href="advanced-pipeline.html">« Setting Up an Advanced Logstash Pipeline</a>
</span>
<span class="next">
<a href="package-repositories.html">Package Repositories »</a>
</span>
</div>
</div>
<!-- end body -->
</div>
<div class="col-12 order-3 col-lg-2 order-lg-3 h-almost-full-lg sticky-top-lg" id="right_col">
<div id="sticky_content">
<!-- The OTP is appended here -->
<div class="row">
<div class="col-0 col-md-4 col-lg-0" id="bottom_left_col"></div>
<div class="col-12 col-md-8 col-lg-12">
<div id="rtpcontainer">
<div class="mktg-promo" id="most-popular">
<p class="aside-heading">Most Popular</p>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&elektra=docs&storm=top-video">
<p class="mb-0">Get Started with Elasticsearch</p>
</a>
</div>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&elektra=docs&storm=top-video">
<p class="mb-0">Intro to Kibana</p>
</a>
</div>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&elektra=docs&storm=top-video">
<p class="mb-0">ELK for Logs & Metrics</p>
</a>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
</div>
<div id='elastic-footer'></div>
<script src='https://www.elastic.co/elastic-footer.js'></script>
<!-- Footer Section end-->
</section>
</div>
<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
window.initial_state = {}</script>
</body>
</html>