forked from elastic/built-docs
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmultiple-input-output-plugins.html
361 lines (343 loc) · 20.9 KB
/
multiple-input-output-plugins.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
<!DOCTYPE html>
<html lang="en-us">
<head>
<meta charset="UTF-8">
<title>Stitching Together Multiple Input and Output Plugins | Logstash Reference [8.0] | Elastic</title>
<meta class="elastic" name="content" content="Stitching Together Multiple Input and Output Plugins | Logstash Reference [8.0]">
<link rel="home" href="index.html" title="Logstash Reference [8.0]"/>
<link rel="up" href="getting-started-with-logstash.html" title="Getting Started with Logstash"/>
<link rel="prev" href="advanced-pipeline.html" title="Parsing Logs with Logstash"/>
<link rel="next" href="pipeline.html" title="How Logstash Works"/>
<meta class="elastic" name="product_version" content="8.0"/>
<meta class="elastic" name="product_name" content="Logstash"/>
<meta class="elastic" name="website_area" content="documentation"/>
<meta name="DC.type" content="Learn/Docs/Logstash/Reference/8.0"/>
<meta name="DC.subject" content="Logstash"/>
<meta name="DC.identifier" content="8.0"/>
<meta name="robots" content="noindex,nofollow"/>
<meta http-equiv="content-type" content="text/html; charset=utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="https://cdn.optimizely.com/js/18132920325.js"></script>
<link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
<link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
<link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
<link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
<link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
<link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
<link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
<link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
<link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
<link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
<link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
<link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
<link rel="manifest" href="/manifest.json">
<meta name="apple-mobile-web-app-title" content="Elastic">
<meta name="application-name" content="Elastic">
<meta name="msapplication-TileColor" content="#ffffff">
<meta name="msapplication-TileImage" content="/mstile-144x144.png">
<meta name="theme-color" content="#ffffff">
<meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd" />
<meta name="yandex-verification" content="d8a47e95d0972434" />
<meta name="localized" content="true" />
<meta name="st:robots" content="follow,index" />
<meta property="og:image" content="https://static-www.elastic.co/v3/assets/bltefdd0b53724fa2ce/blt280217a63b82a734/6202d3378b1f312528798412/elastic-logo.svg" />
<meta property="og:image:width" content="500" />
<meta property="og:image:height" content="172" />
<link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
<link rel="icon" href="/favicon.ico" type="image/x-icon">
<link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
<link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
<link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
<!-- Give IE8 a fighting chance -->
<!--[if lt IE 9]>
<script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
<script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
<![endif]-->
<link rel="stylesheet" type="text/css" href="/guide/static/styles.css" />
</head>
<!--© 2015-2022 Elasticsearch B.V. -->
<!-- All Elastic documentation is licensed under a Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License. -->
<!-- http://creativecommons.org/licenses/by-nc-nd/4.0/ -->
<body>
<!-- Google Tag Manager -->
<script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
<!-- End Google Tag Manager -->
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-12395217-16');
</script>
<!-- Google Tag Manager for GA4 -->
<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-KNJMG2M');</script>
<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KNJMG2M" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
<!-- End Google Tag Manager for GA4-->
<div id='elastic-nav' style="display:none;"></div>
<script src='https://www.elastic.co/elastic-nav.js'></script>
<div class="main-container">
<section id="content" >
<div class="content-wrapper">
<section id="guide" lang="en">
<div class="container-fluid">
<div class="row pb-3">
<div class="col-12 order-2 col-md-4 order-md-1 col-lg-3 h-almost-full-md sticky-top-md" id="left_col">
<!-- The TOC is appended here -->
</div>
<div class="col-12 order-1 col-md-8 order-md-2 col-lg-7 order-lg-2 guide-section" id="middle_col">
<!-- start body -->
<div class="page_header">
<strong>IMPORTANT</strong>: No additional bug fixes or documentation updates
will be released for this version. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="/guide/">Elastic Docs</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="index.html">Logstash Reference [8.0]</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="getting-started-with-logstash.html">Getting Started with Logstash</a></span>
</div>
<div class="navheader">
<span class="prev">
<a href="advanced-pipeline.html">« Parsing Logs with Logstash</a>
</span>
<span class="next">
<a href="pipeline.html">How Logstash Works »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title"><a id="multiple-input-output-plugins"></a>Stitching Together Multiple Input and Output Plugins<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/8.0/docs/static/advanced-pipeline.asciidoc">edit</a></h2>
</div></div></div>
<p>The information you need to manage often comes from several disparate sources, and use cases can require multiple
destinations for your data. Your Logstash pipeline can use multiple input and output plugins to handle these
requirements.</p>
<p>In this section, you create a Logstash pipeline that takes input from a Twitter feed and the Filebeat client, then
sends the information to an Elasticsearch cluster as well as writing the information directly to a file.</p>
<h4><a id="twitter-configuration"></a>Reading from a Twitter Feed<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/8.0/docs/static/advanced-pipeline.asciidoc">edit</a></h4>
<p>To add a Twitter feed, you use the <a href="/guide/en/logstash/8.0/plugins-inputs-twitter.html" class="ulink" target="_top"><code class="literal">twitter</code></a> input plugin. To
configure the plugin, you need several pieces of information:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
A <em>consumer key</em>, which uniquely identifies your Twitter app.
</li>
<li class="listitem">
A <em>consumer secret</em>, which serves as the password for your Twitter app.
</li>
<li class="listitem">
One or more <em>keywords</em> to search in the incoming feed. The example shows using "cloud" as a keyword, but you can use whatever you want.
</li>
<li class="listitem">
An <em>oauth token</em>, which identifies the Twitter account using this app.
</li>
<li class="listitem">
An <em>oauth token secret</em>, which serves as the password of the Twitter account.
</li>
</ul>
</div>
<p>Visit <a href="https://dev.twitter.com/apps" class="ulink" target="_top">https://dev.twitter.com/apps</a> to set up a Twitter account and generate your consumer
key and secret, as well as your access token and secret. See the docs for the <a href="/guide/en/logstash/8.0/plugins-inputs-twitter.html" class="ulink" target="_top"><code class="literal">twitter</code></a> input plugin if you’re not sure how to generate these keys.</p>
<p>Like you did earlier when you worked on <a class="xref" href="advanced-pipeline.html" title="Parsing Logs with Logstash">Parsing Logs with Logstash</a>, create a config file (called <code class="literal">second-pipeline.conf</code>) that
contains the skeleton of a configuration pipeline. If you want, you can reuse the file you created earlier, but make
sure you pass in the correct config file name when you run Logstash.</p>
<p>Add the following lines to the <code class="literal">input</code> section of the <code class="literal">second-pipeline.conf</code> file, substituting your values for the
placeholder values shown here:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json"> twitter {
consumer_key => "enter_your_consumer_key_here"
consumer_secret => "enter_your_secret_here"
keywords => ["cloud"]
oauth_token => "enter_your_access_token_here"
oauth_token_secret => "enter_your_access_token_secret_here"
}</pre>
</div>
<h4><a id="configuring-lsf"></a>Configuring Filebeat to Send Log Lines to Logstash<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/8.0/docs/static/advanced-pipeline.asciidoc">edit</a></h4>
<p>As you learned earlier in <a class="xref" href="advanced-pipeline.html#configuring-filebeat" title="Configuring Filebeat to Send Log Lines to Logstash">Configuring Filebeat to Send Log Lines to Logstash</a>, the <a href="https://github.com/elastic/beats/tree/master/filebeat" class="ulink" target="_top">Filebeat</a>
client is a lightweight, resource-friendly tool that collects logs from files on the server and forwards these logs to your
Logstash instance for processing.</p>
<p>After installing Filebeat, you need to configure it. Open the <code class="literal">filebeat.yml</code> file located in your Filebeat installation
directory, and replace the contents with the following lines. Make sure <code class="literal">paths</code> points to your syslog:</p>
<div class="pre_wrapper lang-shell">
<pre class="programlisting prettyprint lang-shell">filebeat.inputs:
- type: log
paths:
- /var/log/*.log <a id="CO2-1"></a><i class="conum" data-value="1"></i>
fields:
type: syslog <a id="CO2-2"></a><i class="conum" data-value="2"></i>
output.logstash:
hosts: ["localhost:5044"]</pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO2-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Absolute path to the file or files that Filebeat processes.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO2-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>Adds a field called <code class="literal">type</code> with the value <code class="literal">syslog</code> to the event.</p>
</td>
</tr>
</table>
</div>
<p>Save your changes.</p>
<p>To keep the configuration simple, you won’t specify TLS/SSL settings as you would in a real world
scenario.</p>
<p>Configure your Logstash instance to use the Filebeat input plugin by adding the following lines to the <code class="literal">input</code> section
of the <code class="literal">second-pipeline.conf</code> file:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json"> beats {
port => "5044"
}</pre>
</div>
<h4><a id="logstash-file-output"></a>Writing Logstash Data to a File<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/8.0/docs/static/advanced-pipeline.asciidoc">edit</a></h4>
<p>You can configure your Logstash pipeline to write data directly to a file with the
<a href="/guide/en/logstash/8.0/plugins-outputs-file.html" class="ulink" target="_top"><code class="literal">file</code></a> output plugin.</p>
<p>Configure your Logstash instance to use the <code class="literal">file</code> output plugin by adding the following lines to the <code class="literal">output</code> section
of the <code class="literal">second-pipeline.conf</code> file:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json"> file {
path => "/path/to/target/file"
}</pre>
</div>
<h4><a id="multiple-es-nodes"></a>Writing to Multiple Elasticsearch Nodes<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/8.0/docs/static/advanced-pipeline.asciidoc">edit</a></h4>
<p>Writing to multiple Elasticsearch nodes lightens the resource demands on a given Elasticsearch node, as well as
providing redundant points of entry into the cluster when a particular node is unavailable.</p>
<p>To configure your Logstash instance to write to multiple Elasticsearch nodes, edit the <code class="literal">output</code> section of the <code class="literal">second-pipeline.conf</code> file to read:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">output {
elasticsearch {
hosts => ["IP Address 1:port1", "IP Address 2:port2", "IP Address 3"]
}
}</pre>
</div>
<p>Use the IP addresses of three non-master nodes in your Elasticsearch cluster in the host line. When the <code class="literal">hosts</code>
parameter lists multiple IP addresses, Logstash load-balances requests across the list of addresses. Also note that
the default port for Elasticsearch is <code class="literal">9200</code> and can be omitted in the configuration above.</p>
<h5><a id="testing-second-pipeline"></a>Testing the Pipeline<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/logstash/edit/8.0/docs/static/advanced-pipeline.asciidoc">edit</a></h5>
<p>At this point, your <code class="literal">second-pipeline.conf</code> file looks like this:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">input {
twitter {
consumer_key => "enter_your_consumer_key_here"
consumer_secret => "enter_your_secret_here"
keywords => ["cloud"]
oauth_token => "enter_your_access_token_here"
oauth_token_secret => "enter_your_access_token_secret_here"
}
beats {
port => "5044"
}
}
output {
elasticsearch {
hosts => ["IP Address 1:port1", "IP Address 2:port2", "IP Address 3"]
}
file {
path => "/path/to/target/file"
}
}</pre>
</div>
<p>Logstash is consuming data from the Twitter feed you configured, receiving data from Filebeat, and
indexing this information to three nodes in an Elasticsearch cluster as well as writing to a file.</p>
<p>At the data source machine, run Filebeat with the following command:</p>
<div class="pre_wrapper lang-shell">
<pre class="programlisting prettyprint lang-shell">sudo ./filebeat -e -c filebeat.yml -d "publish"</pre>
</div>
<p>Filebeat will attempt to connect on port 5044. Until Logstash starts with an active Beats plugin, there
won’t be any answer on that port, so any messages you see regarding failure to connect on that port are normal for now.</p>
<p>To verify your configuration, run the following command:</p>
<div class="pre_wrapper lang-shell">
<pre class="programlisting prettyprint lang-shell">bin/logstash -f second-pipeline.conf --config.test_and_exit</pre>
</div>
<p>The <code class="literal">--config.test_and_exit</code> option parses your configuration file and reports any errors. When the configuration file
passes the configuration test, start Logstash with the following command:</p>
<div class="pre_wrapper lang-shell">
<pre class="programlisting prettyprint lang-shell">bin/logstash -f second-pipeline.conf</pre>
</div>
<p>Use the <code class="literal">grep</code> utility to search in the target file to verify that information is present:</p>
<div class="pre_wrapper lang-shell">
<pre class="programlisting prettyprint lang-shell">grep syslog /path/to/target/file</pre>
</div>
<p>Run an Elasticsearch query to find the same information in the Elasticsearch cluster:</p>
<div class="pre_wrapper lang-shell">
<pre class="programlisting prettyprint lang-shell">curl -XGET 'localhost:9200/logstash-$DATE/_search?pretty&q=fields.type:syslog'</pre>
</div>
<p>Replace $DATE with the current date, in YYYY.MM.DD format.</p>
<p>To see data from the Twitter feed, try this query:</p>
<div class="pre_wrapper lang-shell">
<pre class="programlisting prettyprint lang-shell">curl -XGET 'http://localhost:9200/logstash-$DATE/_search?pretty&q=client:iphone'</pre>
</div>
<p>Again, remember to replace $DATE with the current date, in YYYY.MM.DD format.</p>
</div>
<div class="navfooter">
<span class="prev">
<a href="advanced-pipeline.html">« Parsing Logs with Logstash</a>
</span>
<span class="next">
<a href="pipeline.html">How Logstash Works »</a>
</span>
</div>
</div>
<!-- end body -->
</div>
<div class="col-12 order-3 col-lg-2 order-lg-3 h-almost-full-lg sticky-top-lg" id="right_col">
<div id="sticky_content">
<!-- The OTP is appended here -->
<div class="row">
<div class="col-0 col-md-4 col-lg-0" id="bottom_left_col"></div>
<div class="col-12 col-md-8 col-lg-12">
<div id="rtpcontainer">
<div class="mktg-promo" id="most-popular">
<p class="aside-heading">Most Popular</p>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&elektra=docs&storm=top-video">
<p class="mb-0">Get Started with Elasticsearch</p>
</a>
</div>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&elektra=docs&storm=top-video">
<p class="mb-0">Intro to Kibana</p>
</a>
</div>
<div class="pb-2">
<p class="media-type">Video</p>
<a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&elektra=docs&storm=top-video">
<p class="mb-0">ELK for Logs & Metrics</p>
</a>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</section>
</div>
<div id='elastic-footer'></div>
<script src='https://www.elastic.co/elastic-footer.js'></script>
<!-- Footer Section end-->
</section>
</div>
<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
window.initial_state = {}</script>
</body>
</html>