html/server/7.11/common-problems.html

<!DOCTYPE html>
<html lang="en-us">
  <head>
    
<meta charset="UTF-8">
<title>Common problems | APM Server Reference [7.11] | Elastic</title>
<meta class="elastic" name="content" content="Common problems | APM Server Reference [7.11]">

<link rel="home" href="index.html" title="APM Server Reference [7.11]"/>
<link rel="up" href="troubleshooting.html" title="Troubleshoot"/>
<link rel="prev" href="troubleshooting.html" title="Troubleshoot"/>
<link rel="next" href="enable-apm-server-debugging.html" title="Debug"/>
<meta class="elastic" name="product_version" content="7.11"/>
<meta class="elastic" name="product_name" content="APM"/>
<meta class="elastic" name="website_area" content="documentation"/>
<meta name="DC.type" content="Learn/Docs/APM Server/Reference/7.11"/>
<meta name="DC.subject" content="APM"/>
<meta name="DC.identifier" content="7.11"/>
<meta name="robots" content="noindex,nofollow"/>

    <meta http-equiv="content-type" content="text/html; charset=utf-8" />
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd" />
    <meta name="yandex-verification" content="d8a47e95d0972434" />
    <meta name="localized" content="true" />
    <meta name="st:robots" content="follow,index" />
    <meta property="og:image" content="https://static-www.elastic.co/v3/assets/bltefdd0b53724fa2ce/blt280217a63b82a734/6202d3378b1f312528798412/elastic-logo.svg" />
    <meta property="og:image:width" content="500" />
    <meta property="og:image:height" content="172" />
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles-v1.css" />
  </head>

  <!--© 2015-2025 Elasticsearch B.V. -->
  <!-- All Elastic documentation is licensed under a Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International License. -->
  <!-- http://creativecommons.org/licenses/by-nc-nd/4.0/ -->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!-- Google Tag Manager for GA4 -->
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src='https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-KNJMG2M');</script>
    <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-KNJMG2M" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <!-- End Google Tag Manager for GA4-->

    <div id='elastic-nav' style="display:none;"></div>
    <script src='https://www.elastic.co/elastic-nav.js'></script>

    <div class="main-container">
      <section id="content" >
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container-fluid">
              <div class="row pb-3">
                <div class="col-12 order-2 col-md-4 order-md-1 col-lg-3 h-almost-full-md sticky-top-md" id="left_col">
                  <!-- The TOC is appended here -->
                </div>

                <div class="col-12 order-1 col-md-8 order-md-2 col-lg-7 order-lg-2 guide-section" id="middle_col">
                  <!-- start body -->
                  
<div class="page_header">
A newer version is available. For the latest information, see the
<a href="../current/index.html">current release documentation</a>.
</div>
<div class="navheader">
<span class="prev">
<a href="troubleshooting.html">« Troubleshoot</a>
</span>
<span class="next">
<a href="enable-apm-server-debugging.html">Debug »</a>
</span>
</div>
<div class="book" lang="en">
<div class="titlepage">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="/guide/">Elastic Docs</a></span>
<span class="chevron-right">›</span><span class="breadcrumb-link">
  <div id="related-products" class="dropdown">
  <div class="related-products-title">APM:</div>
  <div class="dropdown-anchor" tabindex="0">Server Reference<span class="dropdown-icon"></span></div>
  <div class="dropdown-content">
  <ul>
  <li class="dropdown-category">APM</li>
  <ul>
  <li><a href="/guide/en/observability/current/apm.html">Observability › APM</a></li>
  </ul>
  <li class="dropdown-category">APM agents</li>
  <ul>
  <li><a href="/guide/en/apm/agent/android/current/intro.html">Android Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/go/current/introduction.html">Go Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/swift/current/intro.html">iOS Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/java/current/intro.html">Java Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/dotnet/current/intro.html">.NET Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/nodejs/current/intro.html">Node.js Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/php/current/intro.html">PHP Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/python/current/getting-started.html">Python Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/ruby/current/introduction.html">Ruby Agent Reference</a></li>
  <li><a href="/guide/en/apm/agent/rum-js/current/intro.html">Real User Monitoring JavaScript Agent Reference</a></li>
  </ul>
  <li class="dropdown-category">APM extensions</li>
  <ul>
  <li><a href="/guide/en/apm/lambda/current/aws-lambda-arch.html">AWS Lambda extension</a></li>
  <li><a href="/guide/en/apm/attacher/current/apm-attacher.html">Attacher</a></li>
  </ul>
  </ul>
  </div>
  </div>
<span class="chevron-right">›</span><span class="breadcrumb-link"><a href="troubleshooting.html">Troubleshoot</a></span>
</div>
<div>
<div><h1 class="title"><a id="id-1"></a>Common problems</h1><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
</div>
<!--EXTRA-->
</div>
<div id="content">
<div id="url-to-v3" class="version-warning">
    <strong>IMPORTANT</strong>: This documentation is no longer updated. Refer to <a href="https://www.elastic.co/support/eol">Elastic's version policy</a> and the <a href="https://www.elastic.co/docs">latest documentation</a>.
</div>
<div class="chapter">
<div class="titlepage"><div><div>
<div class="position-relative"><h2 class="title"><a id="common-problems"></a>Common problems</h2><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
</div></div></div>
<p>This section describes common problems you might encounter with APM Server.</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<a class="xref" href="common-problems.html#no-data-indexed" title="No data is indexed">No data is indexed</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#bad-request" title="HTTP 400: Data decoding error / Data validation error">HTTP 400: Data decoding error / Data validation error</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#event-too-large" title="HTTP 400: Event too large">HTTP 400: Event too large</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#unauthorized" title="HTTP 401: Invalid token">HTTP 401: Invalid token</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#forbidden" title="HTTP 403: Forbidden request">HTTP 403: Forbidden request</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#queue-full" title="HTTP 503: Queue is full">HTTP 503: Queue is full</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#request-timed-out" title="HTTP 503: Request timed out waiting to be processed">HTTP 503: Request timed out waiting to be processed</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#ssl-client-fails" title="SSL client fails to connect">SSL client fails to connect</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#field-limit-exceeded" title="Field limit exceeded">Field limit exceeded</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#io-timeout" title="I/O Timeout">I/O Timeout</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#server-es-down" title="What happens when APM Server or Elasticsearch is down?">What happens when APM Server or Elasticsearch is down?</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#server-resource-exists-not-alias" title="resource 'apm-7.11.2-$type' exists, but it is not an alias"><code class="literal">resource 'apm-7.11.2-$type' exists, but it is not an alias</code></a>
</li>
</ul>
</div>
<div class="position-relative"><h3><a id="no-data-indexed"></a>No data is indexed</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>If no data shows up in Elasticsearch, first check that the APM components are properly connected.</p>
<p>To ensure that APM Server configuration is valid and it can connect to the configured output, Elasticsearch by default,
run the following commands:</p>
<div class="pre_wrapper lang-sh">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-sh">apm-server test config
apm-server test output</pre>
</div>
<p>To see if the agent can connect to the APM Server, send requests to the instrumented service and look for lines
containing <code class="literal">[request]</code> in the APM Server logs.</p>
<p>If no requests are logged, it might be that SSL is <a class="xref" href="common-problems.html#ssl-client-fails" title="SSL client fails to connect">misconfigured</a> or that the host is wrong.
Particularly, if you are using Docker, ensure to bind to the right interface (for example, set
<code class="literal">apm-server.host = 0.0.0.0:8200</code> to match any IP) and set the <code class="literal">SERVER_URL</code> setting in the agent accordingly.</p>
<p>If you see requests coming through the APM Server but they are not accepted (response code other than <code class="literal">202</code>), consider
the response code to narrow down the possible causes (see sections below).</p>
<p>Another reason for data not showing up is that the agent is not auto-instrumenting something you were expecting, check
the <a href="/guide/en/apm/agent/index.html" class="ulink" target="_top">agent documentation</a> for details on what is automatically instrumented.</p>
<p>APM Server currently relies on Elasticsearch to create indices that do not exist.
As a result, Elasticsearch must be configured to allow <a href="/guide/en/elasticsearch/reference/7.11/docs-index_.html#index-creation" class="ulink" target="_top">automatic index creation</a> for APM indices.</p>
<div class="position-relative"><h3><a id="bad-request"></a>HTTP 400: Data decoding error / Data validation error</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>The most likely cause for this is that you are using incompatible versions of agent and APM Server.
For instance, APM Server 6.2 and 6.5 changed the Intake API spec and require a minimum version of each agent.</p>
<p>View the <a href="/guide/en/apm/get-started/7.11/agent-server-compatibility.html" class="ulink" target="_top">agent/server compatibility matrix</a> for more information.</p>
<div class="position-relative"><h3><a id="event-too-large"></a>HTTP 400: Event too large</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>APM Agents communicate with the APM server by sending events in an HTTP request. Each event is sent as its own line in the HTTP request body. If events are too large, you should consider increasing the <a class="xref" href="configuration-process.html#max_event_size" title="max_event_size"><code class="literal">max_event_size</code></a>
setting in the APM Server, and adjusting relevant settings in the agent.</p>
<div class="position-relative"><h3><a id="unauthorized"></a>HTTP 401: Invalid token</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>The <a class="xref" href="secret-token.html" title="Secret token">secret token</a> in the request header doesn&#8217;t match the configured in the APM Server.</p>
<div class="position-relative"><h3><a id="forbidden"></a>HTTP 403: Forbidden request</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>Either you are sending requests to a <a class="xref" href="configuration-rum.html" title="Configure Real User Monitoring (RUM)">RUM</a> endpoint without RUM enabled, or a request
is coming from an origin not specified in <code class="literal">apm-server.rum.allow_origins</code>. See the <a class="xref" href="configuration-rum.html" title="Configure Real User Monitoring (RUM)">RUM configuration</a>.</p>
<div class="position-relative"><h3><a id="queue-full"></a>HTTP 503: Queue is full</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>APM Server has an internal queue that helps to:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
Buffer data temporarily if Elasticsearch is intermittently unavailable
</li>
<li class="listitem">
Handle sudden large spikes of data
</li>
<li class="listitem">
Send documents to Elasticsearch in bulk, instead of individually
</li>
</ul>
</div>
<p>When the queue has reached the maximum size,
APM Server returns an HTTP 503 status with the message "Queue is full".</p>
<p>A full queue generally means that the agents collect more data than APM server is able to process.
This might happen when APM Server is not configured properly for the size of your Elasticsearch cluster,
or because your Elasticsearch cluster is underpowered or not configured properly for the given workload.</p>
<p>The queue can also fill up if Elasticsearch runs out of disk space.</p>
<p>If the APM Server only returns 503 responses, it indicates that an Elasticsearch disk might be full.
If the APM Server returns interleaved 503 and 202 responses, it indicates that the APM Server can&#8217;t process that much data.</p>
<p>You have a few options to solve this problem:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<a class="xref" href="reduce-storage.html" title="Reduce storage">Reduce storage</a>
</li>
<li class="listitem">
<a class="xref" href="tune-apm-server.html#tune-output-config" title="Tune APM Server output parameters for your Elasticsearch cluster">Tune APM Server output parameters for your Elasticsearch cluster</a>
</li>
<li class="listitem">
<a class="xref" href="tune-apm-server.html#adjust-queue-size" title="Adjust internal queue size">Adjust internal queue size</a>
</li>
<li class="listitem">
<a class="xref" href="tune-es.html" title="Tune Elasticsearch">Tune Elasticsearch for higher ingestion</a>
</li>
</ul>
</div>
<div class="position-relative"><h3><a id="request-timed-out"></a>HTTP 503: Request timed out waiting to be processed</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>This happens when APM Server exceeds the maximum number of requests that it can process concurrently.</p>
<p>To alleviate this problem, you can try to:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<a class="xref" href="reduce-storage.html#reduce-sample-rate" title="Reduce the sample rate">Reduce the sample rate</a>
</li>
<li class="listitem">
<a class="xref" href="reduce-storage.html#reduce-stacktrace" title="Reduce collected stacktrace information">Reduce collected stacktrace information</a>
</li>
<li class="listitem">
<a class="xref" href="tune-apm-server.html#reduce-payload-size" title="Reduce the payload size">Reduce the payload size</a>
</li>
<li class="listitem">
<a class="xref" href="tune-apm-server.html#add-apm-server-instances" title="Add APM Server instances">Add APM Server instances</a>
</li>
</ul>
</div>
<div class="position-relative"><h3><a id="ssl-client-fails"></a>SSL client fails to connect</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>The target host running might be unreachable or the certificate may not be valid. To resolve your issue:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<p>Make sure that server process on the target host is running and you can connect to it.
First, try to ping the target host to verify that you can reach it from the host running APM Server.
Then use either <code class="literal">nc</code> or <code class="literal">telnet</code> to make sure that the port is available. For example:</p>
<div class="pre_wrapper lang-shell">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-shell">ping &lt;hostname or IP&gt;
telnet &lt;hostname or IP&gt; 5044</pre>
</div>
</li>
<li class="listitem">
Verify that the certificate is valid and that the hostname and IP match.
</li>
<li class="listitem">
Use OpenSSL to test connectivity to the target server and diagnose problems.
See the <a href="https://www.openssl.org/docs/manmaster/apps/s_client.html" class="ulink" target="_top">OpenSSL documentation</a> for more info.
</li>
</ul>
</div>
<div class="position-relative"><h4><a id="_common_ssl_related_errors_and_resolutions"></a>Common SSL-Related Errors and Resolutions</h4><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>Here are some common errors and ways to fix them:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<a class="xref" href="common-problems.html#cannot-validate-certificate" title="x509: cannot validate certificate for &lt;IP address&gt; because it doesn&#8217;t contain any IP SANs">x509: cannot validate certificate</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#getsockopt-no-route-to-host" title="getsockopt: no route to host">getsockopt: no route to host</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#getsockopt-connection-refused" title="getsockopt: connection refused">getsockopt: connection refused</a>
</li>
<li class="listitem">
<a class="xref" href="common-problems.html#target-machine-refused-connection" title="No connection could be made because the target machine actively refused it">No connection could be made because the target machine actively refused it</a>
</li>
</ul>
</div>
<div class="position-relative"><h5><a id="cannot-validate-certificate"></a>x509: cannot validate certificate for &lt;IP address&gt; because it doesn&#8217;t contain any IP SANs</h5><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>This happens because your certificate is only valid for the hostname present in the Subject field.</p>
<p>To resolve this problem, try one of these solutions:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
Create a DNS entry for the hostname mapping it to the server&#8217;s IP.
</li>
<li class="listitem">
Create an entry in <code class="literal">/etc/hosts</code> for the hostname. Or on Windows add an entry to
<code class="literal">C:\Windows\System32\drivers\etc\hosts</code>.
</li>
<li class="listitem">
Re-create the server certificate and add a SubjectAltName (SAN) for the IP address of the server. This makes the
server&#8217;s certificate valid for both the hostname and the IP address.
</li>
</ul>
</div>
<div class="position-relative"><h5><a id="getsockopt-no-route-to-host"></a>getsockopt: no route to host</h5><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>This is not an SSL problem. It&#8217;s a networking problem. Make sure the two hosts can communicate.</p>
<div class="position-relative"><h5><a id="getsockopt-connection-refused"></a>getsockopt: connection refused</h5><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>This is not an SSL problem. Make sure that Logstash is running and that there is no firewall blocking the traffic.</p>
<div class="position-relative"><h5><a id="target-machine-refused-connection"></a>No connection could be made because the target machine actively refused it</h5><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>A firewall is refusing the connection. Check if a firewall is blocking the traffic on the client, the network, or the
destination host.</p>
<div class="position-relative"><h3><a id="field-limit-exceeded"></a>Field limit exceeded</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>When adding too many distinct tag keys on a transaction or span,
you risk creating a <a href="/guide/en/elasticsearch/reference/7.11/mapping.html#mapping-limit-settings" class="ulink" target="_top">mapping explosion</a>.</p>
<p>For example,
you should avoid that user-specified data,
like URL parameters,
is used as a tag key.
Likewise,
using the current timestamp or a user ID as a tag key is not a good idea.
However,
tag <span class="strong strong"><strong>values</strong></span> with a high cardinality are not a problem.
Just try to keep the number of distinct tag keys at a minimum.</p>
<p>The symptom of a mapping explosion is that transactions and spans are not indexed anymore after a certain time.
Usually,
on the next day,
the spans and transactions will be indexed again because a new index is created each day.
But as soon as the field limit is reached,
indexing stops again.</p>
<p>In the agent logs,
you won&#8217;t see a sign of failures as the APM server asynchronously sends the data it received from the agents to Elasticsearch.
However,
the APM server and Elasticsearch log a warning like this:</p>
<pre class="screen">{\"type\":\"illegal_argument_exception\",\"reason\":\"Limit of total fields [1000] in index [apm-7.0.0-transaction-2017.05.30] has been exceeded\"}</pre>
<div class="position-relative"><h3><a id="io-timeout"></a>I/O Timeout</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>I/O Timeouts can occur when your timeout settings across the stack are not configured correctly,
especially when using a load balancer.</p>
<p>You may see an error like the one below in the agent logs, and/or a similar error on the APM Server side:</p>
<pre class="screen">[ElasticAPM] APM Server responded with an error:
"read tcp 123.34.22.313:8200-&gt;123.34.22.40:41602: i/o timeout"</pre>
<p>To fix this, ensure timeouts are incrementing from the <a href="/guide/en/apm/agent" class="ulink" target="_top">APM Agent</a>,
through your load balancer, to the <a class="xref" href="configuration-process.html#read_timeout" title="read_timeout">APM Server</a>.</p>
<p>By default, the agent timeouts are set at 10 seconds, and the server timeout is set at 30 seconds.
Your load balancer should be set somewhere between these numbers.</p>
<p>For example:</p>
<pre class="screen">APM Agent --&gt; Load Balancer  --&gt; APM Server
   10s            15s               30s</pre>
<div class="position-relative"><h3><a id="server-es-down"></a>What happens when APM Server or Elasticsearch is down?</h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p><span class="strong strong"><strong>If Elasticsearch is down</strong></span></p>
<p>If Elasticsearch goes down, the APM Server will keep data in memory until Elasticsearch is back up,
or until it runs out of space in its internal in-memory queue.
You can <a class="xref" href="tune-apm-server.html#adjust-queue-size" title="Adjust internal queue size">adjust the internal queue size</a> if necessary.
When the queue becomes full, APM Server will respond with <a class="xref" href="common-problems.html#queue-full" title="HTTP 503: Queue is full"><code class="literal">HTTP 503: Queue is full</code></a>,
and data will be lost.</p>
<p><span class="strong strong"><strong>If APM Server is down</strong></span></p>
<p>Some agents have internal queues or buffers that will temporarily store data if the APM Server goes down.
As a general rule of thumb, queues fill up quickly. Assume data will be lost if APM Server goes down.
Adjusting these queues/buffers can increase the overhead of the agent, so use caution when updating default values.</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
<span class="strong strong"><strong>Go Agent</strong></span> - Circular buffer with configurable size:
<a href="/guide/en/apm/agent/go/current/configuration.html#config-api-buffer-size" class="ulink" target="_top"><code class="literal">ELASTIC_APM_BUFFER_SIZE</code></a>.
</li>
<li class="listitem">
<span class="strong strong"><strong>Java Agent</strong></span> - Internal buffer with configurable size:
<a href="/guide/en/apm/agent/java/current/config-reporter.html#config-max-queue-size" class="ulink" target="_top"><code class="literal">max_queue_size</code></a>.
</li>
<li class="listitem">
<span class="strong strong"><strong>Node.js Agent</strong></span> - No internal queue. Data is lost.
</li>
<li class="listitem">
<span class="strong strong"><strong>Python Agent</strong></span> - Internal <a href="/guide/en/apm/agent/python/current/tuning-and-overhead.html#tuning-queue" class="ulink" target="_top">Transaction queue</a>
with configurable size and time between flushes.
</li>
<li class="listitem">
<span class="strong strong"><strong>Ruby Agent</strong></span> - Internal queue with configurable size:
<a href="/guide/en/apm/agent/ruby/current/configuration.html#config-api-buffer-size" class="ulink" target="_top"><code class="literal">api_buffer_size</code></a>.
</li>
<li class="listitem">
<span class="strong strong"><strong>RUM Agent</strong></span> - No internal queue. Data is lost.
</li>
<li class="listitem">
<span class="strong strong"><strong>.NET Agent</strong></span> - No internal queue. Data is lost.
</li>
</ul>
</div>
<div class="position-relative"><h3><a id="server-resource-exists-not-alias"></a><code class="literal">resource 'apm-7.11.2-$type' exists, but it is not an alias</code></h3><a class="edit_me" rel="nofollow" title="Edit this page on GitHub" href="https://github.com/elastic/apm-server/edit/7.11/docs/common-problems.asciidoc">edit</a></div>
<p>This error occurs when APM Server attempts to write to an index instead of an alias.
One way this can happen, is when indices are manually deleted after APM Server&#8217;s setup process.
Another possibility is that indices were manually set up, but not properly linked with ILM.</p>
<p>To fix this issue, perform the steps below. This example assumes <code class="literal">apm-7.11.2-transaction</code> is the problem index.
Update the steps with the index version and type that are specific to your error.</p>
<div class="olist orderedlist">
<ol class="orderedlist">
<li class="listitem">
<p>Block writes to the index:</p>
<div class="pre_wrapper lang-sh">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-sh">PUT apm-7.11.2-transaction/_settings
{
  "settings": {
    "index.blocks.write": true
  }
}</pre>
</div>
</li>
<li class="listitem">
<p>Clone the index to retain data (optional)</p>
<div class="pre_wrapper lang-sh">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-sh">POST apm-7.11.2-transaction/_clone/apm-7.11.2-transaction-original</pre>
</div>
<p>You can check the progress of the clone with:</p>
<div class="pre_wrapper lang-sh">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-sh">GET _cat/recovery/apm*transaction*?s=index&amp;v=true&amp;h=index,stage</pre>
</div>
<p>When <code class="literal">stage: done</code>, you&#8217;re ready to move on.</p>
</li>
<li class="listitem">
<p>Delete the index that should be a write alias</p>
<div class="pre_wrapper lang-sh">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-sh">DELETE apm-7.11.2-transaction</pre>
</div>
</li>
<li class="listitem">
<p>On the next connection attempt, APM Server will attempt to create a new write alias.
Confirm that APM Server successfully created the write alias with:</p>
<div class="pre_wrapper lang-sh">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-sh">GET _cat/aliases/apm*transaction*?s=index&amp;v=true&amp;h=alias,index,is_write_index</pre>
</div>
<p>If successful, you&#8217;ll see the following:</p>
<div class="pre_wrapper lang-sh">
<div class="console_code_copy" title="Copy to clipboard"></div>
<pre class="programlisting prettyprint lang-sh">alias                 index                        is_write_index
apm-7.11.2-transaction apm-7.11.2-transaction-000001 true</pre>
</div>
</li>
</ol>
</div>
</div>
</div>
</div><div class="navfooter">
<span class="prev">
<a href="troubleshooting.html">« Troubleshoot</a>
</span>
<span class="next">
<a href="enable-apm-server-debugging.html">Debug »</a>
</span>
</div>

                  <!-- end body -->
                </div>

                <div class="col-12 order-3 col-lg-2 order-lg-3 h-almost-full-lg sticky-top-lg" id="right_col">
                  <div id="sticky_content">
                    <!-- The OTP is appended here -->
                    <div class="row">
                      <div class="col-0 col-md-4 col-lg-0" id="bottom_left_col"></div>
                      <div class="col-12 col-md-8 col-lg-12">
                        <div id="rtpcontainer">
                          <div class="mktg-promo" id="most-popular">
                            <p class="aside-heading">Most Popular</p>
                            <div class="pb-2">
                              <p class="media-type">Video</p>
                              <a href="https://www.elastic.co/webinars/getting-started-elasticsearch?page=docs&placement=top-video">
                                <p class="mb-0">Get Started with Elasticsearch</p>
                              </a>
                            </div>
                            <div class="pb-2">
                              <p class="media-type">Video</p>
                              <a href="https://www.elastic.co/webinars/getting-started-kibana?page=docs&placement=top-video">
                                <p class="mb-0">Intro to Kibana</p>
                              </a>
                            </div>
                            <div class="pb-2">
                              <p class="media-type">Video</p>
                              <a href="https://www.elastic.co/webinars/introduction-elk-stack?page=docs&placement=top-video">
                                <p class="mb-0">ELK for Logs & Metrics</p>
                              </a>
                            </div>
                          </div>
                        </div>

                        <!-- Feedback widget -->
                        <div id="feedbackWidgetContainer"></div>
                      </div>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


        <div id='elastic-footer'></div>
        <script src='https://www.elastic.co/elastic-footer.js'></script>
        <!-- Footer Section end-->

      </section>
    </div>

    <!-- Feedback modal -->
    <div id="feedbackModalContainer"></div>

    <script src="/guide/static/jquery.js"></script>
    <script type="text/javascript" src="/guide/static/docs-v1.js"></script>
    <script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>