1.1.0/optim.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta name="robots" content="noindex">
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.optim &mdash; PyTorch master documentation</title>
  

    <link rel="canonical" href="https://pytorch.org/docs/stable/optim.html"/>
  

  <link rel="stylesheet" href="_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="genindex.html" />
    <link rel="search" title="Search" href="search.html" />
    <link rel="next" title="Automatic differentiation package - torch.autograd" href="autograd.html" />
    <link rel="prev" title="torch.nn" href="nn.html" /> 

  
  <script src="_static/js/modernizr.min.js"></script>
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   
    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

                <div class="version">
                  <a href='http://pytorch.org/docs/versions.html'>1.1.0 &#x25BC</a>
                </div>
              
            
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>
<p class="caption"><span class="caption-text">Package Reference</span></p>
<ul class="current">
<li class="toctree-l1"><a class="reference internal" href="torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.html#torch-nn-functional">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="nn.html#torch-nn-init">torch.nn.init</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="multiprocessing.html">torch.multiprocessing</a></li>
<li class="toctree-l1"><a class="reference internal" href="bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="tensorboard.html">torch.utils.tensorboard (experimental)</a></li>
<li class="toctree-l1"><a class="reference internal" href="onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="__config__.html">torch.__config__</a></li>
<li class="toctree-l1"><a class="reference internal" href="distributed_deprecated.html">torch.distributed.deprecated</a></li>
</ul>
<p class="caption"><span class="caption-text">torchvision Reference</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="torchvision/index.html">torchvision</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          

<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
      <li>torch.optim</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
            
            <a href="_sources/optim.rst.txt" rel="nofollow"><img src="_static/images/view-page-source-icon.svg"></a>
          
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <div class="section" id="module-torch.optim">
<span id="torch-optim"></span><h1>torch.optim<a class="headerlink" href="#module-torch.optim" title="Permalink to this headline">¶</a></h1>
<p><a class="reference internal" href="#module-torch.optim" title="torch.optim"><code class="xref py py-mod docutils literal notranslate"><span class="pre">torch.optim</span></code></a> is a package implementing various optimization algorithms.
Most commonly used methods are already supported, and the interface is general
enough, so that more sophisticated ones can be also easily integrated in the
future.</p>
<div class="section" id="how-to-use-an-optimizer">
<h2>How to use an optimizer<a class="headerlink" href="#how-to-use-an-optimizer" title="Permalink to this headline">¶</a></h2>
<p>To use <a class="reference internal" href="#module-torch.optim" title="torch.optim"><code class="xref py py-mod docutils literal notranslate"><span class="pre">torch.optim</span></code></a> you have to construct an optimizer object, that will hold
the current state and will update the parameters based on the computed gradients.</p>
<div class="section" id="constructing-it">
<h3>Constructing it<a class="headerlink" href="#constructing-it" title="Permalink to this headline">¶</a></h3>
<p>To construct an <a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">Optimizer</span></code></a> you have to give it an iterable containing the
parameters (all should be <code class="xref py py-class docutils literal notranslate"><span class="pre">Variable</span></code> s) to optimize. Then,
you can specify optimizer-specific options such as the learning rate, weight decay, etc.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If you need to move a model to GPU via <cite>.cuda()</cite>, please do so before
constructing optimizers for it. Parameters of a model after <cite>.cuda()</cite> will
be different objects with those before the call.</p>
<p>In general, you should make sure that optimized parameters live in
consistent locations when optimizers are constructed and used.</p>
</div>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.01</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
<span class="n">optimizer</span> <span class="o">=</span> <span class="n">optim</span><span class="o">.</span><span class="n">Adam</span><span class="p">([</span><span class="n">var1</span><span class="p">,</span> <span class="n">var2</span><span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.0001</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="section" id="per-parameter-options">
<h3>Per-parameter options<a class="headerlink" href="#per-parameter-options" title="Permalink to this headline">¶</a></h3>
<p><a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">Optimizer</span></code></a> s also support specifying per-parameter options. To do this, instead
of passing an iterable of <code class="xref py py-class docutils literal notranslate"><span class="pre">Variable</span></code> s, pass in an iterable of
<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)"><code class="xref py py-class docutils literal notranslate"><span class="pre">dict</span></code></a> s. Each of them will define a separate parameter group, and should contain
a <code class="docutils literal notranslate"><span class="pre">params</span></code> key, containing a list of parameters belonging to it. Other keys
should match the keyword arguments accepted by the optimizers, and will be used
as optimization options for this group.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>You can still pass options as keyword arguments. They will be used as
defaults, in the groups that didn’t override them. This is useful when you
only want to vary a single option, while keeping all others consistent
between parameter groups.</p>
</div>
<p>For example, this is very useful when one wants to specify per-layer learning rates:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">([</span>
                <span class="p">{</span><span class="s1">&#39;params&#39;</span><span class="p">:</span> <span class="n">model</span><span class="o">.</span><span class="n">base</span><span class="o">.</span><span class="n">parameters</span><span class="p">()},</span>
                <span class="p">{</span><span class="s1">&#39;params&#39;</span><span class="p">:</span> <span class="n">model</span><span class="o">.</span><span class="n">classifier</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="s1">&#39;lr&#39;</span><span class="p">:</span> <span class="mf">1e-3</span><span class="p">}</span>
            <span class="p">],</span> <span class="n">lr</span><span class="o">=</span><span class="mf">1e-2</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
</pre></div>
</div>
<p>This means that <code class="docutils literal notranslate"><span class="pre">model.base</span></code>’s parameters will use the default learning rate of <code class="docutils literal notranslate"><span class="pre">1e-2</span></code>,
<code class="docutils literal notranslate"><span class="pre">model.classifier</span></code>’s parameters will use a learning rate of <code class="docutils literal notranslate"><span class="pre">1e-3</span></code>, and a momentum of
<code class="docutils literal notranslate"><span class="pre">0.9</span></code> will be used for all parameters.</p>
</div>
<div class="section" id="taking-an-optimization-step">
<h3>Taking an optimization step<a class="headerlink" href="#taking-an-optimization-step" title="Permalink to this headline">¶</a></h3>
<p>All optimizers implement a <a class="reference internal" href="#torch.optim.Optimizer.step" title="torch.optim.Optimizer.step"><code class="xref py py-func docutils literal notranslate"><span class="pre">step()</span></code></a> method, that updates the
parameters. It can be used in two ways:</p>
<div class="section" id="optimizer-step">
<h4><code class="docutils literal notranslate"><span class="pre">optimizer.step()</span></code><a class="headerlink" href="#optimizer-step" title="Permalink to this headline">¶</a></h4>
<p>This is a simplified version supported by most optimizers. The function can be
called once the gradients are computed using e.g.
<code class="xref py py-func docutils literal notranslate"><span class="pre">backward()</span></code>.</p>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">for</span> <span class="nb">input</span><span class="p">,</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">dataset</span><span class="p">:</span>
    <span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
    <span class="n">output</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
    <span class="n">loss</span> <span class="o">=</span> <span class="n">loss_fn</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
    <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
    <span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="section" id="optimizer-step-closure">
<h4><code class="docutils literal notranslate"><span class="pre">optimizer.step(closure)</span></code><a class="headerlink" href="#optimizer-step-closure" title="Permalink to this headline">¶</a></h4>
<p>Some optimization algorithms such as Conjugate Gradient and LBFGS need to
reevaluate the function multiple times, so you have to pass in a closure that
allows them to recompute your model. The closure should clear the gradients,
compute the loss, and return it.</p>
<p>Example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="k">for</span> <span class="nb">input</span><span class="p">,</span> <span class="n">target</span> <span class="ow">in</span> <span class="n">dataset</span><span class="p">:</span>
    <span class="k">def</span> <span class="nf">closure</span><span class="p">():</span>
        <span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
        <span class="n">output</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="nb">input</span><span class="p">)</span>
        <span class="n">loss</span> <span class="o">=</span> <span class="n">loss_fn</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">target</span><span class="p">)</span>
        <span class="n">loss</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
        <span class="k">return</span> <span class="n">loss</span>
    <span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="n">closure</span><span class="p">)</span>
</pre></div>
</div>
</div>
</div>
</div>
<div class="section" id="algorithms">
<h2>Algorithms<a class="headerlink" href="#algorithms" title="Permalink to this headline">¶</a></h2>
<dl class="class">
<dt id="torch.optim.Optimizer">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">Optimizer</code><span class="sig-paren">(</span><em>params</em>, <em>defaults</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/optimizer.html#Optimizer"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Optimizer" title="Permalink to this definition">¶</a></dt>
<dd><p>Base class for all optimizers.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Parameters need to be specified as collections that have a deterministic
ordering that is consistent between runs. Examples of objects that don’t
satisfy those properties are sets and iterators over values of dictionaries.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – an iterable of <a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.Tensor</span></code></a> s or
<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)"><code class="xref py py-class docutils literal notranslate"><span class="pre">dict</span></code></a> s. Specifies what Tensors should be optimized.</p></li>
<li><p><strong>defaults</strong> – (dict): a dict containing default values of optimization
options (used when a parameter group doesn’t specify them).</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.Optimizer.add_param_group">
<code class="descname">add_param_group</code><span class="sig-paren">(</span><em>param_group</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/optimizer.html#Optimizer.add_param_group"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Optimizer.add_param_group" title="Permalink to this definition">¶</a></dt>
<dd><p>Add a param group to the <a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">Optimizer</span></code></a> s <cite>param_groups</cite>.</p>
<p>This can be useful when fine tuning a pre-trained network as frozen layers can be made
trainable and added to the <a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><code class="xref py py-class docutils literal notranslate"><span class="pre">Optimizer</span></code></a> as training progresses.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>param_group</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)"><em>dict</em></a>) – Specifies what Tensors should be optimized along with group</p></li>
<li><p><strong>optimization options.</strong> (<em>specific</em>) – </p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.optim.Optimizer.load_state_dict">
<code class="descname">load_state_dict</code><span class="sig-paren">(</span><em>state_dict</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/optimizer.html#Optimizer.load_state_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Optimizer.load_state_dict" title="Permalink to this definition">¶</a></dt>
<dd><p>Loads the optimizer state.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>state_dict</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)"><em>dict</em></a>) – optimizer state. Should be an object returned
from a call to <a class="reference internal" href="#torch.optim.Optimizer.state_dict" title="torch.optim.Optimizer.state_dict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">state_dict()</span></code></a>.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.optim.Optimizer.state_dict">
<code class="descname">state_dict</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/optimizer.html#Optimizer.state_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Optimizer.state_dict" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the state of the optimizer as a <a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)"><code class="xref py py-class docutils literal notranslate"><span class="pre">dict</span></code></a>.</p>
<p>It contains two entries:</p>
<ul class="simple">
<li><dl class="simple">
<dt>state - a dict holding current optimization state. Its content</dt><dd><p>differs between optimizer classes.</p>
</dd>
</dl>
</li>
<li><p>param_groups - a dict containing all parameter groups</p></li>
</ul>
</dd></dl>

<dl class="method">
<dt id="torch.optim.Optimizer.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/optimizer.html#Optimizer.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Optimizer.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step (parameter update).</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em>) – A closure that reevaluates the model and
returns the loss. Optional for most optimizers.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.optim.Optimizer.zero_grad">
<code class="descname">zero_grad</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/optimizer.html#Optimizer.zero_grad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Optimizer.zero_grad" title="Permalink to this definition">¶</a></dt>
<dd><p>Clears the gradients of all optimized <a class="reference internal" href="tensors.html#torch.Tensor" title="torch.Tensor"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.Tensor</span></code></a> s.</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.Adadelta">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">Adadelta</code><span class="sig-paren">(</span><em>params</em>, <em>lr=1.0</em>, <em>rho=0.9</em>, <em>eps=1e-06</em>, <em>weight_decay=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/adadelta.html#Adadelta"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Adadelta" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements Adadelta algorithm.</p>
<p>It has been proposed in <a class="reference external" href="https://arxiv.org/abs/1212.5701">ADADELTA: An Adaptive Learning Rate Method</a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>rho</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – coefficient used for computing a running average
of squared gradients (default: 0.9)</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – term added to the denominator to improve
numerical stability (default: 1e-6)</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – coefficient that scale delta before it is applied
to the parameters (default: 1.0)</p></li>
<li><p><strong>weight_decay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – weight decay (L2 penalty) (default: 0)</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.Adadelta.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/adadelta.html#Adadelta.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Adadelta.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.Adagrad">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">Adagrad</code><span class="sig-paren">(</span><em>params</em>, <em>lr=0.01</em>, <em>lr_decay=0</em>, <em>weight_decay=0</em>, <em>initial_accumulator_value=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/adagrad.html#Adagrad"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Adagrad" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements Adagrad algorithm.</p>
<p>It has been proposed in <a class="reference external" href="http://jmlr.org/papers/v12/duchi11a.html">Adaptive Subgradient Methods for Online Learning
and Stochastic Optimization</a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – learning rate (default: 1e-2)</p></li>
<li><p><strong>lr_decay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – learning rate decay (default: 0)</p></li>
<li><p><strong>weight_decay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – weight decay (L2 penalty) (default: 0)</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.Adagrad.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/adagrad.html#Adagrad.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Adagrad.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.Adam">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">Adam</code><span class="sig-paren">(</span><em>params</em>, <em>lr=0.001</em>, <em>betas=(0.9</em>, <em>0.999)</em>, <em>eps=1e-08</em>, <em>weight_decay=0</em>, <em>amsgrad=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/adam.html#Adam"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Adam" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements Adam algorithm.</p>
<p>It has been proposed in <a class="reference external" href="https://arxiv.org/abs/1412.6980">Adam: A Method for Stochastic Optimization</a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – learning rate (default: 1e-3)</p></li>
<li><p><strong>betas</strong> (<em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em><em>, </em><em>optional</em>) – coefficients used for computing
running averages of gradient and its square (default: (0.9, 0.999))</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – term added to the denominator to improve
numerical stability (default: 1e-8)</p></li>
<li><p><strong>weight_decay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – weight decay (L2 penalty) (default: 0)</p></li>
<li><p><strong>amsgrad</strong> (<em>boolean</em><em>, </em><em>optional</em>) – whether to use the AMSGrad variant of this
algorithm from the paper <a class="reference external" href="https://openreview.net/forum?id=ryQu7f-RZ">On the Convergence of Adam and Beyond</a>
(default: False)</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.Adam.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/adam.html#Adam.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Adam.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.SparseAdam">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">SparseAdam</code><span class="sig-paren">(</span><em>params</em>, <em>lr=0.001</em>, <em>betas=(0.9</em>, <em>0.999)</em>, <em>eps=1e-08</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/sparse_adam.html#SparseAdam"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.SparseAdam" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements lazy version of Adam algorithm suitable for sparse tensors.</p>
<p>In this variant, only moments that show up in the gradient get updated, and
only those portions of the gradient get applied to the parameters.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – learning rate (default: 1e-3)</p></li>
<li><p><strong>betas</strong> (<em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em><em>, </em><em>optional</em>) – coefficients used for computing
running averages of gradient and its square (default: (0.9, 0.999))</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – term added to the denominator to improve
numerical stability (default: 1e-8)</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.SparseAdam.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/sparse_adam.html#SparseAdam.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.SparseAdam.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.Adamax">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">Adamax</code><span class="sig-paren">(</span><em>params</em>, <em>lr=0.002</em>, <em>betas=(0.9</em>, <em>0.999)</em>, <em>eps=1e-08</em>, <em>weight_decay=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/adamax.html#Adamax"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Adamax" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements Adamax algorithm (a variant of Adam based on infinity norm).</p>
<p>It has been proposed in <a class="reference external" href="https://arxiv.org/abs/1412.6980">Adam: A Method for Stochastic Optimization</a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – learning rate (default: 2e-3)</p></li>
<li><p><strong>betas</strong> (<em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em><em>, </em><em>optional</em>) – coefficients used for computing
running averages of gradient and its square</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – term added to the denominator to improve
numerical stability (default: 1e-8)</p></li>
<li><p><strong>weight_decay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – weight decay (L2 penalty) (default: 0)</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.Adamax.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/adamax.html#Adamax.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Adamax.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.ASGD">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">ASGD</code><span class="sig-paren">(</span><em>params</em>, <em>lr=0.01</em>, <em>lambd=0.0001</em>, <em>alpha=0.75</em>, <em>t0=1000000.0</em>, <em>weight_decay=0</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/asgd.html#ASGD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.ASGD" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements Averaged Stochastic Gradient Descent.</p>
<p>It has been proposed in <a class="reference external" href="http://dl.acm.org/citation.cfm?id=131098">Acceleration of stochastic approximation by
averaging</a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – learning rate (default: 1e-2)</p></li>
<li><p><strong>lambd</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – decay term (default: 1e-4)</p></li>
<li><p><strong>alpha</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – power for eta update (default: 0.75)</p></li>
<li><p><strong>t0</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – point at which to start averaging (default: 1e6)</p></li>
<li><p><strong>weight_decay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – weight decay (L2 penalty) (default: 0)</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.ASGD.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/asgd.html#ASGD.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.ASGD.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.LBFGS">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">LBFGS</code><span class="sig-paren">(</span><em>params</em>, <em>lr=1</em>, <em>max_iter=20</em>, <em>max_eval=None</em>, <em>tolerance_grad=1e-05</em>, <em>tolerance_change=1e-09</em>, <em>history_size=100</em>, <em>line_search_fn=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lbfgs.html#LBFGS"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.LBFGS" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements L-BFGS algorithm.</p>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>This optimizer doesn’t support per-parameter options and parameter
groups (there can be only one).</p>
</div>
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Right now all parameters have to be on a single device. This will be
improved in the future.</p>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>This is a very memory intensive optimizer (it requires additional
<code class="docutils literal notranslate"><span class="pre">param_bytes</span> <span class="pre">*</span> <span class="pre">(history_size</span> <span class="pre">+</span> <span class="pre">1)</span></code> bytes). If it doesn’t fit in memory
try reducing the history size, or use a different algorithm.</p>
</div>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – learning rate (default: 1)</p></li>
<li><p><strong>max_iter</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – maximal number of iterations per optimization step
(default: 20)</p></li>
<li><p><strong>max_eval</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – maximal number of function evaluations per optimization
step (default: max_iter * 1.25).</p></li>
<li><p><strong>tolerance_grad</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – termination tolerance on first order optimality
(default: 1e-5).</p></li>
<li><p><strong>tolerance_change</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – termination tolerance on function
value/parameter changes (default: 1e-9).</p></li>
<li><p><strong>history_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – update history size (default: 100).</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.LBFGS.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lbfgs.html#LBFGS.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.LBFGS.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.RMSprop">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">RMSprop</code><span class="sig-paren">(</span><em>params</em>, <em>lr=0.01</em>, <em>alpha=0.99</em>, <em>eps=1e-08</em>, <em>weight_decay=0</em>, <em>momentum=0</em>, <em>centered=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/rmsprop.html#RMSprop"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.RMSprop" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements RMSprop algorithm.</p>
<p>Proposed by G. Hinton in his
<a class="reference external" href="http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf">course</a>.</p>
<p>The centered version first appears in <a class="reference external" href="https://arxiv.org/pdf/1308.0850v5.pdf">Generating Sequences
With Recurrent Neural Networks</a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – learning rate (default: 1e-2)</p></li>
<li><p><strong>momentum</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – momentum factor (default: 0)</p></li>
<li><p><strong>alpha</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – smoothing constant (default: 0.99)</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – term added to the denominator to improve
numerical stability (default: 1e-8)</p></li>
<li><p><strong>centered</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – if <code class="docutils literal notranslate"><span class="pre">True</span></code>, compute the centered RMSProp,
the gradient is normalized by an estimation of its variance</p></li>
<li><p><strong>weight_decay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – weight decay (L2 penalty) (default: 0)</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.RMSprop.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/rmsprop.html#RMSprop.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.RMSprop.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.Rprop">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">Rprop</code><span class="sig-paren">(</span><em>params</em>, <em>lr=0.01</em>, <em>etas=(0.5</em>, <em>1.2)</em>, <em>step_sizes=(1e-06</em>, <em>50)</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/rprop.html#Rprop"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Rprop" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements the resilient backpropagation algorithm.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – learning rate (default: 1e-2)</p></li>
<li><p><strong>etas</strong> (<em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em><em>, </em><em>optional</em>) – pair of (etaminus, etaplis), that
are multiplicative increase and decrease factors
(default: (0.5, 1.2))</p></li>
<li><p><strong>step_sizes</strong> (<em>Tuple</em><em>[</em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>]</em><em>, </em><em>optional</em>) – a pair of minimal and
maximal allowed step sizes (default: (1e-6, 50))</p></li>
</ul>
</dd>
</dl>
<dl class="method">
<dt id="torch.optim.Rprop.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/rprop.html#Rprop.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.Rprop.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.SGD">
<em class="property">class </em><code class="descclassname">torch.optim.</code><code class="descname">SGD</code><span class="sig-paren">(</span><em>params</em>, <em>lr=&lt;required parameter&gt;</em>, <em>momentum=0</em>, <em>dampening=0</em>, <em>weight_decay=0</em>, <em>nesterov=False</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/sgd.html#SGD"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.SGD" title="Permalink to this definition">¶</a></dt>
<dd><p>Implements stochastic gradient descent (optionally with momentum).</p>
<p>Nesterov momentum is based on the formula from
<a class="reference external" href="http://www.cs.toronto.edu/%7Ehinton/absps/momentum.pdf">On the importance of initialization and momentum in deep learning</a>.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>params</strong> (<em>iterable</em>) – iterable of parameters to optimize or dicts defining
parameter groups</p></li>
<li><p><strong>lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – learning rate</p></li>
<li><p><strong>momentum</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – momentum factor (default: 0)</p></li>
<li><p><strong>weight_decay</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – weight decay (L2 penalty) (default: 0)</p></li>
<li><p><strong>dampening</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em>, </em><em>optional</em>) – dampening for momentum (default: 0)</p></li>
<li><p><strong>nesterov</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a><em>, </em><em>optional</em>) – enables Nesterov momentum (default: False)</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">optimizer</span><span class="o">.</span><span class="n">zero_grad</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">loss_fn</span><span class="p">(</span><span class="n">model</span><span class="p">(</span><span class="nb">input</span><span class="p">),</span> <span class="n">target</span><span class="p">)</span><span class="o">.</span><span class="n">backward</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
</pre></div>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The implementation of SGD with Momentum/Nesterov subtly differs from
Sutskever et. al. and implementations in some other frameworks.</p>
<p>Considering the specific case of Momentum, the update can be written as</p>
<div class="math">
\[v = \rho * v + g \\
p = p - lr * v

\]</div>
<p>where p, g, v and <span class="math">\(\rho\)</span> denote the parameters, gradient,
velocity, and momentum respectively.</p>
<p>This is in contrast to Sutskever et. al. and
other frameworks which employ an update of the form</p>
<div class="math">
\[v = \rho * v + lr * g \\
p = p - v

\]</div>
<p>The Nesterov version is analogously modified.</p>
</div>
<dl class="method">
<dt id="torch.optim.SGD.step">
<code class="descname">step</code><span class="sig-paren">(</span><em>closure=None</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/sgd.html#SGD.step"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.SGD.step" title="Permalink to this definition">¶</a></dt>
<dd><p>Performs a single optimization step.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>closure</strong> (<em>callable</em><em>, </em><em>optional</em>) – A closure that reevaluates the model
and returns the loss.</p>
</dd>
</dl>
</dd></dl>

</dd></dl>

</div>
<div class="section" id="how-to-adjust-learning-rate">
<h2>How to adjust Learning Rate<a class="headerlink" href="#how-to-adjust-learning-rate" title="Permalink to this headline">¶</a></h2>
<p><code class="xref py py-mod docutils literal notranslate"><span class="pre">torch.optim.lr_scheduler</span></code> provides several methods to adjust the learning
rate based on the number of epochs. <a class="reference internal" href="#torch.optim.lr_scheduler.ReduceLROnPlateau" title="torch.optim.lr_scheduler.ReduceLROnPlateau"><code class="xref py py-class docutils literal notranslate"><span class="pre">torch.optim.lr_scheduler.ReduceLROnPlateau</span></code></a>
allows dynamic learning rate reducing based on some validation measurements.</p>
<dl class="class">
<dt id="torch.optim.lr_scheduler.LambdaLR">
<em class="property">class </em><code class="descclassname">torch.optim.lr_scheduler.</code><code class="descname">LambdaLR</code><span class="sig-paren">(</span><em>optimizer</em>, <em>lr_lambda</em>, <em>last_epoch=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#LambdaLR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.LambdaLR" title="Permalink to this definition">¶</a></dt>
<dd><p>Sets the learning rate of each parameter group to the initial lr
times a given function. When last_epoch=-1, sets initial lr as lr.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>optimizer</strong> (<a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><em>Optimizer</em></a>) – Wrapped optimizer.</p></li>
<li><p><strong>lr_lambda</strong> (<em>function</em><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a>) – A function which computes a multiplicative
factor given an integer parameter epoch, or a list of such
functions, one for each group in optimizer.param_groups.</p></li>
<li><p><strong>last_epoch</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – The index of last epoch. Default: -1.</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Assuming optimizer has two groups.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">lambda1</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">epoch</span><span class="p">:</span> <span class="n">epoch</span> <span class="o">//</span> <span class="mi">30</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">lambda2</span> <span class="o">=</span> <span class="k">lambda</span> <span class="n">epoch</span><span class="p">:</span> <span class="mf">0.95</span> <span class="o">**</span> <span class="n">epoch</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scheduler</span> <span class="o">=</span> <span class="n">LambdaLR</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">lr_lambda</span><span class="o">=</span><span class="p">[</span><span class="n">lambda1</span><span class="p">,</span> <span class="n">lambda2</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">train</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">validate</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">scheduler</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.optim.lr_scheduler.LambdaLR.load_state_dict">
<code class="descname">load_state_dict</code><span class="sig-paren">(</span><em>state_dict</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#LambdaLR.load_state_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.LambdaLR.load_state_dict" title="Permalink to this definition">¶</a></dt>
<dd><p>Loads the schedulers state.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><p><strong>state_dict</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)"><em>dict</em></a>) – scheduler state. Should be an object returned
from a call to <a class="reference internal" href="#torch.optim.lr_scheduler.LambdaLR.state_dict" title="torch.optim.lr_scheduler.LambdaLR.state_dict"><code class="xref py py-meth docutils literal notranslate"><span class="pre">state_dict()</span></code></a>.</p>
</dd>
</dl>
</dd></dl>

<dl class="method">
<dt id="torch.optim.lr_scheduler.LambdaLR.state_dict">
<code class="descname">state_dict</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#LambdaLR.state_dict"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.LambdaLR.state_dict" title="Permalink to this definition">¶</a></dt>
<dd><p>Returns the state of the scheduler as a <a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#dict" title="(in Python v3.7)"><code class="xref py py-class docutils literal notranslate"><span class="pre">dict</span></code></a>.</p>
<p>It contains an entry for every variable in self.__dict__ which
is not the optimizer.
The learning rate lambda functions will only be saved if they are callable objects
and not if they are functions or lambdas.</p>
</dd></dl>

</dd></dl>

<dl class="class">
<dt id="torch.optim.lr_scheduler.StepLR">
<em class="property">class </em><code class="descclassname">torch.optim.lr_scheduler.</code><code class="descname">StepLR</code><span class="sig-paren">(</span><em>optimizer</em>, <em>step_size</em>, <em>gamma=0.1</em>, <em>last_epoch=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#StepLR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.StepLR" title="Permalink to this definition">¶</a></dt>
<dd><p>Decays the learning rate of each parameter group by gamma every
step_size epochs. Notice that such decay can happen simultaneously with
other changes to the learning rate from outside this scheduler. When
last_epoch=-1, sets initial lr as lr.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>optimizer</strong> (<a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><em>Optimizer</em></a>) – Wrapped optimizer.</p></li>
<li><p><strong>step_size</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Period of learning rate decay.</p></li>
<li><p><strong>gamma</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – Multiplicative factor of learning rate decay.
Default: 0.1.</p></li>
<li><p><strong>last_epoch</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – The index of last epoch. Default: -1.</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Assuming optimizer uses lr = 0.05 for all groups</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># lr = 0.05     if epoch &lt; 30</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># lr = 0.005    if 30 &lt;= epoch &lt; 60</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># lr = 0.0005   if 60 &lt;= epoch &lt; 90</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># ...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scheduler</span> <span class="o">=</span> <span class="n">StepLR</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">step_size</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">0.1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">train</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">validate</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">scheduler</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="class">
<dt id="torch.optim.lr_scheduler.MultiStepLR">
<em class="property">class </em><code class="descclassname">torch.optim.lr_scheduler.</code><code class="descname">MultiStepLR</code><span class="sig-paren">(</span><em>optimizer</em>, <em>milestones</em>, <em>gamma=0.1</em>, <em>last_epoch=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#MultiStepLR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.MultiStepLR" title="Permalink to this definition">¶</a></dt>
<dd><p>Decays the learning rate of each parameter group by gamma once the
number of epoch reaches one of the milestones. Notice that such decay can
happen simultaneously with other changes to the learning rate from outside
this scheduler. When last_epoch=-1, sets initial lr as lr.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>optimizer</strong> (<a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><em>Optimizer</em></a>) – Wrapped optimizer.</p></li>
<li><p><strong>milestones</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a>) – List of epoch indices. Must be increasing.</p></li>
<li><p><strong>gamma</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – Multiplicative factor of learning rate decay.
Default: 0.1.</p></li>
<li><p><strong>last_epoch</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – The index of last epoch. Default: -1.</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="c1"># Assuming optimizer uses lr = 0.05 for all groups</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># lr = 0.05     if epoch &lt; 30</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># lr = 0.005    if 30 &lt;= epoch &lt; 80</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># lr = 0.0005   if epoch &gt;= 80</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scheduler</span> <span class="o">=</span> <span class="n">MultiStepLR</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">milestones</span><span class="o">=</span><span class="p">[</span><span class="mi">30</span><span class="p">,</span><span class="mi">80</span><span class="p">],</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">0.1</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">train</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">validate</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">scheduler</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
</pre></div>
</div>
</dd></dl>

<dl class="class">
<dt id="torch.optim.lr_scheduler.ExponentialLR">
<em class="property">class </em><code class="descclassname">torch.optim.lr_scheduler.</code><code class="descname">ExponentialLR</code><span class="sig-paren">(</span><em>optimizer</em>, <em>gamma</em>, <em>last_epoch=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#ExponentialLR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.ExponentialLR" title="Permalink to this definition">¶</a></dt>
<dd><p>Decays the learning rate of each parameter group by gamma every epoch.
When last_epoch=-1, sets initial lr as lr.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>optimizer</strong> (<a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><em>Optimizer</em></a>) – Wrapped optimizer.</p></li>
<li><p><strong>gamma</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – Multiplicative factor of learning rate decay.</p></li>
<li><p><strong>last_epoch</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – The index of last epoch. Default: -1.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="class">
<dt id="torch.optim.lr_scheduler.CosineAnnealingLR">
<em class="property">class </em><code class="descclassname">torch.optim.lr_scheduler.</code><code class="descname">CosineAnnealingLR</code><span class="sig-paren">(</span><em>optimizer</em>, <em>T_max</em>, <em>eta_min=0</em>, <em>last_epoch=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#CosineAnnealingLR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.CosineAnnealingLR" title="Permalink to this definition">¶</a></dt>
<dd><p>Set the learning rate of each parameter group using a cosine annealing
schedule, where <span class="math">\(\eta_{max}\)</span> is set to the initial lr and
<span class="math">\(T_{cur}\)</span> is the number of epochs since the last restart in SGDR:</p>
<div class="math">
\[\eta_{t+1} = \eta_{min} + (\eta_t - \eta_{min})\frac{1 +
\cos(\frac{T_{cur+1}}{T_{max}}\pi)}{1 + \cos(\frac{T_{cur}}{T_{max}}\pi)},
T_{cur} \neq (2k+1)T_{max};\\
\eta_{t+1} = \eta_{t} + (\eta_{max} - \eta_{min})\frac{1 -
\cos(\frac{1}{T_{max}}\pi)}{2},
T_{cur} = (2k+1)T_{max}.\\

\]</div>
<p>When last_epoch=-1, sets initial lr as lr. Notice that because the schedule
is defined recursively, the learning rate can be simultaneously modified
outside this scheduler by other operators. If the learning rate is set
solely by this scheduler, the learning rate at each step becomes:</p>
<div class="math">
\[\eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})(1 +
\cos(\frac{T_{cur}}{T_{max}}\pi))

\]</div>
<p>It has been proposed in
<a class="reference external" href="https://arxiv.org/abs/1608.03983">SGDR: Stochastic Gradient Descent with Warm Restarts</a>. Note that this only
implements the cosine annealing part of SGDR, and not the restarts.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>optimizer</strong> (<a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><em>Optimizer</em></a>) – Wrapped optimizer.</p></li>
<li><p><strong>T_max</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Maximum number of iterations.</p></li>
<li><p><strong>eta_min</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – Minimum learning rate. Default: 0.</p></li>
<li><p><strong>last_epoch</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – The index of last epoch. Default: -1.</p></li>
</ul>
</dd>
</dl>
</dd></dl>

<dl class="class">
<dt id="torch.optim.lr_scheduler.ReduceLROnPlateau">
<em class="property">class </em><code class="descclassname">torch.optim.lr_scheduler.</code><code class="descname">ReduceLROnPlateau</code><span class="sig-paren">(</span><em>optimizer</em>, <em>mode='min'</em>, <em>factor=0.1</em>, <em>patience=10</em>, <em>verbose=False</em>, <em>threshold=0.0001</em>, <em>threshold_mode='rel'</em>, <em>cooldown=0</em>, <em>min_lr=0</em>, <em>eps=1e-08</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#ReduceLROnPlateau"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.ReduceLROnPlateau" title="Permalink to this definition">¶</a></dt>
<dd><p>Reduce learning rate when a metric has stopped improving.
Models often benefit from reducing the learning rate by a factor
of 2-10 once learning stagnates. This scheduler reads a metrics
quantity and if no improvement is seen for a ‘patience’ number
of epochs, the learning rate is reduced.</p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>optimizer</strong> (<a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><em>Optimizer</em></a>) – Wrapped optimizer.</p></li>
<li><p><strong>mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – One of <cite>min</cite>, <cite>max</cite>. In <cite>min</cite> mode, lr will
be reduced when the quantity monitored has stopped
decreasing; in <cite>max</cite> mode it will be reduced when the
quantity monitored has stopped increasing. Default: ‘min’.</p></li>
<li><p><strong>factor</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – Factor by which the learning rate will be
reduced. new_lr = lr * factor. Default: 0.1.</p></li>
<li><p><strong>patience</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of epochs with no improvement after
which learning rate will be reduced. For example, if
<cite>patience = 2</cite>, then we will ignore the first 2 epochs
with no improvement, and will only decrease the LR after the
3rd epoch if the loss still hasn’t improved then.
Default: 10.</p></li>
<li><p><strong>verbose</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, prints a message to stdout for
each update. Default: <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p></li>
<li><p><strong>threshold</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – Threshold for measuring the new optimum,
to only focus on significant changes. Default: 1e-4.</p></li>
<li><p><strong>threshold_mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – One of <cite>rel</cite>, <cite>abs</cite>. In <cite>rel</cite> mode,
dynamic_threshold = best * ( 1 + threshold ) in ‘max’
mode or best * ( 1 - threshold ) in <cite>min</cite> mode.
In <cite>abs</cite> mode, dynamic_threshold = best + threshold in
<cite>max</cite> mode or best - threshold in <cite>min</cite> mode. Default: ‘rel’.</p></li>
<li><p><strong>cooldown</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of epochs to wait before resuming
normal operation after lr has been reduced. Default: 0.</p></li>
<li><p><strong>min_lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a>) – A scalar or a list of scalars. A
lower bound on the learning rate of all param groups
or each group respectively. Default: 0.</p></li>
<li><p><strong>eps</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – Minimal decay applied to lr. If the difference
between new and old lr is smaller than eps, the update is
ignored. Default: 1e-8.</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scheduler</span> <span class="o">=</span> <span class="n">ReduceLROnPlateau</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="s1">&#39;min&#39;</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">train</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">val_loss</span> <span class="o">=</span> <span class="n">validate</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="c1"># Note that step should be called after validate()</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="n">scheduler</span><span class="o">.</span><span class="n">step</span><span class="p">(</span><span class="n">val_loss</span><span class="p">)</span>
</pre></div>
</div>
</dd></dl>

<dl class="class">
<dt id="torch.optim.lr_scheduler.CyclicLR">
<em class="property">class </em><code class="descclassname">torch.optim.lr_scheduler.</code><code class="descname">CyclicLR</code><span class="sig-paren">(</span><em>optimizer</em>, <em>base_lr</em>, <em>max_lr</em>, <em>step_size_up=2000</em>, <em>step_size_down=None</em>, <em>mode='triangular'</em>, <em>gamma=1.0</em>, <em>scale_fn=None</em>, <em>scale_mode='cycle'</em>, <em>cycle_momentum=True</em>, <em>base_momentum=0.8</em>, <em>max_momentum=0.9</em>, <em>last_epoch=-1</em><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#CyclicLR"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.CyclicLR" title="Permalink to this definition">¶</a></dt>
<dd><p>Sets the learning rate of each parameter group according to
cyclical learning rate policy (CLR). The policy cycles the learning
rate between two boundaries with a constant frequency, as detailed in
the paper <a class="reference external" href="https://arxiv.org/abs/1506.01186">Cyclical Learning Rates for Training Neural Networks</a>.
The distance between the two boundaries can be scaled on a per-iteration
or per-cycle basis.</p>
<p>Cyclical learning rate policy changes the learning rate after every batch.
<cite>step</cite> should be called after a batch has been used for training.</p>
<p>This class has three built-in policies, as put forth in the paper:
“triangular”:</p>
<blockquote>
<div><p>A basic triangular cycle w/ no amplitude scaling.</p>
</div></blockquote>
<dl class="simple">
<dt>“triangular2”:</dt><dd><p>A basic triangular cycle that scales initial amplitude by half each cycle.</p>
</dd>
<dt>“exp_range”:</dt><dd><p>A cycle that scales initial amplitude by gamma**(cycle iterations) at each
cycle iteration.</p>
</dd>
</dl>
<p>This implementation was adapted from the github repo: <a class="reference external" href="https://github.com/bckenstler/CLR">bckenstler/CLR</a></p>
<dl class="field-list simple">
<dt class="field-odd">Parameters</dt>
<dd class="field-odd"><ul class="simple">
<li><p><strong>optimizer</strong> (<a class="reference internal" href="#torch.optim.Optimizer" title="torch.optim.Optimizer"><em>Optimizer</em></a>) – Wrapped optimizer.</p></li>
<li><p><strong>base_lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a>) – Initial learning rate which is the
lower boundary in the cycle for each parameter group.</p></li>
<li><p><strong>max_lr</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a>) – Upper learning rate boundaries in the cycle
for each parameter group. Functionally,
it defines the cycle amplitude (max_lr - base_lr).
The lr at any cycle is the sum of base_lr
and some scaling of the amplitude; therefore
max_lr may not actually be reached depending on
scaling function.</p></li>
<li><p><strong>step_size_up</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of training iterations in the
increasing half of a cycle. Default: 2000</p></li>
<li><p><strong>step_size_down</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – Number of training iterations in the
decreasing half of a cycle. If step_size_down is None,
it is set to step_size_up. Default: None</p></li>
<li><p><strong>mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – One of {triangular, triangular2, exp_range}.
Values correspond to policies detailed above.
If scale_fn is not None, this argument is ignored.
Default: ‘triangular’</p></li>
<li><p><strong>gamma</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a>) – Constant in ‘exp_range’ scaling function:
gamma**(cycle iterations)
Default: 1.0</p></li>
<li><p><strong>scale_fn</strong> (<em>function</em>) – Custom scaling policy defined by a single
argument lambda function, where
0 &lt;= scale_fn(x) &lt;= 1 for all x &gt;= 0.
If specified, then ‘mode’ is ignored.
Default: None</p></li>
<li><p><strong>scale_mode</strong> (<a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#str" title="(in Python v3.7)"><em>str</em></a>) – {‘cycle’, ‘iterations’}.
Defines whether scale_fn is evaluated on
cycle number or cycle iterations (training
iterations since start of cycle).
Default: ‘cycle’</p></li>
<li><p><strong>cycle_momentum</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#bool" title="(in Python v3.7)"><em>bool</em></a>) – If <code class="docutils literal notranslate"><span class="pre">True</span></code>, momentum is cycled inversely
to learning rate between ‘base_momentum’ and ‘max_momentum’.
Default: True</p></li>
<li><p><strong>base_momentum</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a>) – Initial momentum which is the
lower boundary in the cycle for each parameter group.
Default: 0.8</p></li>
<li><p><strong>max_momentum</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#float" title="(in Python v3.7)"><em>float</em></a><em> or </em><a class="reference external" href="https://docs.python.org/3/library/stdtypes.html#list" title="(in Python v3.7)"><em>list</em></a>) – Upper momentum boundaries in the cycle
for each parameter group. Functionally,
it defines the cycle amplitude (max_momentum - base_momentum).
The momentum at any cycle is the difference of max_momentum
and some scaling of the amplitude; therefore
base_momentum may not actually be reached depending on
scaling function. Default: 0.9</p></li>
<li><p><strong>last_epoch</strong> (<a class="reference external" href="https://docs.python.org/3/library/functions.html#int" title="(in Python v3.7)"><em>int</em></a>) – The index of the last batch. This parameter is used when
resuming a training job. Since <cite>step()</cite> should be invoked after each
batch instead of after each epoch, this number represents the total
number of <em>batches</em> computed, not the total number of epochs computed.
When last_epoch=-1, the schedule is started from the beginning.
Default: -1</p></li>
</ul>
</dd>
</dl>
<p class="rubric">Example</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">SGD</span><span class="p">(</span><span class="n">model</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">lr</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scheduler</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">optim</span><span class="o">.</span><span class="n">CyclicLR</span><span class="p">(</span><span class="n">optimizer</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">data_loader</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">utils</span><span class="o">.</span><span class="n">data</span><span class="o">.</span><span class="n">DataLoader</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">for</span> <span class="n">epoch</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">10</span><span class="p">):</span>
<span class="gp">&gt;&gt;&gt; </span>    <span class="k">for</span> <span class="n">batch</span> <span class="ow">in</span> <span class="n">data_loader</span><span class="p">:</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">train_batch</span><span class="p">(</span><span class="o">...</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span>        <span class="n">scheduler</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>
</pre></div>
</div>
<dl class="method">
<dt id="torch.optim.lr_scheduler.CyclicLR.get_lr">
<code class="descname">get_lr</code><span class="sig-paren">(</span><span class="sig-paren">)</span><a class="reference internal" href="_modules/torch/optim/lr_scheduler.html#CyclicLR.get_lr"><span class="viewcode-link">[source]</span></a><a class="headerlink" href="#torch.optim.lr_scheduler.CyclicLR.get_lr" title="Permalink to this definition">¶</a></dt>
<dd><p>Calculates the learning rate at batch index. This function treats
<cite>self.last_epoch</cite> as the last batch index.</p>
<p>If <cite>self.cycle_momentum</cite> is <code class="docutils literal notranslate"><span class="pre">True</span></code>, this function has a side effect of
updating the optimizer’s momentum.</p>
</dd></dl>

</dd></dl>

</div>
</div>


             </article>
             
            </div>
            <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="autograd.html" class="btn btn-neutral float-right" title="Automatic differentiation package - torch.autograd" accesskey="n" rel="next">Next <img src="_static/images/chevron-right-orange.svg" class="next-page"></a>
      
      
        <a href="nn.html" class="btn btn-neutral" title="torch.nn" accesskey="p" rel="prev"><img src="_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
      
    </div>
  

    <hr>

  
  <div role="contentinfo">
    <p>
        &copy; Copyright 2018, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">torch.optim</a><ul>
<li><a class="reference internal" href="#how-to-use-an-optimizer">How to use an optimizer</a><ul>
<li><a class="reference internal" href="#constructing-it">Constructing it</a></li>
<li><a class="reference internal" href="#per-parameter-options">Per-parameter options</a></li>
<li><a class="reference internal" href="#taking-an-optimization-step">Taking an optimization step</a><ul>
<li><a class="reference internal" href="#optimizer-step"><code class="docutils literal notranslate"><span class="pre">optimizer.step()</span></code></a></li>
<li><a class="reference internal" href="#optimizer-step-closure"><code class="docutils literal notranslate"><span class="pre">optimizer.step(closure)</span></code></a></li>
</ul>
</li>
</ul>
</li>
<li><a class="reference internal" href="#algorithms">Algorithms</a></li>
<li><a class="reference internal" href="#how-to-adjust-learning-rate">How to adjust Learning Rate</a></li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>

  
       <script type="text/javascript" id="documentation_options" data-url_root="./" src="_static/documentation_options.js"></script>
         <script type="text/javascript" src="_static/jquery.js"></script>
         <script type="text/javascript" src="_static/underscore.js"></script>
         <script type="text/javascript" src="_static/doctools.js"></script>
         <script type="text/javascript" src="_static/language_data.js"></script>
         <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/katex.min.js"></script>
         <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/contrib/auto-render.min.js"></script>
         <script type="text/javascript" src="_static/katex_autorenderer.js"></script>
     

  <script type="text/javascript" src="_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="_static/js/vendor/bootstrap.min.js"></script>
  <script type="text/javascript" src="_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script> 

  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://pytorch.org/resources">Resources</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/support">Support</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.slack.com" target="_blank">Slack</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md" target="_blank">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Follow Us</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="#">Get Started</a>
          </li>

          <li>
            <a href="#">Features</a>
          </li>

          <li>
            <a href="#">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>