2.6/_modules/torch/autograd.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta name="robots" content="noindex">
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.autograd &mdash; PyTorch 2.6 documentation</title>
  

    <link rel="canonical" href="https://pytorch.org/docs/stable/_modules/torch/autograd.html"/>
  

  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="../../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/copybutton.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/katex-math.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/sphinx-dropdown.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/panels-bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="../../_static/css/custom.css" type="text/css" />
    <link rel="index" title="Index" href="../../genindex.html" />
    <link rel="search" title="Search" href="../../search.html" />


  <!-- Google Tag Manager -->
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
    new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
    j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
    'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
    })(window,document,'script','dataLayer','GTM-T8XT4PS');</script>
    <!-- End Google Tag Manager -->
  

  <script src="../../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>

          <li class="main-menu-item">
          <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Learn
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/get-started">
                  <span class=dropdown-title>Get Started</span>
                  <p>Run PyTorch locally or get started quickly with one of the supported cloud platforms</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tutorials">
                  <span class="dropdown-title">Tutorials</span>
                  <p>Whats new in PyTorch tutorials</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tutorials/beginner/basics/intro.html">
                  <span class="dropdown-title">Learn the Basics</span>
                  <p>Familiarize yourself with PyTorch concepts and modules</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tutorials/recipes/recipes_index.html">
                  <span class="dropdown-title">PyTorch Recipes</span>
                  <p>Bite-size, ready-to-deploy PyTorch code examples</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tutorials/beginner/introyt.html">
                  <span class="dropdown-title">Intro to PyTorch - YouTube Series</span>
                  <p>Master PyTorch basics with our engaging YouTube tutorial series</p>
                </a>
              </div>
            </div>
          </li>

          <li>
          <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Ecosystem
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class="dropdown-title">Tools</span>
                  <p>Learn about the tools and frameworks in the PyTorch Ecosystem</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/#community-module">
                  <span class=dropdown-title>Community</span>
                  <p>Join the PyTorch developer community to contribute, learn, and get your questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://discuss.pytorch.org/" target="_blank">
                  <span class=dropdown-title>Forums</span>
                  <p>A place to discuss PyTorch code, issues, install, research</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/resources">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem/contributor-awards-2023">
                  <span class="dropdown-title">Contributor Awards - 2023</span>
                  <p>Award winners announced at this year's PyTorch Conference</p>
                </a>
              </div>
            </div>
          </li>

          <li>
          <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Edge
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/edge">
                  <span class="dropdown-title">About PyTorch Edge</span>
                  <p>Build innovative and privacy-aware AI experiences for edge devices</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/executorch-overview">
                  <span class="dropdown-title">ExecuTorch</span>
                  <p>End-to-end solution for enabling on-device inference capabilities across mobile and edge devices</p>
                </a>
              </div>
            </div>  
          </li>

          <li class="main-menu-item">
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Docs
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/docs/stable/index.html">
                  <span class="dropdown-title">PyTorch</span>
                  <p>Explore the documentation for comprehensive guidance on how to use PyTorch</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/pytorch-domains">
                  <span class="dropdown-title">PyTorch Domains</span>
                  <p>Read the PyTorch Domains documentation to learn more about domain-specific libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Blogs & News 
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/blog/">
                  <span class="dropdown-title">PyTorch Blog</span>
                  <p>Catch up on the latest technical news and happenings</p>
                </a>
                 <a class="nav-dropdown-item" href="https://pytorch.org/community-blog">
                  <span class="dropdown-title">Community Blog</span>
                  <p>Stories from the PyTorch ecosystem</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/videos">
                  <span class="dropdown-title">Videos</span>
                  <p>Learn about the latest PyTorch tutorials, new, and more </p>
                <a class="nav-dropdown-item" href="https://pytorch.org/community-stories">
                  <span class="dropdown-title">Community Stories</span>
                  <p>Learn how our community solves real, everyday machine learning problems with PyTorch</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/events">
                  <span class="dropdown-title">Events</span>
                  <p>Find events, webinars, and podcasts</p>
                </a>
            </div>
          </li>

          <li>
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                About
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/foundation">
                  <span class="dropdown-title">PyTorch Foundation</span>
                  <p>Learn more about the PyTorch Foundation</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/governing-board">
                  <span class="dropdown-title">Governing Board</span>
                  <p></p>
                </a>
              </div>
            </div>
          </li>

          <li class="main-menu-item">
            <div class="no-dropdown">
              <a href="https://pytorch.org/join" data-cta="join">
                Become a Member
              </a>
            </div>
          </li>
          <li>
           <div class="main-menu-item">
             <a href="https://github.com/pytorch/pytorch" class="github-icon">
             </a>
           </div>
          </li>
          <!--- TODO: This block adds the search icon to the nav bar. We will enable it later. 
          <li>
            <div class="main-menu-item">
             <a href="https://github.com/pytorch/pytorch" class="search-icon">
             </a>
            </div>
          </li>
          --->
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>
  </div>
</div>

<body class="pytorch-body">

   
    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            
    <div class="version">
      <a href='https://pytorch.org/docs/versions.html'>2.6 &#x25BC</a>
    </div>
    <div id="searchBox">
    <div class="searchbox" id="googleSearchBox">
      <script async src="https://cse.google.com/cse.js?cx=e65585f8c3ea1440e"></script>
      <div class="gcse-search"></div>
    </div>
    <div id="sphinxSearchBox" style="display: none;">
      <div role="search">
        <form id="rtd-search-form" class="wy-form" action="../../search.html" method="get">
          <input type="text" name="q" placeholder="Search Docs" />
          <input type="hidden" name="check_keywords" value="yes" />
          <input type="hidden" name="area" value="default" />
        </form>
      </div>
    </div>
  </div>
  <form id="searchForm">
    <label style="margin-bottom: 1rem">
      <input type="radio" name="searchType" value="google" checked>
      Google Search
    </label>
    <label style="margin-bottom: 1rem">
      <input type="radio" name="searchType" value="sphinx">
      Classic Search
    </label>
  </form>

  <script>
     document.addEventListener('DOMContentLoaded', function() {
      const searchForm = document.getElementById('searchForm');
      const googleSearchBox = document.getElementById('googleSearchBox');
      const sphinxSearchBox = document.getElementById('sphinxSearchBox');
      // Function to toggle search box visibility
      function toggleSearchBox(searchType) {
        googleSearchBox.style.display = searchType === 'google' ? 'block' : 'none';
        sphinxSearchBox.style.display = searchType === 'sphinx' ? 'block' : 'none';
      }
      // Determine the default search type
      let defaultSearchType;
      const currentUrl = window.location.href;
      if (currentUrl.startsWith('https://pytorch.org/docs/stable')) {
        // For the stable documentation, default to Google
        defaultSearchType = localStorage.getItem('searchType') || 'google';
      } else {
        // For any other version, including docs-preview, default to Sphinx
        defaultSearchType = 'sphinx';
      }
      // Set the default search type
      document.querySelector(`input[name="searchType"][value="${defaultSearchType}"]`).checked = true;
      toggleSearchBox(defaultSearchType);
      // Event listener for changes in search type
      searchForm.addEventListener('change', function(event) {
        const selectedSearchType = event.target.value;
        localStorage.setItem('searchType', selectedSearchType);
        toggleSearchBox(selectedSearchType);
      });
      // Set placeholder text for Google search box
      window.onload = function() {
        var placeholderText = "Search Docs";
        var googleSearchboxText = document.querySelector("#gsc-i-id1");
        if (googleSearchboxText) {
          googleSearchboxText.placeholder = placeholderText;
          googleSearchboxText.style.fontFamily = 'FreightSans';
          googleSearchboxText.style.fontSize = "1.2rem";
          googleSearchboxText.style.color = '#262626';
        }
      };
    });
  </script>
     <!--temporarily add a link to survey -->
     <div class="survey-link">
       <p><i class="fas fa-poll" aria-hidden="true">&nbsp </i><a href="https://forms.gle/tdrnwJhaQ9tUePxz9">Share Your Feedback</a> about our new search</p>
     </div>

          </div>

          
              <p class="caption" role="heading"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../community/build_ci_governance.html">PyTorch Governance | Build + CI</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../community/design.html">PyTorch Design Philosophy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../community/governance.html">PyTorch Governance | Mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../community/persons_of_interest.html">PyTorch Governance | Maintainers</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Developer Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/custom_operators.html">PyTorch Custom Operators Landing Page</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/extending.func.html">Extending torch.func with autograd.Function</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/fsdp.html">FSDP Notes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/get_start_xpu.html">Getting Started on Intel GPU</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/gradcheck.html">Gradcheck mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/hip.html">HIP (ROCm) semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/modules.html">Modules</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/mps.html">MPS backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/numerical_accuracy.html">Numerical accuracy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../cpp_index.html">C++</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/javadoc/">Javadoc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../deploy.html">torch::deploy</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../amp.html">torch.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../library.html">torch.library</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../accelerator.html">torch.accelerator</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cpu.html">torch.cpu</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../torch_cuda_memory.html">Understanding CUDA Memory Usage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../torch_cuda_memory.html#generating-a-snapshot">Generating a Snapshot</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../torch_cuda_memory.html#using-the-visualizer">Using the visualizer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../torch_cuda_memory.html#snapshot-api-reference">Snapshot API Reference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mps.html">torch.mps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../xpu.html">torch.xpu</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mtia.html">torch.mtia</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mtia.memory.html">torch.mtia.memory</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../meta.html">Meta device</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../backends.html">torch.backends</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../export.html">torch.export</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.tensor.html">torch.distributed.tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.algorithms.join.html">torch.distributed.algorithms.join</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.elastic.html">torch.distributed.elastic</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../fsdp.html">torch.distributed.fsdp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.fsdp.fully_shard.html">torch.distributed.fsdp.fully_shard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.tensor.parallel.html">torch.distributed.tensor.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.optim.html">torch.distributed.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.pipelining.html">torch.distributed.pipelining</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributed.checkpoint.html">torch.distributed.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../torch.compiler.html">torch.compiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../fft.html">torch.fft</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../func.html">torch.func</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../futures.html">torch.futures</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../fx.html">torch.fx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../fx.experimental.html">torch.fx.experimental</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../linalg.html">torch.linalg</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../monitor.html">torch.monitor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../signal.html">torch.signal</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../special.html">torch.special</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../torch.overrides.html">torch.overrides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../package.html">torch.package</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../profiler.html">torch.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../nn.attention.html">torch.nn.attention</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../complex_numbers.html">Complex Numbers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../ddp_comm_hooks.html">DDP Communication Hooks</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../rpc.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../masked.html">torch.masked</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../nested.html">torch.nested</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../size.html">torch.Size</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../testing.html">torch.testing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../utils.html">torch.utils</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../benchmark_utils.html">torch.utils.benchmark</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../deterministic.html">torch.utils.deterministic</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../jit_utils.html">torch.utils.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../mobile_optimizer.html">torch.utils.mobile_optimizer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../module_tracker.html">torch.utils.module_tracker</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../config_mod.html">torch.__config__</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../future_mod.html">torch.__future__</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../logging.html">torch._logging</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../torch_environment_variables.html">Torch Environment Variables</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio/stable">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/data">TorchData</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/torchrec">TorchRec</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text/stable">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/vision/stable">torchvision</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>

            
        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          

<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
          <li><a href="../index.html">Module code</a> &gt;</li>
        
          <li><a href="../torch.html">torch</a> &gt;</li>
        
      <li>torch.autograd</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          <!-- Google Tag Manager (noscript) -->
          <noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-T8XT4PS"
          height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
          <!-- End Google Tag Manager (noscript) -->
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <h1>Source code for torch.autograd</h1><div class="highlight"><pre>
<span></span><span class="c1"># mypy: allow-untyped-defs</span>
<span class="sd">&quot;&quot;&quot;</span>
<span class="sd">``torch.autograd`` provides classes and functions implementing automatic differentiation of arbitrary scalar valued functions.</span>

<span class="sd">It requires minimal changes to the existing code - you only need to declare :class:`Tensor` s</span>
<span class="sd">for which gradients should be computed with the ``requires_grad=True`` keyword.</span>
<span class="sd">As of now, we only support autograd for floating point :class:`Tensor` types (</span>
<span class="sd">half, float, double and bfloat16) and complex :class:`Tensor` types (cfloat, cdouble).</span>
<span class="sd">&quot;&quot;&quot;</span>

<span class="kn">import</span> <span class="nn">warnings</span>
<span class="kn">from</span> <span class="nn">typing</span> <span class="kn">import</span> <span class="n">cast</span><span class="p">,</span> <span class="n">List</span><span class="p">,</span> <span class="n">Optional</span><span class="p">,</span> <span class="n">Sequence</span><span class="p">,</span> <span class="n">Tuple</span><span class="p">,</span> <span class="n">Union</span>

<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">_vmap_internals</span>
<span class="kn">from</span> <span class="nn">torch.overrides</span> <span class="kn">import</span> <span class="n">handle_torch_function</span><span class="p">,</span> <span class="n">has_torch_function</span><span class="p">,</span> <span class="n">is_tensor_like</span>
<span class="kn">from</span> <span class="nn">torch.types</span> <span class="kn">import</span> <span class="n">_size</span><span class="p">,</span> <span class="n">_TensorOrTensors</span><span class="p">,</span> <span class="n">_TensorOrTensorsOrGradEdge</span>

<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">forward_ad</span><span class="p">,</span> <span class="n">functional</span><span class="p">,</span> <span class="n">graph</span>
<span class="kn">from</span> <span class="nn">.anomaly_mode</span> <span class="kn">import</span> <span class="n">detect_anomaly</span><span class="p">,</span> <span class="n">set_detect_anomaly</span>
<span class="kn">from</span> <span class="nn">.function</span> <span class="kn">import</span> <span class="n">Function</span><span class="p">,</span> <span class="n">NestedIOFunction</span>
<span class="kn">from</span> <span class="nn">.grad_mode</span> <span class="kn">import</span> <span class="p">(</span>
    <span class="n">_force_original_view_tracking</span><span class="p">,</span>
    <span class="n">_unsafe_preserve_version_counter</span><span class="p">,</span>
    <span class="n">enable_grad</span><span class="p">,</span>
    <span class="n">inference_mode</span><span class="p">,</span>
    <span class="n">no_grad</span><span class="p">,</span>
    <span class="n">set_grad_enabled</span><span class="p">,</span>
    <span class="n">set_multithreading_enabled</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span> <span class="nn">.gradcheck</span> <span class="kn">import</span> <span class="n">gradcheck</span><span class="p">,</span> <span class="n">gradgradcheck</span>
<span class="kn">from</span> <span class="nn">.graph</span> <span class="kn">import</span> <span class="n">_engine_run_backward</span>
<span class="kn">from</span> <span class="nn">.variable</span> <span class="kn">import</span> <span class="n">Variable</span>


<span class="n">__all__</span> <span class="o">=</span> <span class="p">[</span>
    <span class="s2">&quot;Variable&quot;</span><span class="p">,</span>
    <span class="s2">&quot;Function&quot;</span><span class="p">,</span>
    <span class="s2">&quot;backward&quot;</span><span class="p">,</span>
    <span class="s2">&quot;grad_mode&quot;</span><span class="p">,</span>
    <span class="s2">&quot;NestedIOFunction&quot;</span><span class="p">,</span>
    <span class="s2">&quot;detect_anomaly&quot;</span><span class="p">,</span>
    <span class="s2">&quot;enable_grad&quot;</span><span class="p">,</span>
    <span class="s2">&quot;grad&quot;</span><span class="p">,</span>
    <span class="s2">&quot;gradcheck&quot;</span><span class="p">,</span>
    <span class="s2">&quot;gradgradcheck&quot;</span><span class="p">,</span>
    <span class="s2">&quot;inference_mode&quot;</span><span class="p">,</span>
    <span class="s2">&quot;no_grad&quot;</span><span class="p">,</span>
    <span class="s2">&quot;set_detect_anomaly&quot;</span><span class="p">,</span>
    <span class="s2">&quot;set_grad_enabled&quot;</span><span class="p">,</span>
    <span class="s2">&quot;set_multithreading_enabled&quot;</span><span class="p">,</span>
    <span class="s2">&quot;variable&quot;</span><span class="p">,</span>
<span class="p">]</span>

<span class="n">_OptionalTensor</span> <span class="o">=</span> <span class="n">Optional</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">]</span>
<span class="n">_ShapeorNestedShape</span> <span class="o">=</span> <span class="n">Union</span><span class="p">[</span><span class="n">_size</span><span class="p">,</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">_size</span><span class="p">],</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">]</span>


<span class="k">def</span> <span class="nf">_calculate_shape</span><span class="p">(</span>
    <span class="n">output</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">],</span>
    <span class="n">grad</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span>
    <span class="n">is_grads_batched</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">_ShapeorNestedShape</span><span class="p">,</span> <span class="n">_ShapeorNestedShape</span><span class="p">]:</span>
    <span class="c1"># is_same_size ensures that both tensors are either nested or non nested</span>
    <span class="c1"># circular import</span>
    <span class="kn">from</span> <span class="nn">torch.nested._internal.nested_tensor</span> <span class="kn">import</span> <span class="n">NestedTensor</span>

    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">):</span>
        <span class="c1"># We have already checked that we are not a C++ NestedTensor</span>
        <span class="k">if</span> <span class="n">is_grads_batched</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Batched grads are not supported with GradientEdge&quot;</span><span class="p">)</span>
        <span class="n">out_metadata</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">node</span><span class="o">.</span><span class="n">_input_metadata</span><span class="p">[</span><span class="n">output</span><span class="o">.</span><span class="n">output_nr</span><span class="p">]</span>
        <span class="k">return</span> <span class="n">torch</span><span class="o">.</span><span class="n">Size</span><span class="p">(</span><span class="n">out_metadata</span><span class="o">.</span><span class="n">shape</span><span class="p">),</span> <span class="n">grad</span><span class="o">.</span><span class="n">shape</span>

    <span class="k">if</span> <span class="n">output</span><span class="o">.</span><span class="n">is_nested</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="n">NestedTensor</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">is_grads_batched</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;Batched grads are not supported with Nested Tensor.&quot;</span><span class="p">)</span>
        <span class="n">out_shape</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">_nested_tensor_size</span><span class="p">()</span>
        <span class="n">grad_shape</span> <span class="o">=</span> <span class="n">grad</span><span class="o">.</span><span class="n">_nested_tensor_size</span><span class="p">()</span>

        <span class="k">return</span> <span class="n">out_shape</span><span class="p">,</span> <span class="n">grad_shape</span>

    <span class="n">reg_out_shape</span> <span class="o">=</span> <span class="n">output</span><span class="o">.</span><span class="n">shape</span>
    <span class="n">reg_grad_shape</span> <span class="o">=</span> <span class="n">grad</span><span class="o">.</span><span class="n">shape</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">is_grads_batched</span> <span class="k">else</span> <span class="n">grad</span><span class="o">.</span><span class="n">shape</span><span class="p">[</span><span class="mi">1</span><span class="p">:]</span>
    <span class="k">return</span> <span class="n">reg_out_shape</span><span class="p">,</span> <span class="n">reg_grad_shape</span>


<span class="k">def</span> <span class="nf">_make_grads</span><span class="p">(</span>
    <span class="n">outputs</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">],</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">]],</span>
    <span class="n">grads</span><span class="p">:</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">_OptionalTensor</span><span class="p">],</span>
    <span class="n">is_grads_batched</span><span class="p">:</span> <span class="nb">bool</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">_OptionalTensor</span><span class="p">,</span> <span class="o">...</span><span class="p">]:</span>
    <span class="n">new_grads</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">_OptionalTensor</span><span class="p">]</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">for</span> <span class="n">out</span><span class="p">,</span> <span class="n">grad</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="n">grads</span><span class="p">):</span>
        <span class="n">out</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">Union</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">],</span> <span class="n">out</span><span class="p">)</span>
        <span class="n">out_size</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="n">out_device</span> <span class="o">=</span> <span class="kc">None</span>

        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">):</span>
            <span class="n">out_metadata</span> <span class="o">=</span> <span class="n">out</span><span class="o">.</span><span class="n">node</span><span class="o">.</span><span class="n">_input_metadata</span><span class="p">[</span><span class="n">out</span><span class="o">.</span><span class="n">output_nr</span><span class="p">]</span>
            <span class="n">out_size</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">Size</span><span class="p">(</span><span class="n">out_metadata</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
            <span class="n">out_dtype</span> <span class="o">=</span> <span class="n">out_metadata</span><span class="o">.</span><span class="n">dtype</span>
            <span class="n">out_device</span> <span class="o">=</span> <span class="n">out_metadata</span><span class="o">.</span><span class="n">device</span>
            <span class="n">out_is_nested</span> <span class="o">=</span> <span class="n">out_metadata</span><span class="o">.</span><span class="n">is_nested_tensor</span>
            <span class="k">if</span> <span class="n">out_metadata</span><span class="o">.</span><span class="n">is_cpp_nested_tensor</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                    <span class="s2">&quot;C++ NestedTensor are not supported with GradientEdge&quot;</span>
                <span class="p">)</span>
            <span class="n">out_is_cpp_nested</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="c1"># circular import</span>
            <span class="kn">from</span> <span class="nn">torch.nested._internal.nested_tensor</span> <span class="kn">import</span> <span class="n">NestedTensor</span>

            <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">)</span>
            <span class="n">out_dtype</span> <span class="o">=</span> <span class="n">out</span><span class="o">.</span><span class="n">dtype</span>
            <span class="n">out_is_nested</span> <span class="o">=</span> <span class="n">out</span><span class="o">.</span><span class="n">is_nested</span>
            <span class="n">out_is_cpp_nested</span> <span class="o">=</span> <span class="n">out_is_nested</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">NestedTensor</span><span class="p">)</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="n">out_is_cpp_nested</span><span class="p">:</span>
                <span class="n">out_size</span> <span class="o">=</span> <span class="n">out</span><span class="o">.</span><span class="n">shape</span>

        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">grad</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
            <span class="kn">from</span> <span class="nn">torch.fx.experimental.symbolic_shapes</span> <span class="kn">import</span> <span class="n">expect_true</span><span class="p">,</span> <span class="n">sym_eq</span>

            <span class="n">first_grad</span> <span class="o">=</span> <span class="n">grad</span> <span class="k">if</span> <span class="ow">not</span> <span class="n">is_grads_batched</span> <span class="k">else</span> <span class="n">grad</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>

            <span class="c1"># TODO: We can remove this conditional once we uniformly use</span>
            <span class="c1"># singleton int to represent jagged dimension, so that size() call</span>
            <span class="c1"># on nested tensor works.</span>
            <span class="k">if</span> <span class="n">out_is_cpp_nested</span><span class="p">:</span>
                <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">)</span>
                <span class="n">shape_matches</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">is_same_size</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">first_grad</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="c1"># We need to do a regular size check, without going through</span>
                <span class="c1"># the operator, to be able to handle unbacked symints</span>
                <span class="c1"># (expect_true ensures we can deal with unbacked)</span>
                <span class="k">assert</span> <span class="n">out_size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
                <span class="n">shape_matches</span> <span class="o">=</span> <span class="n">expect_true</span><span class="p">(</span><span class="n">sym_eq</span><span class="p">(</span><span class="n">out_size</span><span class="p">,</span> <span class="n">first_grad</span><span class="o">.</span><span class="n">size</span><span class="p">()))</span>

            <span class="k">if</span> <span class="ow">not</span> <span class="n">shape_matches</span><span class="p">:</span>
                <span class="n">out</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">Union</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">],</span> <span class="n">out</span><span class="p">)</span>
                <span class="n">out_shape</span><span class="p">,</span> <span class="n">grad_shape</span> <span class="o">=</span> <span class="n">_calculate_shape</span><span class="p">(</span>
                    <span class="n">out</span><span class="p">,</span> <span class="n">first_grad</span><span class="p">,</span> <span class="n">is_grads_batched</span>
                <span class="p">)</span>
                <span class="k">if</span> <span class="n">is_grads_batched</span><span class="p">:</span>
                    <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                        <span class="s2">&quot;If `is_grads_batched=True`, we interpret the first &quot;</span>
                        <span class="s2">&quot;dimension of each grad_output as the batch dimension. &quot;</span>
                        <span class="s2">&quot;The sizes of the remaining dimensions are expected to match &quot;</span>
                        <span class="s2">&quot;the shape of corresponding output, but a mismatch &quot;</span>
                        <span class="s2">&quot;was detected: grad_output[&quot;</span>
                        <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">grads</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">grad</span><span class="p">))</span>
                        <span class="o">+</span> <span class="s2">&quot;] has a shape of &quot;</span>
                        <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">grad_shape</span><span class="p">)</span>
                        <span class="o">+</span> <span class="s2">&quot; and output[&quot;</span>
                        <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">outputs</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">out</span><span class="p">))</span>
                        <span class="o">+</span> <span class="s2">&quot;] has a shape of &quot;</span>
                        <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">out_shape</span><span class="p">)</span>
                        <span class="o">+</span> <span class="s2">&quot;. &quot;</span>
                        <span class="s2">&quot;If you only want some tensors in `grad_output` to be considered &quot;</span>
                        <span class="s2">&quot;batched, consider using vmap.&quot;</span>
                    <span class="p">)</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                        <span class="s2">&quot;Mismatch in shape: grad_output[&quot;</span>
                        <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">grads</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">grad</span><span class="p">))</span>
                        <span class="o">+</span> <span class="s2">&quot;] has a shape of &quot;</span>
                        <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">grad_shape</span><span class="p">)</span>
                        <span class="o">+</span> <span class="s2">&quot; and output[&quot;</span>
                        <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">outputs</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">out</span><span class="p">))</span>
                        <span class="o">+</span> <span class="s2">&quot;] has a shape of &quot;</span>
                        <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">out_shape</span><span class="p">)</span>
                        <span class="o">+</span> <span class="s2">&quot;.&quot;</span>
                    <span class="p">)</span>
            <span class="k">if</span> <span class="n">out_dtype</span><span class="o">.</span><span class="n">is_complex</span> <span class="o">!=</span> <span class="n">grad</span><span class="o">.</span><span class="n">dtype</span><span class="o">.</span><span class="n">is_complex</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                    <span class="s2">&quot;For complex Tensors, both grad_output and output&quot;</span>
                    <span class="s2">&quot; are required to have the same dtype.&quot;</span>
                    <span class="s2">&quot; Mismatch in dtype: grad_output[&quot;</span>
                    <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">grads</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">grad</span><span class="p">))</span>
                    <span class="o">+</span> <span class="s2">&quot;] has a dtype of &quot;</span>
                    <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">grad</span><span class="o">.</span><span class="n">dtype</span><span class="p">)</span>
                    <span class="o">+</span> <span class="s2">&quot; and output[&quot;</span>
                    <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">outputs</span><span class="o">.</span><span class="n">index</span><span class="p">(</span><span class="n">out</span><span class="p">))</span>
                    <span class="o">+</span> <span class="s2">&quot;] has a dtype of &quot;</span>
                    <span class="o">+</span> <span class="nb">str</span><span class="p">(</span><span class="n">out_dtype</span><span class="p">)</span>
                    <span class="o">+</span> <span class="s2">&quot;.&quot;</span>
                <span class="p">)</span>
            <span class="n">new_grads</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">grad</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">grad</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">)</span> <span class="ow">or</span> <span class="n">out</span><span class="o">.</span><span class="n">requires_grad</span><span class="p">:</span>  <span class="c1"># type: ignore[attr-defined]</span>
                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">):</span>
                    <span class="k">assert</span> <span class="n">out_size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
                    <span class="n">out_numel_is_1</span> <span class="o">=</span> <span class="nb">all</span><span class="p">(</span><span class="n">o</span> <span class="o">==</span> <span class="mi">1</span> <span class="k">for</span> <span class="n">o</span> <span class="ow">in</span> <span class="n">out_size</span><span class="p">)</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">)</span>
                    <span class="n">out_numel_is_1</span> <span class="o">=</span> <span class="n">out</span><span class="o">.</span><span class="n">numel</span><span class="p">()</span> <span class="o">==</span> <span class="mi">1</span>
                <span class="k">if</span> <span class="ow">not</span> <span class="n">out_numel_is_1</span><span class="p">:</span>
                    <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                        <span class="s2">&quot;grad can be implicitly created only for scalar outputs&quot;</span>
                    <span class="p">)</span>
                <span class="k">if</span> <span class="ow">not</span> <span class="n">out_dtype</span><span class="o">.</span><span class="n">is_floating_point</span><span class="p">:</span>
                    <span class="n">msg</span> <span class="o">=</span> <span class="p">(</span>
                        <span class="s2">&quot;grad can be implicitly created only for real scalar outputs&quot;</span>
                        <span class="sa">f</span><span class="s2">&quot; but got </span><span class="si">{</span><span class="n">out_dtype</span><span class="si">}</span><span class="s2">&quot;</span>
                    <span class="p">)</span>
                    <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>
                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">):</span>
                    <span class="k">assert</span> <span class="n">out_size</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
                    <span class="k">assert</span> <span class="n">out_device</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
                    <span class="n">new_grads</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                        <span class="n">torch</span><span class="o">.</span><span class="n">ones</span><span class="p">(</span>
                            <span class="n">out_size</span><span class="p">,</span>
                            <span class="n">dtype</span><span class="o">=</span><span class="n">out_dtype</span><span class="p">,</span>
                            <span class="n">device</span><span class="o">=</span><span class="n">out_device</span><span class="p">,</span>
                        <span class="p">)</span>
                    <span class="p">)</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="k">assert</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">)</span>
                    <span class="n">new_grads</span><span class="o">.</span><span class="n">append</span><span class="p">(</span>
                        <span class="n">torch</span><span class="o">.</span><span class="n">ones_like</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="n">memory_format</span><span class="o">=</span><span class="n">torch</span><span class="o">.</span><span class="n">preserve_format</span><span class="p">)</span>
                    <span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">new_grads</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="kc">None</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
                <span class="s2">&quot;gradients can be either Tensors or None, but got &quot;</span>
                <span class="o">+</span> <span class="nb">type</span><span class="p">(</span><span class="n">grad</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span>
            <span class="p">)</span>
    <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">new_grads</span><span class="p">)</span>


<span class="k">def</span> <span class="nf">_tensor_or_tensors_to_tuple</span><span class="p">(</span>
    <span class="n">tensors</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_TensorOrTensors</span><span class="p">],</span> <span class="n">length</span><span class="p">:</span> <span class="nb">int</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">_OptionalTensor</span><span class="p">,</span> <span class="o">...</span><span class="p">]:</span>
    <span class="k">if</span> <span class="n">tensors</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">return</span> <span class="p">(</span><span class="kc">None</span><span class="p">,)</span> <span class="o">*</span> <span class="n">length</span>
    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">tensors</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
        <span class="k">return</span> <span class="p">(</span><span class="n">tensors</span><span class="p">,)</span>
    <span class="k">return</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">tensors</span><span class="p">)</span>


<div class="viewcode-block" id="backward"><a class="viewcode-back" href="../../generated/torch.autograd.backward.html#torch.autograd.backward">[docs]</a><span class="k">def</span> <span class="nf">backward</span><span class="p">(</span>
    <span class="n">tensors</span><span class="p">:</span> <span class="n">_TensorOrTensors</span><span class="p">,</span>
    <span class="n">grad_tensors</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_TensorOrTensors</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="n">retain_graph</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="n">create_graph</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
    <span class="n">grad_variables</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_TensorOrTensors</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="n">inputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_TensorOrTensorsOrGradEdge</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Compute the sum of gradients of given tensors with respect to graph leaves.</span>

<span class="sd">    The graph is differentiated using the chain rule. If any of ``tensors``</span>
<span class="sd">    are non-scalar (i.e. their data has more than one element) and require</span>
<span class="sd">    gradient, then the Jacobian-vector product would be computed, in this</span>
<span class="sd">    case the function additionally requires specifying ``grad_tensors``.</span>
<span class="sd">    It should be a sequence of matching length, that contains the &quot;vector&quot;</span>
<span class="sd">    in the Jacobian-vector product, usually the gradient of the differentiated</span>
<span class="sd">    function w.r.t. corresponding tensors (``None`` is an acceptable value for</span>
<span class="sd">    all tensors that don&#39;t need gradient tensors).</span>

<span class="sd">    This function accumulates gradients in the leaves - you might need to zero</span>
<span class="sd">    ``.grad`` attributes or set them to ``None`` before calling it.</span>
<span class="sd">    See :ref:`Default gradient layouts&lt;default-grad-layouts&gt;`</span>
<span class="sd">    for details on the memory layout of accumulated gradients.</span>

<span class="sd">    .. note::</span>
<span class="sd">        Using this method with ``create_graph=True`` will create a reference cycle</span>
<span class="sd">        between the parameter and its gradient which can cause a memory leak.</span>
<span class="sd">        We recommend using ``autograd.grad`` when creating the graph to avoid this.</span>
<span class="sd">        If you have to use this function, make sure to reset the ``.grad`` fields of your</span>
<span class="sd">        parameters to ``None`` after use to break the cycle and avoid the leak.</span>

<span class="sd">    .. note::</span>

<span class="sd">        If you run any forward ops, create ``grad_tensors``, and/or call ``backward``</span>
<span class="sd">        in a user-specified CUDA stream context, see</span>
<span class="sd">        :ref:`Stream semantics of backward passes&lt;bwd-cuda-stream-semantics&gt;`.</span>

<span class="sd">    .. note::</span>

<span class="sd">        When ``inputs`` are provided and a given input is not a leaf,</span>
<span class="sd">        the current implementation will call its grad_fn (even though it is not strictly needed to get this gradients).</span>
<span class="sd">        It is an implementation detail on which the user should not rely.</span>
<span class="sd">        See https://github.com/pytorch/pytorch/pull/60521#issuecomment-867061780 for more details.</span>

<span class="sd">    Args:</span>
<span class="sd">        tensors (Sequence[Tensor] or Tensor): Tensors of which the derivative will be</span>
<span class="sd">            computed.</span>
<span class="sd">        grad_tensors (Sequence[Tensor or None] or Tensor, optional): The &quot;vector&quot; in</span>
<span class="sd">            the Jacobian-vector product, usually gradients w.r.t. each element of</span>
<span class="sd">            corresponding tensors. None values can be specified for scalar Tensors or</span>
<span class="sd">            ones that don&#39;t require grad. If a None value would be acceptable for all</span>
<span class="sd">            grad_tensors, then this argument is optional.</span>
<span class="sd">        retain_graph (bool, optional): If ``False``, the graph used to compute the grad</span>
<span class="sd">            will be freed. Note that in nearly all cases setting this option to ``True``</span>
<span class="sd">            is not needed and often can be worked around in a much more efficient</span>
<span class="sd">            way. Defaults to the value of ``create_graph``.</span>
<span class="sd">        create_graph (bool, optional): If ``True``, graph of the derivative will</span>
<span class="sd">            be constructed, allowing to compute higher order derivative products.</span>
<span class="sd">            Defaults to ``False``.</span>
<span class="sd">        inputs (Sequence[Tensor] or Tensor or Sequence[GradientEdge], optional): Inputs w.r.t. which the gradient</span>
<span class="sd">            be will accumulated into ``.grad``. All other Tensors will be ignored. If</span>
<span class="sd">            not provided, the gradient is accumulated into all the leaf Tensors that</span>
<span class="sd">            were used to compute the :attr:`tensors`.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_are_functorch_transforms_active</span><span class="p">():</span>
        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
            <span class="s2">&quot;backward() called inside a functorch transform. This is not &quot;</span>
            <span class="s2">&quot;supported, please use functorch.grad or functorch.vjp instead &quot;</span>
            <span class="s2">&quot;or call backward() outside of functorch transforms.&quot;</span>
        <span class="p">)</span>

    <span class="k">if</span> <span class="n">grad_variables</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
            <span class="s2">&quot;`grad_variables` is deprecated. Use `grad_tensors` instead.&quot;</span><span class="p">,</span>
            <span class="ne">FutureWarning</span><span class="p">,</span>
            <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
        <span class="p">)</span>
        <span class="k">if</span> <span class="n">grad_tensors</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">grad_tensors</span> <span class="o">=</span> <span class="n">grad_variables</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                <span class="s2">&quot;`grad_tensors` and `grad_variables` (deprecated) &quot;</span>
                <span class="s2">&quot;arguments both passed to `backward()`. Please only &quot;</span>
                <span class="s2">&quot;use `grad_tensors`.&quot;</span>
            <span class="p">)</span>
    <span class="k">if</span> <span class="n">inputs</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;`inputs` argument to `backward()` cannot be empty.&quot;</span><span class="p">)</span>

    <span class="n">tensors</span> <span class="o">=</span> <span class="p">(</span><span class="n">tensors</span><span class="p">,)</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">tensors</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">)</span> <span class="k">else</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">tensors</span><span class="p">)</span>
    <span class="n">inputs</span> <span class="o">=</span> <span class="p">(</span>
        <span class="p">(</span><span class="n">inputs</span><span class="p">,)</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">))</span>
        <span class="k">else</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">inputs</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
        <span class="k">else</span> <span class="p">()</span>
    <span class="p">)</span>

    <span class="n">grad_tensors_</span> <span class="o">=</span> <span class="n">_tensor_or_tensors_to_tuple</span><span class="p">(</span><span class="n">grad_tensors</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">tensors</span><span class="p">))</span>
    <span class="n">grad_tensors_</span> <span class="o">=</span> <span class="n">_make_grads</span><span class="p">(</span><span class="n">tensors</span><span class="p">,</span> <span class="n">grad_tensors_</span><span class="p">,</span> <span class="n">is_grads_batched</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">retain_graph</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">retain_graph</span> <span class="o">=</span> <span class="n">create_graph</span>

    <span class="c1"># The reason we repeat the same comment below is that</span>
    <span class="c1"># some Python versions print out the first line of a multi-line function</span>
    <span class="c1"># calls in the traceback and some print out the last line</span>
    <span class="n">_engine_run_backward</span><span class="p">(</span>
        <span class="n">tensors</span><span class="p">,</span>
        <span class="n">grad_tensors_</span><span class="p">,</span>
        <span class="n">retain_graph</span><span class="p">,</span>
        <span class="n">create_graph</span><span class="p">,</span>
        <span class="n">inputs</span><span class="p">,</span>
        <span class="n">allow_unreachable</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
        <span class="n">accumulate_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
    <span class="p">)</span></div>


<div class="viewcode-block" id="grad"><a class="viewcode-back" href="../../generated/torch.autograd.grad.html#torch.autograd.grad">[docs]</a><span class="k">def</span> <span class="nf">grad</span><span class="p">(</span>
    <span class="n">outputs</span><span class="p">:</span> <span class="n">_TensorOrTensorsOrGradEdge</span><span class="p">,</span>
    <span class="n">inputs</span><span class="p">:</span> <span class="n">_TensorOrTensorsOrGradEdge</span><span class="p">,</span>
    <span class="n">grad_outputs</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">_TensorOrTensors</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="n">retain_graph</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="n">create_graph</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
    <span class="n">only_inputs</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">True</span><span class="p">,</span>
    <span class="n">allow_unused</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">bool</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
    <span class="n">is_grads_batched</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
    <span class="n">materialize_grads</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
<span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Tuple</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">,</span> <span class="o">...</span><span class="p">]:</span>
<span class="w">    </span><span class="sa">r</span><span class="sd">&quot;&quot;&quot;Compute and return the sum of gradients of outputs with respect to the inputs.</span>

<span class="sd">    ``grad_outputs`` should be a sequence of length matching ``output``</span>
<span class="sd">    containing the &quot;vector&quot; in vector-Jacobian product, usually the pre-computed</span>
<span class="sd">    gradients w.r.t. each of the outputs. If an output doesn&#39;t require_grad,</span>
<span class="sd">    then the gradient can be ``None``).</span>

<span class="sd">    .. note::</span>

<span class="sd">        If you run any forward ops, create ``grad_outputs``, and/or call ``grad``</span>
<span class="sd">        in a user-specified CUDA stream context, see</span>
<span class="sd">        :ref:`Stream semantics of backward passes&lt;bwd-cuda-stream-semantics&gt;`.</span>

<span class="sd">    .. note::</span>

<span class="sd">        ``only_inputs`` argument is deprecated and is ignored now (defaults to ``True``).</span>
<span class="sd">        To accumulate gradient for other parts of the graph, please use</span>
<span class="sd">        ``torch.autograd.backward``.</span>

<span class="sd">    Args:</span>
<span class="sd">        outputs (sequence of Tensor or GradientEdge): outputs of the differentiated function.</span>
<span class="sd">        inputs (sequence of Tensor or GradientEdge): Inputs w.r.t. which the gradient will be</span>
<span class="sd">            returned (and not accumulated into ``.grad``).</span>
<span class="sd">        grad_outputs (sequence of Tensor): The &quot;vector&quot; in the vector-Jacobian product.</span>
<span class="sd">            Usually gradients w.r.t. each output. None values can be specified for scalar</span>
<span class="sd">            Tensors or ones that don&#39;t require grad. If a None value would be acceptable</span>
<span class="sd">            for all grad_tensors, then this argument is optional. Default: None.</span>
<span class="sd">        retain_graph (bool, optional): If ``False``, the graph used to compute the grad</span>
<span class="sd">            will be freed. Note that in nearly all cases setting this option to ``True``</span>
<span class="sd">            is not needed and often can be worked around in a much more efficient</span>
<span class="sd">            way. Defaults to the value of ``create_graph``.</span>
<span class="sd">        create_graph (bool, optional): If ``True``, graph of the derivative will</span>
<span class="sd">            be constructed, allowing to compute higher order derivative products.</span>
<span class="sd">            Default: ``False``.</span>
<span class="sd">        allow_unused (Optional[bool], optional): If ``False``, specifying inputs</span>
<span class="sd">            that were not used when computing outputs (and therefore their grad is</span>
<span class="sd">            always zero) is an error. Defaults to the value of ``materialize_grads``.</span>
<span class="sd">        is_grads_batched (bool, optional): If ``True``, the first dimension of each</span>
<span class="sd">            tensor in ``grad_outputs`` will be interpreted as the batch dimension.</span>
<span class="sd">            Instead of computing a single vector-Jacobian product, we compute a</span>
<span class="sd">            batch of vector-Jacobian products for each &quot;vector&quot; in the batch.</span>
<span class="sd">            We use the vmap prototype feature as the backend to vectorize calls</span>
<span class="sd">            to the autograd engine so that this computation can be performed in a</span>
<span class="sd">            single call. This should lead to performance improvements when compared</span>
<span class="sd">            to manually looping and performing backward multiple times. Note that</span>
<span class="sd">            due to this feature being experimental, there may be performance</span>
<span class="sd">            cliffs. Please use ``torch._C._debug_only_display_vmap_fallback_warnings(True)``</span>
<span class="sd">            to show any performance warnings and file an issue on github if warnings exist</span>
<span class="sd">            for your use case. Defaults to ``False``.</span>
<span class="sd">        materialize_grads (bool, optional): If ``True``, set the gradient for unused inputs</span>
<span class="sd">            to zero instead of None. This is useful when computing higher-order derivatives.</span>
<span class="sd">            If ``materialize_grads`` is ``True`` and ``allow_unused`` is ``False``, an error</span>
<span class="sd">            will be raised. Defaults to ``False``.</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">materialize_grads</span> <span class="ow">and</span> <span class="n">allow_unused</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
            <span class="s2">&quot;Expected allow_unused to be True or not passed when materialize_grads=True, &quot;</span>
            <span class="s2">&quot;but got: allow_unused=False.&quot;</span>
        <span class="p">)</span>
    <span class="k">if</span> <span class="n">allow_unused</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">allow_unused</span> <span class="o">=</span> <span class="n">materialize_grads</span>
    <span class="k">if</span> <span class="n">is_tensor_like</span><span class="p">(</span><span class="n">outputs</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">):</span>
        <span class="n">outputs</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span>
            <span class="n">Union</span><span class="p">[</span><span class="n">Sequence</span><span class="p">[</span><span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">],</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">]],</span> <span class="p">(</span><span class="n">outputs</span><span class="p">,)</span>
        <span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">outputs</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">outputs</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">is_tensor_like</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">graph</span><span class="o">.</span><span class="n">GradientEdge</span><span class="p">):</span>
        <span class="n">inputs</span> <span class="o">=</span> <span class="n">cast</span><span class="p">(</span><span class="n">_TensorOrTensorsOrGradEdge</span><span class="p">,</span> <span class="p">(</span><span class="n">inputs</span><span class="p">,))</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">inputs</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
    <span class="n">t_outputs</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">outputs</span> <span class="k">if</span> <span class="n">is_tensor_like</span><span class="p">(</span><span class="n">i</span><span class="p">))</span>
    <span class="n">t_inputs</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span><span class="n">i</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="n">inputs</span> <span class="k">if</span> <span class="n">is_tensor_like</span><span class="p">(</span><span class="n">i</span><span class="p">))</span>
    <span class="n">overridable_args</span> <span class="o">=</span> <span class="n">t_outputs</span> <span class="o">+</span> <span class="n">t_inputs</span>
    <span class="k">if</span> <span class="n">has_torch_function</span><span class="p">(</span><span class="n">overridable_args</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">handle_torch_function</span><span class="p">(</span>
            <span class="n">grad</span><span class="p">,</span>
            <span class="n">overridable_args</span><span class="p">,</span>
            <span class="n">outputs</span><span class="p">,</span>
            <span class="n">inputs</span><span class="p">,</span>
            <span class="n">grad_outputs</span><span class="o">=</span><span class="n">grad_outputs</span><span class="p">,</span>
            <span class="n">retain_graph</span><span class="o">=</span><span class="n">retain_graph</span><span class="p">,</span>
            <span class="n">create_graph</span><span class="o">=</span><span class="n">create_graph</span><span class="p">,</span>
            <span class="n">only_inputs</span><span class="o">=</span><span class="n">only_inputs</span><span class="p">,</span>
            <span class="n">allow_unused</span><span class="o">=</span><span class="n">allow_unused</span><span class="p">,</span>
            <span class="n">is_grads_batched</span><span class="o">=</span><span class="n">is_grads_batched</span><span class="p">,</span>
            <span class="n">materialize_grads</span><span class="o">=</span><span class="n">materialize_grads</span><span class="p">,</span>
        <span class="p">)</span>

    <span class="k">if</span> <span class="ow">not</span> <span class="n">only_inputs</span><span class="p">:</span>
        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
            <span class="s2">&quot;only_inputs argument is deprecated and is ignored now &quot;</span>
            <span class="s2">&quot;(defaults to True). To accumulate gradient for other &quot;</span>
            <span class="s2">&quot;parts of the graph, please use torch.autograd.backward.&quot;</span><span class="p">,</span>
            <span class="ne">FutureWarning</span><span class="p">,</span>
            <span class="n">stacklevel</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span>
        <span class="p">)</span>

    <span class="n">grad_outputs_</span> <span class="o">=</span> <span class="n">_tensor_or_tensors_to_tuple</span><span class="p">(</span><span class="n">grad_outputs</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">outputs</span><span class="p">))</span>
    <span class="n">grad_outputs_</span> <span class="o">=</span> <span class="n">_make_grads</span><span class="p">(</span>
        <span class="n">outputs</span><span class="p">,</span> <span class="n">grad_outputs_</span><span class="p">,</span> <span class="n">is_grads_batched</span><span class="o">=</span><span class="n">is_grads_batched</span>
    <span class="p">)</span>

    <span class="k">if</span> <span class="n">retain_graph</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">retain_graph</span> <span class="o">=</span> <span class="n">create_graph</span>

    <span class="c1"># The reason we repeat the same comment several times below is because</span>
    <span class="c1"># some Python versions print out the first line of multi-line function</span>
    <span class="c1"># calls in the traceback and some print out the last line</span>
    <span class="k">if</span> <span class="n">is_grads_batched</span><span class="p">:</span>

        <span class="k">def</span> <span class="nf">vjp</span><span class="p">(</span><span class="n">gO</span><span class="p">):</span>
            <span class="k">return</span> <span class="n">_engine_run_backward</span><span class="p">(</span>
                <span class="n">outputs</span><span class="p">,</span>
                <span class="n">gO</span><span class="p">,</span>
                <span class="n">retain_graph</span><span class="p">,</span>
                <span class="n">create_graph</span><span class="p">,</span>
                <span class="n">inputs</span><span class="p">,</span>
                <span class="n">allow_unused</span><span class="p">,</span>
                <span class="n">accumulate_grad</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
            <span class="p">)</span>

        <span class="n">result</span> <span class="o">=</span> <span class="n">_vmap_internals</span><span class="o">.</span><span class="n">_vmap</span><span class="p">(</span><span class="n">vjp</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="n">allow_none_pass_through</span><span class="o">=</span><span class="kc">True</span><span class="p">)(</span>
            <span class="n">grad_outputs_</span>
        <span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">result</span> <span class="o">=</span> <span class="n">_engine_run_backward</span><span class="p">(</span>
            <span class="n">outputs</span><span class="p">,</span>
            <span class="n">grad_outputs_</span><span class="p">,</span>
            <span class="n">retain_graph</span><span class="p">,</span>
            <span class="n">create_graph</span><span class="p">,</span>
            <span class="n">inputs</span><span class="p">,</span>
            <span class="n">allow_unused</span><span class="p">,</span>
            <span class="n">accumulate_grad</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
        <span class="p">)</span>
    <span class="k">if</span> <span class="n">materialize_grads</span><span class="p">:</span>
        <span class="k">if</span> <span class="nb">any</span><span class="p">(</span>
            <span class="n">result</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">is_tensor_like</span><span class="p">(</span><span class="n">inputs</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
            <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">))</span>
        <span class="p">):</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                <span class="s2">&quot;materialize_grads cannot be used when the given input is a GradientEdge&quot;</span>
            <span class="p">)</span>
        <span class="n">result</span> <span class="o">=</span> <span class="nb">tuple</span><span class="p">(</span>
            <span class="n">output</span>
            <span class="k">if</span> <span class="n">output</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
            <span class="k">else</span> <span class="n">torch</span><span class="o">.</span><span class="n">zeros_like</span><span class="p">(</span><span class="nb">input</span><span class="p">,</span> <span class="n">requires_grad</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
            <span class="k">for</span> <span class="p">(</span><span class="n">output</span><span class="p">,</span> <span class="nb">input</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">result</span><span class="p">,</span> <span class="n">inputs</span><span class="p">)</span>
        <span class="p">)</span>
    <span class="k">return</span> <span class="n">result</span></div>


<span class="c1"># This function applies in case of gradient checkpointing for memory</span>
<span class="c1"># optimization. Currently, gradient checkpointing is supported only if the</span>
<span class="c1"># execution engine is invoked through torch.autograd.backward() and its</span>
<span class="c1"># inputs argument is not passed. It is not supported for torch.autograd.grad().</span>
<span class="c1"># This is because if inputs are specified, the gradient won&#39;t be calculated for</span>
<span class="c1"># anything else e.g. model parameters like weights, bias etc.</span>
<span class="c1">#</span>
<span class="c1"># This function returns whether the checkpointing is valid i.e. torch.autograd.backward</span>
<span class="c1"># or not i.e. torch.autograd.grad. The implementation works by maintaining a thread</span>
<span class="c1"># local variable in torch/csrc/autograd/engine.cpp which looks at the NodeTask</span>
<span class="c1"># in the stack and before a NodeTask is executed in evaluate_function, it</span>
<span class="c1"># checks for whether reentrant backwards is imperative or not.</span>
<span class="c1"># See https://github.com/pytorch/pytorch/pull/4594 for more discussion/context</span>
<span class="k">def</span> <span class="nf">_is_checkpoint_valid</span><span class="p">():</span>
    <span class="k">return</span> <span class="n">Variable</span><span class="o">.</span><span class="n">_execution_engine</span><span class="o">.</span><span class="n">is_checkpoint_valid</span><span class="p">()</span>


<span class="k">def</span> <span class="nf">variable</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>  <span class="c1"># noqa: D103</span>
    <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
        <span class="s2">&quot;torch.autograd.variable(...) is deprecated, use torch.tensor(...) instead&quot;</span>
    <span class="p">)</span>


<span class="c1"># Monkey patching variable.Variable to fix FX codegen. FX generates a call by roughly doing</span>
<span class="c1"># f&quot;{fn.__module__}.{fn.__name__}(...). This yields torch.autograd.variable.Variable(...) in the</span>
<span class="c1"># output of an FX graph.  Unfortunately the module name torch.autograd.variable is shadowed by the</span>
<span class="c1"># deprecated function - variable(...).</span>
<span class="n">variable</span><span class="o">.</span><span class="n">Variable</span> <span class="o">=</span> <span class="n">Variable</span>  <span class="c1"># type: ignore[attr-defined]</span>

<span class="k">if</span> <span class="ow">not</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_autograd_init</span><span class="p">():</span>
    <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;autograd initialization failed&quot;</span><span class="p">)</span>

<span class="c1"># Import all native method/classes</span>
<span class="kn">from</span> <span class="nn">torch._C._autograd</span> <span class="kn">import</span> <span class="p">(</span>
    <span class="n">_add_metadata_json</span><span class="p">,</span>
    <span class="n">_disable_profiler</span><span class="p">,</span>
    <span class="n">_disable_profiler_legacy</span><span class="p">,</span>
    <span class="n">_enable_profiler</span><span class="p">,</span>
    <span class="n">_enable_profiler_legacy</span><span class="p">,</span>
    <span class="n">_enable_record_function</span><span class="p">,</span>
    <span class="n">_get_sequence_nr</span><span class="p">,</span>
    <span class="n">_kineto_step</span><span class="p">,</span>
    <span class="n">_KinetoEvent</span><span class="p">,</span>
    <span class="n">_pop_saved_tensors_default_hooks</span><span class="p">,</span>
    <span class="n">_prepare_profiler</span><span class="p">,</span>
    <span class="n">_profiler_enabled</span><span class="p">,</span>
    <span class="n">_ProfilerResult</span><span class="p">,</span>
    <span class="n">_push_saved_tensors_default_hooks</span><span class="p">,</span>
    <span class="n">_record_function_with_args_enter</span><span class="p">,</span>
    <span class="n">_record_function_with_args_exit</span><span class="p">,</span>
    <span class="n">_set_empty_test_observer</span><span class="p">,</span>
    <span class="n">_supported_activities</span><span class="p">,</span>
    <span class="n">_toggle_collection_dynamic</span><span class="p">,</span>
    <span class="n">DeviceType</span><span class="p">,</span>
    <span class="n">kineto_available</span><span class="p">,</span>
    <span class="n">ProfilerEvent</span><span class="p">,</span>
    <span class="n">SavedTensor</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span> <span class="nn">torch._C._profiler</span> <span class="kn">import</span> <span class="n">ProfilerActivity</span><span class="p">,</span> <span class="n">ProfilerConfig</span><span class="p">,</span> <span class="n">ProfilerState</span>

<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="n">profiler</span>


<span class="k">def</span> <span class="nf">_register_py_tensor_class_for_device</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="bp">cls</span><span class="p">):</span>
    <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="nb">type</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;cls isn&#39;t a typeinfo object&quot;</span><span class="p">)</span>
    <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_register_py_class_for_device</span><span class="p">(</span><span class="n">device</span><span class="p">,</span> <span class="bp">cls</span><span class="p">)</span>


<span class="n">is_multithreading_enabled</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_is_multithreading_enabled</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_add_docstr</span><span class="p">(</span>
    <span class="n">is_multithreading_enabled</span><span class="p">,</span> <span class="s2">&quot;Returns True if multithreading is currently enabled.&quot;</span>
<span class="p">)</span>

<span class="n">is_view_replay_enabled</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_is_view_replay_enabled</span>
<span class="n">torch</span><span class="o">.</span><span class="n">_C</span><span class="o">.</span><span class="n">_add_docstr</span><span class="p">(</span>
    <span class="n">is_view_replay_enabled</span><span class="p">,</span> <span class="s2">&quot;Returns True if view-replay is currently enabled.&quot;</span>
<span class="p">)</span>
</pre></div>

             </article>
             
            </div>
            <footer>
  

    <hr>

  
  <div role="contentinfo">
    <p>
        &copy; Copyright 2024, PyTorch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
<script>

var match = window.location.href.match(/\/_[a-zA-Z0-9_]*.html|_dynamo/gi);
var url = window.location.href.lastIndexOf(match[match.length-1]);

if (url)
  {
    var div = '<div class="admonition note"><p class="admonition-title">Note</p><p><i class="fa fa-exclamation-circle" aria-hidden="true">&nbsp</i> This page describes an internal API which is not intended to be used outside of the PyTorch codebase and can be modified or removed without notice.</p></div>'
    document.getElementById("pytorch-article").insertAdjacentHTML('afterBegin', div)
  }
</script>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              
            </div>
          </div>
        </div>
      </section>
    </div>

  
       <script type="text/javascript" id="documentation_options" data-url_root="../../" src="../../_static/documentation_options.js"></script>
         <script data-url_root="../../" id="documentation_options" src="../../_static/documentation_options.js"></script>
         <script src="../../_static/jquery.js"></script>
         <script src="../../_static/underscore.js"></script>
         <script src="../../_static/_sphinx_javascript_frameworks_compat.js"></script>
         <script src="../../_static/doctools.js"></script>
         <script src="../../_static/sphinx_highlight.js"></script>
         <script src="../../_static/clipboard.min.js"></script>
         <script src="../../_static/copybutton.js"></script>
     

  <script type="text/javascript" src="../../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script script type="text/javascript">
  var collapsedSections = ['Developer Notes', 'Language Bindings', 'Libraries', 'Community'];
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title">Stay up to date</li>
            <li><a href="https://www.facebook.com/pytorch" target="_blank">Facebook</a></li>
            <li><a href="https://twitter.com/pytorch" target="_blank">Twitter</a></li>
            <li><a href="https://www.youtube.com/pytorch" target="_blank">YouTube</a></li>
            <li><a href="https://www.linkedin.com/company/pytorch" target="_blank">LinkedIn</a></li>
          </ul>  
          </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title">PyTorch Podcasts</li>
            <li><a href="https://open.spotify.com/show/6UzHKeiy368jKfQMKKvJY5" target="_blank">Spotify</a></li>
            <li><a href="https://podcasts.apple.com/us/podcast/pytorch-developer-podcast/id1566080008" target="_blank">Apple</a></li>
            <li><a href="https://www.google.com/podcasts?feed=aHR0cHM6Ly9mZWVkcy5zaW1wbGVjYXN0LmNvbS9PQjVGa0lsOA%3D%3D" target="_blank">Google</a></li>
            <li><a href="https://music.amazon.com/podcasts/7a4e6f0e-26c2-49e9-a478-41bd244197d0/PyTorch-Developer-Podcast?" target="_blank">Amazon</a></li>
          </ul>
         </div>
        </div>
        
        <div class="privacy-policy">
          <ul>
            <li class="privacy-policy-links"><a href="https://www.linuxfoundation.org/terms/" target="_blank">Terms</a></li>
            <li class="privacy-policy-links">|</li>
            <li class="privacy-policy-links"><a href="https://www.linuxfoundation.org/privacy-policy/" target="_blank">Privacy</a></li>
          </ul>
        </div>
        <div class="copyright">
        <p>© Copyright The Linux Foundation. The PyTorch Foundation is a project of The Linux Foundation.
          For web site terms of use, trademark policy and other policies applicable to The PyTorch Foundation please see
          <a href="https://www.linuxfoundation.org/policies/">www.linuxfoundation.org/policies/</a>. The PyTorch Foundation supports the PyTorch open source
          project, which has been established as PyTorch Project a Series of LF Projects, LLC. For policies applicable to the PyTorch Project a Series of LF Projects, LLC,
          please see <a href="https://www.lfprojects.org/policies/">www.lfprojects.org/policies/</a>.</p>
      </div>
     </div>

  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
           <li class="resources-mobile-menu-title">
             <a>Learn</a>
           </li>
           <ul class="resources-mobile-menu-items">
             <li>
               <a href="https://pytorch.org/get-started">Get Started</a>
             </li>
             <li>
               <a href="https://pytorch.org/tutorials">Tutorials</a>
             </li>
             <li>
               <a href="https://pytorch.org/tutorials/beginner/basics/intro.html">Learn the Basics</a>
             </li>
             <li>
               <a href="https://pytorch.org/tutorials/recipes/recipes_index.html">PyTorch Recipes</a>
             </li>
             <li>
               <a href="https://pytorch.org/tutorials/beginner/introyt.html">Introduction to PyTorch - YouTube Series</a>
             </li>
           </ul>
           <li class="resources-mobile-menu-title">
             <a>Ecosystem</a>
           </li>
           <ul class="resources-mobile-menu-items">
             <li>
               <a href="https://pytorch.org/ecosystem">Tools</a>
             </li>
             <li>
               <a href="https://pytorch.org/#community-module">Community</a>
             </li>
             <li>
               <a href="https://discuss.pytorch.org/">Forums</a>
             </li>
             <li>
               <a href="https://pytorch.org/resources">Developer Resources</a>
             </li>
             <li>
               <a href="https://pytorch.org/ecosystem/contributor-awards-2023">Contributor Awards - 2023</a>
             </li>
           </ul>

           <li class="resources-mobile-menu-title">
             <a>Edge</a>
           </li>

           <ul class="resources-mobile-menu-items">
             <li>
               <a href="https://pytorch.org/edge">About PyTorch Edge</a>
             </li>
             
             <li>
               <a href="https://pytorch.org/executorch-overview">ExecuTorch</a>
             </li>
           </ul>

           <li class="resources-mobile-menu-title">
             <a>Docs</a>
           </li>

           <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/docs/stable/index.html">PyTorch</a>
            </li>

            <li>
              <a href="https://pytorch.org/pytorch-domains">PyTorch Domains</a>
            </li>
          </ul>

          <li class="resources-mobile-menu-title">
            <a>Blog & News</a>
          </li>
            
           <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/blog/">PyTorch Blog</a>
            </li>
            <li>
              <a href="https://pytorch.org/community-blog">Community Blog</a>
            </li>

            <li>
              <a href="https://pytorch.org/videos">Videos</a>
            </li>

            <li>
              <a href="https://pytorch.org/community-stories">Community Stories</a>
            </li>
            <li>
              <a href="https://pytorch.org/events">Events</a>
            </li>
          </ul>
          
          <li class="resources-mobile-menu-title">
            <a>About</a>
          </li>

          <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/foundation">PyTorch Foundation</a>
            </li>
            <li>
              <a href="https://pytorch.org/governing-board">Governing Board</a>
            </li>
          </ul>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>