dev/modules/kernel_ridge.html



<!DOCTYPE html>
<!-- data-theme below is forced to be "light" but should be changed if we use pydata-theme-sphinx in the future -->
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" data-content_root="../" data-theme="light"> <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" data-content_root="../" data-theme="light"> <!--<![endif]-->
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1" />
<meta property="og:title" content="1.3. Kernel ridge regression" />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://scikit-learn/stable/modules/kernel_ridge.html" />
<meta property="og:site_name" content="scikit-learn" />
<meta property="og:description" content="Kernel ridge regression (KRR)[M2012] combines Ridge regression and classification(linear least squares with l2-norm regularization) with the kernel trick. It thus learns a linear function in the sp..." />
<meta property="og:image" content="https://scikit-learn/stable/_images/sphx_glr_plot_kernel_ridge_regression_001.png" />
<meta property="og:image:alt" content="scikit-learn" />
<meta name="description" content="Kernel ridge regression (KRR)[M2012] combines Ridge regression and classification(linear least squares with l2-norm regularization) with the kernel trick. It thus learns a linear function in the sp..." />

  <meta name="viewport" content="width=device-width, initial-scale=1.0">

  
  <title>1.3. Kernel ridge regression &mdash; scikit-learn 1.5.dev0 documentation</title>
  
  <link rel="canonical" href="http://scikit-learn.org/stable/modules/kernel_ridge.html" />

  
  <link rel="shortcut icon" href="../_static/favicon.ico"/>
  

  <link rel="stylesheet" href="../_static/css/vendor/bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/copybutton.css" type="text/css" />
  <link rel="stylesheet" href="../_static/plot_directive.css" type="text/css" />
  <link rel="stylesheet" href="../https://fonts.googleapis.com/css?family=Vibur" type="text/css" />
  <link rel="stylesheet" href="../_static/jupyterlite_sphinx.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-binder.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-dataframe.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-rendered-html.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<script id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
<script src="../_static/js/vendor/jquery-3.6.3.slim.min.js"></script>
<script src="../_static/js/details-permalink.js"></script> 
</head>
<body>


<nav id="navbar" class="sk-docs-navbar navbar navbar-expand-md navbar-light bg-light py-0">
  <div class="container-fluid sk-docs-container px-0">
      <a class="navbar-brand py-0" href="../index.html">
        <img
          class="sk-brand-img"
          src="../_static/scikit-learn-logo-small.png"
          alt="logo"/>
      </a>
    <button
      id="sk-navbar-toggler"
      class="navbar-toggler"
      type="button"
      data-toggle="collapse"
      data-target="#navbarSupportedContent"
      aria-controls="navbarSupportedContent"
      aria-expanded="false"
      aria-label="Toggle navigation"
    >
      <span class="navbar-toggler-icon"></span>
    </button>

    <div class="sk-navbar-collapse collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav mr-auto">
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../install.html">Install</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../user_guide.html">User Guide</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="classes.html">API</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../auto_examples/index.html">Examples</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" target="_blank" rel="noopener noreferrer" href="https://blog.scikit-learn.org/">Community</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../getting_started.html" >Getting Started</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../tutorial/index.html" >Tutorial</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../whats_new/v1.5.html" >What's new</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../glossary.html" >Glossary</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../developers/index.html" >Development</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../faq.html" >FAQ</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../support.html" >Support</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../related_projects.html" >Related packages</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../roadmap.html" >Roadmap</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../governance.html" >Governance</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../about.html" >About us</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://github.com/scikit-learn/scikit-learn" >GitHub</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://scikit-learn.org/dev/versions.html" >Other Versions and Download</a>
        </li>
        <li class="nav-item dropdown nav-more-item-dropdown">
          <a class="sk-nav-link nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
          <div class="dropdown-menu" aria-labelledby="navbarDropdown">
              <a class="sk-nav-dropdown-item dropdown-item" href="../getting_started.html" >Getting Started</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../tutorial/index.html" >Tutorial</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../whats_new/v1.5.html" >What's new</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../glossary.html" >Glossary</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../developers/index.html" >Development</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../faq.html" >FAQ</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../support.html" >Support</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../related_projects.html" >Related packages</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../roadmap.html" >Roadmap</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../governance.html" >Governance</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../about.html" >About us</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://github.com/scikit-learn/scikit-learn" >GitHub</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://scikit-learn.org/dev/versions.html" >Other Versions and Download</a>
          </div>
        </li>
      </ul>
      <div id="searchbox" role="search">
          <div class="searchformwrapper">
          <form class="search" action="../search.html" method="get">
            <input class="sk-search-text-input" type="text" name="q" aria-labelledby="searchlabel" />
            <input class="sk-search-text-btn" type="submit" value="Go" />
          </form>
          </div>
      </div>
    </div>
  </div>
</nav>
<div class="d-flex" id="sk-doc-wrapper">
    <input type="checkbox" name="sk-toggle-checkbox" id="sk-toggle-checkbox">
    <label id="sk-sidemenu-toggle" class="sk-btn-toggle-toc btn sk-btn-primary" for="sk-toggle-checkbox">Toggle Menu</label>
    <div id="sk-sidebar-wrapper" class="border-right">
      <div class="sk-sidebar-toc-wrapper">
        <div class="btn-group w-100 mb-2" role="group" aria-label="rellinks">
            <a href="lda_qda.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1.2. Linear and Quadratic Discriminant Analysis">Prev</a><a href="../supervised_learning.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1. Supervised learning">Up</a>
            <a href="svm.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1.4. Support Vector Machines">Next</a>
        </div>
        <div class="alert alert-danger p-1 mb-2" role="alert">
          <p class="text-center mb-0">
          <strong>scikit-learn 1.5.dev0</strong><br/>
          <a href="http://scikit-learn.org/dev/versions.html">Other versions</a>
          </p>
        </div>
        <div class="alert alert-warning p-1 mb-2" role="alert">
          <p class="text-center mb-0">
            Please <a class="font-weight-bold" href="../about.html#citing-scikit-learn"><string>cite us</string></a> if you use the software.
          </p>
        </div>
            <div class="sk-sidebar-toc">
              <ul>
<li><a class="reference internal" href="#">1.3. Kernel ridge regression</a></li>
</ul>

            </div>
      </div>
    </div>
    <div id="sk-page-content-wrapper">
      <div class="sk-page-content container-fluid body px-md-3" role="main">
        
  <section id="kernel-ridge-regression">
<span id="kernel-ridge"></span><h1><span class="section-number">1.3. </span>Kernel ridge regression<a class="headerlink" href="#kernel-ridge-regression" title="Link to this heading">¶</a></h1>
<p>Kernel ridge regression (KRR) <a class="reference internal" href="#m2012" id="id1"><span>[M2012]</span></a> combines <a class="reference internal" href="linear_model.html#ridge-regression"><span class="std std-ref">Ridge regression and classification</span></a>
(linear least squares with l2-norm regularization) with the <a class="reference external" href="https://en.wikipedia.org/wiki/Kernel_method">kernel trick</a>. It thus learns a linear
function in the space induced by the respective kernel and the data. For
non-linear kernels, this corresponds to a non-linear function in the original
space.</p>
<p>The form of the model learned by <a class="reference internal" href="generated/sklearn.kernel_ridge.KernelRidge.html#sklearn.kernel_ridge.KernelRidge" title="sklearn.kernel_ridge.KernelRidge"><code class="xref py py-class docutils literal notranslate"><span class="pre">KernelRidge</span></code></a> is identical to support
vector regression (<a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a>). However, different loss
functions are used: KRR uses squared error loss while support vector
regression uses <span class="math notranslate nohighlight">\(\epsilon\)</span>-insensitive loss, both combined with l2
regularization. In contrast to <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a>, fitting
<a class="reference internal" href="generated/sklearn.kernel_ridge.KernelRidge.html#sklearn.kernel_ridge.KernelRidge" title="sklearn.kernel_ridge.KernelRidge"><code class="xref py py-class docutils literal notranslate"><span class="pre">KernelRidge</span></code></a> can be done in closed-form and is typically faster for
medium-sized datasets. On the other hand, the learned model is non-sparse and
thus slower than <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a>, which learns a sparse model for
<span class="math notranslate nohighlight">\(\epsilon &gt; 0\)</span>, at prediction-time.</p>
<p>The following figure compares <a class="reference internal" href="generated/sklearn.kernel_ridge.KernelRidge.html#sklearn.kernel_ridge.KernelRidge" title="sklearn.kernel_ridge.KernelRidge"><code class="xref py py-class docutils literal notranslate"><span class="pre">KernelRidge</span></code></a> and
<a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a> on an artificial dataset, which consists of a
sinusoidal target function and strong noise added to every fifth datapoint.
The learned model of <a class="reference internal" href="generated/sklearn.kernel_ridge.KernelRidge.html#sklearn.kernel_ridge.KernelRidge" title="sklearn.kernel_ridge.KernelRidge"><code class="xref py py-class docutils literal notranslate"><span class="pre">KernelRidge</span></code></a> and <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a> is
plotted, where both complexity/regularization and bandwidth of the RBF kernel
have been optimized using grid-search. The learned functions are very
similar; however, fitting <a class="reference internal" href="generated/sklearn.kernel_ridge.KernelRidge.html#sklearn.kernel_ridge.KernelRidge" title="sklearn.kernel_ridge.KernelRidge"><code class="xref py py-class docutils literal notranslate"><span class="pre">KernelRidge</span></code></a> is approximately seven times
faster than fitting <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a> (both with grid-search).
However, prediction of 100000 target values is more than three times faster
with <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a> since it has learned a sparse model using only
approximately 1/3 of the 100 training datapoints as support vectors.</p>
<figure class="align-center">
<a class="reference external image-reference" href="../auto_examples/miscellaneous/plot_kernel_ridge_regression.html"><img alt="../_images/sphx_glr_plot_kernel_ridge_regression_001.png" src="../_images/sphx_glr_plot_kernel_ridge_regression_001.png" /></a>
</figure>
<p>The next figure compares the time for fitting and prediction of
<a class="reference internal" href="generated/sklearn.kernel_ridge.KernelRidge.html#sklearn.kernel_ridge.KernelRidge" title="sklearn.kernel_ridge.KernelRidge"><code class="xref py py-class docutils literal notranslate"><span class="pre">KernelRidge</span></code></a> and <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a> for different sizes of the
training set. Fitting <a class="reference internal" href="generated/sklearn.kernel_ridge.KernelRidge.html#sklearn.kernel_ridge.KernelRidge" title="sklearn.kernel_ridge.KernelRidge"><code class="xref py py-class docutils literal notranslate"><span class="pre">KernelRidge</span></code></a> is faster than
<a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a> for medium-sized training sets (less than 1000
samples); however, for larger training sets <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a> scales
better. With regard to prediction time, <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a> is faster
than <a class="reference internal" href="generated/sklearn.kernel_ridge.KernelRidge.html#sklearn.kernel_ridge.KernelRidge" title="sklearn.kernel_ridge.KernelRidge"><code class="xref py py-class docutils literal notranslate"><span class="pre">KernelRidge</span></code></a> for all sizes of the training set because of the
learned sparse solution. Note that the degree of sparsity and thus the
prediction time depends on the parameters <span class="math notranslate nohighlight">\(\epsilon\)</span> and <span class="math notranslate nohighlight">\(C\)</span> of
the <a class="reference internal" href="generated/sklearn.svm.SVR.html#sklearn.svm.SVR" title="sklearn.svm.SVR"><code class="xref py py-class docutils literal notranslate"><span class="pre">SVR</span></code></a>; <span class="math notranslate nohighlight">\(\epsilon = 0\)</span> would correspond to a
dense model.</p>
<figure class="align-center">
<a class="reference external image-reference" href="../auto_examples/miscellaneous/plot_kernel_ridge_regression.html"><img alt="../_images/sphx_glr_plot_kernel_ridge_regression_002.png" src="../_images/sphx_glr_plot_kernel_ridge_regression_002.png" /></a>
</figure>
<aside class="topic">
<p class="topic-title">References:</p>
<div role="list" class="citation-list">
<div class="citation" id="m2012" role="doc-biblioentry">
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id1">M2012</a><span class="fn-bracket">]</span></span>
<p>“Machine Learning: A Probabilistic Perspective”
Murphy, K. P. - chapter 14.4.3, pp. 492-493, The MIT Press, 2012</p>
</div>
</div>
</aside>
</section>


      </div>
    <div class="container">
      <footer class="sk-content-footer">
            &copy; 2007 - 2023, scikit-learn developers (BSD License).
          <a href="../_sources/modules/kernel_ridge.rst.txt" rel="nofollow">Show this page source</a>
      </footer>
    </div>
  </div>
</div>
<script src="../_static/js/vendor/bootstrap.min.js"></script>

<script>
    window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
    ga('create', 'UA-22606712-2', 'auto');
    ga('set', 'anonymizeIp', true);
    ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>


<script defer data-domain="scikit-learn.org" src="https://views.scientific-python.org/js/script.js">
</script>


<script src="../_static/clipboard.min.js"></script>
<script src="../_static/copybutton.js"></script>

<script>
$(document).ready(function() {
    /* Add a [>>>] button on the top-right corner of code samples to hide
     * the >>> and ... prompts and the output and thus make the code
     * copyable. */
    var div = $('.highlight-python .highlight,' +
                '.highlight-python3 .highlight,' +
                '.highlight-pycon .highlight,' +
		'.highlight-default .highlight')
    var pre = div.find('pre');

    // get the styles from the current theme
    pre.parent().parent().css('position', 'relative');

    // create and add the button to all the code blocks that contain >>>
    div.each(function(index) {
        var jthis = $(this);
        // tracebacks (.gt) contain bare text elements that need to be
        // wrapped in a span to work with .nextUntil() (see later)
        jthis.find('pre:has(.gt)').contents().filter(function() {
            return ((this.nodeType == 3) && (this.data.trim().length > 0));
        }).wrap('<span>');
    });

	/*** Add permalink buttons next to glossary terms ***/
	$('dl.glossary > dt[id]').append(function() {
		return ('<a class="headerlink" href="#' +
			    this.getAttribute('id') +
			    '" title="Permalink to this term">¶</a>');
	});
});

</script>
    
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js"></script>
    
    <script src="https://scikit-learn.org/versionwarning.js"></script>
</body>
</html>