1.3/modules/lda_qda.html



<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  <meta name="generator" content="Docutils 0.19: https://docutils.sourceforge.io/" />
<meta property="og:title" content="1.2. Linear and Quadratic Discriminant Analysis" />
<meta property="og:type" content="website" />
<meta property="og:url" content="https://scikit-learn/stable/modules/lda_qda.html" />
<meta property="og:site_name" content="scikit-learn" />
<meta property="og:description" content="Linear Discriminant Analysis ( LinearDiscriminantAnalysis) and Quadratic Discriminant Analysis ( QuadraticDiscriminantAnalysis) are two classic classifiers, with, as their names suggest, a linear a..." />
<meta property="og:image" content="https://scikit-learn/stable/modules/auto_examples/classification/images/sphx_glr_plot_lda_qda_001.png" />
<meta property="og:image:alt" content="ldaqda" />
<meta name="description" content="Linear Discriminant Analysis ( LinearDiscriminantAnalysis) and Quadratic Discriminant Analysis ( QuadraticDiscriminantAnalysis) are two classic classifiers, with, as their names suggest, a linear a..." />

  <meta name="viewport" content="width=device-width, initial-scale=1.0">

  
  <title>1.2. Linear and Quadratic Discriminant Analysis &mdash; scikit-learn 1.3.2 documentation</title>
  
  <link rel="canonical" href="http://scikit-learn.org/stable/modules/lda_qda.html" />

  
  <link rel="shortcut icon" href="../_static/favicon.ico"/>
  

  <link rel="stylesheet" href="../_static/css/vendor/bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/copybutton.css" type="text/css" />
  <link rel="stylesheet" href="../_static/plot_directive.css" type="text/css" />
  <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Vibur" type="text/css" />
  <link rel="stylesheet" href="../_static/jupyterlite_sphinx.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-binder.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-dataframe.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-rendered-html.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<script id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
<script src="../_static/js/vendor/jquery-3.6.3.slim.min.js"></script> 
</head>
<body>


<nav id="navbar" class="sk-docs-navbar navbar navbar-expand-md navbar-light bg-light py-0">
  <div class="container-fluid sk-docs-container px-0">
      <a class="navbar-brand py-0" href="../index.html">
        <img
          class="sk-brand-img"
          src="../_static/scikit-learn-logo-small.png"
          alt="logo"/>
      </a>
    <button
      id="sk-navbar-toggler"
      class="navbar-toggler"
      type="button"
      data-toggle="collapse"
      data-target="#navbarSupportedContent"
      aria-controls="navbarSupportedContent"
      aria-expanded="false"
      aria-label="Toggle navigation"
    >
      <span class="navbar-toggler-icon"></span>
    </button>

    <div class="sk-navbar-collapse collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav mr-auto">
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../install.html">Install</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../user_guide.html">User Guide</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="classes.html">API</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../auto_examples/index.html">Examples</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" target="_blank" rel="noopener noreferrer" href="https://blog.scikit-learn.org/">Community</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../getting_started.html" >Getting Started</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../tutorial/index.html" >Tutorial</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../whats_new/v1.3.html" >What's new</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../glossary.html" >Glossary</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://scikit-learn.org/dev/developers/index.html" target="_blank" rel="noopener noreferrer">Development</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../faq.html" >FAQ</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../support.html" >Support</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../related_projects.html" >Related packages</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../roadmap.html" >Roadmap</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../governance.html" >Governance</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../about.html" >About us</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://github.com/scikit-learn/scikit-learn" >GitHub</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://scikit-learn.org/dev/versions.html" >Other Versions and Download</a>
        </li>
        <li class="nav-item dropdown nav-more-item-dropdown">
          <a class="sk-nav-link nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
          <div class="dropdown-menu" aria-labelledby="navbarDropdown">
              <a class="sk-nav-dropdown-item dropdown-item" href="../getting_started.html" >Getting Started</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../tutorial/index.html" >Tutorial</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../whats_new/v1.3.html" >What's new</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../glossary.html" >Glossary</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://scikit-learn.org/dev/developers/index.html" target="_blank" rel="noopener noreferrer">Development</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../faq.html" >FAQ</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../support.html" >Support</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../related_projects.html" >Related packages</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../roadmap.html" >Roadmap</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../governance.html" >Governance</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../about.html" >About us</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://github.com/scikit-learn/scikit-learn" >GitHub</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://scikit-learn.org/dev/versions.html" >Other Versions and Download</a>
          </div>
        </li>
      </ul>
      <div id="searchbox" role="search">
          <div class="searchformwrapper">
          <form class="search" action="../search.html" method="get">
            <input class="sk-search-text-input" type="text" name="q" aria-labelledby="searchlabel" />
            <input class="sk-search-text-btn" type="submit" value="Go" />
          </form>
          </div>
      </div>
    </div>
  </div>
</nav>
<div class="d-flex" id="sk-doc-wrapper">
    <input type="checkbox" name="sk-toggle-checkbox" id="sk-toggle-checkbox">
    <label id="sk-sidemenu-toggle" class="sk-btn-toggle-toc btn sk-btn-primary" for="sk-toggle-checkbox">Toggle Menu</label>
    <div id="sk-sidebar-wrapper" class="border-right">
      <div class="sk-sidebar-toc-wrapper">
        <div class="btn-group w-100 mb-2" role="group" aria-label="rellinks">
            <a href="linear_model.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1.1. Linear Models">Prev</a><a href="../supervised_learning.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1. Supervised learning">Up</a>
            <a href="kernel_ridge.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="1.3. Kernel ridge regression">Next</a>
        </div>
        <div class="alert alert-danger p-1 mb-2" role="alert">
          <p class="text-center mb-0">
          <strong>scikit-learn 1.3.2</strong><br/>
          <a href="http://scikit-learn.org/dev/versions.html">Other versions</a>
          </p>
        </div>
        <div class="alert alert-warning p-1 mb-2" role="alert">
          <p class="text-center mb-0">
            Please <a class="font-weight-bold" href="../about.html#citing-scikit-learn"><string>cite us</string></a> if you use the software.
          </p>
        </div>
            <div class="sk-sidebar-toc">
              <ul>
<li><a class="reference internal" href="#">1.2. Linear and Quadratic Discriminant Analysis</a><ul>
<li><a class="reference internal" href="#dimensionality-reduction-using-linear-discriminant-analysis">1.2.1. Dimensionality reduction using Linear Discriminant Analysis</a></li>
<li><a class="reference internal" href="#mathematical-formulation-of-the-lda-and-qda-classifiers">1.2.2. Mathematical formulation of the LDA and QDA classifiers</a><ul>
<li><a class="reference internal" href="#qda">1.2.2.1. QDA</a></li>
<li><a class="reference internal" href="#lda">1.2.2.2. LDA</a></li>
</ul>
</li>
<li><a class="reference internal" href="#mathematical-formulation-of-lda-dimensionality-reduction">1.2.3. Mathematical formulation of LDA dimensionality reduction</a></li>
<li><a class="reference internal" href="#shrinkage-and-covariance-estimator">1.2.4. Shrinkage and Covariance Estimator</a></li>
<li><a class="reference internal" href="#estimation-algorithms">1.2.5. Estimation algorithms</a></li>
</ul>
</li>
</ul>

            </div>
      </div>
    </div>
    <div id="sk-page-content-wrapper">
      <div class="sk-page-content container-fluid body px-md-3" role="main">
        
  <section id="linear-and-quadratic-discriminant-analysis">
<span id="lda-qda"></span><h1><span class="section-number">1.2. </span>Linear and Quadratic Discriminant Analysis<a class="headerlink" href="#linear-and-quadratic-discriminant-analysis" title="Permalink to this heading">¶</a></h1>
<p>Linear Discriminant Analysis
(<a class="reference internal" href="generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis" title="sklearn.discriminant_analysis.LinearDiscriminantAnalysis"><code class="xref py py-class docutils literal notranslate"><span class="pre">LinearDiscriminantAnalysis</span></code></a>) and Quadratic
Discriminant Analysis
(<a class="reference internal" href="generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html#sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis" title="sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis"><code class="xref py py-class docutils literal notranslate"><span class="pre">QuadraticDiscriminantAnalysis</span></code></a>) are two classic
classifiers, with, as their names suggest, a linear and a quadratic decision
surface, respectively.</p>
<p>These classifiers are attractive because they have closed-form solutions that
can be easily computed, are inherently multiclass, have proven to work well in
practice, and have no hyperparameters to tune.</p>
<p class="centered">
<strong><a class="reference external" href="../auto_examples/classification/plot_lda_qda.html"><img alt="ldaqda" src="../_images/sphx_glr_plot_lda_qda_001.png" style="width: 800.0px; height: 640.0px;" /></a></strong></p><p>The plot shows decision boundaries for Linear Discriminant Analysis and
Quadratic Discriminant Analysis. The bottom row demonstrates that Linear
Discriminant Analysis can only learn linear boundaries, while Quadratic
Discriminant Analysis can learn quadratic boundaries and is therefore more
flexible.</p>
<aside class="topic">
<p class="topic-title">Examples:</p>
<p><a class="reference internal" href="../auto_examples/classification/plot_lda_qda.html#sphx-glr-auto-examples-classification-plot-lda-qda-py"><span class="std std-ref">Linear and Quadratic Discriminant Analysis with covariance ellipsoid</span></a>: Comparison of LDA and QDA
on synthetic data.</p>
</aside>
<section id="dimensionality-reduction-using-linear-discriminant-analysis">
<h2><span class="section-number">1.2.1. </span>Dimensionality reduction using Linear Discriminant Analysis<a class="headerlink" href="#dimensionality-reduction-using-linear-discriminant-analysis" title="Permalink to this heading">¶</a></h2>
<p><a class="reference internal" href="generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis" title="sklearn.discriminant_analysis.LinearDiscriminantAnalysis"><code class="xref py py-class docutils literal notranslate"><span class="pre">LinearDiscriminantAnalysis</span></code></a> can be used to
perform supervised dimensionality reduction, by projecting the input data to a
linear subspace consisting of the directions which maximize the separation
between classes (in a precise sense discussed in the mathematics section
below). The dimension of the output is necessarily less than the number of
classes, so this is in general a rather strong dimensionality reduction, and
only makes sense in a multiclass setting.</p>
<p>This is implemented in the <code class="docutils literal notranslate"><span class="pre">transform</span></code> method. The desired dimensionality can
be set using the <code class="docutils literal notranslate"><span class="pre">n_components</span></code> parameter. This parameter has no influence
on the <code class="docutils literal notranslate"><span class="pre">fit</span></code> and <code class="docutils literal notranslate"><span class="pre">predict</span></code> methods.</p>
<aside class="topic">
<p class="topic-title">Examples:</p>
<p><a class="reference internal" href="../auto_examples/decomposition/plot_pca_vs_lda.html#sphx-glr-auto-examples-decomposition-plot-pca-vs-lda-py"><span class="std std-ref">Comparison of LDA and PCA 2D projection of Iris dataset</span></a>: Comparison of LDA and PCA
for dimensionality reduction of the Iris dataset</p>
</aside>
</section>
<section id="mathematical-formulation-of-the-lda-and-qda-classifiers">
<span id="lda-qda-math"></span><h2><span class="section-number">1.2.2. </span>Mathematical formulation of the LDA and QDA classifiers<a class="headerlink" href="#mathematical-formulation-of-the-lda-and-qda-classifiers" title="Permalink to this heading">¶</a></h2>
<p>Both LDA and QDA can be derived from simple probabilistic models which model
the class conditional distribution of the data <span class="math notranslate nohighlight">\(P(X|y=k)\)</span> for each class
<span class="math notranslate nohighlight">\(k\)</span>. Predictions can then be obtained by using Bayes’ rule, for each
training sample <span class="math notranslate nohighlight">\(x \in \mathcal{R}^d\)</span>:</p>
<div class="math notranslate nohighlight">
\[P(y=k | x) = \frac{P(x | y=k) P(y=k)}{P(x)} = \frac{P(x | y=k) P(y = k)}{ \sum_{l} P(x | y=l) \cdot P(y=l)}\]</div>
<p>and we select the class <span class="math notranslate nohighlight">\(k\)</span> which maximizes this posterior probability.</p>
<p>More specifically, for linear and quadratic discriminant analysis,
<span class="math notranslate nohighlight">\(P(x|y)\)</span> is modeled as a multivariate Gaussian distribution with
density:</p>
<div class="math notranslate nohighlight">
\[P(x | y=k) = \frac{1}{(2\pi)^{d/2} |\Sigma_k|^{1/2}}\exp\left(-\frac{1}{2} (x-\mu_k)^t \Sigma_k^{-1} (x-\mu_k)\right)\]</div>
<p>where <span class="math notranslate nohighlight">\(d\)</span> is the number of features.</p>
<section id="qda">
<h3><span class="section-number">1.2.2.1. </span>QDA<a class="headerlink" href="#qda" title="Permalink to this heading">¶</a></h3>
<p>According to the model above, the log of the posterior is:</p>
<div class="math notranslate nohighlight">
\[\begin{split}\log P(y=k | x) &amp;= \log P(x | y=k) + \log P(y = k) + Cst \\
&amp;= -\frac{1}{2} \log |\Sigma_k| -\frac{1}{2} (x-\mu_k)^t \Sigma_k^{-1} (x-\mu_k) + \log P(y = k) + Cst,\end{split}\]</div>
<p>where the constant term <span class="math notranslate nohighlight">\(Cst\)</span> corresponds to the denominator
<span class="math notranslate nohighlight">\(P(x)\)</span>, in addition to other constant terms from the Gaussian. The
predicted class is the one that maximises this log-posterior.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><strong>Relation with Gaussian Naive Bayes</strong></p>
<p>If in the QDA model one assumes that the covariance matrices are diagonal,
then the inputs are assumed to be conditionally independent in each class,
and the resulting classifier is equivalent to the Gaussian Naive Bayes
classifier <a class="reference internal" href="generated/sklearn.naive_bayes.GaussianNB.html#sklearn.naive_bayes.GaussianNB" title="sklearn.naive_bayes.GaussianNB"><code class="xref py py-class docutils literal notranslate"><span class="pre">naive_bayes.GaussianNB</span></code></a>.</p>
</div>
</section>
<section id="lda">
<h3><span class="section-number">1.2.2.2. </span>LDA<a class="headerlink" href="#lda" title="Permalink to this heading">¶</a></h3>
<p>LDA is a special case of QDA, where the Gaussians for each class are assumed
to share the same covariance matrix: <span class="math notranslate nohighlight">\(\Sigma_k = \Sigma\)</span> for all
<span class="math notranslate nohighlight">\(k\)</span>. This reduces the log posterior to:</p>
<div class="math notranslate nohighlight">
\[\log P(y=k | x) = -\frac{1}{2} (x-\mu_k)^t \Sigma^{-1} (x-\mu_k) + \log P(y = k) + Cst.\]</div>
<p>The term <span class="math notranslate nohighlight">\((x-\mu_k)^t \Sigma^{-1} (x-\mu_k)\)</span> corresponds to the
<a class="reference external" href="https://en.wikipedia.org/wiki/Mahalanobis_distance">Mahalanobis Distance</a>
between the sample <span class="math notranslate nohighlight">\(x\)</span> and the mean <span class="math notranslate nohighlight">\(\mu_k\)</span>. The Mahalanobis
distance tells how close <span class="math notranslate nohighlight">\(x\)</span> is from <span class="math notranslate nohighlight">\(\mu_k\)</span>, while also
accounting for the variance of each feature. We can thus interpret LDA as
assigning <span class="math notranslate nohighlight">\(x\)</span> to the class whose mean is the closest in terms of
Mahalanobis distance, while also accounting for the class prior
probabilities.</p>
<p>The log-posterior of LDA can also be written <a class="footnote-reference brackets" href="#id7" id="id1" role="doc-noteref"><span class="fn-bracket">[</span>3<span class="fn-bracket">]</span></a> as:</p>
<div class="math notranslate nohighlight">
\[\log P(y=k | x) = \omega_k^t x + \omega_{k0} + Cst.\]</div>
<p>where <span class="math notranslate nohighlight">\(\omega_k = \Sigma^{-1} \mu_k\)</span> and <span class="math notranslate nohighlight">\(\omega_{k0} =
-\frac{1}{2} \mu_k^t\Sigma^{-1}\mu_k + \log P (y = k)\)</span>. These quantities
correspond to the <code class="docutils literal notranslate"><span class="pre">coef_</span></code> and <code class="docutils literal notranslate"><span class="pre">intercept_</span></code> attributes, respectively.</p>
<p>From the above formula, it is clear that LDA has a linear decision surface.
In the case of QDA, there are no assumptions on the covariance matrices
<span class="math notranslate nohighlight">\(\Sigma_k\)</span> of the Gaussians, leading to quadratic decision surfaces.
See <a class="footnote-reference brackets" href="#id5" id="id2" role="doc-noteref"><span class="fn-bracket">[</span>1<span class="fn-bracket">]</span></a> for more details.</p>
</section>
</section>
<section id="mathematical-formulation-of-lda-dimensionality-reduction">
<h2><span class="section-number">1.2.3. </span>Mathematical formulation of LDA dimensionality reduction<a class="headerlink" href="#mathematical-formulation-of-lda-dimensionality-reduction" title="Permalink to this heading">¶</a></h2>
<p>First note that the K means <span class="math notranslate nohighlight">\(\mu_k\)</span> are vectors in
<span class="math notranslate nohighlight">\(\mathcal{R}^d\)</span>, and they lie in an affine subspace <span class="math notranslate nohighlight">\(H\)</span> of
dimension at most <span class="math notranslate nohighlight">\(K - 1\)</span> (2 points lie on a line, 3 points lie on a
plane, etc.).</p>
<p>As mentioned above, we can interpret LDA as assigning <span class="math notranslate nohighlight">\(x\)</span> to the class
whose mean <span class="math notranslate nohighlight">\(\mu_k\)</span> is the closest in terms of Mahalanobis distance,
while also accounting for the class prior probabilities. Alternatively, LDA
is equivalent to first <em>sphering</em> the data so that the covariance matrix is
the identity, and then assigning <span class="math notranslate nohighlight">\(x\)</span> to the closest mean in terms of
Euclidean distance (still accounting for the class priors).</p>
<p>Computing Euclidean distances in this d-dimensional space is equivalent to
first projecting the data points into <span class="math notranslate nohighlight">\(H\)</span>, and computing the distances
there (since the other dimensions will contribute equally to each class in
terms of distance). In other words, if <span class="math notranslate nohighlight">\(x\)</span> is closest to <span class="math notranslate nohighlight">\(\mu_k\)</span>
in the original space, it will also be the case in <span class="math notranslate nohighlight">\(H\)</span>.
This shows that, implicit in the LDA
classifier, there is a dimensionality reduction by linear projection onto a
<span class="math notranslate nohighlight">\(K-1\)</span> dimensional space.</p>
<p>We can reduce the dimension even more, to a chosen <span class="math notranslate nohighlight">\(L\)</span>, by projecting
onto the linear subspace <span class="math notranslate nohighlight">\(H_L\)</span> which maximizes the variance of the
<span class="math notranslate nohighlight">\(\mu^*_k\)</span> after projection (in effect, we are doing a form of PCA for the
transformed class means <span class="math notranslate nohighlight">\(\mu^*_k\)</span>). This <span class="math notranslate nohighlight">\(L\)</span> corresponds to the
<code class="docutils literal notranslate"><span class="pre">n_components</span></code> parameter used in the
<a class="reference internal" href="generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis.transform" title="sklearn.discriminant_analysis.LinearDiscriminantAnalysis.transform"><code class="xref py py-func docutils literal notranslate"><span class="pre">transform</span></code></a> method. See
<a class="footnote-reference brackets" href="#id5" id="id3" role="doc-noteref"><span class="fn-bracket">[</span>1<span class="fn-bracket">]</span></a> for more details.</p>
</section>
<section id="shrinkage-and-covariance-estimator">
<h2><span class="section-number">1.2.4. </span>Shrinkage and Covariance Estimator<a class="headerlink" href="#shrinkage-and-covariance-estimator" title="Permalink to this heading">¶</a></h2>
<p>Shrinkage is a form of regularization used to improve the estimation of
covariance matrices in situations where the number of training samples is
small compared to the number of features.
In this scenario, the empirical sample covariance is a poor
estimator, and shrinkage helps improving the generalization performance of
the classifier.
Shrinkage LDA can be used by setting the <code class="docutils literal notranslate"><span class="pre">shrinkage</span></code> parameter of
the <a class="reference internal" href="generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis" title="sklearn.discriminant_analysis.LinearDiscriminantAnalysis"><code class="xref py py-class docutils literal notranslate"><span class="pre">LinearDiscriminantAnalysis</span></code></a> class to ‘auto’.
This automatically determines the optimal shrinkage parameter in an analytic
way following the lemma introduced by Ledoit and Wolf <a class="footnote-reference brackets" href="#id6" id="id4" role="doc-noteref"><span class="fn-bracket">[</span>2<span class="fn-bracket">]</span></a>. Note that
currently shrinkage only works when setting the <code class="docutils literal notranslate"><span class="pre">solver</span></code> parameter to ‘lsqr’
or ‘eigen’.</p>
<p>The <code class="docutils literal notranslate"><span class="pre">shrinkage</span></code> parameter can also be manually set between 0 and 1. In
particular, a value of 0 corresponds to no shrinkage (which means the empirical
covariance matrix will be used) and a value of 1 corresponds to complete
shrinkage (which means that the diagonal matrix of variances will be used as
an estimate for the covariance matrix). Setting this parameter to a value
between these two extrema will estimate a shrunk version of the covariance
matrix.</p>
<p>The shrunk Ledoit and Wolf estimator of covariance may not always be the
best choice. For example if the distribution of the data
is normally distributed, the
Oracle Shrinkage Approximating estimator <a class="reference internal" href="generated/sklearn.covariance.OAS.html#sklearn.covariance.OAS" title="sklearn.covariance.OAS"><code class="xref py py-class docutils literal notranslate"><span class="pre">sklearn.covariance.OAS</span></code></a>
yields a smaller Mean Squared Error than the one given by Ledoit and Wolf’s
formula used with shrinkage=”auto”. In LDA, the data are assumed to be gaussian
conditionally to the class. If these assumptions hold, using LDA with
the OAS estimator of covariance will yield a better classification
accuracy than if Ledoit and Wolf or the empirical covariance estimator is used.</p>
<p>The covariance estimator can be chosen using with the <code class="docutils literal notranslate"><span class="pre">covariance_estimator</span></code>
parameter of the <a class="reference internal" href="generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis" title="sklearn.discriminant_analysis.LinearDiscriminantAnalysis"><code class="xref py py-class docutils literal notranslate"><span class="pre">discriminant_analysis.LinearDiscriminantAnalysis</span></code></a>
class. A covariance estimator should have a <a class="reference internal" href="../glossary.html#term-fit"><span class="xref std std-term">fit</span></a> method and a
<code class="docutils literal notranslate"><span class="pre">covariance_</span></code> attribute like all covariance estimators in the
<a class="reference internal" href="classes.html#module-sklearn.covariance" title="sklearn.covariance"><code class="xref py py-mod docutils literal notranslate"><span class="pre">sklearn.covariance</span></code></a> module.</p>
<p class="centered">
<strong><a class="reference external" href="../auto_examples/classification/plot_lda.html"><img alt="shrinkage" src="../_images/sphx_glr_plot_lda_001.png" style="width: 480.0px; height: 360.0px;" /></a></strong></p><aside class="topic">
<p class="topic-title">Examples:</p>
<p><a class="reference internal" href="../auto_examples/classification/plot_lda.html#sphx-glr-auto-examples-classification-plot-lda-py"><span class="std std-ref">Normal, Ledoit-Wolf and OAS Linear Discriminant Analysis for classification</span></a>: Comparison of LDA classifiers
with Empirical, Ledoit Wolf and OAS covariance estimator.</p>
</aside>
</section>
<section id="estimation-algorithms">
<h2><span class="section-number">1.2.5. </span>Estimation algorithms<a class="headerlink" href="#estimation-algorithms" title="Permalink to this heading">¶</a></h2>
<p>Using LDA and QDA requires computing the log-posterior which depends on the
class priors <span class="math notranslate nohighlight">\(P(y=k)\)</span>, the class means <span class="math notranslate nohighlight">\(\mu_k\)</span>, and the
covariance matrices.</p>
<p>The ‘svd’ solver is the default solver used for
<a class="reference internal" href="generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis" title="sklearn.discriminant_analysis.LinearDiscriminantAnalysis"><code class="xref py py-class docutils literal notranslate"><span class="pre">LinearDiscriminantAnalysis</span></code></a>, and it is
the only available solver for
<a class="reference internal" href="generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html#sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis" title="sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis"><code class="xref py py-class docutils literal notranslate"><span class="pre">QuadraticDiscriminantAnalysis</span></code></a>.
It can perform both classification and transform (for LDA).
As it does not rely on the calculation of the covariance matrix, the ‘svd’
solver may be preferable in situations where the number of features is large.
The ‘svd’ solver cannot be used with shrinkage.
For QDA, the use of the SVD solver relies on the fact that the covariance
matrix <span class="math notranslate nohighlight">\(\Sigma_k\)</span> is, by definition, equal to <span class="math notranslate nohighlight">\(\frac{1}{n - 1}
X_k^tX_k = \frac{1}{n - 1} V S^2 V^t\)</span> where <span class="math notranslate nohighlight">\(V\)</span> comes from the SVD of the (centered)
matrix: <span class="math notranslate nohighlight">\(X_k = U S V^t\)</span>. It turns out that we can compute the
log-posterior above without having to explicitly compute <span class="math notranslate nohighlight">\(\Sigma\)</span>:
computing <span class="math notranslate nohighlight">\(S\)</span> and <span class="math notranslate nohighlight">\(V\)</span> via the SVD of <span class="math notranslate nohighlight">\(X\)</span> is enough. For
LDA, two SVDs are computed: the SVD of the centered input matrix <span class="math notranslate nohighlight">\(X\)</span>
and the SVD of the class-wise mean vectors.</p>
<p>The ‘lsqr’ solver is an efficient algorithm that only works for
classification. It needs to explicitly compute the covariance matrix
<span class="math notranslate nohighlight">\(\Sigma\)</span>, and supports shrinkage and custom covariance estimators.
This solver computes the coefficients
<span class="math notranslate nohighlight">\(\omega_k = \Sigma^{-1}\mu_k\)</span> by solving for <span class="math notranslate nohighlight">\(\Sigma \omega =
\mu_k\)</span>, thus avoiding the explicit computation of the inverse
<span class="math notranslate nohighlight">\(\Sigma^{-1}\)</span>.</p>
<p>The ‘eigen’ solver is based on the optimization of the between class scatter to
within class scatter ratio. It can be used for both classification and
transform, and it supports shrinkage. However, the ‘eigen’ solver needs to
compute the covariance matrix, so it might not be suitable for situations with
a high number of features.</p>
<aside class="topic">
<p class="topic-title">References:</p>
<aside class="footnote-list brackets">
<aside class="footnote brackets" id="id5" role="note">
<span class="label"><span class="fn-bracket">[</span>1<span class="fn-bracket">]</span></span>
<span class="backrefs">(<a role="doc-backlink" href="#id2">1</a>,<a role="doc-backlink" href="#id3">2</a>)</span>
<p>“The Elements of Statistical Learning”, Hastie T., Tibshirani R.,
Friedman J., Section 4.3, p.106-119, 2008.</p>
</aside>
<aside class="footnote brackets" id="id6" role="note">
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id4">2</a><span class="fn-bracket">]</span></span>
<p>Ledoit O, Wolf M. Honey, I Shrunk the Sample Covariance Matrix.
The Journal of Portfolio Management 30(4), 110-119, 2004.</p>
</aside>
<aside class="footnote brackets" id="id7" role="note">
<span class="label"><span class="fn-bracket">[</span><a role="doc-backlink" href="#id1">3</a><span class="fn-bracket">]</span></span>
<p>R. O. Duda, P. E. Hart, D. G. Stork. Pattern Classification
(Second Edition), section 2.6.2.</p>
</aside>
</aside>
</aside>
</section>
</section>


      </div>
    <div class="container">
      <footer class="sk-content-footer">
            &copy; 2007 - 2023, scikit-learn developers (BSD License).
          <a href="../_sources/modules/lda_qda.rst.txt" rel="nofollow">Show this page source</a>
      </footer>
    </div>
  </div>
</div>
<script src="../_static/js/vendor/bootstrap.min.js"></script>

<script>
    window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
    ga('create', 'UA-22606712-2', 'auto');
    ga('set', 'anonymizeIp', true);
    ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>


<script defer data-domain="scikit-learn.org" src="https://views.scientific-python.org/js/script.js">
</script>


<script src="../_static/clipboard.min.js"></script>
<script src="../_static/copybutton.js"></script>

<script>
$(document).ready(function() {
    /* Add a [>>>] button on the top-right corner of code samples to hide
     * the >>> and ... prompts and the output and thus make the code
     * copyable. */
    var div = $('.highlight-python .highlight,' +
                '.highlight-python3 .highlight,' +
                '.highlight-pycon .highlight,' +
		'.highlight-default .highlight')
    var pre = div.find('pre');

    // get the styles from the current theme
    pre.parent().parent().css('position', 'relative');

    // create and add the button to all the code blocks that contain >>>
    div.each(function(index) {
        var jthis = $(this);
        // tracebacks (.gt) contain bare text elements that need to be
        // wrapped in a span to work with .nextUntil() (see later)
        jthis.find('pre:has(.gt)').contents().filter(function() {
            return ((this.nodeType == 3) && (this.data.trim().length > 0));
        }).wrap('<span>');
    });

	/*** Add permalink buttons next to glossary terms ***/
	$('dl.glossary > dt[id]').append(function() {
		return ('<a class="headerlink" href="#' +
			    this.getAttribute('id') +
			    '" title="Permalink to this term">¶</a>');
	});
});

</script>
    
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js"></script>
    
    <script src="https://scikit-learn.org/versionwarning.js"></script>
</body>
</html>