forked from scikit-learn/scikit-learn.github.io
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathcalibration.html
430 lines (396 loc) · 23.6 KB
/
calibration.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>1.16. Probability calibration — scikit-learn 0.20.dev0 documentation</title>
<!-- htmltitle is before nature.css - we use this hack to load bootstrap first -->
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="../_static/css/bootstrap.min.css" media="screen" />
<link rel="stylesheet" href="../_static/css/bootstrap-responsive.css"/>
<link rel="stylesheet" href="../_static/nature.css" type="text/css" />
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../',
VERSION: '0.20.dev0',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true,
SOURCELINK_SUFFIX: '.txt'
};
</script>
<script type="text/javascript" src="../_static/jquery.js"></script>
<script type="text/javascript" src="../_static/underscore.js"></script>
<script type="text/javascript" src="../_static/doctools.js"></script>
<script type="text/javascript" src="../_static/js/copybutton.js"></script>
<script type="text/javascript" src="../_static/js/extra.js"></script>
<script type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.0/MathJax.js?config=TeX-AMS_SVG"></script>
<link rel="shortcut icon" href="../_static/favicon.ico"/>
<link rel="author" title="About these documents" href="../about.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="1.17. Neural network models (supervised)" href="neural_networks_supervised.html" />
<link rel="prev" title="1.15. Isotonic regression" href="isotonic.html" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<script src="../_static/js/bootstrap.min.js" type="text/javascript"></script>
<script>
VERSION_SUBDIR = (function(groups) {
return groups ? groups[1] : null;
})(location.href.match(/^https?:\/\/scikit-learn.org\/([^\/]+)/));
</script>
<link rel="canonical" href="http://scikit-learn.org/stable/modules/calibration.html" />
<script type="text/javascript">
$("div.buttonNext, div.buttonPrevious").hover(
function () {
$(this).css('background-color', '#FF9C34');
},
function () {
$(this).css('background-color', '#A7D6E2');
}
);
function showMenu() {
var topNav = document.getElementById("scikit-navbar");
if (topNav.className === "navbar") {
topNav.className += " responsive";
} else {
topNav.className = "navbar";
}
};
</script>
</head>
<body>
<div class="header-wrapper">
<div class="header">
<p class="logo"><a href="../index.html">
<img src="../_static/scikit-learn-logo-small.png" alt="Logo"/>
</a>
</p><div class="navbar" id="scikit-navbar">
<ul>
<li><a href="../index.html">Home</a></li>
<li><a href="../install.html">Installation</a></li>
<li class="btn-li"><div class="btn-group">
<a href="../documentation.html">Documentation</a>
<a class="btn dropdown-toggle" data-toggle="dropdown">
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li class="link-title">Scikit-learn <script>document.write(DOCUMENTATION_OPTIONS.VERSION + (VERSION_SUBDIR ? " (" + VERSION_SUBDIR + ")" : ""));</script></li>
<li><a href="../tutorial/index.html">Tutorials</a></li>
<li><a href="../user_guide.html">User guide</a></li>
<li><a href="classes.html">API</a></li>
<li><a href="../glossary.html">Glossary</a></li>
<li><a href="../faq.html">FAQ</a></li>
<li><a href="../developers/contributing.html">Contributing</a></li>
<li class="divider"></li>
<script>if (VERSION_SUBDIR != "stable") document.write('<li><a href="http://scikit-learn.org/stable/documentation.html">Stable version</a></li>')</script>
<script>if (VERSION_SUBDIR != "dev") document.write('<li><a href="http://scikit-learn.org/dev/documentation.html">Development version</a></li>')</script>
<li><a href="http://scikit-learn.org/dev/versions.html">All available versions</a></li>
<li><a href="../_downloads/scikit-learn-docs.pdf">PDF documentation</a></li>
</ul>
</div>
</li>
<li><a href="../auto_examples/index.html">Examples</a></li>
</ul>
<a href="javascript:void(0);" onclick="showMenu()">
<div class="nav-icon">
<div class="hamburger-line"></div>
<div class="hamburger-line"></div>
<div class="hamburger-line"></div>
</div>
</a>
<div class="search_form">
<div class="gcse-search" id="cse" style="width: 100%;"></div>
</div>
</div> <!-- end navbar --></div>
</div>
<!-- GitHub "fork me" ribbon -->
<a href="https://github.com/scikit-learn/scikit-learn">
<img class="fork-me"
style="position: absolute; top: 0; right: 0; border: 0;"
src="../_static/img/forkme.png"
alt="Fork me on GitHub" />
</a>
<div class="content-wrapper">
<div class="sphinxsidebar">
<div class="sphinxsidebarwrapper">
<div class="rel">
<div class="rellink">
<a href="isotonic.html"
accesskey="P">Previous
<br/>
<span class="smallrellink">
1.15. Isotoni...
</span>
<span class="hiddenrellink">
1.15. Isotonic regression
</span>
</a>
</div>
<div class="spacer">
</div>
<div class="rellink">
<a href="neural_networks_supervised.html"
accesskey="N">Next
<br/>
<span class="smallrellink">
1.17. Neural ...
</span>
<span class="hiddenrellink">
1.17. Neural network models (supervised)
</span>
</a>
</div>
<!-- Ad a link to the 'up' page -->
<div class="spacer">
</div>
<div class="rellink">
<a href="../supervised_learning.html">
Up
<br/>
<span class="smallrellink">
1. Supervised...
</span>
<span class="hiddenrellink">
1. Supervised learning
</span>
</a>
</div>
</div>
<p class="doc-version"><b>scikit-learn v0.20.dev0</b><br/>
<a href="http://scikit-learn.org/stable/support.html#documentation-resources">Other versions</a></p>
<p class="citing">Please <b><a href="../about.html#citing-scikit-learn" style="font-size: 110%;">cite us </a></b>if you use the software.</p>
<ul>
<li><a class="reference internal" href="#">1.16. Probability calibration</a></li>
</ul>
</div>
</div>
<input type="checkbox" id="nav-trigger" class="nav-trigger" checked />
<label for="nav-trigger"></label>
<div class="content">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<div class="section" id="probability-calibration">
<span id="calibration"></span><h1>1.16. Probability calibration<a class="headerlink" href="#probability-calibration" title="Permalink to this headline">¶</a></h1>
<p>When performing classification you often want not only to predict the class
label, but also obtain a probability of the respective label. This probability
gives you some kind of confidence on the prediction. Some models can give you
poor estimates of the class probabilities and some even do not support
probability prediction. The calibration module allows you to better calibrate
the probabilities of a given model, or to add support for probability
prediction.</p>
<p>Well calibrated classifiers are probabilistic classifiers for which the output
of the predict_proba method can be directly interpreted as a confidence level.
For instance, a well calibrated (binary) classifier should classify the samples
such that among the samples to which it gave a predict_proba value close to 0.8,
approximately 80% actually belong to the positive class. The following plot compares
how well the probabilistic predictions of different classifiers are calibrated:</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/calibration/plot_compare_calibration.html"><img alt="../_images/sphx_glr_plot_compare_calibration_0011.png" src="../_images/sphx_glr_plot_compare_calibration_0011.png" /></a>
</div>
<p><a class="reference internal" href="generated/sklearn.linear_model.LogisticRegression.html#sklearn.linear_model.LogisticRegression" title="sklearn.linear_model.LogisticRegression"><code class="xref py py-class docutils literal"><span class="pre">LogisticRegression</span></code></a> returns well calibrated predictions by default as it directly
optimizes log-loss. In contrast, the other methods return biased probabilities;
with different biases per method:</p>
<ul class="simple">
<li><a class="reference internal" href="generated/sklearn.naive_bayes.GaussianNB.html#sklearn.naive_bayes.GaussianNB" title="sklearn.naive_bayes.GaussianNB"><code class="xref py py-class docutils literal"><span class="pre">GaussianNB</span></code></a> tends to push probabilities to 0 or 1 (note the
counts in the histograms). This is mainly because it makes the assumption
that features are conditionally independent given the class, which is not
the case in this dataset which contains 2 redundant features.</li>
</ul>
<ul class="simple">
<li><a class="reference internal" href="generated/sklearn.ensemble.RandomForestClassifier.html#sklearn.ensemble.RandomForestClassifier" title="sklearn.ensemble.RandomForestClassifier"><code class="xref py py-class docutils literal"><span class="pre">RandomForestClassifier</span></code></a> shows the opposite behavior: the histograms
show peaks at approximately 0.2 and 0.9 probability, while probabilities close to
0 or 1 are very rare. An explanation for this is given by Niculescu-Mizil
and Caruana <a class="footnote-reference" href="#id4" id="id1">[4]</a>: “Methods such as bagging and random forests that average
predictions from a base set of models can have difficulty making predictions
near 0 and 1 because variance in the underlying base models will bias
predictions that should be near zero or one away from these values. Because
predictions are restricted to the interval [0,1], errors caused by variance
tend to be one-sided near zero and one. For example, if a model should
predict p = 0 for a case, the only way bagging can achieve this is if all
bagged trees predict zero. If we add noise to the trees that bagging is
averaging over, this noise will cause some trees to predict values larger
than 0 for this case, thus moving the average prediction of the bagged
ensemble away from 0. We observe this effect most strongly with random
forests because the base-level trees trained with random forests have
relatively high variance due to feature subsetting.” As a result, the
calibration curve also referred to as the reliability diagram (Wilks 1995 <a class="footnote-reference" href="#id5" id="id2">[5]</a>) shows a
characteristic sigmoid shape, indicating that the classifier could trust its
“intuition” more and return probabilities closer to 0 or 1 typically.</li>
</ul>
<ul class="simple">
<li>Linear Support Vector Classification (<a class="reference internal" href="generated/sklearn.svm.LinearSVC.html#sklearn.svm.LinearSVC" title="sklearn.svm.LinearSVC"><code class="xref py py-class docutils literal"><span class="pre">LinearSVC</span></code></a>) shows an even more sigmoid curve
as the RandomForestClassifier, which is typical for maximum-margin methods
(compare Niculescu-Mizil and Caruana <a class="footnote-reference" href="#id4" id="id3">[4]</a>), which focus on hard samples
that are close to the decision boundary (the support vectors).</li>
</ul>
<p>Two approaches for performing calibration of probabilistic predictions are
provided: a parametric approach based on Platt’s sigmoid model and a
non-parametric approach based on isotonic regression (<a class="reference internal" href="classes.html#module-sklearn.isotonic" title="sklearn.isotonic"><code class="xref py py-mod docutils literal"><span class="pre">sklearn.isotonic</span></code></a>).
Probability calibration should be done on new data not used for model fitting.
The class <a class="reference internal" href="generated/sklearn.calibration.CalibratedClassifierCV.html#sklearn.calibration.CalibratedClassifierCV" title="sklearn.calibration.CalibratedClassifierCV"><code class="xref py py-class docutils literal"><span class="pre">CalibratedClassifierCV</span></code></a> uses a cross-validation generator and
estimates for each split the model parameter on the train samples and the
calibration of the test samples. The probabilities predicted for the
folds are then averaged. Already fitted classifiers can be calibrated by
<a class="reference internal" href="generated/sklearn.calibration.CalibratedClassifierCV.html#sklearn.calibration.CalibratedClassifierCV" title="sklearn.calibration.CalibratedClassifierCV"><code class="xref py py-class docutils literal"><span class="pre">CalibratedClassifierCV</span></code></a> via the parameter cv=”prefit”. In this case,
the user has to take care manually that data for model fitting and calibration
are disjoint.</p>
<p>The following images demonstrate the benefit of probability calibration.
The first image present a dataset with 2 classes and 3 blobs of
data. The blob in the middle contains random samples of each class.
The probability for the samples in this blob should be 0.5.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/calibration/plot_calibration.html"><img alt="../_images/sphx_glr_plot_calibration_0011.png" src="../_images/sphx_glr_plot_calibration_0011.png" /></a>
</div>
<p>The following image shows on the data above the estimated probability
using a Gaussian naive Bayes classifier without calibration,
with a sigmoid calibration and with a non-parametric isotonic
calibration. One can observe that the non-parametric model
provides the most accurate probability estimates for samples
in the middle, i.e., 0.5.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/calibration/plot_calibration.html"><img alt="../_images/sphx_glr_plot_calibration_0021.png" src="../_images/sphx_glr_plot_calibration_0021.png" /></a>
</div>
<p>The following experiment is performed on an artificial dataset for binary
classification with 100,000 samples (1,000 of them are used for model fitting)
with 20 features. Of the 20 features, only 2 are informative and 10 are
redundant. The figure shows the estimated probabilities obtained with
logistic regression, a linear support-vector classifier (SVC), and linear SVC with
both isotonic calibration and sigmoid calibration.
The Brier score is a metric which is a combination of calibration loss and refinement loss,
<a class="reference internal" href="generated/sklearn.metrics.brier_score_loss.html#sklearn.metrics.brier_score_loss" title="sklearn.metrics.brier_score_loss"><code class="xref py py-func docutils literal"><span class="pre">brier_score_loss</span></code></a>, reported in the legend (the smaller the better).
Calibration loss is defined as the mean squared deviation from empirical probabilities
derived from the slope of ROC segments. Refinement loss can be defined as the expected
optimal loss as measured by the area under the optimal cost curve.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/calibration/plot_calibration_curve.html"><img alt="../_images/sphx_glr_plot_calibration_curve_0021.png" src="../_images/sphx_glr_plot_calibration_curve_0021.png" /></a>
</div>
<p>One can observe here that logistic regression is well calibrated as its curve is
nearly diagonal. Linear SVC’s calibration curve or reliability diagram has a
sigmoid curve, which is typical for an under-confident classifier. In the case of
LinearSVC, this is caused by the margin property of the hinge loss, which lets
the model focus on hard samples that are close to the decision boundary
(the support vectors). Both kinds of calibration can fix this issue and yield
nearly identical results. The next figure shows the calibration curve of
Gaussian naive Bayes on the same data, with both kinds of calibration and also
without calibration.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/calibration/plot_calibration_curve.html"><img alt="../_images/sphx_glr_plot_calibration_curve_0011.png" src="../_images/sphx_glr_plot_calibration_curve_0011.png" /></a>
</div>
<p>One can see that Gaussian naive Bayes performs very badly but does so in an
other way than linear SVC: While linear SVC exhibited a sigmoid calibration
curve, Gaussian naive Bayes’ calibration curve has a transposed-sigmoid shape.
This is typical for an over-confident classifier. In this case, the classifier’s
overconfidence is caused by the redundant features which violate the naive Bayes
assumption of feature-independence.</p>
<p>Calibration of the probabilities of Gaussian naive Bayes with isotonic
regression can fix this issue as can be seen from the nearly diagonal
calibration curve. Sigmoid calibration also improves the brier score slightly,
albeit not as strongly as the non-parametric isotonic calibration. This is an
intrinsic limitation of sigmoid calibration, whose parametric form assumes a
sigmoid rather than a transposed-sigmoid curve. The non-parametric isotonic
calibration model, however, makes no such strong assumptions and can deal with
either shape, provided that there is sufficient calibration data. In general,
sigmoid calibration is preferable in cases where the calibration curve is sigmoid
and where there is limited calibration data, while isotonic calibration is
preferable for non-sigmoid calibration curves and in situations where large
amounts of data are available for calibration.</p>
<p><a class="reference internal" href="generated/sklearn.calibration.CalibratedClassifierCV.html#sklearn.calibration.CalibratedClassifierCV" title="sklearn.calibration.CalibratedClassifierCV"><code class="xref py py-class docutils literal"><span class="pre">CalibratedClassifierCV</span></code></a> can also deal with classification tasks that
involve more than two classes if the base estimator can do so. In this case,
the classifier is calibrated first for each class separately in an one-vs-rest
fashion. When predicting probabilities for unseen data, the calibrated
probabilities for each class are predicted separately. As those probabilities
do not necessarily sum to one, a postprocessing is performed to normalize them.</p>
<p>The next image illustrates how sigmoid calibration changes predicted
probabilities for a 3-class classification problem. Illustrated is the standard
2-simplex, where the three corners correspond to the three classes. Arrows point
from the probability vectors predicted by an uncalibrated classifier to the
probability vectors predicted by the same classifier after sigmoid calibration
on a hold-out validation set. Colors indicate the true class of an instance
(red: class 1, green: class 2, blue: class 3).</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/calibration/plot_calibration_multiclass.html"><img alt="../_images/sphx_glr_plot_calibration_multiclass_0001.png" src="../_images/sphx_glr_plot_calibration_multiclass_0001.png" /></a>
</div>
<p>The base classifier is a random forest classifier with 25 base estimators
(trees). If this classifier is trained on all 800 training datapoints, it is
overly confident in its predictions and thus incurs a large log-loss.
Calibrating an identical classifier, which was trained on 600 datapoints, with
method=’sigmoid’ on the remaining 200 datapoints reduces the confidence of the
predictions, i.e., moves the probability vectors from the edges of the simplex
towards the center:</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/calibration/plot_calibration_multiclass.html"><img alt="../_images/sphx_glr_plot_calibration_multiclass_0011.png" src="../_images/sphx_glr_plot_calibration_multiclass_0011.png" /></a>
</div>
<p>This calibration results in a lower log-loss. Note that an alternative would
have been to increase the number of base estimators which would have resulted in
a similar decrease in log-loss.</p>
<div class="topic">
<p class="topic-title first">References:</p>
<ul class="simple">
<li>Obtaining calibrated probability estimates from decision trees
and naive Bayesian classifiers, B. Zadrozny & C. Elkan, ICML 2001</li>
<li>Transforming Classifier Scores into Accurate Multiclass
Probability Estimates, B. Zadrozny & C. Elkan, (KDD 2002)</li>
<li>Probabilistic Outputs for Support Vector Machines and Comparisons to
Regularized Likelihood Methods, J. Platt, (1999)</li>
</ul>
<table class="docutils footnote" frame="void" id="id4" rules="none">
<colgroup><col class="label" /><col /></colgroup>
<tbody valign="top">
<tr><td class="label">[4]</td><td><em>(<a class="fn-backref" href="#id1">1</a>, <a class="fn-backref" href="#id3">2</a>)</em> Predicting Good Probabilities with Supervised Learning,
A. Niculescu-Mizil & R. Caruana, ICML 2005</td></tr>
</tbody>
</table>
<table class="docutils footnote" frame="void" id="id5" rules="none">
<colgroup><col class="label" /><col /></colgroup>
<tbody valign="top">
<tr><td class="label"><a class="fn-backref" href="#id2">[5]</a></td><td>On the combination of forecast probabilities for
consecutive precipitation periods. Wea. Forecasting, 5, 640–650.,
Wilks, D. S., 1990a</td></tr>
</tbody>
</table>
</div>
</div>
</div>
</div>
</div>
<div class="clearer"></div>
</div>
</div>
<div class="footer">
© 2007 - 2018, scikit-learn developers (BSD License).
<a href="../_sources/modules/calibration.rst.txt" rel="nofollow">Show this page source</a>
</div>
<div class="rel">
<div class="buttonPrevious">
<a href="isotonic.html">Previous
</a>
</div>
<div class="buttonNext">
<a href="neural_networks_supervised.html">Next
</a>
</div>
</div>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-22606712-2']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
<script>
(function() {
var cx = '016639176250731907682:tjtqbvtvij0';
var gcse = document.createElement('script'); gcse.type = 'text/javascript'; gcse.async = true;
gcse.src = 'https://cse.google.com/cse.js?cx=' + cx;
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(gcse, s);
})();
</script>
</body>
</html>