-
Notifications
You must be signed in to change notification settings - Fork 81
/
Copy pathneighbors.html
839 lines (799 loc) · 76.5 KB
/
neighbors.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>1.6. Nearest Neighbors — scikit-learn 0.17.1 documentation</title>
<!-- htmltitle is before nature.css - we use this hack to load bootstrap first -->
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="../_static/css/bootstrap.min.css" media="screen" />
<link rel="stylesheet" href="../_static/css/bootstrap-responsive.css"/>
<link rel="stylesheet" href="../_static/nature.css" type="text/css" />
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../',
VERSION: '0.17.1',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../_static/jquery.js"></script>
<script type="text/javascript" src="../_static/underscore.js"></script>
<script type="text/javascript" src="../_static/doctools.js"></script>
<script type="text/javascript" src="../_static/js/copybutton.js"></script>
<link rel="shortcut icon" href="../_static/favicon.ico"/>
<link rel="author" title="About these documents" href="../about.html" />
<link rel="top" title="scikit-learn 0.17.1 documentation" href="../index.html" />
<link rel="up" title="1. Supervised learning" href="../supervised_learning.html" />
<link rel="next" title="1.7. Gaussian Processes" href="gaussian_process.html" />
<link rel="prev" title="1.5. Stochastic Gradient Descent" href="sgd.html" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<script src="../_static/js/bootstrap.min.js" type="text/javascript"></script>
<link rel="canonical" href="https://scikit-learn.org/stable/modules/neighbors.html" />
<script type="text/javascript">
$("div.buttonNext, div.buttonPrevious").hover(
function () {
$(this).css('background-color', '#FF9C34');
},
function () {
$(this).css('background-color', '#A7D6E2');
}
);
</script>
</head>
<body role="document">
<div class="header-wrapper">
<div class="header">
<p class="logo"><a href="../index.html">
<img src="../_static/scikit-learn-logo-small.png" alt="Logo"/>
</a>
</p><div class="navbar">
<ul>
<li><a href="../index.html">Home</a></li>
<li><a href="../install.html">Installation</a></li>
<li class="btn-li"><div class="btn-group">
<a href="../documentation.html">Documentation</a>
<a class="btn dropdown-toggle" data-toggle="dropdown">
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li class="link-title">Scikit-learn 0.17 (stable)</li>
<li><a href="../tutorial/index.html">Tutorials</a></li>
<li><a href="../user_guide.html">User guide</a></li>
<li><a href="classes.html">API</a></li>
<li><a href="../faq.html">FAQ</a></li>
<li><a href="../developers/contributing.html">Contributing</a></li>
<li class="divider"></li>
<li><a href="http://scikit-learn.org/dev/documentation.html">Scikit-learn 0.18 (development)</a></li>
<li><a href="http://scikit-learn.org/0.16/documentation.html">Scikit-learn 0.16</a></li>
<li><a href="../_downloads/scikit-learn-docs.pdf">PDF documentation</a></li>
</ul>
</div>
</li>
<li><a href="../auto_examples/index.html">Examples</a></li>
</ul>
<div class="search_form">
<div id="cse" style="width: 100%;"></div>
</div>
</div> <!-- end navbar --></div>
</div>
<!-- Github "fork me" ribbon -->
<a href="https://github.com/scikit-learn/scikit-learn">
<img class="fork-me"
style="position: absolute; top: 0; right: 0; border: 0;"
src="../_static/img/forkme.png"
alt="Fork me on GitHub" />
</a>
<div class="content-wrapper">
<div class="sphinxsidebar">
<div class="sphinxsidebarwrapper">
<div class="rel">
<!-- rellinks[1:] is an ugly hack to avoid link to module
index -->
<div class="rellink">
<a href="sgd.html"
accesskey="P">Previous
<br/>
<span class="smallrellink">
1.5. Stochast...
</span>
<span class="hiddenrellink">
1.5. Stochastic Gradient Descent
</span>
</a>
</div>
<!-- Ad a link to the 'up' page -->
<div class="spacer">
</div>
<div class="rellink">
<a href="../supervised_learning.html">
Up
<br/>
<span class="smallrellink">
1. Supervised...
</span>
<span class="hiddenrellink">
1. Supervised learning
</span>
</a>
</div>
</div>
<p class="doc-version">This documentation is for scikit-learn <strong>version 0.17.1</strong> — <a href="http://scikit-learn.org/stable/support.html#documentation-resources">Other versions</a></p>
<p class="citing">If you use the software, please consider <a href="../about.html#citing-scikit-learn">citing scikit-learn</a>.</p>
<ul>
<li><a class="reference internal" href="#">1.6. Nearest Neighbors</a><ul>
<li><a class="reference internal" href="#unsupervised-nearest-neighbors">1.6.1. Unsupervised Nearest Neighbors</a><ul>
<li><a class="reference internal" href="#finding-the-nearest-neighbors">1.6.1.1. Finding the Nearest Neighbors</a></li>
<li><a class="reference internal" href="#kdtree-and-balltree-classes">1.6.1.2. KDTree and BallTree Classes</a></li>
</ul>
</li>
<li><a class="reference internal" href="#nearest-neighbors-classification">1.6.2. Nearest Neighbors Classification</a></li>
<li><a class="reference internal" href="#nearest-neighbors-regression">1.6.3. Nearest Neighbors Regression</a></li>
<li><a class="reference internal" href="#nearest-neighbor-algorithms">1.6.4. Nearest Neighbor Algorithms</a><ul>
<li><a class="reference internal" href="#brute-force">1.6.4.1. Brute Force</a></li>
<li><a class="reference internal" href="#k-d-tree">1.6.4.2. K-D Tree</a></li>
<li><a class="reference internal" href="#ball-tree">1.6.4.3. Ball Tree</a></li>
<li><a class="reference internal" href="#choice-of-nearest-neighbors-algorithm">1.6.4.4. Choice of Nearest Neighbors Algorithm</a></li>
<li><a class="reference internal" href="#effect-of-leaf-size">1.6.4.5. Effect of <code class="docutils literal"><span class="pre">leaf_size</span></code></a></li>
</ul>
</li>
<li><a class="reference internal" href="#nearest-centroid-classifier">1.6.5. Nearest Centroid Classifier</a><ul>
<li><a class="reference internal" href="#nearest-shrunken-centroid">1.6.5.1. Nearest Shrunken Centroid</a></li>
</ul>
</li>
<li><a class="reference internal" href="#approximate-nearest-neighbors">1.6.6. Approximate Nearest Neighbors</a><ul>
<li><a class="reference internal" href="#locality-sensitive-hashing-forest">1.6.6.1. Locality Sensitive Hashing Forest</a></li>
<li><a class="reference internal" href="#mathematical-description-of-locality-sensitive-hashing">1.6.6.2. Mathematical description of Locality Sensitive Hashing</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<input type="checkbox" id="nav-trigger" class="nav-trigger" checked />
<label for="nav-trigger"></label>
<div class="content">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<div class="section" id="nearest-neighbors">
<span id="neighbors"></span><h1>1.6. Nearest Neighbors<a class="headerlink" href="#nearest-neighbors" title="Permalink to this headline">¶</a></h1>
<p><a class="reference internal" href="classes.html#module-sklearn.neighbors" title="sklearn.neighbors"><code class="xref py py-mod docutils literal"><span class="pre">sklearn.neighbors</span></code></a> provides functionality for unsupervised and
supervised neighbors-based learning methods. Unsupervised nearest neighbors
is the foundation of many other learning methods,
notably manifold learning and spectral clustering. Supervised neighbors-based
learning comes in two flavors: <a class="reference internal" href="#classification">classification</a> for data with
discrete labels, and <a class="reference internal" href="#regression">regression</a> for data with continuous labels.</p>
<p>The principle behind nearest neighbor methods is to find a predefined number
of training samples closest in distance to the new point, and
predict the label from these. The number of samples can be a user-defined
constant (k-nearest neighbor learning), or vary based
on the local density of points (radius-based neighbor learning).
The distance can, in general, be any metric measure: standard Euclidean
distance is the most common choice.
Neighbors-based methods are known as <em>non-generalizing</em> machine
learning methods, since they simply “remember” all of its training data
(possibly transformed into a fast indexing structure such as a
<a class="reference internal" href="#ball-tree"><span>Ball Tree</span></a> or <a class="reference internal" href="#kd-tree"><span>KD Tree</span></a>.).</p>
<p>Despite its simplicity, nearest neighbors has been successful in a
large number of classification and regression problems, including
handwritten digits or satellite image scenes. Being a non-parametric method,
it is often successful in classification situations where the decision
boundary is very irregular.</p>
<p>The classes in <a class="reference internal" href="classes.html#module-sklearn.neighbors" title="sklearn.neighbors"><code class="xref py py-mod docutils literal"><span class="pre">sklearn.neighbors</span></code></a> can handle either Numpy arrays or
<cite>scipy.sparse</cite> matrices as input. For dense matrices, a large number of
possible distance metrics are supported. For sparse matrices, arbitrary
Minkowski metrics are supported for searches.</p>
<p>There are many learning routines which rely on nearest neighbors at their
core. One example is <a class="reference internal" href="density.html#kernel-density"><span>kernel density estimation</span></a>,
discussed in the <a class="reference internal" href="density.html#density-estimation"><span>density estimation</span></a> section.</p>
<div class="section" id="unsupervised-nearest-neighbors">
<span id="unsupervised-neighbors"></span><h2>1.6.1. Unsupervised Nearest Neighbors<a class="headerlink" href="#unsupervised-nearest-neighbors" title="Permalink to this headline">¶</a></h2>
<p><a class="reference internal" href="generated/sklearn.neighbors.NearestNeighbors.html#sklearn.neighbors.NearestNeighbors" title="sklearn.neighbors.NearestNeighbors"><code class="xref py py-class docutils literal"><span class="pre">NearestNeighbors</span></code></a> implements unsupervised nearest neighbors learning.
It acts as a uniform interface to three different nearest neighbors
algorithms: <a class="reference internal" href="generated/sklearn.neighbors.BallTree.html#sklearn.neighbors.BallTree" title="sklearn.neighbors.BallTree"><code class="xref py py-class docutils literal"><span class="pre">BallTree</span></code></a>, <a class="reference internal" href="generated/sklearn.neighbors.KDTree.html#sklearn.neighbors.KDTree" title="sklearn.neighbors.KDTree"><code class="xref py py-class docutils literal"><span class="pre">KDTree</span></code></a>, and a
brute-force algorithm based on routines in <a class="reference internal" href="classes.html#module-sklearn.metrics.pairwise" title="sklearn.metrics.pairwise"><code class="xref py py-mod docutils literal"><span class="pre">sklearn.metrics.pairwise</span></code></a>.
The choice of neighbors search algorithm is controlled through the keyword
<code class="docutils literal"><span class="pre">'algorithm'</span></code>, which must be one of
<code class="docutils literal"><span class="pre">['auto',</span> <span class="pre">'ball_tree',</span> <span class="pre">'kd_tree',</span> <span class="pre">'brute']</span></code>. When the default value
<code class="docutils literal"><span class="pre">'auto'</span></code> is passed, the algorithm attempts to determine the best approach
from the training data. For a discussion of the strengths and weaknesses
of each option, see <a class="reference internal" href="#nearest-neighbor-algorithms">Nearest Neighbor Algorithms</a>.</p>
<blockquote>
<div><div class="admonition warning">
<p class="first admonition-title">Warning</p>
<p class="last">Regarding the Nearest Neighbors algorithms, if two
neighbors, neighbor <img class="math" src="../_images/math/3494d6c050dcba63488def5cd4e3c183f1234306.png" alt="k+1"/> and <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/>, have identical distances
but different labels, the results will depend on the ordering of the
training data.</p>
</div>
</div></blockquote>
<div class="section" id="finding-the-nearest-neighbors">
<h3>1.6.1.1. Finding the Nearest Neighbors<a class="headerlink" href="#finding-the-nearest-neighbors" title="Permalink to this headline">¶</a></h3>
<p>For the simple task of finding the nearest neighbors between two sets of
data, the unsupervised algorithms within <a class="reference internal" href="classes.html#module-sklearn.neighbors" title="sklearn.neighbors"><code class="xref py py-mod docutils literal"><span class="pre">sklearn.neighbors</span></code></a> can be
used:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">sklearn.neighbors</span> <span class="kn">import</span> <span class="n">NearestNeighbors</span>
<span class="gp">>>> </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="gp">>>> </span><span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">2</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">]])</span>
<span class="gp">>>> </span><span class="n">nbrs</span> <span class="o">=</span> <span class="n">NearestNeighbors</span><span class="p">(</span><span class="n">n_neighbors</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">algorithm</span><span class="o">=</span><span class="s1">'ball_tree'</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">distances</span><span class="p">,</span> <span class="n">indices</span> <span class="o">=</span> <span class="n">nbrs</span><span class="o">.</span><span class="n">kneighbors</span><span class="p">(</span><span class="n">X</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">indices</span>
<span class="go">array([[0, 1],</span>
<span class="go"> [1, 0],</span>
<span class="go"> [2, 1],</span>
<span class="go"> [3, 4],</span>
<span class="go"> [4, 3],</span>
<span class="go"> [5, 4]]...)</span>
<span class="gp">>>> </span><span class="n">distances</span>
<span class="go">array([[ 0. , 1. ],</span>
<span class="go"> [ 0. , 1. ],</span>
<span class="go"> [ 0. , 1.41421356],</span>
<span class="go"> [ 0. , 1. ],</span>
<span class="go"> [ 0. , 1. ],</span>
<span class="go"> [ 0. , 1.41421356]])</span>
</pre></div>
</div>
<p>Because the query set matches the training set, the nearest neighbor of each
point is the point itself, at a distance of zero.</p>
<p>It is also possible to efficiently produce a sparse graph showing the
connections between neighboring points:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="n">nbrs</span><span class="o">.</span><span class="n">kneighbors_graph</span><span class="p">(</span><span class="n">X</span><span class="p">)</span><span class="o">.</span><span class="n">toarray</span><span class="p">()</span>
<span class="go">array([[ 1., 1., 0., 0., 0., 0.],</span>
<span class="go"> [ 1., 1., 0., 0., 0., 0.],</span>
<span class="go"> [ 0., 1., 1., 0., 0., 0.],</span>
<span class="go"> [ 0., 0., 0., 1., 1., 0.],</span>
<span class="go"> [ 0., 0., 0., 1., 1., 0.],</span>
<span class="go"> [ 0., 0., 0., 0., 1., 1.]])</span>
</pre></div>
</div>
<p>Our dataset is structured such that points nearby in index order are nearby
in parameter space, leading to an approximately block-diagonal matrix of
K-nearest neighbors. Such a sparse graph is useful in a variety of
circumstances which make use of spatial relationships between points for
unsupervised learning: in particular, see <a class="reference internal" href="generated/sklearn.manifold.Isomap.html#sklearn.manifold.Isomap" title="sklearn.manifold.Isomap"><code class="xref py py-class docutils literal"><span class="pre">sklearn.manifold.Isomap</span></code></a>,
<a class="reference internal" href="generated/sklearn.manifold.LocallyLinearEmbedding.html#sklearn.manifold.LocallyLinearEmbedding" title="sklearn.manifold.LocallyLinearEmbedding"><code class="xref py py-class docutils literal"><span class="pre">sklearn.manifold.LocallyLinearEmbedding</span></code></a>, and
<a class="reference internal" href="generated/sklearn.cluster.SpectralClustering.html#sklearn.cluster.SpectralClustering" title="sklearn.cluster.SpectralClustering"><code class="xref py py-class docutils literal"><span class="pre">sklearn.cluster.SpectralClustering</span></code></a>.</p>
</div>
<div class="section" id="kdtree-and-balltree-classes">
<h3>1.6.1.2. KDTree and BallTree Classes<a class="headerlink" href="#kdtree-and-balltree-classes" title="Permalink to this headline">¶</a></h3>
<p>Alternatively, one can use the <a class="reference internal" href="generated/sklearn.neighbors.KDTree.html#sklearn.neighbors.KDTree" title="sklearn.neighbors.KDTree"><code class="xref py py-class docutils literal"><span class="pre">KDTree</span></code></a> or <a class="reference internal" href="generated/sklearn.neighbors.BallTree.html#sklearn.neighbors.BallTree" title="sklearn.neighbors.BallTree"><code class="xref py py-class docutils literal"><span class="pre">BallTree</span></code></a> classes
directly to find nearest neighbors. This is the functionality wrapped by
the <a class="reference internal" href="generated/sklearn.neighbors.NearestNeighbors.html#sklearn.neighbors.NearestNeighbors" title="sklearn.neighbors.NearestNeighbors"><code class="xref py py-class docutils literal"><span class="pre">NearestNeighbors</span></code></a> class used above. The Ball Tree and KD Tree
have the same interface; we’ll show an example of using the KD Tree here:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">sklearn.neighbors</span> <span class="kn">import</span> <span class="n">KDTree</span>
<span class="gp">>>> </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="gp">>>> </span><span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">2</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">]])</span>
<span class="gp">>>> </span><span class="n">kdt</span> <span class="o">=</span> <span class="n">KDTree</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">leaf_size</span><span class="o">=</span><span class="mi">30</span><span class="p">,</span> <span class="n">metric</span><span class="o">=</span><span class="s1">'euclidean'</span><span class="p">)</span>
<span class="gp">>>> </span><span class="n">kdt</span><span class="o">.</span><span class="n">query</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">k</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">return_distance</span><span class="o">=</span><span class="bp">False</span><span class="p">)</span>
<span class="go">array([[0, 1],</span>
<span class="go"> [1, 0],</span>
<span class="go"> [2, 1],</span>
<span class="go"> [3, 4],</span>
<span class="go"> [4, 3],</span>
<span class="go"> [5, 4]]...)</span>
</pre></div>
</div>
<p>Refer to the <a class="reference internal" href="generated/sklearn.neighbors.KDTree.html#sklearn.neighbors.KDTree" title="sklearn.neighbors.KDTree"><code class="xref py py-class docutils literal"><span class="pre">KDTree</span></code></a> and <a class="reference internal" href="generated/sklearn.neighbors.BallTree.html#sklearn.neighbors.BallTree" title="sklearn.neighbors.BallTree"><code class="xref py py-class docutils literal"><span class="pre">BallTree</span></code></a> class documentation
for more information on the options available for neighbors searches,
including specification of query strategies, of various distance metrics, etc.
For a list of available metrics, see the documentation of the
<a class="reference internal" href="generated/sklearn.neighbors.DistanceMetric.html#sklearn.neighbors.DistanceMetric" title="sklearn.neighbors.DistanceMetric"><code class="xref py py-class docutils literal"><span class="pre">DistanceMetric</span></code></a> class.</p>
</div>
</div>
<div class="section" id="nearest-neighbors-classification">
<span id="classification"></span><h2>1.6.2. Nearest Neighbors Classification<a class="headerlink" href="#nearest-neighbors-classification" title="Permalink to this headline">¶</a></h2>
<p>Neighbors-based classification is a type of <em>instance-based learning</em> or
<em>non-generalizing learning</em>: it does not attempt to construct a general
internal model, but simply stores instances of the training data.
Classification is computed from a simple majority vote of the nearest
neighbors of each point: a query point is assigned the data class which
has the most representatives within the nearest neighbors of the point.</p>
<p>scikit-learn implements two different nearest neighbors classifiers:
<a class="reference internal" href="generated/sklearn.neighbors.KNeighborsClassifier.html#sklearn.neighbors.KNeighborsClassifier" title="sklearn.neighbors.KNeighborsClassifier"><code class="xref py py-class docutils literal"><span class="pre">KNeighborsClassifier</span></code></a> implements learning based on the <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/>
nearest neighbors of each query point, where <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> is an integer value
specified by the user. <a class="reference internal" href="generated/sklearn.neighbors.RadiusNeighborsClassifier.html#sklearn.neighbors.RadiusNeighborsClassifier" title="sklearn.neighbors.RadiusNeighborsClassifier"><code class="xref py py-class docutils literal"><span class="pre">RadiusNeighborsClassifier</span></code></a> implements learning
based on the number of neighbors within a fixed radius <img class="math" src="../_images/math/2ede365ad144ab396916ec60458da03860803078.png" alt="r"/> of each
training point, where <img class="math" src="../_images/math/2ede365ad144ab396916ec60458da03860803078.png" alt="r"/> is a floating-point value specified by
the user.</p>
<p>The <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/>-neighbors classification in <a class="reference internal" href="generated/sklearn.neighbors.KNeighborsClassifier.html#sklearn.neighbors.KNeighborsClassifier" title="sklearn.neighbors.KNeighborsClassifier"><code class="xref py py-class docutils literal"><span class="pre">KNeighborsClassifier</span></code></a>
is the more commonly used of the two techniques. The
optimal choice of the value <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> is highly data-dependent: in general
a larger <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> suppresses the effects of noise, but makes the
classification boundaries less distinct.</p>
<p>In cases where the data is not uniformly sampled, radius-based neighbors
classification in <a class="reference internal" href="generated/sklearn.neighbors.RadiusNeighborsClassifier.html#sklearn.neighbors.RadiusNeighborsClassifier" title="sklearn.neighbors.RadiusNeighborsClassifier"><code class="xref py py-class docutils literal"><span class="pre">RadiusNeighborsClassifier</span></code></a> can be a better choice.
The user specifies a fixed radius <img class="math" src="../_images/math/2ede365ad144ab396916ec60458da03860803078.png" alt="r"/>, such that points in sparser
neighborhoods use fewer nearest neighbors for the classification. For
high-dimensional parameter spaces, this method becomes less effective due
to the so-called “curse of dimensionality”.</p>
<p>The basic nearest neighbors classification uses uniform weights: that is, the
value assigned to a query point is computed from a simple majority vote of
the nearest neighbors. Under some circumstances, it is better to weight the
neighbors such that nearer neighbors contribute more to the fit. This can
be accomplished through the <code class="docutils literal"><span class="pre">weights</span></code> keyword. The default value,
<code class="docutils literal"><span class="pre">weights</span> <span class="pre">=</span> <span class="pre">'uniform'</span></code>, assigns uniform weights to each neighbor.
<code class="docutils literal"><span class="pre">weights</span> <span class="pre">=</span> <span class="pre">'distance'</span></code> assigns weights proportional to the inverse of the
distance from the query point. Alternatively, a user-defined function of the
distance can be supplied which is used to compute the weights.</p>
<p class="centered">
<strong><a class="reference external" href="../auto_examples/neighbors/plot_classification.html"><img alt="classification_1" src="../_images/plot_classification_0011.png" style="width: 400.0px; height: 300.0px;" /></a> <a class="reference external" href="../auto_examples/neighbors/plot_classification.html"><img alt="classification_2" src="../_images/plot_classification_0021.png" style="width: 400.0px; height: 300.0px;" /></a></strong></p><div class="topic">
<p class="topic-title first">Examples:</p>
<ul class="simple">
<li><a class="reference internal" href="../auto_examples/neighbors/plot_classification.html#example-neighbors-plot-classification-py"><span>Nearest Neighbors Classification</span></a>: an example of
classification using nearest neighbors.</li>
</ul>
</div>
</div>
<div class="section" id="nearest-neighbors-regression">
<span id="regression"></span><h2>1.6.3. Nearest Neighbors Regression<a class="headerlink" href="#nearest-neighbors-regression" title="Permalink to this headline">¶</a></h2>
<p>Neighbors-based regression can be used in cases where the data labels are
continuous rather than discrete variables. The label assigned to a query
point is computed based the mean of the labels of its nearest neighbors.</p>
<p>scikit-learn implements two different neighbors regressors:
<a class="reference internal" href="generated/sklearn.neighbors.KNeighborsRegressor.html#sklearn.neighbors.KNeighborsRegressor" title="sklearn.neighbors.KNeighborsRegressor"><code class="xref py py-class docutils literal"><span class="pre">KNeighborsRegressor</span></code></a> implements learning based on the <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/>
nearest neighbors of each query point, where <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> is an integer
value specified by the user. <a class="reference internal" href="generated/sklearn.neighbors.RadiusNeighborsRegressor.html#sklearn.neighbors.RadiusNeighborsRegressor" title="sklearn.neighbors.RadiusNeighborsRegressor"><code class="xref py py-class docutils literal"><span class="pre">RadiusNeighborsRegressor</span></code></a> implements
learning based on the neighbors within a fixed radius <img class="math" src="../_images/math/2ede365ad144ab396916ec60458da03860803078.png" alt="r"/> of the
query point, where <img class="math" src="../_images/math/2ede365ad144ab396916ec60458da03860803078.png" alt="r"/> is a floating-point value specified by the
user.</p>
<p>The basic nearest neighbors regression uses uniform weights: that is,
each point in the local neighborhood contributes uniformly to the
classification of a query point. Under some circumstances, it can be
advantageous to weight points such that nearby points contribute more
to the regression than faraway points. This can be accomplished through
the <code class="docutils literal"><span class="pre">weights</span></code> keyword. The default value, <code class="docutils literal"><span class="pre">weights</span> <span class="pre">=</span> <span class="pre">'uniform'</span></code>,
assigns equal weights to all points. <code class="docutils literal"><span class="pre">weights</span> <span class="pre">=</span> <span class="pre">'distance'</span></code> assigns
weights proportional to the inverse of the distance from the query point.
Alternatively, a user-defined function of the distance can be supplied,
which will be used to compute the weights.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/neighbors/plot_regression.html"><img alt="../_images/plot_regression_0011.png" src="../_images/plot_regression_0011.png" style="width: 600.0px; height: 450.0px;" /></a>
</div>
<p>The use of multi-output nearest neighbors for regression is demonstrated in
<a class="reference internal" href="../auto_examples/plot_multioutput_face_completion.html#example-plot-multioutput-face-completion-py"><span>Face completion with a multi-output estimators</span></a>. In this example, the inputs
X are the pixels of the upper half of faces and the outputs Y are the pixels of
the lower half of those faces.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/plot_multioutput_face_completion.html"><img alt="../_images/plot_multioutput_face_completion_0011.png" src="../_images/plot_multioutput_face_completion_0011.png" style="width: 750.0px; height: 847.5px;" /></a>
</div>
<div class="topic">
<p class="topic-title first">Examples:</p>
<ul class="simple">
<li><a class="reference internal" href="../auto_examples/neighbors/plot_regression.html#example-neighbors-plot-regression-py"><span>Nearest Neighbors regression</span></a>: an example of regression
using nearest neighbors.</li>
<li><a class="reference internal" href="../auto_examples/plot_multioutput_face_completion.html#example-plot-multioutput-face-completion-py"><span>Face completion with a multi-output estimators</span></a>: an example of
multi-output regression using nearest neighbors.</li>
</ul>
</div>
</div>
<div class="section" id="nearest-neighbor-algorithms">
<h2>1.6.4. Nearest Neighbor Algorithms<a class="headerlink" href="#nearest-neighbor-algorithms" title="Permalink to this headline">¶</a></h2>
<div class="section" id="brute-force">
<span id="id1"></span><h3>1.6.4.1. Brute Force<a class="headerlink" href="#brute-force" title="Permalink to this headline">¶</a></h3>
<p>Fast computation of nearest neighbors is an active area of research in
machine learning. The most naive neighbor search implementation involves
the brute-force computation of distances between all pairs of points in the
dataset: for <img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/> samples in <img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/> dimensions, this approach scales
as <img class="math" src="../_images/math/95247227810a3f2251f6560767129543ba6703d8.png" alt="O[D N^2]"/>. Efficient brute-force neighbors searches can be very
competitive for small data samples.
However, as the number of samples <img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/> grows, the brute-force
approach quickly becomes infeasible. In the classes within
<a class="reference internal" href="classes.html#module-sklearn.neighbors" title="sklearn.neighbors"><code class="xref py py-mod docutils literal"><span class="pre">sklearn.neighbors</span></code></a>, brute-force neighbors searches are specified
using the keyword <code class="docutils literal"><span class="pre">algorithm</span> <span class="pre">=</span> <span class="pre">'brute'</span></code>, and are computed using the
routines available in <a class="reference internal" href="classes.html#module-sklearn.metrics.pairwise" title="sklearn.metrics.pairwise"><code class="xref py py-mod docutils literal"><span class="pre">sklearn.metrics.pairwise</span></code></a>.</p>
</div>
<div class="section" id="k-d-tree">
<span id="kd-tree"></span><h3>1.6.4.2. K-D Tree<a class="headerlink" href="#k-d-tree" title="Permalink to this headline">¶</a></h3>
<p>To address the computational inefficiencies of the brute-force approach, a
variety of tree-based data structures have been invented. In general, these
structures attempt to reduce the required number of distance calculations
by efficiently encoding aggregate distance information for the sample.
The basic idea is that if point <img class="math" src="../_images/math/0acafa529182e79b4f56165ec677554fba7fcf98.png" alt="A"/> is very distant from point
<img class="math" src="../_images/math/83956e92fcc80dee17fce864543216939a3c9da7.png" alt="B"/>, and point <img class="math" src="../_images/math/83956e92fcc80dee17fce864543216939a3c9da7.png" alt="B"/> is very close to point <img class="math" src="../_images/math/2bcc65482aa8e15cd4c9e9f2542451fb4e971a91.png" alt="C"/>,
then we know that points <img class="math" src="../_images/math/0acafa529182e79b4f56165ec677554fba7fcf98.png" alt="A"/> and <img class="math" src="../_images/math/2bcc65482aa8e15cd4c9e9f2542451fb4e971a91.png" alt="C"/>
are very distant, <em>without having to explicitly calculate their distance</em>.
In this way, the computational cost of a nearest neighbors search can be
reduced to <img class="math" src="../_images/math/03d3d364bd2e391e05067b1741b2f2b4912f4acd.png" alt="O[D N \log(N)]"/> or better. This is a significant
improvement over brute-force for large <img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/>.</p>
<p>An early approach to taking advantage of this aggregate information was
the <em>KD tree</em> data structure (short for <em>K-dimensional tree</em>), which
generalizes two-dimensional <em>Quad-trees</em> and 3-dimensional <em>Oct-trees</em>
to an arbitrary number of dimensions. The KD tree is a binary tree
structure which recursively partitions the parameter space along the data
axes, dividing it into nested orthotopic regions into which data points
are filed. The construction of a KD tree is very fast: because partitioning
is performed only along the data axes, no <img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/>-dimensional distances
need to be computed. Once constructed, the nearest neighbor of a query
point can be determined with only <img class="math" src="../_images/math/2a36ca75fbd0817c7a2020f6009239a7829b1aa7.png" alt="O[\log(N)]"/> distance computations.
Though the KD tree approach is very fast for low-dimensional (<img class="math" src="../_images/math/e477a6f58d3e85cc0e8c8487ff7c5b0e683ac950.png" alt="D < 20"/>)
neighbors searches, it becomes inefficient as <img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/> grows very large:
this is one manifestation of the so-called “curse of dimensionality”.
In scikit-learn, KD tree neighbors searches are specified using the
keyword <code class="docutils literal"><span class="pre">algorithm</span> <span class="pre">=</span> <span class="pre">'kd_tree'</span></code>, and are computed using the class
<a class="reference internal" href="generated/sklearn.neighbors.KDTree.html#sklearn.neighbors.KDTree" title="sklearn.neighbors.KDTree"><code class="xref py py-class docutils literal"><span class="pre">KDTree</span></code></a>.</p>
<div class="topic">
<p class="topic-title first">References:</p>
<ul class="simple">
<li><a class="reference external" href="http://dl.acm.org/citation.cfm?doid=361002.361007">“Multidimensional binary search trees used for associative searching”</a>,
Bentley, J.L., Communications of the ACM (1975)</li>
</ul>
</div>
</div>
<div class="section" id="ball-tree">
<span id="id2"></span><h3>1.6.4.3. Ball Tree<a class="headerlink" href="#ball-tree" title="Permalink to this headline">¶</a></h3>
<p>To address the inefficiencies of KD Trees in higher dimensions, the <em>ball tree</em>
data structure was developed. Where KD trees partition data along
Cartesian axes, ball trees partition data in a series of nesting
hyper-spheres. This makes tree construction more costly than that of the
KD tree, but
results in a data structure which can be very efficient on highly-structured
data, even in very high dimensions.</p>
<p>A ball tree recursively divides the data into
nodes defined by a centroid <img class="math" src="../_images/math/2bcc65482aa8e15cd4c9e9f2542451fb4e971a91.png" alt="C"/> and radius <img class="math" src="../_images/math/2ede365ad144ab396916ec60458da03860803078.png" alt="r"/>, such that each
point in the node lies within the hyper-sphere defined by <img class="math" src="../_images/math/2ede365ad144ab396916ec60458da03860803078.png" alt="r"/> and
<img class="math" src="../_images/math/2bcc65482aa8e15cd4c9e9f2542451fb4e971a91.png" alt="C"/>. The number of candidate points for a neighbor search
is reduced through use of the <em>triangle inequality</em>:</p>
<div class="math">
<p><img src="../_images/math/20c99f81ba0757651f17ff9b10f01f5400f989d9.png" alt="|x+y| \leq |x| + |y|"/></p>
</div><p>With this setup, a single distance calculation between a test point and
the centroid is sufficient to determine a lower and upper bound on the
distance to all points within the node.
Because of the spherical geometry of the ball tree nodes, it can out-perform
a <em>KD-tree</em> in high dimensions, though the actual performance is highly
dependent on the structure of the training data.
In scikit-learn, ball-tree-based
neighbors searches are specified using the keyword <code class="docutils literal"><span class="pre">algorithm</span> <span class="pre">=</span> <span class="pre">'ball_tree'</span></code>,
and are computed using the class <a class="reference internal" href="generated/sklearn.neighbors.BallTree.html#sklearn.neighbors.BallTree" title="sklearn.neighbors.BallTree"><code class="xref py py-class docutils literal"><span class="pre">sklearn.neighbors.BallTree</span></code></a>.
Alternatively, the user can work with the <a class="reference internal" href="generated/sklearn.neighbors.BallTree.html#sklearn.neighbors.BallTree" title="sklearn.neighbors.BallTree"><code class="xref py py-class docutils literal"><span class="pre">BallTree</span></code></a> class directly.</p>
<div class="topic">
<p class="topic-title first">References:</p>
<ul class="simple">
<li><a class="reference external" href="http://citeseer.ist.psu.edu/viewdoc/summary?doi=10.1.1.91.8209">“Five balltree construction algorithms”</a>,
Omohundro, S.M., International Computer Science Institute
Technical Report (1989)</li>
</ul>
</div>
</div>
<div class="section" id="choice-of-nearest-neighbors-algorithm">
<h3>1.6.4.4. Choice of Nearest Neighbors Algorithm<a class="headerlink" href="#choice-of-nearest-neighbors-algorithm" title="Permalink to this headline">¶</a></h3>
<p>The optimal algorithm for a given dataset is a complicated choice, and
depends on a number of factors:</p>
<ul>
<li><p class="first">number of samples <img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/> (i.e. <code class="docutils literal"><span class="pre">n_samples</span></code>) and dimensionality
<img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/> (i.e. <code class="docutils literal"><span class="pre">n_features</span></code>).</p>
<ul class="simple">
<li><em>Brute force</em> query time grows as <img class="math" src="../_images/math/f5ee3e6dd2b03108b0874de08173a2312a061e0c.png" alt="O[D N]"/></li>
<li><em>Ball tree</em> query time grows as approximately <img class="math" src="../_images/math/d692a352d052c59e2ea79129c89cbeae6e4571c1.png" alt="O[D \log(N)]"/></li>
<li><em>KD tree</em> query time changes with <img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/> in a way that is difficult
to precisely characterise. For small <img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/> (less than 20 or so)
the cost is approximately <img class="math" src="../_images/math/f26b81fda0dc2a941a15a041e21408577e7f5480.png" alt="O[D\log(N)]"/>, and the KD tree
query can be very efficient.
For larger <img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/>, the cost increases to nearly <img class="math" src="../_images/math/83c93e5fdd9103cdb2a328c3dff63ebddd0fe67f.png" alt="O[DN]"/>, and
the overhead due to the tree
structure can lead to queries which are slower than brute force.</li>
</ul>
<p>For small data sets (<img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/> less than 30 or so), <img class="math" src="../_images/math/53be6b6975d2852e3f88ae15c7d0140ed7f19a51.png" alt="\log(N)"/> is
comparable to <img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/>, and brute force algorithms can be more efficient
than a tree-based approach. Both <a class="reference internal" href="generated/sklearn.neighbors.KDTree.html#sklearn.neighbors.KDTree" title="sklearn.neighbors.KDTree"><code class="xref py py-class docutils literal"><span class="pre">KDTree</span></code></a> and <a class="reference internal" href="generated/sklearn.neighbors.BallTree.html#sklearn.neighbors.BallTree" title="sklearn.neighbors.BallTree"><code class="xref py py-class docutils literal"><span class="pre">BallTree</span></code></a>
address this through providing a <em>leaf size</em> parameter: this controls the
number of samples at which a query switches to brute-force. This allows both
algorithms to approach the efficiency of a brute-force computation for small
<img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/>.</p>
</li>
<li><p class="first">data structure: <em>intrinsic dimensionality</em> of the data and/or <em>sparsity</em>
of the data. Intrinsic dimensionality refers to the dimension
<img class="math" src="../_images/math/f8f4f6c615ba73bfa56ee8121d9e72708924c579.png" alt="d \le D"/> of a manifold on which the data lies, which can be linearly
or non-linearly embedded in the parameter space. Sparsity refers to the
degree to which the data fills the parameter space (this is to be
distinguished from the concept as used in “sparse” matrices. The data
matrix may have no zero entries, but the <strong>structure</strong> can still be
“sparse” in this sense).</p>
<ul class="simple">
<li><em>Brute force</em> query time is unchanged by data structure.</li>
<li><em>Ball tree</em> and <em>KD tree</em> query times can be greatly influenced
by data structure. In general, sparser data with a smaller intrinsic
dimensionality leads to faster query times. Because the KD tree
internal representation is aligned with the parameter axes, it will not
generally show as much improvement as ball tree for arbitrarily
structured data.</li>
</ul>
<p>Datasets used in machine learning tend to be very structured, and are
very well-suited for tree-based queries.</p>
</li>
<li><p class="first">number of neighbors <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> requested for a query point.</p>
<ul class="simple">
<li><em>Brute force</em> query time is largely unaffected by the value of <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/></li>
<li><em>Ball tree</em> and <em>KD tree</em> query time will become slower as <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/>
increases. This is due to two effects: first, a larger <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> leads
to the necessity to search a larger portion of the parameter space.
Second, using <img class="math" src="../_images/math/53ebe25152a7cee2742919c6996cef9c6003a412.png" alt="k > 1"/> requires internal queueing of results
as the tree is traversed.</li>
</ul>
<p>As <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> becomes large compared to <img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/>, the ability to prune
branches in a tree-based query is reduced. In this situation, Brute force
queries can be more efficient.</p>
</li>
<li><p class="first">number of query points. Both the ball tree and the KD Tree
require a construction phase. The cost of this construction becomes
negligible when amortized over many queries. If only a small number of
queries will be performed, however, the construction can make up
a significant fraction of the total cost. If very few query points
will be required, brute force is better than a tree-based method.</p>
</li>
</ul>
<p>Currently, <code class="docutils literal"><span class="pre">algorithm</span> <span class="pre">=</span> <span class="pre">'auto'</span></code> selects <code class="docutils literal"><span class="pre">'kd_tree'</span></code> if <img class="math" src="../_images/math/106c968682dbc48c2e3c7b95f4c5e5928ad93589.png" alt="k < N/2"/>
and the <code class="docutils literal"><span class="pre">'effective_metric_'</span></code> is in the <code class="docutils literal"><span class="pre">'VALID_METRICS'</span></code> list of
<code class="docutils literal"><span class="pre">'kd_tree'</span></code>. It selects <code class="docutils literal"><span class="pre">'ball_tree'</span></code> if <img class="math" src="../_images/math/106c968682dbc48c2e3c7b95f4c5e5928ad93589.png" alt="k < N/2"/> and the
<code class="docutils literal"><span class="pre">'effective_metric_'</span></code> is not in the <code class="docutils literal"><span class="pre">'VALID_METRICS'</span></code> list of
<code class="docutils literal"><span class="pre">'kd_tree'</span></code>. It selects <code class="docutils literal"><span class="pre">'brute'</span></code> if <img class="math" src="../_images/math/94984f2ef131beb5a8c81144728e6858fd19573b.png" alt="k >= N/2"/>. This choice is based on the assumption that the number of query points is at least the
same order as the number of training points, and that <code class="docutils literal"><span class="pre">leaf_size</span></code> is
close to its default value of <code class="docutils literal"><span class="pre">30</span></code>.</p>
</div>
<div class="section" id="effect-of-leaf-size">
<h3>1.6.4.5. Effect of <code class="docutils literal"><span class="pre">leaf_size</span></code><a class="headerlink" href="#effect-of-leaf-size" title="Permalink to this headline">¶</a></h3>
<p>As noted above, for small sample sizes a brute force search can be more
efficient than a tree-based query. This fact is accounted for in the ball
tree and KD tree by internally switching to brute force searches within
leaf nodes. The level of this switch can be specified with the parameter
<code class="docutils literal"><span class="pre">leaf_size</span></code>. This parameter choice has many effects:</p>
<dl class="docutils">
<dt><strong>construction time</strong></dt>
<dd>A larger <code class="docutils literal"><span class="pre">leaf_size</span></code> leads to a faster tree construction time, because
fewer nodes need to be created</dd>
<dt><strong>query time</strong></dt>
<dd>Both a large or small <code class="docutils literal"><span class="pre">leaf_size</span></code> can lead to suboptimal query cost.
For <code class="docutils literal"><span class="pre">leaf_size</span></code> approaching 1, the overhead involved in traversing
nodes can significantly slow query times. For <code class="docutils literal"><span class="pre">leaf_size</span></code> approaching
the size of the training set, queries become essentially brute force.
A good compromise between these is <code class="docutils literal"><span class="pre">leaf_size</span> <span class="pre">=</span> <span class="pre">30</span></code>, the default value
of the parameter.</dd>
<dt><strong>memory</strong></dt>
<dd>As <code class="docutils literal"><span class="pre">leaf_size</span></code> increases, the memory required to store a tree structure
decreases. This is especially important in the case of ball tree, which
stores a <img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/>-dimensional centroid for each node. The required
storage space for <a class="reference internal" href="generated/sklearn.neighbors.BallTree.html#sklearn.neighbors.BallTree" title="sklearn.neighbors.BallTree"><code class="xref py py-class docutils literal"><span class="pre">BallTree</span></code></a> is approximately <code class="docutils literal"><span class="pre">1</span> <span class="pre">/</span> <span class="pre">leaf_size</span></code> times
the size of the training set.</dd>
</dl>
<p><code class="docutils literal"><span class="pre">leaf_size</span></code> is not referenced for brute force queries.</p>
</div>
</div>
<div class="section" id="nearest-centroid-classifier">
<span id="id3"></span><h2>1.6.5. Nearest Centroid Classifier<a class="headerlink" href="#nearest-centroid-classifier" title="Permalink to this headline">¶</a></h2>
<p>The <a class="reference internal" href="generated/sklearn.neighbors.NearestCentroid.html#sklearn.neighbors.NearestCentroid" title="sklearn.neighbors.NearestCentroid"><code class="xref py py-class docutils literal"><span class="pre">NearestCentroid</span></code></a> classifier is a simple algorithm that represents
each class by the centroid of its members. In effect, this makes it
similar to the label updating phase of the <code class="xref py py-class docutils literal"><span class="pre">sklearn.KMeans</span></code> algorithm.
It also has no parameters to choose, making it a good baseline classifier. It
does, however, suffer on non-convex classes, as well as when classes have
drastically different variances, as equal variance in all dimensions is
assumed. See Linear Discriminant Analysis (<a class="reference internal" href="generated/sklearn.discriminant_analysis.LinearDiscriminantAnalysis.html#sklearn.discriminant_analysis.LinearDiscriminantAnalysis" title="sklearn.discriminant_analysis.LinearDiscriminantAnalysis"><code class="xref py py-class docutils literal"><span class="pre">sklearn.discriminant_analysis.LinearDiscriminantAnalysis</span></code></a>)
and Quadratic Discriminant Analysis (<a class="reference internal" href="generated/sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis.html#sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis" title="sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis"><code class="xref py py-class docutils literal"><span class="pre">sklearn.discriminant_analysis.QuadraticDiscriminantAnalysis</span></code></a>)
for more complex methods that do not make this assumption. Usage of the default
<a class="reference internal" href="generated/sklearn.neighbors.NearestCentroid.html#sklearn.neighbors.NearestCentroid" title="sklearn.neighbors.NearestCentroid"><code class="xref py py-class docutils literal"><span class="pre">NearestCentroid</span></code></a> is simple:</p>
<div class="highlight-python"><div class="highlight"><pre><span class="gp">>>> </span><span class="kn">from</span> <span class="nn">sklearn.neighbors.nearest_centroid</span> <span class="kn">import</span> <span class="n">NearestCentroid</span>
<span class="gp">>>> </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="kn">as</span> <span class="nn">np</span>
<span class="gp">>>> </span><span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">2</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">]])</span>
<span class="gp">>>> </span><span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>
<span class="gp">>>> </span><span class="n">clf</span> <span class="o">=</span> <span class="n">NearestCentroid</span><span class="p">()</span>
<span class="gp">>>> </span><span class="n">clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="go">NearestCentroid(metric='euclidean', shrink_threshold=None)</span>
<span class="gp">>>> </span><span class="k">print</span><span class="p">(</span><span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">([[</span><span class="o">-</span><span class="mf">0.8</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">]]))</span>
<span class="go">[1]</span>
</pre></div>
</div>
<div class="section" id="nearest-shrunken-centroid">
<h3>1.6.5.1. Nearest Shrunken Centroid<a class="headerlink" href="#nearest-shrunken-centroid" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.neighbors.NearestCentroid.html#sklearn.neighbors.NearestCentroid" title="sklearn.neighbors.NearestCentroid"><code class="xref py py-class docutils literal"><span class="pre">NearestCentroid</span></code></a> classifier has a <code class="docutils literal"><span class="pre">shrink_threshold</span></code> parameter,
which implements the nearest shrunken centroid classifier. In effect, the value
of each feature for each centroid is divided by the within-class variance of
that feature. The feature values are then reduced by <code class="docutils literal"><span class="pre">shrink_threshold</span></code>. Most
notably, if a particular feature value crosses zero, it is set
to zero. In effect, this removes the feature from affecting the classification.
This is useful, for example, for removing noisy features.</p>
<p>In the example below, using a small shrink threshold increases the accuracy of
the model from 0.81 to 0.82.</p>
<p class="centered">
<strong><a class="reference external" href="../auto_examples/neighbors/plot_classification.html"><img alt="nearest_centroid_1" src="../_images/plot_nearest_centroid_0011.png" style="width: 400.0px; height: 300.0px;" /></a> <a class="reference external" href="../auto_examples/neighbors/plot_classification.html"><img alt="nearest_centroid_2" src="../_images/plot_nearest_centroid_0021.png" style="width: 400.0px; height: 300.0px;" /></a></strong></p><div class="topic">
<p class="topic-title first">Examples:</p>
<ul class="simple">
<li><a class="reference internal" href="../auto_examples/neighbors/plot_nearest_centroid.html#example-neighbors-plot-nearest-centroid-py"><span>Nearest Centroid Classification</span></a>: an example of
classification using nearest centroid with different shrink thresholds.</li>
</ul>
</div>
</div>
</div>
<div class="section" id="approximate-nearest-neighbors">
<span id="id4"></span><h2>1.6.6. Approximate Nearest Neighbors<a class="headerlink" href="#approximate-nearest-neighbors" title="Permalink to this headline">¶</a></h2>
<p>There are many efficient exact nearest neighbor search algorithms for low
dimensions <img class="math" src="../_images/math/425d86ba2f2979d75b7535c2bcf92c33ed6b285a.png" alt="d"/> (approximately 50). However these algorithms perform poorly
with respect to space and query time when <img class="math" src="../_images/math/425d86ba2f2979d75b7535c2bcf92c33ed6b285a.png" alt="d"/> increases. These algorithms
are not any better than comparing query point to each point from the database in
a high dimension (see <a class="reference internal" href="#brute-force"><span>Brute Force</span></a>). This is a well-known consequence of
the phenomenon called “The Curse of Dimensionality”.</p>
<p>There are certain applications where we do not need the exact nearest neighbors
but having a “good guess” would suffice. When answers do not have to be exact,
the <a class="reference internal" href="generated/sklearn.neighbors.LSHForest.html#sklearn.neighbors.LSHForest" title="sklearn.neighbors.LSHForest"><code class="xref py py-class docutils literal"><span class="pre">LSHForest</span></code></a> class implements an approximate nearest neighbor search.
Approximate nearest neighbor search methods have been designed to try to speedup
query time with high dimensional data. These techniques are useful when the aim
is to characterize the neighborhood rather than identifying the exact neighbors
themselves (eg: k-nearest neighbors classification and regression). Some of the
most popular approximate nearest neighbor search techniques are locality
sensitive hashing, best bin fit and balanced box-decomposition tree based
search.</p>
<div class="section" id="locality-sensitive-hashing-forest">
<h3>1.6.6.1. Locality Sensitive Hashing Forest<a class="headerlink" href="#locality-sensitive-hashing-forest" title="Permalink to this headline">¶</a></h3>
<p>The vanilla implementation of locality sensitive hashing has a hyper-parameter
that is hard to tune in practice, therefore scikit-learn implements a variant
called <a class="reference internal" href="generated/sklearn.neighbors.LSHForest.html#sklearn.neighbors.LSHForest" title="sklearn.neighbors.LSHForest"><code class="xref py py-class docutils literal"><span class="pre">LSHForest</span></code></a> that has more reasonable hyperparameters.
Both methods use internally random hyperplanes to index the samples into
buckets and actual cosine similarities are only computed for samples that
collide with the query hence achieving sublinear scaling.
(see <a class="reference internal" href="#mathematical-description-of-lsh"><span>Mathematical description of Locality Sensitive
Hashing</span></a>).</p>
<p><a class="reference internal" href="generated/sklearn.neighbors.LSHForest.html#sklearn.neighbors.LSHForest" title="sklearn.neighbors.LSHForest"><code class="xref py py-class docutils literal"><span class="pre">LSHForest</span></code></a> has two main hyper-parameters: <code class="docutils literal"><span class="pre">n_estimators</span></code> and
<code class="docutils literal"><span class="pre">n_candidates</span></code>. The accuracy of queries can be controlled using these
parameters as demonstrated in the following plots:</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/neighbors/plot_approximate_nearest_neighbors_hyperparameters.html"><img alt="../_images/plot_approximate_nearest_neighbors_hyperparameters_0011.png" src="../_images/plot_approximate_nearest_neighbors_hyperparameters_0011.png" style="width: 400.0px; height: 300.0px;" /></a>
</div>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/neighbors/plot_approximate_nearest_neighbors_hyperparameters.html"><img alt="../_images/plot_approximate_nearest_neighbors_hyperparameters_0021.png" src="../_images/plot_approximate_nearest_neighbors_hyperparameters_0021.png" style="width: 400.0px; height: 300.0px;" /></a>
</div>
<p>As a rule of thumb, a user can set <code class="docutils literal"><span class="pre">n_estimators</span></code> to a large enough value
(e.g. between 10 and 50) and then adjust <code class="docutils literal"><span class="pre">n_candidates</span></code> to trade off accuracy
for query time.</p>
<p>For small data sets, the brute force method for exact nearest neighbor search
can be faster than LSH Forest. However LSH Forest has a sub-linear query time
scalability with the index size. The exact break even point where LSH Forest
queries become faster than brute force depends on the dimensionality, structure
of the dataset, required level of precision, characteristics of the runtime
environment such as availability of BLAS optimizations, number of CPU cores and
size of the CPU caches. Following graphs depict scalability of LSHForest queries
with index size.</p>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/neighbors/plot_approximate_nearest_neighbors_scalability.html"><img alt="../_images/plot_approximate_nearest_neighbors_scalability_0011.png" src="../_images/plot_approximate_nearest_neighbors_scalability_0011.png" style="width: 400.0px; height: 300.0px;" /></a>
</div>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/neighbors/plot_approximate_nearest_neighbors_scalability.html"><img alt="../_images/plot_approximate_nearest_neighbors_scalability_0021.png" src="../_images/plot_approximate_nearest_neighbors_scalability_0021.png" style="width: 400.0px; height: 300.0px;" /></a>
</div>
<div class="figure align-center">
<a class="reference external image-reference" href="../auto_examples/neighbors/plot_approximate_nearest_neighbors_scalability.html"><img alt="../_images/plot_approximate_nearest_neighbors_scalability_0031.png" src="../_images/plot_approximate_nearest_neighbors_scalability_0031.png" style="width: 400.0px; height: 300.0px;" /></a>
</div>
<p>For fixed <a class="reference internal" href="generated/sklearn.neighbors.LSHForest.html#sklearn.neighbors.LSHForest" title="sklearn.neighbors.LSHForest"><code class="xref py py-class docutils literal"><span class="pre">LSHForest</span></code></a> parameters, the accuracy of queries tends to slowly
decrease with larger datasets. The error bars on the previous plots represent
standard deviation across different queries.</p>
<div class="topic">
<p class="topic-title first">Examples:</p>
<ul class="simple">
<li><a class="reference internal" href="../auto_examples/neighbors/plot_approximate_nearest_neighbors_hyperparameters.html#example-neighbors-plot-approximate-nearest-neighbors-hyperparameters-py"><span>Hyper-parameters of Approximate Nearest Neighbors</span></a>: an example of
the behavior of hyperparameters of approximate nearest neighbor search using LSH Forest.</li>
<li><a class="reference internal" href="../auto_examples/neighbors/plot_approximate_nearest_neighbors_scalability.html#example-neighbors-plot-approximate-nearest-neighbors-scalability-py"><span>Scalability of Approximate Nearest Neighbors</span></a>: an example of
scalability of approximate nearest neighbor search using LSH Forest.</li>
</ul>
</div>
</div>
<div class="section" id="mathematical-description-of-locality-sensitive-hashing">
<span id="mathematical-description-of-lsh"></span><h3>1.6.6.2. Mathematical description of Locality Sensitive Hashing<a class="headerlink" href="#mathematical-description-of-locality-sensitive-hashing" title="Permalink to this headline">¶</a></h3>
<p>Locality sensitive hashing (LSH) techniques have been used in many areas where
nearest neighbor search is performed in high dimensions. The main concept
behind LSH is to hash each data point in the database using multiple
(often simple) hash functions to form a digest (also called a <em>hash</em>). At this
point the probability of collision - where two objects have similar digests
- is much higher for the points which are close to each other than that of the
distant points. We describe the requirements for a hash function family to be
locality sensitive as follows.</p>
<p>A family <img class="math" src="../_images/math/48f75e0b463f2fdb7fca9566cffdc439cae6967f.png" alt="H"/> of functions from a domain <img class="math" src="../_images/math/11a85f3c69ae6702cb1d99d3de451913b8f84c04.png" alt="S"/> to a range <img class="math" src="../_images/math/73f5e249c88b2b3068263480f576b051cb5c4f6e.png" alt="U"/>
is called <img class="math" src="../_images/math/3d1267ec4c47329a50e586bb0a7cd5fb528a117f.png" alt="(r, e , p1 , p2 )"/>-sensitive, with <img class="math" src="../_images/math/0c23542ac8b511252ba44a5ce8d0767b977699a8.png" alt="r, e > 0"/>,
<img class="math" src="../_images/math/2283f7ea4e07f0f115b4aaa6a817ab1810efdbea.png" alt="p_1 > p_2 > 0"/>, if for any <img class="math" src="../_images/math/f5edd8e9205dfc62dfbaa2de49e1b86a2e076038.png" alt="p, q \in S"/>, the following conditions
hold (<img class="math" src="../_images/math/9dfa31437b58c0473299320aa638151cd88cf61b.png" alt="D"/> is the distance function):</p>
<ul class="simple">
<li>If <img class="math" src="../_images/math/c923f188c5cc014df88dc9de60d6bb69fb68ce16.png" alt="D(p,q) <= r"/> then <img class="math" src="../_images/math/12a7743572f61fae6a99a07bc1117776e819981d.png" alt="P_H[h(p) = h(q)] >= p_1"/>,</li>
<li>If <img class="math" src="../_images/math/f683a5dcec20d2e17aea5cbe6836d64e05107cb2.png" alt="D(p,q) > r(1 + e)"/> then <img class="math" src="../_images/math/552aaf9d5e8d55bbfb58a9cb38448b0e47b15ece.png" alt="P_H[h(p) = h(q)] <= p_2"/>.</li>
</ul>
<p>As defined, nearby points within a distance of <img class="math" src="../_images/math/2ede365ad144ab396916ec60458da03860803078.png" alt="r"/> to each other are
likely to collide with probability <img class="math" src="../_images/math/35aa292c643aceacdb42eb4eff352a2f5c58a710.png" alt="p_1"/>. In contrast, distant points
which are located with the distance more than <img class="math" src="../_images/math/37f1b269585e35f9d313e5218393fd9196cfc081.png" alt="r(1 + e)"/> have a small
probability of <img class="math" src="../_images/math/40cdb0e5fc0c7cf6e3d3248c29f7de379c0f6abb.png" alt="p_2"/> of collision. Suppose there is a family of LSH
function <img class="math" src="../_images/math/48f75e0b463f2fdb7fca9566cffdc439cae6967f.png" alt="H"/>. An <em>LSH index</em> is built as follows:</p>
<ol class="arabic simple">
<li>Choose <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> functions <img class="math" src="../_images/math/5f8624741cd6a4e53c43692c6e4bbd56bd5b561b.png" alt="h_1, h_2, … h_k"/> uniformly at
random (with replacement) from <img class="math" src="../_images/math/48f75e0b463f2fdb7fca9566cffdc439cae6967f.png" alt="H"/>. For any <img class="math" src="../_images/math/56868f91fc4163062ac1c061410c88eb585228de.png" alt="p \in S"/>, place
<img class="math" src="../_images/math/3eca8557203e86160952e1c0f735f7417f3285b1.png" alt="p"/> in the bucket with label
<img class="math" src="../_images/math/fcd6f632794fa15a93253fc5edddb129eae8d43e.png" alt="g(p) = (h_1(p), h_2(p), … h_k(p))"/>. Observe that if
each <img class="math" src="../_images/math/672b8ad37aa193efaa4a26c04891131142ad96fc.png" alt="h_i"/> outputs one “digit”, each bucket has a k-digit label.</li>
<li>Independently perform step 1 <img class="math" src="../_images/math/d99efde75c84ef402a92cd4497530d2366fb112f.png" alt="l"/> times to construct <img class="math" src="../_images/math/d99efde75c84ef402a92cd4497530d2366fb112f.png" alt="l"/>
separate estimators, with hash functions <img class="math" src="../_images/math/87eb940f6dc61884b47853459bf0609474e91dba.png" alt="g_1, g_2, … g_l"/>.</li>
</ol>
<p>The reason to concatenate hash functions in the step 1 is to decrease the
probability of the collision of distant points as much as possible. The
probability drops from <img class="math" src="../_images/math/40cdb0e5fc0c7cf6e3d3248c29f7de379c0f6abb.png" alt="p_2"/> to <img class="math" src="../_images/math/eee75f0de1ffa68af07e423cdc89f87dd4a0c32a.png" alt="p_2^k"/> which is negligibly
small for large <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/>. The choice of <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> is strongly dependent on
the data set size and structure and is therefore hard to tune in practice.
There is a side effect of having a large <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/>; it has the potential of
decreasing the chance of nearby points getting collided. To address this
issue, multiple estimators are constructed in step 2.</p>
<p>The requirement to tune <img class="math" src="../_images/math/e9203da50e1059455123460d4e716c9c7f440cc3.png" alt="k"/> for a given dataset makes classical LSH
cumbersome to use in practice. The LSH Forest variant has benn designed to
alleviate this requirement by automatically adjusting the number of digits
used to hash the samples.</p>
<p>LSH Forest is formulated with prefix trees with each leaf of
a tree corresponding to an actual data point in the database. There are
<img class="math" src="../_images/math/d99efde75c84ef402a92cd4497530d2366fb112f.png" alt="l"/> such trees which compose the forest and they are constructed using
independently drawn random sequence of hash functions from <img class="math" src="../_images/math/48f75e0b463f2fdb7fca9566cffdc439cae6967f.png" alt="H"/>. In this
implementation, “Random Projections” is being used as the LSH technique which
is an approximation for the cosine distance. The length of the sequence of
hash functions is kept fixed at 32. Moreover, a prefix tree is implemented
using sorted arrays and binary search.</p>
<p>There are two phases of tree traversals used in order to answer a query to find
the <img class="math" src="../_images/math/c4bb40dd65eae6c11b325989b14e0b8d35e4e3ef.png" alt="m"/> nearest neighbors of a point <img class="math" src="../_images/math/23f1b45408e5b4130c0f940fcbfcec54492cbdcd.png" alt="q"/>. First, a top-down
traversal is performed using a binary search to identify the leaf having the
longest prefix match (maximum depth) with <img class="math" src="../_images/math/23f1b45408e5b4130c0f940fcbfcec54492cbdcd.png" alt="q"/>‘s label after subjecting
<img class="math" src="../_images/math/23f1b45408e5b4130c0f940fcbfcec54492cbdcd.png" alt="q"/> to the same hash functions. <img class="math" src="../_images/math/79f68af53194633b5cdca0c04d0b3779a5751a90.png" alt="M >> m"/> points (total candidates)
are extracted from the forest, moving up from the previously found maximum
depth towards the root synchronously across all trees in the bottom-up
traversal. <cite>M</cite> is set to <img class="math" src="../_images/math/63a6f23c6e8455055f6590d49baed24be96c3f2b.png" alt="cl"/> where <img class="math" src="../_images/math/65868d23a5bfe5b3b2d819386b19c14fa36af134.png" alt="c"/>, the number of candidates
extracted from each tree, is a constant. Finally, the similarity of each of
these <img class="math" src="../_images/math/2de38926392ba23abeac75effc66c37a5f5be5fd.png" alt="M"/> points against point <img class="math" src="../_images/math/23f1b45408e5b4130c0f940fcbfcec54492cbdcd.png" alt="q"/> is calculated and the top
<img class="math" src="../_images/math/c4bb40dd65eae6c11b325989b14e0b8d35e4e3ef.png" alt="m"/> points are returned as the nearest neighbors of <img class="math" src="../_images/math/23f1b45408e5b4130c0f940fcbfcec54492cbdcd.png" alt="q"/>. Since
most of the time in these queries is spent calculating the distances to
candidates, the speedup compared to brute force search is approximately
<img class="math" src="../_images/math/4ae960fb7de7cf651ceddc646dee4f5dcae8f897.png" alt="N/M"/>, where <img class="math" src="../_images/math/75e27f04188974063be3230dca208cd495b77ce1.png" alt="N"/> is the number of points in database.</p>
<div class="topic">
<p class="topic-title first">References:</p>
<ul class="simple">
<li><a class="reference external" href="http://web.mit.edu/andoni/www/papers/cSquared.pdf">“Near-Optimal Hashing Algorithms for Approximate Nearest Neighbor in
High Dimensions”</a>,
Alexandr, A., Indyk, P., Foundations of Computer Science, 2006. FOCS
‘06. 47th Annual IEEE Symposium</li>
<li><a class="reference external" href="http://www2005.org/docs/p651.pdf">“LSH Forest: Self-Tuning Indexes for Similarity Search”</a>,
Bawa, M., Condie, T., Ganesan, P., WWW ‘05 Proceedings of the 14th
international conference on World Wide Web Pages 651-660</li>
</ul>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
<div class="clearer"></div>
</div>
</div>
<div class="footer">
© 2010 - 2014, scikit-learn developers (BSD License).
<a href="../_sources/modules/neighbors.txt" rel="nofollow">Show this page source</a>
</div>
<div class="rel">
<div class="buttonPrevious">
<a href="sgd.html">Previous
</a>
</div>
</div>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-22606712-2']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
<script src="http://www.google.com/jsapi" type="text/javascript"></script>
<script type="text/javascript"> google.load('search', '1',
{language : 'en'}); google.setOnLoadCallback(function() {
var customSearchControl = new
google.search.CustomSearchControl('016639176250731907682:tjtqbvtvij0');
customSearchControl.setResultSetSize(google.search.Search.FILTERED_CSE_RESULTSET);
var options = new google.search.DrawOptions();
options.setAutoComplete(true);
customSearchControl.draw('cse', options); }, true);
</script>
<script src="https://scikit-learn.org/versionwarning.js"></script>
</body>
</html>