-
Notifications
You must be signed in to change notification settings - Fork 81
/
Copy pathdp-derivation.html
585 lines (545 loc) · 33.9 KB
/
dp-derivation.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>2.1.3.2.1. Variational Gaussian Mixture Models — scikit-learn 0.17.1 documentation</title>
<!-- htmltitle is before nature.css - we use this hack to load bootstrap first -->
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="stylesheet" href="../_static/css/bootstrap.min.css" media="screen" />
<link rel="stylesheet" href="../_static/css/bootstrap-responsive.css"/>
<link rel="stylesheet" href="../_static/nature.css" type="text/css" />
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
<script type="text/javascript">
var DOCUMENTATION_OPTIONS = {
URL_ROOT: '../',
VERSION: '0.17.1',
COLLAPSE_INDEX: false,
FILE_SUFFIX: '.html',
HAS_SOURCE: true
};
</script>
<script type="text/javascript" src="../_static/jquery.js"></script>
<script type="text/javascript" src="../_static/underscore.js"></script>
<script type="text/javascript" src="../_static/doctools.js"></script>
<script type="text/javascript" src="../_static/js/copybutton.js"></script>
<link rel="shortcut icon" href="../_static/favicon.ico"/>
<link rel="author" title="About these documents" href="../about.html" />
<link rel="top" title="scikit-learn 0.17.1 documentation" href="../index.html" />
<link rel="up" title="2.1. Gaussian mixture models" href="mixture.html" />
<link rel="next" title="2.2. Manifold learning" href="manifold.html" />
<link rel="prev" title="2.1. Gaussian mixture models" href="mixture.html" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<script src="../_static/js/bootstrap.min.js" type="text/javascript"></script>
<link rel="canonical" href="https://scikit-learn.org/stable/modules/dp-derivation.html" />
<script type="text/javascript">
$("div.buttonNext, div.buttonPrevious").hover(
function () {
$(this).css('background-color', '#FF9C34');
},
function () {
$(this).css('background-color', '#A7D6E2');
}
);
</script>
</head>
<body role="document">
<div class="header-wrapper">
<div class="header">
<p class="logo"><a href="../index.html">
<img src="../_static/scikit-learn-logo-small.png" alt="Logo"/>
</a>
</p><div class="navbar">
<ul>
<li><a href="../index.html">Home</a></li>
<li><a href="../install.html">Installation</a></li>
<li class="btn-li"><div class="btn-group">
<a href="../documentation.html">Documentation</a>
<a class="btn dropdown-toggle" data-toggle="dropdown">
<span class="caret"></span>
</a>
<ul class="dropdown-menu">
<li class="link-title">Scikit-learn 0.17 (stable)</li>
<li><a href="../tutorial/index.html">Tutorials</a></li>
<li><a href="../user_guide.html">User guide</a></li>
<li><a href="classes.html">API</a></li>
<li><a href="../faq.html">FAQ</a></li>
<li><a href="../developers/contributing.html">Contributing</a></li>
<li class="divider"></li>
<li><a href="http://scikit-learn.org/dev/documentation.html">Scikit-learn 0.18 (development)</a></li>
<li><a href="http://scikit-learn.org/0.16/documentation.html">Scikit-learn 0.16</a></li>
<li><a href="../_downloads/scikit-learn-docs.pdf">PDF documentation</a></li>
</ul>
</div>
</li>
<li><a href="../auto_examples/index.html">Examples</a></li>
</ul>
<div class="search_form">
<div id="cse" style="width: 100%;"></div>
</div>
</div> <!-- end navbar --></div>
</div>
<!-- Github "fork me" ribbon -->
<a href="https://github.com/scikit-learn/scikit-learn">
<img class="fork-me"
style="position: absolute; top: 0; right: 0; border: 0;"
src="../_static/img/forkme.png"
alt="Fork me on GitHub" />
</a>
<div class="content-wrapper">
<div class="sphinxsidebar">
<div class="sphinxsidebarwrapper">
<div class="rel">
<!-- rellinks[1:] is an ugly hack to avoid link to module
index -->
<div class="rellink">
<a href="mixture.html"
accesskey="P">Previous
<br/>
<span class="smallrellink">
2.1. Gaussian...
</span>
<span class="hiddenrellink">
2.1. Gaussian mixture models
</span>
</a>
</div>
<!-- Ad a link to the 'up' page -->
<div class="spacer">
</div>
<div class="rellink">
<a href="mixture.html">
Up
<br/>
<span class="smallrellink">
2.1. Gaussian...
</span>
<span class="hiddenrellink">
2.1. Gaussian mixture models
</span>
</a>
</div>
</div>
<p class="doc-version">This documentation is for scikit-learn <strong>version 0.17.1</strong> — <a href="http://scikit-learn.org/stable/support.html#documentation-resources">Other versions</a></p>
<p class="citing">If you use the software, please consider <a href="../about.html#citing-scikit-learn">citing scikit-learn</a>.</p>
<ul>
<li><a class="reference internal" href="#">2.1.3.2.1. Variational Gaussian Mixture Models</a></li>
<li><a class="reference internal" href="#update-rules-for-vb-inference">2.1.3.2.2. Update rules for VB inference</a><ul>
<li><a class="reference internal" href="#the-spherical-model">2.1.3.2.2.1. The spherical model</a><ul>
<li><a class="reference internal" href="#the-bound">2.1.3.2.2.1.1. The bound</a></li>
<li><a class="reference internal" href="#the-updates">2.1.3.2.2.1.2. The updates</a></li>
</ul>
</li>
<li><a class="reference internal" href="#the-diagonal-model">2.1.3.2.2.2. The diagonal model</a><ul>
<li><a class="reference internal" href="#the-lower-bound">2.1.3.2.2.2.1. The lower bound</a></li>
<li><a class="reference internal" href="#id1">2.1.3.2.2.2.2. The updates</a></li>
</ul>
</li>
<li><a class="reference internal" href="#the-tied-model">2.1.3.2.2.3. The tied model</a><ul>
<li><a class="reference internal" href="#id2">2.1.3.2.2.3.1. The lower bound</a></li>
<li><a class="reference internal" href="#id3">2.1.3.2.2.3.2. The updates</a></li>
</ul>
</li>
<li><a class="reference internal" href="#the-full-model">2.1.3.2.2.4. The full model</a><ul>
<li><a class="reference internal" href="#id4">2.1.3.2.2.4.1. The lower bound</a></li>
<li><a class="reference internal" href="#id5">2.1.3.2.2.4.2. The updates</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</div>
</div>
<input type="checkbox" id="nav-trigger" class="nav-trigger" checked />
<label for="nav-trigger"></label>
<div class="content">
<div class="documentwrapper">
<div class="bodywrapper">
<div class="body" role="main">
<div class="section" id="variational-gaussian-mixture-models">
<span id="dpgmm-derivation"></span><h1>2.1.3.2.1. Variational Gaussian Mixture Models<a class="headerlink" href="#variational-gaussian-mixture-models" title="Permalink to this headline">¶</a></h1>
<p>The API is identical to that of the <a class="reference internal" href="generated/sklearn.mixture.GMM.html#sklearn.mixture.GMM" title="sklearn.mixture.GMM"><code class="xref py py-class docutils literal"><span class="pre">GMM</span></code></a> class, the main
difference being that it offers access to precision matrices as well
as covariance matrices.</p>
<p>The inference algorithm is the one from the following paper:</p>
<blockquote>
<div><ul class="simple">
<li><a class="reference external" href="http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.61.4467&rep=rep1&type=pdf">Variational Inference for Dirichlet Process Mixtures</a>
David Blei, Michael Jordan. Bayesian Analysis, 2006</li>
</ul>
</div></blockquote>
<p>While this paper presents the parts of the inference algorithm that
are concerned with the structure of the dirichlet process, it does not
go into detail in the mixture modeling part, which can be just as
complex, or even more. For this reason we present here a full
derivation of the inference algorithm and all the update and
lower-bound equations. If you’re not interested in learning how to
derive similar algorithms yourself and you’re not interested in
changing/debugging the implementation in the scikit this document is
not for you.</p>
<p>The complexity of this implementation is linear in the number of
mixture components and data points. With regards to the
dimensionality, it is linear when using <code class="docutils literal"><span class="pre">spherical</span></code> or <code class="docutils literal"><span class="pre">diag</span></code> and
quadratic/cubic when using <code class="docutils literal"><span class="pre">tied</span></code> or <code class="docutils literal"><span class="pre">full</span></code>. For <code class="docutils literal"><span class="pre">spherical</span></code> or <code class="docutils literal"><span class="pre">diag</span></code>
it is O(n_states * n_points * dimension) and for <code class="docutils literal"><span class="pre">tied</span></code> or <code class="docutils literal"><span class="pre">full</span></code> it
is O(n_states * n_points * dimension^2 + n_states * dimension^3) (it
is necessary to invert the covariance/precision matrices and compute
its determinant, hence the cubic term).</p>
<p>This implementation is expected to scale at least as well as EM for
the mixture of Gaussians.</p>
</div>
<div class="section" id="update-rules-for-vb-inference">
<h1>2.1.3.2.2. Update rules for VB inference<a class="headerlink" href="#update-rules-for-vb-inference" title="Permalink to this headline">¶</a></h1>
<p>Here the full mathematical derivation of the Variational Bayes update
rules for Gaussian Mixture Models is given. The main parameters of the
model, defined for any class <img class="math" src="../_images/math/5d90819185ae56b9e0d1c08df4bd719b6aa06b03.png" alt="k \in [1..K]"/> are the class
proportion <img class="math" src="../_images/math/34ea3b8fe46ef695984759103d7bcdecd553f321.png" alt="\phi_k"/>, the mean parameters <img class="math" src="../_images/math/e0f4c204f26e9afa4f5bbeda60af5262e624c148.png" alt="\mu_k"/>, the
covariance parameters <img class="math" src="../_images/math/07b943c6dd1061bde470bebfa3e9b5e373216870.png" alt="\Sigma_k"/>, which is characterized by
variational Wishart density, <img class="math" src="../_images/math/24a5a6643f900bf129cdd7a03a6e161d1524ebfc.png" alt="Wishart(a_k, \mathbf{B_k})"/>, where
<img class="math" src="../_images/math/7dd2a5ea01fbd72ad2a58dd1f3d6ecbfde6208a1.png" alt="a"/> is the degrees of freedom, and <img class="math" src="../_images/math/83956e92fcc80dee17fce864543216939a3c9da7.png" alt="B"/> is the
scale matrix. Depending on the covariance parametrization,
<img class="math" src="../_images/math/1a826fd4e37c0ceccbb19369141c92a7c1743e04.png" alt="B_k"/> can be a positive scalar, a positive vector or a Symmetric
Positive Definite matrix.</p>
<div class="section" id="the-spherical-model">
<h2>2.1.3.2.2.1. The spherical model<a class="headerlink" href="#the-spherical-model" title="Permalink to this headline">¶</a></h2>
<p>The model then is</p>
<div class="math">
<p><img src="../_images/math/338694bf0978bd05178b8a7e238c866a14f3516f.png" alt="\begin{array}{rcl}
\phi_k &\sim& Beta(1, \alpha_1) \\
\mu_k &\sim& Normal(0, \mathbf{I}) \\
\sigma_k &\sim& Gamma(1, 1) \\
z_{i} &\sim& SBP(\phi) \\
X_t &\sim& Normal(\mu_{z_i}, \frac{1}{\sigma_{z_i}} \mathbf{I})
\end{array}"/></p>
</div><p>The variational distribution we’ll use is</p>
<div class="math">
<p><img src="../_images/math/6da4b9b5dc6b2670c9f936c72ee7ee803389523f.png" alt="\begin{array}{rcl}
\phi_k &\sim& Beta(\gamma_{k,1}, \gamma_{k,2}) \\
\mu_k &\sim& Normal(\nu_{\mu_k}, \mathbf{I}) \\
\sigma_k &\sim& Gamma(a_{k}, b_{k}) \\
z_{i} &\sim& Discrete(\nu_{z_i}) \\
\end{array}"/></p>
</div><div class="section" id="the-bound">
<h3>2.1.3.2.2.1.1. The bound<a class="headerlink" href="#the-bound" title="Permalink to this headline">¶</a></h3>
<p>The variational bound is</p>
<div class="math">
<p><img src="../_images/math/b654eee1dcbceb1e8500e575796fbfea6f5518bb.png" alt="\begin{array}{rcl}
\log P(X) &\ge&
\sum_k (E_q[\log P(\phi_k)] - E_q[\log Q(\phi_k)]) \\
&&
+\sum_k \left( E_q[\log P(\mu_k)] - E_q[\log Q(\mu_k)] \right) \\
&&
+\sum_k \left( E_q[\log P(\sigma_k)] - E_q[\log Q(\sigma_k)] \right) \\
&&
+\sum_i \left( E_q[\log P(z_i)] - E_q[\log Q(z_i)] \right) \\
&&
+\sum_i E_q[\log P(X_t)]
\end{array}"/></p>
</div><p><strong>The bound for</strong> <img class="math" src="../_images/math/34ea3b8fe46ef695984759103d7bcdecd553f321.png" alt="\phi_k"/></p>
<div class="math">
<p><img src="../_images/math/8a794bac91917159e451b8cfc65d3014cffcf45a.png" alt="\begin{array}{rcl}
E_q[\log Beta(1,\alpha)] - E[\log Beta(\gamma_{k,1},\gamma_{k,2})]
&=&
\log \Gamma(1+\alpha) - \log \Gamma(\alpha) \\ &&
+(\alpha-1)(\Psi(\gamma_{k,2})-\Psi(\gamma_{k,1}+\gamma_{k,2})) \\ &&
- \log \Gamma(\gamma_{k,1}+\gamma_{k,2}) + \log \Gamma(\gamma_{k,1}) +
\log \Gamma(\gamma_{k,2}) \\ &&
-
(\gamma_{k,1}-1)(\Psi(\gamma_{k,1})-\Psi(\gamma_{k,1}+\gamma_{k,2}))
\\ &&
-
(\gamma_{k,2}-1)(\Psi(\gamma_{k,2})-\Psi(\gamma_{k,1}+\gamma_{k,2}))
\end{array}"/></p>
</div><p><strong>The bound for</strong> <img class="math" src="../_images/math/e0f4c204f26e9afa4f5bbeda60af5262e624c148.png" alt="\mu_k"/></p>
<div class="math">
<p><img src="../_images/math/6249d7686a865f94d4173d4351dc919df639f900.png" alt="\begin{array}{rcl}
&& E_q[\log P(\mu_k)] - E_q[\log Q(\mu_k)] \\
&=&
\int\!d\mu_f q(\mu_f) \log P(\mu_f)
- \int\!d\mu_f q(\mu_f) \log Q(\mu_f) \\
&=&
- \frac{D}{2}\log 2\pi - \frac{1}{2} ||\nu_{\mu_k}||^2 - \frac{D}{2}
+ \frac{D}{2} \log 2\pi e
\end{array}"/></p>
</div><p><strong>The bound for</strong> <img class="math" src="../_images/math/088138d180d3b322617921d408597bf140875237.png" alt="\sigma_k"/></p>
<p>Here I’ll use the inverse scale parametrization of the gamma
distribution.</p>
<div class="math">
<p><img src="../_images/math/82c2a519749bba8b5cdf13fac6f28b6645b044ab.png" alt="\begin{array}{rcl}
&& E_q[\log P(\sigma_k)] - E_q [\log Q(\sigma_k)] \\ &=&
\log \Gamma (a_k) - (a_k-1)\Psi(a_k) -\log b_k + a_k - \frac{a_k}{b_k}
\end{array}"/></p>
</div><p><strong>The bound for z</strong></p>
<div class="math">
<p><img src="../_images/math/ea7883239d280b3403de364242bc5eaa7c043fa3.png" alt="\begin{array}{rcl}
&& E_q[\log P(z)] - E_q[\log Q(z)] \\
&=&
\sum_{k} \left(
\left(\sum_{j=k+1}^K \nu_{z_{i,j}}\right)(\Psi(\gamma_{k,2})-\Psi(\gamma_{k,1}+\gamma_{k,2}))
+ \nu_{z_{i,k}}(\Psi(\gamma_{k,1})-\Psi(\gamma_{k,1}+\gamma_{k,2}))
- \log \nu_{z_{i,k}} \right)
\end{array}"/></p>
</div><p><strong>The bound for</strong> <img class="math" src="../_images/math/f026aecf11ec7f6141ab863f260d395f94b10f51.png" alt="X"/></p>
<p>Recall that there is no need for a <img class="math" src="../_images/math/debe2ff09239bb9b81160d734bd2d97cf7058162.png" alt="Q(X)"/> so this bound is just</p>
<div class="math">
<p><img src="../_images/math/5ec40441786a0fb92ebe6de43edbc7d6f5b6bab2.png" alt="\begin{array}{rcl}
E_q[\log P(X_i)] &=& \sum_k \nu_{z_k} \left( - \frac{D}{2}\log 2\pi
+\frac{D}{2} (\Psi(a_k) - \log(b_k))
-\frac{a_k}{2b_k} (||X_i - \nu_{\mu_k}||^2+D) - \log 2 \pi e \right)
\end{array}"/></p>
</div><p>For simplicity I’ll later call the term inside the parenthesis <img class="math" src="../_images/math/e43d9cd0fe15ae82f739a60dd2cd5be2c65c1d99.png" alt="E_q[\log P(X_i|z_i=k)]"/></p>
</div>
<div class="section" id="the-updates">
<h3>2.1.3.2.2.1.2. The updates<a class="headerlink" href="#the-updates" title="Permalink to this headline">¶</a></h3>
<p><strong>Updating</strong> <img class="math" src="../_images/math/0ebb67342b546ca42a1c634b1ef03c893c4cdedb.png" alt="\gamma"/></p>
<div class="math">
<p><img src="../_images/math/eeed3159db3c4592236053169e26a6aa2cbca30d.png" alt="\begin{array}{rcl}
\gamma_{k,1} &=& 1+\sum_i \nu_{z_{i,k}} \\
\gamma_{k,2} &=& \alpha + \sum_i \sum_{j > k} \nu_{z_{i,j}}.
\end{array}"/></p>
</div><p><strong>Updating</strong> <img class="math" src="../_images/math/126e84ba38f7dece5f0ad64e929b9588b20f6440.png" alt="\mu"/></p>
<p>The updates for mu essentially are just weighted expectations of
<img class="math" src="../_images/math/f026aecf11ec7f6141ab863f260d395f94b10f51.png" alt="X"/> regularized by the prior. We can see this by taking the
gradient of the bound with regards to <img class="math" src="../_images/math/d7b9c9300675caab903407dce0551c0ff4e67663.png" alt="\nu_{\mu}"/> and setting it to zero.
The gradient is</p>
<div class="math">
<p><img src="../_images/math/24b02b1bfa811241ff9b1e8dbdbf425ef7851750.png" alt="\nabla L = -\nu_{\mu_k} + \sum_i \frac{\nu_{z_{i,k}}b_k}{a_k}(X_i + -\nu_{\mu})"/></p>
</div><p>so the update is</p>
<div class="math">
<p><img src="../_images/math/430e80b89227bed7145710b8999c77815de5ee99.png" alt="\nu_{\mu_k} = \frac{\sum_i \frac{\nu_{z_{i,k}}b_k}{a_k}X_i}{1+\sum_i \frac{\nu_{z_{i,k}}b_k}{a_k}}"/></p>
</div><p><strong>Updating</strong> <img class="math" src="../_images/math/7dd2a5ea01fbd72ad2a58dd1f3d6ecbfde6208a1.png" alt="a"/> <strong>and</strong> <img class="math" src="../_images/math/5e87bf41a96deddf6cb485ff530f153f2590e9cc.png" alt="b"/></p>
<p>For some odd reason it doesn’t really work when you derive the updates
for a and b using the gradients of the lower bound (terms involving the
<img class="math" src="../_images/math/df391fc64254ea60aabeb28d65a2f40966bc23f9.png" alt="\Psi'"/> function show up and <img class="math" src="../_images/math/7dd2a5ea01fbd72ad2a58dd1f3d6ecbfde6208a1.png" alt="a"/> is hard to isolate).
However, we can use the other formula,</p>
<div class="math">
<p><img src="../_images/math/5e0afb293a769e69c218750f2c321c3f81e12fba.png" alt="\log Q(\sigma_k) = E_{v \ne \sigma_k}[\log P] + const"/></p>
</div><p>All the terms not involving <img class="math" src="../_images/math/088138d180d3b322617921d408597bf140875237.png" alt="\sigma_k"/> get folded over into the
constant and we get two terms: the prior and the probability of
<img class="math" src="../_images/math/f026aecf11ec7f6141ab863f260d395f94b10f51.png" alt="X"/>. This gives us</p>
<div class="math">
<p><img src="../_images/math/7d6c7ac8c23d999a54a206610dc62438dc13fd4c.png" alt="\log Q(\sigma_k) = -\sigma_k + \frac{D}{2} \sum_i \nu_{z_{i,k}}\log \sigma_k - \frac{\sigma_k}{2}\sum_i \nu_{z_{i,k}} (||X_i-\mu_k||^2 + D)"/></p>
</div><p>This is the log of a gamma distribution, with <img class="math" src="../_images/math/62be32aea2502a24649da8d7cc427e3b6bb402a0.png" alt="a_k = 1+\frac{D}{2}\sum_i \nu_{z_{i,k}}"/> and</p>
<div class="math">
<p><img src="../_images/math/5b60001364caf7a350cd44edec70e2e3019dbc12.png" alt="b_k = 1 + \frac{1}{2}\sum_i \nu_{z_{i,k}} (||X_i-\mu_k||^2 + D)."/></p>
</div><p>You can verify this by normalizing the previous term.</p>
<p><strong>Updating</strong> <img class="math" src="../_images/math/84d7271dd9e78c1e05d6c3c6ecb60309ef7dfc73.png" alt="z"/></p>
<div class="math">
<p><img src="../_images/math/fd50a0c15b891077a575799eeb6d2dbcb4ed3cf2.png" alt="\log \nu_{z_{i,k}} \propto \Psi(\gamma_{k,1}) -
\Psi(\gamma_{k,1} + \gamma_{k,2}) + E_q[\log P(X_i|z_i=k)] +
\sum_{j < k} \left (\Psi(\gamma_{j,2}) -
\Psi(\gamma_{j,1}+\gamma_{j,2})\right)."/></p>
</div></div>
</div>
<div class="section" id="the-diagonal-model">
<h2>2.1.3.2.2.2. The diagonal model<a class="headerlink" href="#the-diagonal-model" title="Permalink to this headline">¶</a></h2>
<p>The model then is</p>
<div class="math">
<p><img src="../_images/math/93457a375ad774ed2e9a4d3e58e264e32f391f01.png" alt="\begin{array}{rcl}
\phi_k &\sim& Beta(1, \alpha_1) \\
\mu_k &\sim& Normal(0, \mathbf{I}) \\
\sigma_{k,d} &\sim& Gamma(1, 1) \\
z_{i} &\sim& SBP(\phi) \\
X_t &\sim& Normal(\mu_{z_i}, \bm{\sigma_{z_i}}^{-1})
\end{array}"/></p>
</div><p>Tha variational distribution we’ll use is</p>
<div class="math">
<p><img src="../_images/math/75ffb8a1ebecd4726c681ab02b0af9d3ce0f5388.png" alt="\begin{array}{rcl}
\phi_k &\sim& Beta(\gamma_{k,1}, \gamma_{k,2}) \\
\mu_k &\sim& Normal(\nu_{\mu_k}, \mathbf{I}) \\
\sigma_{k,d} &\sim& Gamma(a_{k,d}, b_{k,d}) \\
z_{i} &\sim& Discrete(\nu_{z_i}) \\
\end{array}"/></p>
</div><div class="section" id="the-lower-bound">
<h3>2.1.3.2.2.2.1. The lower bound<a class="headerlink" href="#the-lower-bound" title="Permalink to this headline">¶</a></h3>
<p>The changes in this lower bound from the previous model are in the
distributions of <img class="math" src="../_images/math/2298cf1485084afe72757a9c8483af49a138d81f.png" alt="\sigma"/> (as there are a lot more <img class="math" src="../_images/math/2298cf1485084afe72757a9c8483af49a138d81f.png" alt="\sigma"/> s
now) and <img class="math" src="../_images/math/f026aecf11ec7f6141ab863f260d395f94b10f51.png" alt="X"/>.</p>
<p>The bound for <img class="math" src="../_images/math/98960fec06a81f158a2b33b6351e2e87c58861f1.png" alt="\sigma_{k,d}"/> is the same bound for <img class="math" src="../_images/math/088138d180d3b322617921d408597bf140875237.png" alt="\sigma_k"/> and can
be safely omitted.</p>
<p><strong>The bound for</strong> <img class="math" src="../_images/math/f026aecf11ec7f6141ab863f260d395f94b10f51.png" alt="X"/> :</p>
<p>The main difference here is that the precision matrix <img class="math" src="../_images/math/9523e30d2c532ce5ce84b3c272f599e7c7b86953.png" alt="\bm{\sigma_k}"/>
scales the norm, so we have an extra term after computing the
expectation of <img class="math" src="../_images/math/22f4ab6468e7a706bc27b9e5b2769965b9e2991a.png" alt="\mu_k^T\bm{\sigma_k}\mu_k"/>, which is
<img class="math" src="../_images/math/2e6c422c84607919c1467f3f18f81ac44daf48f2.png" alt="\nu_{\mu_k}^T\bm{\sigma_k}\nu_{\mu_k} + \sum_d \sigma_{k,d}"/>. We then
have</p>
<div class="math">
<p><img src="../_images/math/27fb9b03759b81490d29723a983e444e58902355.png" alt="\begin{array}{rcl}
E_q[\log P(X_i)] &=& \sum_k \nu_{z_k} \Big( - \frac{D}{2}\log 2\pi
+\frac{1}{2}\sum_d (\Psi(a_{k,d}) - \log(b_{k,d})) \\
&&
-\frac{1}{2}((X_i - \nu_{\mu_k})^T\bm{\frac{a_k}{b_k}}(X_i - \nu_{\mu_k})+ \sum_d \sigma_{k,d})- \log 2 \pi e \Big)
\end{array}"/></p>
</div></div>
<div class="section" id="id1">
<h3>2.1.3.2.2.2.2. The updates<a class="headerlink" href="#id1" title="Permalink to this headline">¶</a></h3>
<p>The updates only chance for <img class="math" src="../_images/math/126e84ba38f7dece5f0ad64e929b9588b20f6440.png" alt="\mu"/> (to weight them with the new
<img class="math" src="../_images/math/2298cf1485084afe72757a9c8483af49a138d81f.png" alt="\sigma"/>), <img class="math" src="../_images/math/84d7271dd9e78c1e05d6c3c6ecb60309ef7dfc73.png" alt="z"/> (but the change is all folded into the
<img class="math" src="../_images/math/8c59c344ff9ba0a15e07b4bf3158a3b9d8123072.png" alt="E_q[P(X_i|z_i=k)]"/> term), and the <img class="math" src="../_images/math/7dd2a5ea01fbd72ad2a58dd1f3d6ecbfde6208a1.png" alt="a"/> and <img class="math" src="../_images/math/5e87bf41a96deddf6cb485ff530f153f2590e9cc.png" alt="b"/> variables themselves.</p>
<p><strong>The update for</strong> <img class="math" src="../_images/math/126e84ba38f7dece5f0ad64e929b9588b20f6440.png" alt="\mu"/></p>
<div class="math">
<p><img src="../_images/math/2e71c7804cd1c5edb6aa75451a295c86d3488be6.png" alt="\nu_{\mu_k} = \left(\mathbf{I}+\sum_i \frac{\nu_{z_{i,k}}\mathbf{b_k}}{\mathbf{a_k}}\right)^{-1}\left(\sum_i \frac{\nu_{z_{i,k}}b_k}{a_k}X_i\right)"/></p>
</div><p><strong>The updates for a and b</strong></p>
<p>Here we’ll do something very similar to the spheric model. The main
difference is that now each <img class="math" src="../_images/math/98960fec06a81f158a2b33b6351e2e87c58861f1.png" alt="\sigma_{k,d}"/> controls only one dimension
of the bound:</p>
<div class="math">
<p><img src="../_images/math/435990955e5bf931b5a6c2904ac75afaf4f20930.png" alt="\log Q(\sigma_{k,d}) = -\sigma_{k,d} + \sum_i \nu_{z_{i,k}}\frac{1}{2}\log \sigma_{k,d}
- \frac{\sigma_{k,d}}{2}\sum_i \nu_{z_{i,k}} ((X_{i,d}-\mu_{k,d})^2 + 1)"/></p>
</div><p>Hence</p>
<div class="math">
<p><img src="../_images/math/92bc70f6f13ec15f8fb46a2100f7273dc8bdf993.png" alt="a_{k,d} = 1 + \frac{1}{2} \sum_i \nu_{z_{i,k}}"/></p>
</div><div class="math">
<p><img src="../_images/math/bcc380de2952077d4d650c00bf936484d23bb854.png" alt="b_{k,d} = 1 + \frac{1}{2} \sum_i \nu_{z_{i,k}}((X_{i,d}-\mu_{k,d})^2 + 1)"/></p>
</div></div>
</div>
<div class="section" id="the-tied-model">
<h2>2.1.3.2.2.3. The tied model<a class="headerlink" href="#the-tied-model" title="Permalink to this headline">¶</a></h2>
<p>The model then is</p>
<div class="math">
<p><img src="../_images/math/ea669d9510e28549cf732e326579f743f2d9c4bb.png" alt="\begin{array}{rcl}
\phi_k &\sim& Beta(1, \alpha_1) \\
\mu_k &\sim& Normal(0, \mathbf{I}) \\
\Sigma &\sim& Wishart(D, \mathbf{I}) \\
z_{i} &\sim& SBP(\phi) \\
X_t &\sim& Normal(\mu_{z_i}, \Sigma^{-1})
\end{array}"/></p>
</div><p>Tha variational distribution we’ll use is</p>
<div class="math">
<p><img src="../_images/math/a6c82f4798ded3445ff7923a021649b5133cc5b3.png" alt="\begin{array}{rcl}
\phi_k &\sim& Beta(\gamma_{k,1}, \gamma_{k,2}) \\
\mu_k &\sim& Normal(\nu_{\mu_k}, \mathbf{I}) \\
\Sigma &\sim& Wishart(a, \mathbf{B}) \\
z_{i} &\sim& Discrete(\nu_{z_i}) \\
\end{array}"/></p>
</div><div class="section" id="id2">
<h3>2.1.3.2.2.3.1. The lower bound<a class="headerlink" href="#id2" title="Permalink to this headline">¶</a></h3>
<p>There are two changes in the lower-bound: for <img class="math" src="../_images/math/19ac15bf260b22dcb61a1042c60259e4b0bfbd64.png" alt="\Sigma"/> and for <img class="math" src="../_images/math/f026aecf11ec7f6141ab863f260d395f94b10f51.png" alt="X"/>.</p>
<p><strong>The bound for</strong> <img class="math" src="../_images/math/19ac15bf260b22dcb61a1042c60259e4b0bfbd64.png" alt="\Sigma"/></p>
<div class="math">
<p><img src="../_images/math/1df05e599583852d227514ff84ea614fc5159fba.png" alt="\begin{array}{rcl}
\frac{D^2}{2}\log 2 + \sum_d \log \Gamma(\frac{D+1-d}{2}) \\
- \frac{aD}{2}\log 2 + \frac{a}{2} \log |\mathbf{B}| + \sum_d \log \Gamma(\frac{a+1-d}{2}) \\
+ \frac{a-D}{2}\left(\sum_d \Psi\left(\frac{a+1-d}{2}\right)
+ D \log 2 + \log |\mathbf{B}|\right) \\
+ \frac{1}{2} a \mathbf{tr}[\mathbf{B}-\mathbf{I}]
\end{array}"/></p>
</div><p><strong>The bound for X</strong></p>
<div class="math">
<p><img src="../_images/math/a1658cc847336e3efa70e0601d507a3f78cbefe8.png" alt="\begin{array}{rcl}
E_q[\log P(X_i)] &=& \sum_k \nu_{z_k} \Big( - \frac{D}{2}\log 2\pi
+\frac{1}{2}\left(\sum_d \Psi\left(\frac{a+1-d}{2}\right)
+ D \log 2 + \log |\mathbf{B}|\right) \\
&&
-\frac{1}{2}((X_i - \nu_{\mu_k})a\mathbf{B}(X_i - \nu_{\mu_k})+ a\mathbf{tr}(\mathbf{B}))- \log 2 \pi e \Big)
\end{array}"/></p>
</div></div>
<div class="section" id="id3">
<h3>2.1.3.2.2.3.2. The updates<a class="headerlink" href="#id3" title="Permalink to this headline">¶</a></h3>
<p>As in the last setting, what changes are the trivial update for <img class="math" src="../_images/math/84d7271dd9e78c1e05d6c3c6ecb60309ef7dfc73.png" alt="z"/>,
the update for <img class="math" src="../_images/math/126e84ba38f7dece5f0ad64e929b9588b20f6440.png" alt="\mu"/> and the update for <img class="math" src="../_images/math/7dd2a5ea01fbd72ad2a58dd1f3d6ecbfde6208a1.png" alt="a"/> and <img class="math" src="../_images/math/fb542518f2c3338d27234e14932fd1ab5a311d8b.png" alt="\mathbf{B}"/>.</p>
<p><strong>The update for</strong> <img class="math" src="../_images/math/126e84ba38f7dece5f0ad64e929b9588b20f6440.png" alt="\mu"/></p>
<div class="math">
<p><img src="../_images/math/a906029eb7b8918858b4d8262c430f3d59b748e7.png" alt="\nu_{\mu_k} = \left(\mathbf{I}+ a\mathbf{B}\sum_i \nu_{z_{i,k}}\right)^{-1}
\left(a\mathbf{B}\sum_i \nu_{z_{i,k}} X_i\right)"/></p>
</div><p><strong>The update for</strong> <img class="math" src="../_images/math/7dd2a5ea01fbd72ad2a58dd1f3d6ecbfde6208a1.png" alt="a"/> <strong>and</strong> <img class="math" src="../_images/math/83956e92fcc80dee17fce864543216939a3c9da7.png" alt="B"/></p>
<p>As this distribution is far too complicated I’m not even going to try
going at it the gradient way.</p>
<div class="math">
<p><img src="../_images/math/6cfe4cdaf5ef73a55c04bb0fcb95103837383bc8.png" alt="\log Q(\Sigma) = +\frac{1}{2}\log |\Sigma| - \frac{1}{2} \mathbf{tr}[\Sigma]
+ \sum_i \sum_k \nu_{z_{i,k}} \left( +\frac{1}{2}\log |\Sigma| - \frac{1}{2}((X_i-\nu_{\mu_k})^T\Sigma(X_i-\nu_{\mu_k})+\mathbf{tr}[\Sigma]) \right)"/></p>
</div><p>which non-trivially (seeing that the quadratic form with <img class="math" src="../_images/math/19ac15bf260b22dcb61a1042c60259e4b0bfbd64.png" alt="\Sigma"/> in
the middle can be expressed as the trace of something) reduces to</p>
<div class="math">
<p><img src="../_images/math/15c553f2cfb7fa130f468d1a1bf31553d617f0fc.png" alt="\log Q(\Sigma) = +\frac{1}{2}\log |\Sigma| - \frac{1}{2} \mathbf{tr}[\Sigma]
+ \sum_i \sum_k \nu_{z_{i,k}} \left( +\frac{1}{2}\log |\Sigma| - \frac{1}{2}(\mathbf{tr}[(X_i-\nu_{\mu_k})(X_i-\nu_{\mu_k})^T\Sigma]+\mathbf{tr}[I \Sigma]) \right)"/></p>
</div><p>hence this (with a bit of squinting) looks like a wishart with parameters</p>
<div class="math">
<p><img src="../_images/math/1b10bb021bdcab80e9a192bc46f57eaac675bff0.png" alt="a = 2 + D + T"/></p>
</div><p>and</p>
<div class="math">
<p><img src="../_images/math/b2c9abe9dcb37c7fe576baebfcc34f1dc74d4dde.png" alt="\mathbf{B} = \left(\mathbf{I} + \sum_i \sum_k \nu_{z_{i,k}}(X_i-\nu_{\mu_k})(X_i-\nu_{\mu_k})^T\right)^{-1}"/></p>
</div></div>
</div>
<div class="section" id="the-full-model">
<h2>2.1.3.2.2.4. The full model<a class="headerlink" href="#the-full-model" title="Permalink to this headline">¶</a></h2>
<blockquote>
<div>The model then is</div></blockquote>
<div class="math">
<p><img src="../_images/math/78a38be674bdf4202a26ae17ef1935fbdbe23682.png" alt="\begin{array}{rcl}
\phi_k &\sim& Beta(1, \alpha_1) \\
\mu_k &\sim& Normal(0, \mathbf{I}) \\
\Sigma_k &\sim& Wishart(D, \mathbf{I}) \\
z_{i} &\sim& SBP(\phi) \\
X_t &\sim& Normal(\mu_{z_i}, \Sigma_{z,i}^{-1})
\end{array}"/></p>
</div><p>The variational distribution we’ll use is</p>
<div class="math">
<p><img src="../_images/math/78ec2f9210cf8a4c33c1501f9e90b1472fe920ed.png" alt="\begin{array}{rcl}
\phi_k &\sim& Beta(\gamma_{k,1}, \gamma_{k,2}) \\
\mu_k &\sim& Normal(\nu_{\mu_k}, \mathbf{I}) \\
\Sigma_k &\sim& Wishart(a_k, \mathbf{B_k}) \\
z_{i} &\sim& Discrete(\nu_{z_i}) \\
\end{array}"/></p>
</div><div class="section" id="id4">
<h3>2.1.3.2.2.4.1. The lower bound<a class="headerlink" href="#id4" title="Permalink to this headline">¶</a></h3>
<p>All that changes in this lower bound in comparison to the previous one
is that there are K priors on different <img class="math" src="../_images/math/19ac15bf260b22dcb61a1042c60259e4b0bfbd64.png" alt="\Sigma"/> precision matrices
and there are the correct indices on the bound for X.</p>
</div>
<div class="section" id="id5">
<h3>2.1.3.2.2.4.2. The updates<a class="headerlink" href="#id5" title="Permalink to this headline">¶</a></h3>
<p>All that changes in the updates is that the update for mu uses only
the proper sigma and the updates for a and B don’t have a sum over K, so</p>
<div class="math">
<p><img src="../_images/math/3c8b8b6a9aef0e43ed0a2c0772a5db285ba66de6.png" alt="\nu_{\mu_k} = \left(\mathbf{I}+ a_k\mathbf{B_k}\sum_i \nu_{z_{i,k}}\right)^{-1}
\left(a_k\mathbf{B_k}\sum_i \nu_{z_{i,k}} X_i\right)"/></p>
</div><div class="math">
<p><img src="../_images/math/b2e267f8737a7dc04a95656aa5ffff5779b29de1.png" alt="a_k = 2 + D + \sum_i \nu_{z_{i,k}}"/></p>
</div><p>and</p>
<div class="math">
<p><img src="../_images/math/49ab0ef6841fb842550e844d69ee90d528252557.png" alt="\mathbf{B} = \left(\left(\sum_i\nu_{z_{i,k}}+1\right)\mathbf{I} + \sum_i \nu_{z_{i,k}}(X_i-\nu_{\mu_k})(X_i-\nu_{\mu_k})^T\right)^{-1}"/></p>
</div></div>
</div>
</div>
</div>
</div>
</div>
<div class="clearer"></div>
</div>
</div>
<div class="footer">
© 2010 - 2014, scikit-learn developers (BSD License).
<a href="../_sources/modules/dp-derivation.txt" rel="nofollow">Show this page source</a>
</div>
<div class="rel">
<div class="buttonPrevious">
<a href="mixture.html">Previous
</a>
</div>
</div>
<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-22606712-2']);
_gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>
<script src="http://www.google.com/jsapi" type="text/javascript"></script>
<script type="text/javascript"> google.load('search', '1',
{language : 'en'}); google.setOnLoadCallback(function() {
var customSearchControl = new
google.search.CustomSearchControl('016639176250731907682:tjtqbvtvij0');
customSearchControl.setResultSetSize(google.search.Search.FILTERED_CSE_RESULTSET);
var options = new google.search.DrawOptions();
options.setAutoComplete(true);
customSearchControl.draw('cse', options); }, true);
</script>
<script src="https://scikit-learn.org/versionwarning.js"></script>
</body>
</html>