215/24, 109M —_producton-gradescope-uploads.s2-us-est-2.amazonaws.comluploads(text_fifle40277343/gmm.py7X-Amz-Algoritim=AWS4- import numpy as_np from tqdn import tadn from kneans import kMeans SIGMA_CONST = 1e-6 LOG_CONST = 1¢-32 FULL_MATRIX = True # Set False if the covariance matrix is a diagonal matrix class GHM(object) def _init_(self, x, K, max_iters=100): # No need to change Angs: X: the observations/datapoints, N x D nunpy array K: number of clusters/components max maxinun nunber of iterations (used in EM implementation) self.points = x self.max_iters = max_iters self.N = self.points.shape[0] # nunber of observations self.D = self.points.shape[1] # number of features self.k = K # number of conponents/clusters 4 Helper function for you to implement def softmax(self, logit): # [Spts] args: logit: N x D numpy array Return prob: Nx D numpy array. See the above function. rue in your np.sum() function to avoid broadcast error. maxelt = np.anax(logit, axi logit = logit - maxelt # print(np.sum(np.exp(logit), axis=-1, keepdins-True)) return np.exp(logit)/np.sum(np.exp(logit), axise-1, keepdims=True) 1) -reshape((-1,1)) def 1 unexp(self, logit): # [Spts] args: logit: Nx D numpy array Return si Nx 1 array where [4,0] = logsumexp(logit[i,:]). See the above function Hint: ‘The keepdims paraneter could be handy maxelt = np.amax(logit, axis= logit = logit - maxelt ans = np.log(np.sum(np.exp(logit), axis=-1, keepdins= True)) return ans + maxelt 1) .reshape((-1,1)) 4 for undergraduate student def nornalPOF(self, points, mui, signai): # [5pts] args: points: Nx D nunpy array mu_i: (D,) numpy array, the center for the ith gaussian. sigma_i: OxD numpy array, the covariance matrix of the ith gaussian. Return pdf: (N,) numpy array, the probability density value of N data for the ith gaussian ntps: production -gradescope-uploads s3-us-west-2.amazoraws.comiuploadstext_flefle!402773438igmm.py?X-Amz-Algorthm=AWSS-HMAC-SHA... 1/5215/24, 209M —_producton-gradescope-uploads.s2-us-est-2.amazonaws.comluploads(text_flfle402773438/gmm,py7X-Amz-Algoritim=AWS4- Hint: np.diagonal() should be handy. raise NotImplenentedérror # for grad students def multinornalPOF(self, points, mui, sigmai): # [Spts] args: points: Nx D nunpy array my_i: (D,) numpy array, the center for the ith gaussian. signa_i: DxD numpy array, the covariance matrix of the ith gaussian. Return normal_pdf: (N,) numpy array, the probability density value of N data for the ith gaussian Hint: 1. np-Linalg.det() and np.linalg.inv() should be handy. 2. The value in self.D may be outdated and not correspond to the current dataset, ‘try using another method involving the current arguments to get the value of D D = points. shape[1] constant = 1/np.power(2*np.pi, D/2) try: inv = np.linalg.inv(signa_i) det = np.linalg-det(sigma_i) except inv = np.Lnalg.inv(signa_i + SIGMA_CONST) det = np.linalg.det(signa_i + SIGMA_CONST) diff = points - mu_i[np.newaxis, :] term = np.matmul (diff, inv) # print(terml. shape) ‘tern? = np.sum(terml.T*(diff.1), axis=0) # print(term2.shape) return constant*np.power(det, -1/2)*np.exp(-(1/2)*term2) def _init_conponents(self, **kwargs): # [Spts] args: kwargs: any other arguments you want Return pi: numpy array of length k, prior mu: KxD numpy array, the center for each gaussian signa: KxOxD numpy array, the diagonal standard deviation of each gaussian. You will have KxDxD nunpy array for full covariance matrix case Hint: np-random.seed(5) may be used at the start of this function to ensure consistent outputs. np.random.seed(S) #00 Not Remove Seed pi = np.full(self.k, 1/self.k) my = self.points[np.randon.randint(@, self.N, self.k)] signa = np.repeat(np.eye(self.0)[np-newaxis, :, :], self.k, axi: return pi, mu, sigma nips: production -gradescope-uploads s3-us-west-2.amazoraws.comiuploadsitext_lefle!402773438igmm.py?X-Amz-Algorthm=AWSS-HMAC-SHA.. 2/5,215/24, 209M —_producton-gradescope-uploads.s2-us-est-2.amazonaws.comluploads(text_flfle402773438/gmm,py7X-Amz-Algoritim=AWS4- det 1_joint(self, pi, mu, signa, full_matrix-FULL_MATRIX, **kwargs): # [10 pts] ngs: pi: np array of length K, the prior of each conponent mu: KxD numpy array, the center for each gaussian signa: KxOxD numpy array, the diagonal standard deviation of each gaussian. You will have KxOxD numpy array for full covariance matrix case full_matrix: whether we use full covariance matrix in Normal POF or not. Default is True. Return L1(1og-Likelihood): NxK array, where 11(i, k) = log pi(k) + log NormalPOF(points_i | uk], signa[k]) # == graduate implenentation wif full_matrix is True: # === undergraduate inplenentation wif full_matrix is False: #. small_const = 1e-32 # Llenp.array([]) if full_matrix is True: Llsnp.array([)) for k in range(self.k) normal = self.nultinormalPDF(self.points, mu[k], signa[k]) b = np-log(pi[k] + small_const) + np.log(normal + smal1_const) # print(11.shape, b.shape) # print(11) if Ten(11): np.column_stack((11, b[:, np-newaxis])) np.append(11, b) # print("returning l= ", 11) return 11 def _£ tep(self, pi, mu, sigma, full_matrix = FULL_MATRIX , **kwargs): # [Spts] Angs: pi: np array of length K, the prior of each component mu: KxD numpy array, the center for each gaussian signa: KxOxD numpy array, the diagonal standard deviation of each gaussian.You will have KxOxD numpy array for full covariance matrix case full_matrix: whether we use full covariance matrix in Normal POF or not. Default is True. Return ganma(tau): NxK array, the posterior distribution (a.k.a, the soft cluster assignment) for each observation Hint: You should be able to do this with just a few lines of code by using _11_joint() and softmax() defined above. # se= graduate implementation #if full_matrix is True: #. # =s= undergraduate inplenentation #if full_matrix is False: # nips: production -gradescope-uploads s3-us-west-2.amazoraws.comiuploadstext_flefle!402773438igmm.py?X-Amz-Algorthm=AWSS-HMAC-SHA... 315,215/24, 209M —_producton-gradescope-uploads.s2-us-est-2.amazonaws.comluploads(text_flfle402773438/gmm,py7X-Amz-Algoritim=AWS4- if full_matrix is True: self._11_joint(pi, mu, sigma, full_matrix) # print("logit= “, logit) return self. softmax(logit) def _Mstep(self, gamma, full_matrix=FULL_MATRIX, **kwargs): # [1epts] args: ‘gamma(tau): NxK array, the posterior distribution (a.k.a, the soft cluster assignment) for each observation. full_matrix: whether we use full covariance matrix in Normal POF or not. Default is True. Return pi: np array of length K, the prior of each conponent mu: KxD numpy array, the center for each gaussian signa: KxOxD nunpy array, the diagonal standard deviation of each gaussian. You will have KxDxD numpy array for full covariance matrix case Hint: There are formulas in the slides and in the Jupyter Notebook. Undergrads: To simplify your calculation in signa, make sure to only take the diagonal terms in your covariance matrix graduate implenentation #if full_matrix is True: #. # <= undergraduate inplementation #if full_matrix is False: #. Af full_matrix is True: pis np.zeros(self.k) mu = np.zeros((self.K, self.D)) sigma = np.zeros((self.k, self.D, self.D)) for k in range(self.k) ganma_k = ganna[:, k] Nk = np.sum(ganna_k) uk] = np.dot( (ganma_k), self. points) /Nk Pilk]= Nk/self.N Signa[k] = (1/Nk)*np.dot(ganna_k.T*(self.points - mufk]).T, (self.points - mu(k])) return pi, mu, signa def _call_(self, full_matri to change ULL_MATRTX, abs_tol=1e-16, rel_to je-16, **kwargs): # No need args: abs_tol: convergence criteria w.r.t absolute change of loss rel_tol: convergence criteria w.r.t relative change of loss kwargs: any additional arguments you want Return ganna(tau): NxK array, the posterior distribution (a.k.a, the soft cluster assignnent) for each observation. (pi, mu, sigma): (1xk np array, KxD nunpy array, KxDXD aunpy array) Hint: You do not need to change it. For each iteration, we process & and M steps, then update the paramters. pi, mu, signa = self._init_components(**kwargs) pbar = tadm(range(sel¥.max_iters)) for it in pbar: ntps:production-gradescope-uploads s3-us-west-2.amazoraws.comiuploadsitext_ flefle!402773438igmm.py?X-Amz-Algorthm=AWSS-HMAC-SHA... 8/5215/24, 209M —_producton-gradescope-uploads.s2-us-est-2.amazonaws.comluploads(text_flfle402773438/gmm,py7X-Amz-Algoritim=AWS4- # Estep ganna self.£ step(pi, mu, signa, full_natrix) # Mstep pi, mu, signa = self._M_step(ganma, full_natrix) # calculate the negative log-Likelinood of observation joint_1l = self._ll_joint(pi, mu, sigma, full_matrix) loss = -np.sum(self.logsumexp(joint_11)) sf it: diff = np.abs(prev_loss - loss) Sf Giff < abs_tol and ciff / prev loss < rel_tol: break prev_loss = loss pbarsset. description( ‘iter Xd, loss: X.4f* X (it, loss) return ganna, (pi, mu, signa) nips: production -gradescope-uploads s3-us-west-2.amazoraws.comiuploadsitext flefle!402773438igmm.py?X-Amz-Algorthm=AWSS-HMAC-SHA... 5/5