GMM_em_routing

Hence

\begin{aligned} \ln (p (X, Z | θ)) & = \sum_{n = 1}^{N} a_{n} \ln p (x_{n} | z_{n}, Σ, μ) + \sum_{n = 1}^{N} a_{n} \ln p (z_{n} | π) \\ = \sum_{n = 1}^{N} a_{n} \sum_{k = 1}^{K} (\ln π_{k}) z_{k n} + \sum_{n = 1}^{N} a_{n} \sum_{k = 1}^{K} (\ln ϕ_{k} (x_{n})) z_{k n} \\ = \ln π^{T} \cdot Z \cdot a + \\ \sum_{n = 1}^{N} a_{n} \sum_{k = 1}^{K} [- \frac{l}{2} \ln (2 π) - \frac{1}{2} \ln | Σ_{k} | - \frac{1}{2} (x_{n} - μ_{k})^{T} Σ_{k}^{- 1} (x_{n} - μ_{k})] z_{k n} \\ = (\ln π^{T} - \frac{1}{2} | Σ |^{T}) \cdot Z \cdot a - \frac{l}{2} \ln (2 π) o n e s (1, K) \cdot Z \cdot a \\ - \frac{1}{2} \sum_{k = 1}^{K} [tr ((X - μ_{k} \cdot 1^{T})^{T} Σ_{k}^{- 1} (X - μ_{k} \cdot 1^{T}) \cdot diag (a ⊙ z_{k}))] \end{aligned}

Now find close form of Q

$\sum_\limits{\boldsymbol{Z}} p\left(\boldsymbol{Z} | \boldsymbol{X}, \boldsymbol{\theta}^{\text { old }}\right)$ of $Z$ [probability distribution] random variable

$q^k = p\left(\boldsymbol{Z} | \boldsymbol{X}, \boldsymbol{\theta}^{k}\right) = p\left(\boldsymbol{Z} | \boldsymbol{X}, \boldsymbol{\theta}^{\text { old }}\right)$

\begin{aligned} Q (θ, θ^{old}) & \equiv \sum_{Z} p (Z | X, θ^{old}) \ln p (X, Z | θ) \\ = {[\sum_{n = 1}^{N} a_{n} \sum_{z_{n}} p (z_{n} | x_{n}, θ^{old}) z_{n}^{T}] (\ln π - \frac{1}{2} | Σ | - \frac{l}{2} \ln (2 π) \cdot 1.)} \\ - \frac{1}{2} {\sum_{n = 1}^{N} a_{n} \sum_{k = 1}^{K} [\sum_{z_{n}} p (z_{n} | x_{n}, θ^{old}) (x_{n} - μ_{k})^{T} Σ_{k}^{- 1} (x_{n} - μ_{k}) z_{k n}]} \\ = [γ \cdot a]^{T} (\ln π - \frac{1}{2} | Σ | - \frac{l}{2} \ln (2 π) \cdot 1.) \\ - \frac{1}{2} \sum_{k = 1}^{K} tr (d i a g (a ⊙ γ_{k}) \cdot (X - μ_{k} \cdot 1^{T})^{T} Σ_{k}^{- 1} (X - μ_{k} \cdot 1^{T})) \end{aligned}

Here

p (z_{n} | x_{n}, θ^{old}) = \frac{p (x_{n} | z_{n}, θ^{old}) p (z_{n} | θ^{old})}{\sum_{z_{n}} p (x_{n} | z_{n}, θ^{old}) p (z_{n} | θ^{old})}

$1.^T \boldsymbol{\gamma}=ones(1, N)$

\begin{matrix} p (z_{n} = k | x_{n}, θ^{old}) = \frac{ϕ_{k}^{old} (x_{n}) π_{k}^{old}}{\sum_{k = 1}^{K} ϕ_{k}^{old} (x_{n}) π_{k}^{old}} \equiv γ_{k n} \\ γ = ϕ^{old} (X) ⊙ (π_{k}^{old} \cdot {1.}^{T}) ⊙ (\frac{1.}{1. \cdot (π^{old})^{T} ϕ^{old} (X)}) \end{matrix}

maximize Q

To sum up

\begin{aligned} π & = \frac{[γ \cdot a]}{{1.}^{T} [γ \cdot a]} = \frac{[γ \cdot a]}{{1.}^{T} a} \\ μ_{k} & = \frac{X (a ⊙ γ_{k})}{{1.}^{T} (a ⊙ γ_{k})} \\ Σ_{k} & = \frac{(X - μ_{k} \cdot 1^{T}) d i a g (a ⊙ γ_{k}) \cdot (X - μ_{k} \cdot 1^{T})^{T}}{{1.}^{T} (a ⊙ γ_{k})} \end{aligned}

$1.^T \boldsymbol{\gamma}=ones(1, N)$

\begin{matrix} p (z_{n} = k | x_{n}, θ^{old}) = \frac{ϕ_{k}^{old} (x_{n}) π_{k}^{old}}{\sum_{k = 1}^{K} ϕ_{k}^{old} (x_{n}) π_{k}^{old}} \equiv γ_{k n} \\ γ = ϕ^{old} (X) ⊙ (π_{k}^{old} \cdot {1.}^{T}) ⊙ (\frac{1.}{1. \cdot (π^{old})^{T} ϕ^{old} (X)}) \end{matrix}

$\boldsymbol{\gamma}_k$ is (N x 1) matrix

\begin{matrix} γ = [\begin{matrix} ⋮ \\ γ_{k}^{T} \\ ⋮ \end{matrix}] \end{matrix}

$\boldsymbol{\pi}$

π_{k} = sigmoid (λ (β_{a} - (β_{u} + \ln | Σ_{k} |) (a^{T} γ_{k})))