<html><head><meta name="color-scheme" content="light dark"></head><body><pre style="word-wrap: break-word; white-space: pre-wrap;">% Homework template for Inference and Information
% UPDATE: September 26, 2017 by Xiangxiang
\documentclass[a4paper]{article}
\usepackage{ctex}
\ctexset{
proofname = \heiti{è¯æ˜Ž}
}
\usepackage{amsmath, amssymb, amsthm}
% amsmath: equation*, amssymb: mathbb, amsthm: proof
\usepackage{moreenum}
\usepackage{mathtools}
\usepackage{url}
\usepackage{bm}
\usepackage{enumitem}
\usepackage{graphicx}
\usepackage{subcaption}
\usepackage{booktabs} % toprule
\usepackage[mathcal]{eucal}
\usepackage[thehwcnt = 1]{iidef}

\thecourseinstitute{æ¸…åŽå¤§å­¦æ·±åœ³ç&nbsp;”ç©¶ç”Ÿé™¢}
\thecoursename{åº”ç”¨ä¿¡æ¯è®º}
\theterm{2018å¹´æ˜¥å­£å­¦æœŸ}
\hwname{ä½œä¸š}
\slname{\heiti{è§£}}
\begin{document}
\courseheader
\name{YOUR NAME}

\begin{enumerate}
  \setlength{\itemsep}{3\parskip}

  \item è®¾$X$å’Œ$Y$æ˜¯å„æœ‰å‡å€¼$m_x,m_y$,æ–¹å·®ä¸º$\sigma_x^2,\sigma_y^2$ï¼Œä¸”ç›¸äº’ç‹¬ç«‹çš„é«˜æ–¯éšæœºå˜é‡ï¼Œå·²çŸ¥$U=X+Y,V=X-Y$ã€‚è¯•æ±‚$I(U;V)$ã€‚
\begin{solution}
$U,V$çš„è”åˆåˆ†å¸ƒæ˜¯å‡å€¼ä¸º$[\mu_x+\mu_y,\mu_x-\mu_y]$ï¼Œåæ–¹å·®çŸ©é˜µä¸º
$$\Lambda_{U,V}=\begin{bmatrix}
1 &amp; 1\\
1 &amp; -1\\
\end{bmatrix}
\begin{bmatrix}
\sigma_x^2 &amp; 0\\
0 &amp; \sigma_x^2\\
\end{bmatrix}
\begin{bmatrix}
1 &amp; 1\\
1 &amp; -1\\
\end{bmatrix}^T
=\begin{bmatrix}
\sigma_x^2+\sigma_y^2 &amp; \sigma_x^2-\sigma_y^2\\
\sigma_x^2-\sigma_y^2 &amp; \sigma_x^2+\sigma_y^2\\
\end{bmatrix}
$$
ç”±å¤šå…ƒé«˜æ–¯åˆ†å¸ƒå¾®åˆ†ç†µçš„å…¬å¼
$$
h(U)=\frac{1}{2}\log ((2\pi e)^2 |\Lambda_{U,V}|)=\frac{1}{2}\log(16\pi^2 e^2 \sigma^2_x\sigma^2_y)
$$
$U|V=v$ä¹Ÿæ˜¯é«˜æ–¯åˆ†å¸ƒ,æ–¹å·®ä¸º$\frac{4\sigma_x^2\sigma_y^2}{\sigma_x^2+\sigma_y^2}$ï¼Œä¸Ž$v$æ—&nbsp;å…³ï¼Œå›&nbsp;æ­¤
$$
h(U|V)=\E_{V}[h(U|V=v)]=\frac{1}{2}\log(2\pi e \frac{4\sigma_x^2\sigma_y^2}{\sigma_x^2+\sigma_y^2})\Rightarrow
$$
\begin{align*}
I(U;V)=&amp; h(U)-h(U|V)\\
=&amp;\frac{1}{2}\log(16\pi^2 e^2 \sigma^2_x\sigma^
2_y)-\frac{1}{2}\log(2\pi e \frac{4\sigma_x^2\sigma_y^2}{\sigma_x^2+\sigma_y^2})\\
=&amp;\frac{1}{2}\log(2\pi e (\sigma_x^2+\sigma_y^2))
\end{align*}

%$$
%p(u,v)=\frac{1}{4\pi \sigma_x\sigma_y}\exp(-\frac{(\frac{u+v}{2}-\mu_x)^2+(\frac{u+v}{2}-\mu_y)^2}{2})
%$$

\end{solution}
\item è®¾æœ‰éšæœºå˜é‡$X,Y,Z$å‡å–å€¼äºŽ$\{0,1\}$,å·²çŸ¥$I(X;Y)=0,I(X;Y|Z)=1$ã€‚æ±‚è¯$H(Z)=1,H(X,Y,Z)=2$
\begin{proof}
$I(X;Y|Z)=H(X|Z)-H(X|Y,Z)\leq H(X|Z)\leq H(X)\leq \log(2)=1$
æ‰€ä»¥ç­‰å·å…¨éƒ½æˆç«‹$\Rightarrow X\sim B(\frac{1}{2})$ã€‚
åŒç†å¯çŸ¥$Y\sim B(\frac{1}{2})$ã€‚
å¦å¤–$H(Y|Z)=H(Y)\Rightarrow I(Y;Z)=0\Rightarrow H(Z|Y)=H(Z)$
\begin{align*}
&amp;H(X|Y,Z)=0 \\
\iff &amp;H(X,Y,Z)=H(Y,Z)\\
\iff &amp;H(X,Y)+H(Z|X,Y)=H(Y)+H(Z|Y)\\
\iff &amp;2+H(Z|X,Y)=1+H(Z)\\
\iff &amp;H(Z)=1+H(Z|X,Y)
\end{align*}
ç”±ä¸Šå¼æŽ¨å‡º$H(Z)\geq 1$,åˆ$H(Z)\leq 1\Rightarrow H(Z)=1\Rightarrow H(X,Y,Z)=2$
\end{proof}
\item
è®¾æœ‰ä¿¡å·$X$ç»è¿‡å¤„ç†å™¨$A$åŽèŽ·è¾“å‡º$Y$,$Y$å†ç»å¤„ç†å™¨$B$åŽèŽ·è¾“å‡º$Z$ã€‚å·²çŸ¥å¤„ç†å™¨$A$å’Œ$B$
åˆ†åˆ«ç‹¬ç«‹å¤„ç†$X$å’Œ$Y$ã€‚è¯•è¯ï¼š$I(X;Z)\leq I(X;Y)$
\begin{proof}
$I(X;Z)=H(Z)-H(Z|X)=H(Z);I(Y;Z)=H(Y)$å›&nbsp;ä¸º$Z$æ˜¯$Y$çš„å‡½æ•°$\Rightarrow H(Z)\leq H(Y) \Rightarrow I(X;Z)\leq I(X;Y)$
\end{proof}
\item å·²çŸ¥éšæœºå˜é‡$X$å’Œ$Y$çš„è”åˆæ¦‚çŽ‡å¯†åº¦$p(a_k,b_j)$æ»¡è¶³
$$
p(a_1)=\frac{1}{2},p(a_2)=p(a_3)=\frac{1}{4},p(b_1)=\frac{2}{3},p(b_2)=p(b_3)=\frac{1}{6}
$$
è¯•æ±‚èƒ½ä½¿$H(X,Y)$å–å¾—æœ€å¤§å€¼çš„è”åˆæ¦‚çŽ‡å¯†åº¦åˆ†å¸ƒã€‚
\begin{solution}
$H(X,Y)=H(X)+H(Y)-I(X;Y)\leq H(X)+H(Y)=\frac{7}{6}+\log 3$
ç­‰å·æˆç«‹å½“ä¸”ä»…å½“$X,Y$ç›¸äº’ç‹¬ç«‹$\Rightarrow p(x,y)=p(x)p(y)$
\end{solution}
\item è®¾éšæœºå˜é‡$X,Y,Z$æ»¡è¶³$p(x,y,z)=p(x)p(y|x)p(z|y)$ã€‚æ±‚è¯$I(X;Y)\geq I(X;Y|Z)$
\begin{proof}
å›&nbsp;ä¸º$p(x,y,z)=p(x)p(y|x)p(z|y,x)\Rightarrow p(z|y,x)=p(z|x)\Rightarrow x$ ä¸Ž$z$å…³äºŽ$y$æ¡ä»¶ç‹¬ç«‹$\Rightarrow I(X;Y|Z)=H(X|Z)-H(X|Y,Z)= H(X|Z)-H(X|Y)\leq H(X)-H(X|Y) =I(X;Y)$
\end{proof}
\item æ±‚è¯$I(X;Y;Z)=H(X,Y,Z)-H(X)-H(Y)-H(Z)+I(X;Y)+I(Y;Z)+I(Z;X)$,å…¶ä¸­
$I(X;Y;Z)\triangleq I(X;Y)-I(X;Y|Z)$
\begin{proof}
\begin{align*}
I(X;Y;Z) =&amp; I(X;Y)-I(X;Y|Z) \\
=&amp; H(X)+H(Y)-H(X,Y)-(H(X|Z)-H(X|Y,Z))\\
=&amp; H(X)+H(Y)-H(X,Y)-(H(X,Z)-H(Z))+H(X,Y,Z)-H(Y,Z)\\
=&amp; H(X,Y,Z)-H(X)-H(Y)-H(Z)+(H(X)+H(Y)-H(X,Y))\\
+&amp;(H(Y)+H(Z)-H(Y,Z))+(H(Z)+H(X)-H(X,Z))\\
=&amp; H(X,Y,Z)-H(X)-H(Y)-H(Z)+I(X;Y)+I(Y;Z)+I(Z;X)
\end{align*}
\end{proof}
\item ä»¤$p=(p_1,p_2,\dots,p_a)$æ˜¯ä¸€ä¸ªæ¦‚çŽ‡åˆ†å¸ƒï¼Œæ»¡è¶³$p_1\geq p_2\geq \dots p_a$,å‡è®¾$\epsilon &gt;0 $ä½¿å¾—$p_1-\epsilon \geq p_2+\epsilon$æˆç«‹ï¼Œè¯æ˜Žï¼š$H(p_1,p_2,\dots,p_a)
\leq H(p_1-\epsilon,p_2+\epsilon,p_3,\dots,p_a)$
\begin{proof}
è®¾$f(\epsilon)=(p_1-\epsilon)\log(p_1-\epsilon)+(p_2+\epsilon)\log(p_2+\epsilon)$
ç”±å·²çŸ¥$0\leq \epsilon \frac{p_2-p_1}{2}$
$f'(\epsilon)=\log\frac{p_2+\epsilon}{p_1-\epsilon}\leq 0$
$\Rightarrow f(\epsilon)\leq f(0)\Rightarrow H(p_1,p_2,\dots,p_a)\leq H(p_1-\epsilon,p_2+\epsilon,p_3,\dots,p_a)$
\end{proof}
\item è®¾$p_i(x)\sim N(\mu_i,\sigma_i^2)$,è¯•æ±‚ç›¸å¯¹ç†µ$D(p_2||p_1)$
\begin{solution}
\begin{align*}
D(p_2||p_1)=&amp; \int_{\mathbb{R}} p_2(x) \log \frac{p_2(x)}{p_1(x)}dx\\
=&amp; \int_{\mathbb{R}} p_2(x) \left(\log \frac{\sigma_1^2}{\sigma_2^2}+\frac{1}{2}((x-\mu_1)^2-(x-\mu_2)^2)\log e\right)dx\\
=&amp; 2\log \frac{\sigma_1}{\sigma_2}+\frac{1}{2}(\mu_1^2-\mu_2^2)\log e+(\mu_2-\mu_1)\mu_2\log e\\
=&amp; 2\log \frac{\sigma_1}{\sigma_2}+\frac{1}{2}(\mu_1-\mu_2)^2\log e
\end{align*}
\end{solution}
\item è‹¥$f(x)$åˆ†åˆ«æ˜¯åŒºé—´$(0,0.01),(0,0.5),(0,1),(0,2),(0,5)$ä¸Šå‡åŒ€åˆ†å¸ƒçš„åˆ†å¸ƒå‡½æ•°ï¼Œè®¡ç®—$f(x)$çš„å¾®åˆ†ç†µã€‚
\begin{solution}
è®¾$U_t$æ˜¯$(0,t)$ä¸Šçš„å‡åŒ€åˆ†å¸ƒï¼Œåˆ™$h(U_t)=\log t$
\begin{itemize}
\item $h(U_{0.01})=\log 0.01$
\item $h(U_{0.5})=-1$
\item $h(U_{1})=0$
\item $h(U_{2})=1$
\item $h(U_{5})=\log 5$
\end{itemize}
\end{solution}
\item è®¾
\begin{align*}
p_1(x,y)=&amp; \frac{1}{2\pi \sigma_x\sigma_y}\exp[-\frac{1}{2}(\frac{x^2}{\sigma_x^2}+\frac{y^2}{\sigma_y^2})]\\
p_2(x,y)=&amp; \frac{1}{2\pi \sigma_x\sigma_y\sqrt{1-\rho^2}}\exp[-\frac{1}{2(1-\rho^2)}(\frac{x^2}{\sigma_x^2}-2\rho\frac{xy}{\sigma_x\sigma_y}+\frac{y^2}{\sigma_y^2})]
\end{align*}
è¯•æ±‚$D(p_2||p_1)$å’Œ$I(X;Y)$ï¼Œå…¶ä¸­$X,Y\sim p_2$
\begin{solution}
\begin{align*}
D(p_2||p_1) = &amp; \iint_{\mathbb{R}^2} p_2(x,y)\log \frac{p_2(x,y)}{p_1(x,y)}dxdy \\
 -&amp;\frac{1}{2}\log(1-\rho^2)\\
-&amp;\frac{1}{2}(\log e)\iint_{\mathbb{R}^2} p_2(x,y)\left[\frac{\rho^2 x^2}{\sigma_x^2(1-\rho^2)}+\frac{\rho^2 y^2}{\sigma_y^2(1-\rho^2)}-\frac{2\rho xy}{(1-\rho^2)\sigma_x\sigma_y}\right]dxdy\\
=&amp;-\frac{1}{2}\log(1-\rho^2)
\end{align*}
$X|Y=y$æœä»Žé«˜æ–¯åˆ†å¸ƒï¼Œæ–¹å·®ä¸º$(1-\rho^2)\sigma_x^2$
\begin{align*}
I(X;Y) = &amp; h(X)-h(X|Y)\\
= &amp; \frac{1}{2}\log(2\pi e \sigma_x^2) - \frac{1}{2}\log(2\pi e \sigma_x^2(1-\rho^2))\\
= &amp; \frac{1}{2}\log(\frac{2\pi e}{1-\rho^2})
\end{align*}

\end{solution}

\end{enumerate}
\end{document}
\begin{equation}
\end{equation}

%%% Local Variables:
%%% mode: late\rvx
%%% TeX-master: t
%%% End:
</pre></body></html>