\documentclass{article}
\usepackage{fullpage, epsf, setspace, latexsym, cite}
\usepackage{xspace, amsmath, amssymb, subfigure, hyperref}
\usepackage{graphicx}

\newtheorem{thm}{Theorem}
\newtheorem{prop}{Proposition}
\newtheorem{cor}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{ex}{Example}
\newtheorem{exse}{Exercise}
\newtheorem{defn}{Definition}

\def\QED{\mbox{\rule[0pt]{1.5ex}{1.5ex}}}
\def\endproof{\hspace*{\fill}~\QED\par\endtrivlist\unskip}
\newenvironment{pf}{\begin{verse}{\bf Proof:}\rm}{\endproof \end{verse}}
\newcommand{\1}[1]{\ensuremath{{\bf 1}_{\{#1\}}}}

\newcommand{\lecture}[4]{
   \pagestyle{myheadings}
   \thispagestyle{plain}
   \newpage
   \setcounter{page}{1}
   \noindent
   \begin{center}
   \framebox{
      \vbox{\vspace{2mm}
    \hbox to 6.28in { {\bf ECE901 Spring 2004 Statistical Regularization and Learning Theory \hfill Lecture: #4} }
       \vspace{6mm}
       \hbox to 6.28in { {\Large \hfill #1  \hfill} }
       \vspace{6mm}
       \hbox to 6.28in { {\it Lecturer: #2 \hfill Scribe: #3} }
      \vspace{2mm}}
   }
   \end{center}
   \markboth{#1}{#1}
   \setlength{\headsep}{10mm}
   \vspace*{4mm}
}

%notation

\def\X{\ensuremath{{\mathcal X}}\xspace}
\def\Y{\ensuremath{{\mathcal Y}}\xspace}
\def\F{\ensuremath{{\mathcal F}}\xspace}
\def\R{\ensuremath{{\bf R}}\xspace}
\def\E{{E}}
\def\sign{\ensuremath{\mbox{sign}}}
\def\ie{{\em i.e., }}
\def\eg{{\em e.g., }}
\def\as{\mbox{ as }}
\def\dist_iid{\stackrel{i.i.d.}{\sim}}
\def\h{\widehat}

%Lecture dependent definitions, can be erased if necessary in other lecture notes
\def\I{{\mbox{I}_{j,m}}}

\begin{document}

\lecture{Maximum Likelihood Estimation and Complexity Regularization}{Rob
  Nowak}{Pam Limpiti}{14}

\section{Review : Maximum Likelihood Estimation}

In the last lecture, we have $n$ i.i.d observations drawn from an unknown distribution\\
\begin{eqnarray*}
Y_i\dist_iid p_{\theta^*} \quad ,\ i=\{1,\ldots,n\}
\end{eqnarray*}
\[ \mbox{where $\theta^*\in \Theta$.} \]
With {\it loss function} defined as $l(\theta,Y_i)=-\log p_\theta(Y_i)$, the {\bf empirical risk} is $$\hat{R_n}=-\frac{1}{n}\sum_{i=1}^n\log p_\theta(Y_i).$$
Essentially, we want to choose a distribution from the collection of distributions within the parameter space that minimizes the empirical risk,\ie we would like to select
$$p_{\hat{\theta_n}}\in \mathcal{P}=\{p_\theta\}_{\theta \in \Theta}$$

where $$\hat{\theta_n}=\arg \, \min_{\theta \in \Theta} \quad -\sum_{i=1}^n\log p_\theta(Y_i).$$
The {\bf risk} is defined as $$R(\theta)=\E[l(\theta,Y)]= -\E[\log p_\theta(Y)].$$

Note that $\theta^*$ minimizes $R(\theta)$ over $\Theta$.
\begin{eqnarray*}
\theta^*&=&\arg\, \min_{\theta \in \Theta} \quad -\E[\log p_\theta(Y)]\\
&=&\arg \min_{\theta \in \Theta} \quad - \int \log p_\theta(y)\cdot p_{\theta^*}(y)\,\mathrm{d}y.
\end{eqnarray*}
Finally, the {\bf excess risk} of $\theta$ is defined as 
$$R(\theta)-R(\theta^*)=\int \log \frac{p_{\theta^*}(y)}{p_\theta(y)}\,p_{\theta^*}(y)\,\mathrm{d}y \equiv K(p_\theta,p_{\theta^*})\,.$$

We recognized that the excess risk corresponding to this loss function is simply the \textit{Kullback-Leibler (KL) Divergence} or \textit{Relative Entropy}, denoted by $K(p_{\theta_1},p_{\theta_2})$. It is easy to see that $K(p_{\theta_1},p_{\theta_2})$ is always non-negative and is zero if and only if $p_{\theta_1}=p_{\theta_2}$. KL divergence measures how different two probability distributions are and therefore is natural to measure convergence of the maximum likelihood procedures. However, $K(p_{\theta_1},p_{\theta_2})$ is not a distance metric because it is not symmetric and does not satisfy the triangle inequality. For this reason, two other quantities play a key role in maximum likelihood estimation, namely \textit{Hellinger Distance} and \textit{Affinity}. 
\pagebreak \\
The {\bf Hellinger distance} is defined as $$H(p_{\theta_1},p_{\theta_2})=\left(\int \left(\sqrt{p_{\theta_1}(y)}-\sqrt{p_{\theta_2}(y)}\right)^2\,\mathrm{d}y\right)^{\frac{1}{2}}\,.$$

We proved that the squared Hellinger distance lower bounds the KL divergence:
\begin{eqnarray*}
H^2(p_{\theta_1},p_{\theta_2})&\le&K(p_{\theta_1},p_{\theta_2})\\
H^2(p_{\theta_1},p_{\theta_2})&\le&K(p_{\theta_2},p_{\theta_1})\,.
\end{eqnarray*}
The {\bf affinity} is defined as 
$$A(p_{\theta_1},p_{\theta_2})=\int \sqrt{p_{\theta_1}\cdot p_{\theta_2}(y)}\,\mathrm{d}y\,.$$

we also proved that $$H^2(p_{\theta_1},p_{\theta_2})\le-2\log \left(A(p_{\theta_1},p_{\theta_2})\right)\,.$$\\


\begin{ex}[Gaussian Distribution] Y is Gaussian with mean $\theta$ and variance $\sigma^2$.
 $$p_\theta(y)=\frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(y-\theta)^2}{2\sigma^2}}$$
\end{ex}
First, look at $$\log\,\frac{p_{\theta_2}}{p_{\theta_1}}=\frac{1}{2\sigma^2}[(\theta_1^2-\theta_2^2)-2(\theta_1-\theta_2)y]$$
Then,
\begin{eqnarray*}
K(p_{\theta_1},p_{\theta_2})&=&\E_{\theta_2}\left[\log\,\frac{p_{\theta_2}}{p_{\theta_1}}\right]\\
&=&\frac{\theta_1^2-\theta_2^2}{2\sigma^2}-\frac{2(\theta_1-\theta_2)}{2\sigma^2}\underbrace{\int y\cdot p_{\theta_2}(y)\,\mathrm{d}y}_{\E[Y]=\theta_2}\\
&=&\frac{1}{2\sigma^2}(\theta_1^2+\theta_2^2-2\theta_1\theta_2) \quad = \frac{(\theta_1^2-\theta_2)^2}{2\sigma^2}.\\
-2\log\,A(p_{\theta_1},p_{\theta_2})&=&-2\log\left(\int\left(\frac{1}{\sqrt{2\pi\sigma^2}}\,e^{-\frac{(y-\theta_1)^2}{2\sigma^2}}\right)^{1/2}\cdot\left(\frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(y-\theta_2)^2}{2\sigma^2}}\right)^{1/2}\,\mathrm{d}y\right)\\
&=&-2\log\left(\int\frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(y-\theta_1)^2}{4\sigma^2}-\frac{(y-\theta_2)^2}{4\sigma^2}}\,\mathrm{d}y\right)\\
&=&-2\log\left(\int\frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{1}{2\sigma^2}\left[\left(y-\frac{\theta_1+\theta_2}{2}\right)^2+\left(\frac{\theta_1-\theta_2}{2}\right)^2\right]}\,\mathrm{d}y\right)\\
&=&-2\log\,e^{-\frac{\left(\frac{\theta_1-\theta_2}{2}\right)^2}{2\sigma^2}}\\
&=&\frac{(\theta_1-\theta_2)^2}{4\sigma^2}\quad =\frac{1}{2}K(p_{\theta_1},p_{\theta_2})\quad \ge H^2(p_{\theta_1},p_{\theta_2})\,.
\end{eqnarray*}

\pagebreak
\section{Maximum likelihood estimation and Complexity regularization}

Suppose that we have n i.i.d training samples,  $\{X_i,Y_i\}_{i=1}^n \dist_iid p_{XY}$\,.\\
Using conditional probability, $p_{XY}$ can be written as $$p_{XY}(x,y) = p_X(x)\cdot p_{Y|X=x}(y)\,.$$
Let's assume for the moment that $p_X$ is completely unknown, but $p_{Y|X=x}(y)$ has a special form:$$p_{Y|X=x}(y) = p_{f^*(x)}(y)$$

where $p_{Y|X=x}(y)$ is a known parametric density function with parameter $f^*(x)$.\\
\begin{ex}[Signal-plus-noise observation model]
$$Y_i = f^*(X_i) + W_i \quad , i=1,\ldots,n$$

where $W_i \dist_iid \mathcal{N}(0,\sigma^2)$ and $X_i \dist_iid p_X$\,.\\
$$p_{f^*(x)}(y) = \frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(y-f^*(x))^2}{2\sigma^2}}$$

$Y|X=x \sim$ Poisson$(f^*(x))$\\
$$p_{f^*(x)}(y)=e^{-f^*(x)}\frac{[f^*(x)]^y}{y!}\,.$$
\end{ex}
The {\bf likelihood loss function} is
\begin{eqnarray*}
l(f(x),y) &=& -\log\,p_{XY}(X,Y)\\
&=&-\log\,p_X(X)-\log\,p_{Y|X}(Y|X)\\
&=&-\log\,p_X(X)-\log\,p_{f(X)}(Y)\,.
\end{eqnarray*}
The \textit{expected loss} is
\begin{eqnarray*}
\E[l(f(X),Y)]&=&\E_X\left[\E_{Y|X}[l(f(X),Y)|X=x]\right]\\
&=&\E_X[\,\E_{Y|X}[-\log\,p_X(x)-\log\,p_{f(x)}(Y)|X=x]\,]\\
&=&-\E_X[\,\log\,p_X(X)\,]-\E_X[\,\E_{Y|X}[\,\log\,p_{f(x)}(Y)|X=x\,]\,]\\
&=&-\E_X[\,\log\,p_X(X)\,]-\E[\,\log\,p_{f(X)}(Y)\,]\,.
\end{eqnarray*}
Notice that the first term is a constant with respect to $f$.\\
Hence, we define our {\bf risk} to be
\begin{eqnarray*}
R(f)&=&-\E[\,\log\,p_{f(X)}(Y)\,]\\
&=&-\E_X[\,\E_{Y|X}[\log\,p_{f(x)}(Y)|X=x]\,]\\
&=&-\int\left(\int\log\,p_{f(x)}(y)\cdot p_{f^*(x)}(y)\,\mathrm{d}y\right)\,p_X(x)\,\mathrm{d}x\,.
\end{eqnarray*}

The function $f^*$ minimizes this risk since $f(x)=f^*(x)$ minimizes the integrand.\\
Our {\bf empirical risk} is the negative log-likelihood of the training samples:
$$\hat{R_n}(f) = \frac{1}{n}\sum_{i=1}^n\,-\log\,p_{f(X_i)}(Y_i)$$

The value $\frac{1}{n}$ is the \textit{empirical} probability of observing $X=X_i$.\\

Often in function estimation, we have control over where we sample X. Let's assume that $\X=[0,1]^d$ and $\Y=\R$. Suppose we sample $\X$ uniformly with $n=m^d$ samples for some positive integer $m$ (\ie,take $m$ evenly spaced samples in each coordinate).\\

Let $x_i$ ,$i=1,\ldots,n$ denote these sample points, and assume that $Y_i \sim p_{f^*(x_i)}(y)$. Then, our empirical risk is 
$$\hat{R_n}(f)=\frac{1}{n}\sum_{i=1}^n l(f(x_i),Y_i)=\frac{1}{n}\sum_{i=1}^n\,-\log\,p_{f(x_i)}(Y_i)\,.$$ Note that $x_i$ is now a deterministic quantity.\\
Our {\bf risk} is 
\begin{eqnarray*}
R(f)&=&-\frac{1}{n}\sum_{i=1}^n\,\E\left[\log\,p_{f(x_i)}(Y_i)\right]\\
&=&-\frac{1}{n}\sum_{i=1}^n\left[\int \log\,p_{f(x_i)}(y_i)\cdot p_{f^*(x_i)}(y_i)\,\mathrm{d}y_i\right]\,.
\end{eqnarray*}
The risk is minimized by $f^*$. However, $f^*$ is not a unique minimizer. Any $f$ that agrees with $f^*$ at the point $\{x_i,Y_i\}$ also minimizes this risk.\\

Now, we will make use of the following vector and shorthand notation. The uppercase $Y$ denotes a random variable, while the lowercase $y$ and $x$ denote deterministic quantities. \\
$$Y=\left[\begin{array}{c}Y_1\\Y_2\\ \vdots \\Y_n \end{array}\right] \quad y=\left[\begin{array}{c}y_1\\y_2\\ \vdots \\y_n \end{array}\right] \quad x=\left[\begin{array}{c}x_1\\x_2\\ \vdots \\x_n \end{array}\right]$$\\
Then,\\

$p_f(Y)=\prod^n_{i=1}p\left(Y_i|f(x_i)\right)$ \quad (random)\\

$p_f(y)=\prod^n_{i=1}p\left(y_i|f(x_i)\right)$ \quad (deterministic)\,.\\

With this notation, the empirical risk and the true risk can be written as
\begin{eqnarray*}
\hat{R_n}(f)&=&-\frac{1}{n}\,\log p_f(Y)\,.\\
R(f)&=&-\frac{1}{n}\E[\log p_f(Y)]\\
&=&-\frac{1}{n}\int\log p_f(y)\cdot p_{f^*}(y)\,\mathrm{d}y\,.
\end{eqnarray*}

\section{Error Bound}

Suppose that we have a pool of candidate functions $\F$, and we want to select a function $f$ from $\F$ using the training data. Our usual approach is to show that the distribution of $\hat{R_n}(f)$ concentrates about its mean as $n$ grows. First, we assign a complexity $c(f) > 0$ to each $f \in \F$ so that $\sum 2^{-c(f)} \le 1$. Then, apply the union bound to get a \textit{uniform} concentration inequality holding for all models in $\F$. Finally, we use this concentration inequality to bound the expected risk of our selected model.

We will essentially accomplish the same result here, but avoid the need for explicit concentration inequalities and instead make use of the information-theoretic bounds.\\

We would like to select an $f \in \F$ so that the excess risk is small.
\begin{eqnarray*}
0 &\le& R(f) - R(f^*)\\
&=&\frac{1}{n}\E[\log p_{f^*}(Y)-\log p_f(Y)]\\
&=&\frac{1}{n}\E\left[\log \frac{p_{f^*}(Y)}{p_f(Y)}\right]\\
&\equiv&\frac{1}{n}K(p_f,p_{f^*})
\end{eqnarray*}
where
$$K(p_f,p_{f^*})=\sum_{i=1}^n\underbrace{\left(\int \log \frac{p_{f^*(x_i)}(y_i)}{p_{f(x_i)}(y_i)}\cdot p_{f^*(x_i)}(y_i)\,\mathrm{d}y_i\right)}_{K(p_{f(x_i)},p_{f^*(x_i)})}$$
is again the KL divergence.\\

Unfortunately, as mentioned before, $K(p_f,p_{f^*})$ is not a true distance. So instead we will focus on the expected squared Hellinger distance as our measure of performance. We will get a bound on
$$\frac{1}{n}\E\left[H^2(p_f(Y),p_{f^*}(Y))\right]=\frac{1}{n}\sum_{i=1}^n\left(\int\left(\sqrt{p_{f(x_i)}(y_i)}-\sqrt{p_{f^*(x_i)}(y_i)}\right)^2\,\mathrm{d}y_i\right).$$

\section{Maximum Complexity-Regularized Likelihood Estimation}
\begin{thm}[Li-Barron 2000, Kolaczyk-Nowak 2002]

Let $\{x_i,Y_i\}_{i=1}^n$ be a random sample of training data with $\{Y_i\}$ independent,
$$Y_i \sim p_{f^*(x_i)}(y_i) \quad, i=1,\ldots,n$$
for some unknown function $f^*$.\\ Suppose we have a collection of candidate functions $\F$, and complexities $c(f)>0, f \in \F$, satisfying $$\sum_{f \in\F} 2^{-c(f)} \le 1.$$
Define the complexity-regularized estimator
$$\hat{f_n}\equiv \arg \,\min_{f \in \F}\left\{-\frac{1}{n}\sum_{i=1}^n\log\,p_f(Y_i)+\frac{2c(f)\log2}{n}\right\}\,.$$
Then,
\begin{eqnarray*}
\frac{1}{n}\E\left[H^2(p_f(Y),p_{f^*}(Y))\right] &\le& -\frac{2}{n}\E\left[\log\left(A(p_f(Y),p_{f^*}(Y))\right)\right]\\
&\le& \min_{f \in\F}\left\{\frac{1}{n}K(p_f,p_{f^*})+\frac{2c(f)\log2}{n}\right\}\,.
\end{eqnarray*}
\end{thm}
Before proving the theorem, let's look at a special case.
\pagebreak
\begin{ex}[Gaussian noise] Suppose $Y_i = f(x_i) + W_i \quad , W_i \dist_iid \mathcal{N}(0,\sigma^2)$.
$$p_{f(x_i)}(y_i)=\frac{1}{\sqrt{2\pi\sigma^2}}e^{-\frac{(y_i-f(x_i))^2}{2\sigma^2}}$$
Using results from example 1, we have
\begin{eqnarray*}
-2\log A\left(p_{\hat{f_n}}(Y),p_{f^*}(Y)\right)&=&\sum_{i=1}^n\,-2\log A\left(p_{\hat{f_n}(x_i)}(Y_i),p_{f^*(x_i)}(Y_i)\right)\\
&=&\sum_{i=1}^n\,-2\log\int\sqrt{p_{\hat{f_n}(x_i)}(y_i)\cdot p_{f^*(x_i)}(y_i)}\,\mathrm{d}y_i\\
&=&\frac{1}{4\sigma^2}\sum_{i=1}^n\left(\hat{f_n}(x_i)-f^*(x_i)\right)^2.
\end{eqnarray*}
Then,
$$-\frac{2}{n}\E\left[\log\,A(p_{\hat{f_n}},p_{f^*})\right]=\frac{1}{4\sigma^2n}\sum_{i=1}^n\E\left[\left(\hat{f_n}(x_i)-f^*(x_i)\right)^2\right].$$
We also have,
\begin{eqnarray*}
\frac{1}{n}K(p_f,p_{f^*})&=&\frac{1}{n}\sum_{i=1}^n\frac{\left(f(x_i)-f^*(x_i)\right)^2}{2\sigma^2}\\
-\log\,p_f(Y)&=&\sum_{i=1}^n\frac{(Y_i-f(X_i))^2}{2\sigma^2}\,.
\end{eqnarray*}
Combine everything together to get
$$\hat{f_n}=\arg \min_{f\in\F} \left\{\frac{1}{n}\sum_{i=1}^n\frac{(Y_i-f(X_i))^2}{2\sigma^2}+\frac{2c(f)\log2}{n}\right\}\,.$$
The theorem tells us that
$$\frac{1}{4n}\sum_{i=1}^n\E\left[\frac{\left(\hat{f_n}(x_i)-f^*(x_i)\right)^2}{\sigma^2}\right]\le \min_{f\in\F}\left\{\frac{1}{n}\sum_{i=1}^n\frac{\left(f(x_i)-f^*(x_i)\right)^2}{2\sigma^2} + \frac{2c(f)\log2}{n}\right\}$$
or
$$\frac{1}{n}\sum_{i=1}^n\E\left[\left(\hat{f_n}(x_i)-f^*(x_i)\right)^2\right]\le \min_{f\in\F}\left\{\frac{2}{n}\sum_{i=1}^n\left(f(x_i)-f^*(x_i)\right)^2 + \frac{8\sigma^2c(f)\log2}{n}\right\}.$$
\end{ex}
Now let's come back to the proof.\\
\begin{pf}
\begin{eqnarray*}
H^2\left(p_{\hat{f_n}},p_{f^*}\right)&=&\int\left(\sqrt{p_{\hat{f_n}}(y)}-\sqrt{p_{f^*}(y)}\right)^2\,\mathrm{d}y\\
&\le&-2\log\underbrace{\left(\int\sqrt{p_{\hat{f_n}}(y)\cdot p_{f^*}(y)}\,\mathrm{d}y\right)}_{affinity}
\end{eqnarray*}
$\Rightarrow$
$$\E\left[H^2\left(p_{\hat{f_n}},p_{f^*}\right)\right]\le 2\,\E\left[\log\left(\frac{1}{\int\sqrt{p_{\hat{f_n}}(y)\cdot p_{f^*}(y)}\,\mathrm{d}y}\right)\right]\,.$$
Now, define the theoretical analog of $\hat{f_n}$: 
$$f_n=\arg \min_{f\in\F}\left\{\frac{1}{n}K\left(p_f,p_{f^*}\right)+\frac{2c(f)\log2}{n}\right\}.$$
Since 
\begin{eqnarray*}
\hat{f_n}&=&\arg\min_{f\in\F}\left\{-\frac{1}{n}\log p_f(Y)+\frac{2c(f)\log2}{n}\right\}\\
&=&\arg\max_{f\in\F}\left\{\frac{1}{n}\left(\log p_f(Y)-2c(f)\log2\right)\right\}\\
&=&\arg\max_{f\in\F}\left\{\frac{1}{2}\left(\log p_f(Y)-2c(f)\log2\right)\right\}\\
&=&\arg\max_{f\in\F}\left\{\log\left(\sqrt{p_f(Y)}\cdot e^{-c(f)\log2}\right)\right\}\\
&=&\arg\max_{f\in\F}\left\{\sqrt{p_f(Y)}\cdot e^{-c(f)\log2}\right\}
\end{eqnarray*}
we can see that
$$\frac{\sqrt{p_{\hat{f_n}}(Y)}e^{-c(\hat{f_n})\log2}}{\sqrt{p_{f_n}(Y)}e^{-c(f_n)\log2}}\ge 1\,.$$ 
Then can write
\begin{eqnarray*}
\E\left[H^2\left(p_{\hat{f_n}},p_{f^*}\right)\right]&\le& 2\,\E\left[\log\left(\frac{1}{\int\sqrt{p_{\hat{f_n}}(y)\cdot p_{f^*}(y)}\,\mathrm{d}y}\right)\right]\\
&\le&2\,\E\left[\log\left(\frac{\sqrt{p_{\hat{f_n}}(Y)}e^{-c(\hat{f_n})\log2}}{\sqrt{p_{f_n}(Y)}e^{-c(f_n)\log2}}\cdot\frac{1}{\int\sqrt{p_{\hat{f_n}}\cdot p_{f^*}}\,\mathrm{d}y}\right)\right]\,.
\end{eqnarray*}
Now, simply multiply the argument inside the $\log$ by $\sqrt{\frac{p_{f^*}(Y)}{p_{f^*}(Y)}}$ to get
\begin{eqnarray*}
\E\left[H^2\left(p_{\hat{f_n}},p_{f^*}\right)\right]&\le&2\,\E\left[\log\left(\frac{\sqrt{p_{f^*}(Y)}}{\sqrt{p_{f_n}(Y)}}\frac{\sqrt{p_{\hat{f_n}}(Y)}}{\sqrt{p_{f^*}(Y)}}\frac{e^{-c(\hat{f_n})\log2}}{e^{-c(f_n)\log2}}\cdot\frac{1}{\int\sqrt{p_{\hat{f_n}}(y)\cdot p_{f^*}(y)}\,\mathrm{d}y}\right)\right]\\
&=&\E\left[\log\left(\frac{p_{f^*}(Y)}{p_{f_n}(Y)}\right)\right] + 2c(f_n)\log2 \\ & & \ \ \ \ + 2\E\left[\log\left(\frac{\sqrt{p_{\hat{f_n}}(Y)}}{\sqrt{p_{f^*}(Y)}}\cdot \frac{e^{-c(\hat{f_n})\log2}}{\int\sqrt{p_{\hat{f_n}}(y)\cdot p_{f^*}(y)}\,\mathrm{d}y}\right)\right]\\
&=&K\left(p_{f_n},p_{f^*}\right) + 2c(f_n)\log2 \\ & & \ \ \ + 2\E\left[\log\left(\frac{\sqrt{p_{\hat{f_n}}(Y)}}{\sqrt{p_{f^*}(Y)}}\cdot \frac{e^{-c(\hat{f_n})\log2}}{\int\sqrt{p_{\hat{f_n}}(y)\cdot p_{f^*}(y)}\,\mathrm{d}y}\right)\right]\,.
\end{eqnarray*}
The terms $K\left(p_{f_n},p_{f^*}\right) + 2c(f_n)\log2$ are precisely
what we wanted for the upper bound of the theorem. So, to finish the
proof we only need to show that the last term is non-positive.
Applying Jensen's inequality, we get
$$2\E\left[\log\left(\frac{\sqrt{p_{\hat{f_n}}(Y)}}{\sqrt{p_{f^*}(Y)}}\cdot
    \frac{e^{-c(\hat{f_n})\log2}}{\int\sqrt{p_{\hat{f_n}}(y)\cdot
        p_{f^*}(y)}\,\mathrm{d}y}\right)\right] \le
2\log\left(\E\left[e^{-c(\hat{f_n})\log2}\cdot\frac{\sqrt{\frac{p_{\hat{f_n}}(Y)}{p_{f^*}(Y)}}}{\int\sqrt{p_{\hat{f_n}}(y)\cdot
        p_{f^*}(y)}\,\mathrm{d}y}\right]\right)\,.$$
Both $Y$ and
$\hat{f_n}$ are random, which makes the expectation difficult to
compute. However, we can simplify the problem using the union bound,
which eliminates the dependence on $\hat{f_n}$:
\begin{eqnarray*}
 2\E\left[\log\left(\frac{\sqrt{p_{\hat{f_n}}(Y)}}{\sqrt{p_{f^*}(Y)}}\cdot \frac{e^{-c(\hat{f_n})\log2}}{\int\sqrt{p_{\hat{f_n}}(y)\cdot p_{f^*}(y)}\,\mathrm{d}y}\right)\right]&\le&  2\log\left(\E\left[\sum_{f\in\F}e^{-c(f)\log2}\cdot\frac{\sqrt{\frac{p_f(Y)}{p_{f^*}(Y)}}}{\int\sqrt{p_f(y)\cdot p_{f^*}(y)}\,\mathrm{d}y}\right]\right)\\
&=&2\log\left(\sum_{f\in\F}2^{-c(f)}\frac{\E\left[\sqrt{\frac{p_f(Y)}{p_{f^*}(Y)}}\right]}{\int\sqrt{p_f(y)\cdot p_{f^*}(y)}\,\mathrm{d}y}\right)\\
&=&2\log\left(\sum_{f\in\F}2^{-c(f)}\right)\\
&\le&0.
\end{eqnarray*}
where the last two lines come from 
$$\E\left[\sqrt{\frac{p_f(Y)}{p_{f^*}(Y)}}\right]=\int\sqrt{\frac{p_f(y)}{p_{f^*}(y)}}\cdot
p_{f^*}(y)\,\mathrm{d}y=\int\sqrt{p_f(y)\cdot
  p_{f^*}(y)}\,\mathrm{d}y$$
and
$$\sum_{f\in\F}2^{-c(f)}\le 1.$$

\end{pf}

\end{document}
