\documentclass{article}
\usepackage{fullpage, epsf, setspace, latexsym, cite}
\usepackage{xspace, amsmath, amssymb, subfigure, hyperref}
\usepackage{graphicx}

\newtheorem{thm}{Theorem}
\newtheorem{prop}{Proposition}
\newtheorem{cor}{Corollary}
\newtheorem{lemma}{Lemma}
\newtheorem{ex}{Example}
\newtheorem{exse}{Exercise}
\newtheorem{defn}{Definition}

\def\QED{\mbox{\rule[0pt]{1.5ex}{1.5ex}}}
\def\endproof{\hspace*{\fill}~\QED\par\endtrivlist\unskip}
\newenvironment{pf}{\begin{verse}{\bf Proof:}\rm}{\endproof \end{verse}}
\newcommand{\1}[1]{\ensuremath{{\bf 1}_{\{#1\}}}}

\newcommand{\lecture}[4]{
   \pagestyle{myheadings}
   \thispagestyle{plain}
   \newpage
   \setcounter{page}{1}
   \noindent
   \begin{center}
   \framebox{
      \vbox{\vspace{2mm}
    \hbox to 6.28in { {\bf ECE901 Spring 2004 Statistical Regularization and Learning Theory \hfill Lecture: #4} }
       \vspace{6mm}
       \hbox to 6.28in { {\Large \hfill #1  \hfill} }
       \vspace{6mm}
       \hbox to 6.28in { {\it Lecturer: #2 \hfill Scribe: #3} }
      \vspace{2mm}}
   }
   \end{center}
   \markboth{#1}{#1}
   \setlength{\headsep}{10mm}
   \vspace*{4mm}
}

%notation

\def\X{\ensuremath{{\mathcal X}}\xspace}
\def\Y{\ensuremath{{\mathcal Y}}\xspace}
\def\F{\ensuremath{{\mathcal F}}\xspace}
\def\R{\ensuremath{{\bf R}}\xspace}
\def\E{{E}}
\def\sign{\ensuremath{\mbox{sign}}}
\def\ie{{\em i.e., }}
\def\eg{{\em e.g., }}
\def\as{\mbox{ as }}
\def\dist_iid{\stackrel{i.i.d.}{\sim}}
\def\h{\widehat}

%Lecture dependent definitions, can be erased if necessary in other lecture notes
\def\I{{\mbox{I}_{j,m}}}

\begin{document}

\lecture{Denoising in Smooth Function Spaces II-Adapting to
Unknown Smoothness}{Rob Nowak}{Mike Nowak}{15}

\section{Review: Denoising in Smooth Function Spaces I \\ - Method of Sieves}

Suppose we make noisy measurements of a smooth function:
$$Y_i=f^*(x_i)+W_i, \ i=\{1,\ldots,n\},$$ where
$$W_i\dist_iid N\left(0,\sigma^2\right)$$ and $$x_{i}=\left(\frac{i}{n}\right).$$
The unknown function $f^{*}$ is a map $$f^*:[0,1]\rightarrow\R$$
In Lecture 4, we consider this problem in the case where $f^{*}$
was Lipschitz on $[0,1].$  That is, $f^{*}$ satisfied
$$|f^*(t)-f^*(s)|\leq L|t-s|,\ \hspace{2mm} \forall t,s\in [0,1]$$
where $L>0$ is a constant. In that case, we showed that by using a
piecewise constant function on a partition of $n^{\frac{1}{3}}$
equal-size bins (Figure~\ref{fig:sieve}) we were able to obtain an
estimator $\h{f}_n$ whose mean square error was
$$E\left[\|f^*-\h{f}_n\|^2\right] = O\left(n^{-\frac{2}{3}}\right)$$
\begin{figure}[h]\label{fig:sieve}
\centering
\includegraphics[width=7cm]{sieve.pdf}\\
\caption{Example of the piecewise constant approximation of $f^*$}
\end{figure}

\noindent In this lecture we will use the Maximum
Complexity-Regularized Likelihood Estimation result we derived in
Lecture 14 to extend our denoising scheme in several important
ways.

\vspace*{5mm} \noindent To begin with let's consider a broader
class of functions.

\section{H\"{o}lder Spaces}
For $0 < \alpha < 1,$ define the space of functions
$$H^{\alpha}(C_{\alpha}) =\left\{
|f|<C_{\alpha} : \hspace{1 mm} \sup_{x,h}
\frac{|f(x+h)-f(x)|}{|h|^{\alpha}} \leq C_{\alpha}\right\}$$ for
some constant $C_{\alpha} < \infty $ and where $f \in L_{\infty}.$
$H^{\alpha}$ above contains functions that are bounded, but less
smooth than Lipschitz functions.  Indeed, the space of Lipschitz
functions can be defined as $H^{1}$ ($\alpha=1$)
$$H^{1}(C_{1}) =\left\{
|f|<C_{1} : \hspace{1 mm} \sup_{x,h} \frac{|f(x+h)-f(x)|}{|h|}
\leq C_{1}\right\}$$ for $C_{1} < \infty .$  Functions in $H^{1}$
are continuous, but those in $H^{\alpha}, \alpha < 1,$ are not in
general.

\vspace*{5mm} \noindent Let's also consider functions that are
smoother than Lipschitz. If $\alpha = 1 + \beta,$ where $0 < \beta
< 1,$ then define
$$H^{\alpha}(C_{\alpha}) =\left\{f \in H^{1}(C_{\alpha}) :
\frac{\partial f}{\partial x} \in H^{\beta}(C_{\alpha}) \right\}$$
In other words, $H^{\alpha}, 1 < \alpha < 2,$ contains Lipschitz
functions that are also differentiable \textit{\textbf{and}} their
derivatives are H\"{o}lder smooth with smoothness $\beta =
\alpha-1.$

\vspace*{5mm} \noindent And finally, let
$$H^{2}(C_{2}) =\left\{f :
\frac{\partial f}{\partial x} \in H^{1}(C_{2}) \right\}$$ contain
functions that have continuous derivatives, but that are not
necessarily twice-differentiable.  

 \vspace*{5mm} \noindent If $f
\in H^{\alpha}(C_{\alpha})$, $0 < \alpha \leq 2,$ then we say that
$f$ is H\"{o}lder$- \alpha$  smooth with H\"{o}lder constant
$C_{\alpha}.$  The notion of H\"{o}lder smoothness can also be
extended to $\alpha >2$ in a straightforward way.

 \vspace*{5mm} \noindent  \textbf{Note:} If $\alpha_{1} < \alpha_{2}$ then
$$ f \in H^{\alpha_{2}} \Rightarrow f \in H^{\alpha_{1}} $$

\noindent Summarizing, we can describe H\"{o}lder spaces as
follows. If $f^{*} \in H^{\alpha}\left(C_{\alpha}\right)$ for some
$0 < \alpha \leq 2$ and $C_{\alpha} < \infty,$ then
\begin{description}
\item[(i)] $ 0 < \alpha \leq 1 $ \hspace{25mm}
$|f^{*}(t)-f^{*}(s)| \leq C_{\alpha} |t-s|^{\alpha}$
 \item[(ii)]
$ 1 < \alpha \leq 2$ \hspace{25mm} $\left|\frac{\partial
f^{*}}{\partial x}(t) - \frac{\partial f^{*}}{\partial
x}(s)\right| \leq C_{\alpha} |t-s|^{\alpha -1}$
\end{description}
Note that in general there is a natural relationship between the
H\"{o}lder space containing the function and the approximation class
used to estimate the function.  Here we will consider functions which
are H\"{o}lder$-\alpha$ smooth where $ 0 < \alpha \leq 2 $ and work
with piecewise linear approximations. If we were to consider smoother
functions, $ \alpha > 2 $ we would need consider higher order
approximation functions, i.e. quadratic, cubic, etc.

\section{Denoising Example for Signal-plus-Gaussian Noise Observation Model}
Now let's assume $f^{*} \in H^{\alpha}(C_{\alpha})$ for some
\textbf{\textit{unknown}} $\alpha$ ($ 0 < \alpha \leq 2) $; i.e.
we don't know how smooth $f^{*}$ is.  We will use our observations
$$Y_i=f^*(x_i)+W_i, \ i=\{1,\ldots,n\},$$ to construct an
estimator $\h{f}_{n}.$  Intuitively, the smoother $f^{*}$ is, the
better we should be able to estimate it.  Can we take advantage of
extra smoothness in $f^{*}$ if we don't know how smooth it is? The
smoother $f^{*}$ is, the more averaging we can perform to reduce
noise.  In other words for smoother $f^{*}$ we should average over
larger bins.  Also, we will need to exploit the extra smoothness
in our approximation of $f^{*}.$  To that end, we will consider
candidate functions that are piecewise \textbf{\textit{linear}}
functions on uniform partitions of $[0,1].$ Let
$$\F_k = \left\{ \begin{array}{ll}
|f|\leq C: & \mbox{\textit{ f is piecewise linear on }}
\left[0,\frac{1}{k}\right),\left[\frac{1}{k},\frac{2}{k}\right),\ldots
\left[\frac{k-1}{k},1\right) \mbox{\textit{ and the }}\\
 & \mbox{\textit{ coefficients of each line segment are quantized
 to}} \frac{1}{2}\log n \mbox{\textit{ bits.}} \end{array} \right\} $$

\begin{figure}[htb]
\centering
\includegraphics[width=9cm]{Quantized_f_Fig.pdf}\\
 \caption{Example on the quantization of $f$ on interval
$\left[\frac{i-1}{k},\frac{i}{k}\right)$}
\label{fig:Quantized_f_Fig}
\end{figure}

\noindent The start and end points of each line segment are each
one of $\sqrt{n}$ discrete values, as indicated in
Figure~\ref{fig:Quantized_f_Fig}. Since each line may start at any
of the $\sqrt{n}$ levels and terminate at any of the $\sqrt{n}$
levels, there are a total of $n$ possible lines for each segment.



\noindent Given that there are $k$ intervals we have
$$ \left| \F_k \right| = n^{k} \Rightarrow \log \left| \F_k
\right|= k \log n $$ Therefore we can use $k \log n $ bits to
describe a function $f \in \F_k$.

\noindent Let $$\F = \bigcup_{k \geq 1} \F_k .$$ Construct a
prefix code for every $f \in \F$ by
$$ \begin{array}{ll}
\mbox{(i)} &  \mbox{Use } \underbrace{000 \dots 1}_{k
\hspace{.5mm} bits} \mbox{ to encode the smallest } k \mbox{ such
that } f \in \F_k
\vspace*{2mm} \\
\mbox{(ii)} & \mbox{Use } k \log n \mbox{ bits to
encode which element of } \F_k \mbox{ we
 are considering.}
 \end{array} $$

\noindent Thus, if $f \in \F_k$, then the prefix code associated
with $f$ has codeword length $$ c(f) = k + k \log n = k(1 + \log
n)$$ which satisfies the Kraft Inequality $$ \sum_{f \in \F}
2^{-c(f)} \leq 1.$$

\noindent Now we will apply our complexity regularization result
to select a function $\widehat{f}_{n}$ from $\F$ and bound its risk.
We are assuming Gaussian errors, so
$$ -\log p_{f}(Y_i) = \frac{\left(Y_i - f\left(\frac{i}{n}\right)\right)^2}{2\sigma^{2}}
+ constant.$$ We can ignore the constant term and so our empirical
selection is
$$ \widehat{f}_n = \arg \min_{f \in \F} \left\{ \frac{1}{n}
\sum_{i=1}^{n}  \frac{\left(Y_i -
f\left(\frac{i}{n}\right)\right)^2}{2\sigma^{2}} +
\frac{2c(f)\log2}{n} \right \}$$ We can compute $\widehat{f}_n$
according to:

\vspace*{3mm} \noindent For $k=1,\ldots, n$
$$ \widehat{f}_n^{(k)} = \arg \min_{f \in \F_k} \hat{R_n}(f)
=\arg \min_{f \in \F_k} \frac{1}{n} \sum_{i=1}^{n} \frac{\left(Y_i
- f\left(\frac{i}{n}\right)\right)^2}{2\sigma^{2}} $$ then select
$$ \hat{k}=\arg \min_{k=1,\ldots,n} \left\{
\hat{R_n}\left(\widehat{f}_n^{(k)}\right)+ \frac{2k(1 + \log
n)\log2}{n} \right\}$$ and finally
$$\widehat{f}_n=\widehat{f}_n^{(\hat{k})}.$$

\vspace*{5mm} \noindent Because the KL divergence and $-2\log$
\textit{affinity} simply reduce to squared error in the Gaussian
case (Lecture 14), we arrive at a relatively simple bound on the
mean square error of $\widehat{f}_n$
$$ \frac{1}{n} \sum_{i=1}^n E \left[ \left(
\widehat{f}_n\left(\frac{i}{n}\right) - f^*\left(\frac{i}{n}\right)
\right)^2 \right] \leq \min_{f \in \F} \left\{ \frac{2}{n}
\sum_{i=1}^n \left( f \left(\frac{i}{n}\right) -
f^*\left(\frac{i}{n}\right) \right)^2 + \frac{8\sigma^2 c(f)
\log2}{n} \right\}$$ The first term in the brackets above is
related to the error incurred by approximating $f^*$ by an element
of $\F.$  The second term is related to the estimation error
involved with the model selection process.

\vspace*{5mm} \noindent Let's focus on the approximation error.
First, suppose $f^{*} \in H^{\alpha}\left( C_{\alpha}\right)$ for
$1 < \alpha \leq 2.$ Let $f^{*}_{k}$ be the ``best" piecewise
linear approximation to $f^{*}$, with $k$ pieces on intervals $
\left[0,\frac{1}{k}\right),\left[\frac{1}{k},\frac{2}{k}\right),\ldots
\left[\frac{k-1}{k},1\right).$   Consider the difference between
$f^*$ and $f_{k}^{*}$ on one such interval, say
$\left[\frac{i-1}{k},\frac{i}{k}\right).$  By applying Taylor's
theorem with remainder we have
$$ f^*(t) = f^* \left( \frac{i}{k} \right) + \frac{\partial
f^*}{\partial x} (t') \left(t-\frac{i}{k}\right)$$ for $t \in
\left[\frac{i-1}{k},\frac{i}{k}\right)$ and some $ t' \in \left[
t, \frac{i}{k} \right] $ . Define
$$
f_{k}^*(t) \equiv f^* \left( \frac{i}{k} \right) + \frac{\partial
  f^*}{\partial x} \left( \frac{i}{k} \right)
\left(t-\frac{i}{k}\right).$$
Note that $f_{k}^*(t)$ is not
necessarily the best piecewise linear approximation to $f^{*}$, just
good enough for our purposes.  Then using the fact that $f^* \in
H^{\alpha}\left( C_{\alpha}\right)$, for $t\in [i-1/k,i/k)$ we have
\begin{eqnarray*}
\left| f^*(t) - f_{k}^{*}(t) \right| &=& \left| \frac{\partial
f^*}{\partial x}(t')\left(t-\frac{i}{k}\right) - \frac{\partial
f^*}{\partial x} \left(\frac{i}{k}\right) \left(t - \frac{i}{k}
\right) \right|\\
&\leq& \frac{1}{k}  \left| \frac{\partial f^*}{\partial x}(t') -
\frac{\partial f^*}{\partial x} \left(\frac{i}{k}\right) \right|
\\
&\leq& \frac{1}{k} C_{\alpha} \left| t' - \frac{i}{k}
\right|^{\alpha-1}\\
&\leq& \frac{1}{k} C_{\alpha} \left( \frac{1}{k}
\right)^{\alpha-1} = C_{\alpha} k^{-\alpha}.
\end{eqnarray*}

\noindent So, for all $t \in [0,1]$
$$ \left|f^*(t) - f_{k}^{*}(t) \right| \leq C_{\alpha}
k^{-\alpha}.$$

\vspace*{5mm} \noindent Now let $f_{k}$ be the element of $\F_{k}$
closest to $f_{k}^{*}$ ($f_k$ is the quantized version of $f^*_k$)
\begin{eqnarray*}
\left|f^*(t) - f_{k}(t) \right| &=& \left|f^*(t) - f^{*}_{k}(t) +
f^{*}_{k}(t) - f_{k}(t) \right|\\
& \leq & \left|f^*(t) - f^{*}_{k}(t)\right| + \left| f^{*}_{k}(t)
- f_{k}(t) \right|\\
& \leq & C_{\alpha} k^{-\alpha} + \frac{1}{\sqrt{n}}
\end{eqnarray*} since we used $\frac{1}{2} \log n$ bits to quantize
the endpoints of each line segment. Consequently, \begin{eqnarray*}
  \left| f^*(t) - f_{k}^{*}(t) \right|^{2}& \leq & \left| f^*(t) -
    f_{k}^{*}(t) \right|^{2} + 2 \left| f^*(t) - f_{k}^{*}(t) \right|
  \left| f_{k}^{*}(t) - f_{k}(t) \right| + \left| f_{k}^{*}(t) -
    f_{k}(t) \right|^{2}\\
  & \leq & C_{\alpha}^{2}k^{-2 \alpha} + 2 C_{\alpha}
  \frac{k^{-\alpha}}{\sqrt{n}} + \frac{1}{n}. \end{eqnarray*} Thus it
follows that $$
\min_{f \in \F_{k}} \left\{ \frac{2}{n} \sum_{i=1}^n
  \left( f (i/n) - f^*(i/n) \right)^2 + \frac{8 \sigma^{2} c(f) \log
    2}{n} \right\} \leq 2 C^{2}_{\alpha} k^{-2 \alpha} +
\frac{4C_{\alpha}k^{-\alpha}}{\sqrt{n}} + \frac{2}{n} +
\frac{8\sigma^2k(\log n + 1)\log2}{n}.$$
The first and last terms
dominate the above expression. Therefore, the upper bound is minimized
when $k^{-2\alpha}$ and $\frac{k}{n}$ are balanced.  This is
accomplished by choosing $k = \lfloor n^{\frac{1}{2 \alpha +
    1}}\rfloor.$ Then it follows that
$$ \min_{f \in \F_{k}} \left\{ \frac{2}{n} \sum_{i=1}^n \left(
f \left( \frac{i}{n}\right) - f^*\left(\frac{i}{n}\right)
\right)^2 + \frac{8\sigma^{2} c(f)\log 2}{n} \right\} = O \left(
n^{-\frac{2 \alpha}{2 \alpha + 1}} \log n \right).$$ If $\alpha
=2$ then we have
$$ \frac{1}{n} \sum_{i=1}^n E \left[ \left(
\widehat{f}_n\left(\frac{i}{n}\right) - f^*\left(\frac{i}{n}\right)
\right)^2 \right] = O \left( n^{-\frac{4}{5}} \log n \right)$$

\vspace*{5mm} \noindent If $f^* \in H^{\alpha} \left( C_{\alpha}
\right)$ for $ 0 < \alpha \leq 1$, let $f^{*}_{k}$ be the following
piecewise constant approximation to $f^{*}.$  Let
$$ f^{*}_{k}(t) \equiv f^{*}\left( \frac{i}{n} \right) \mbox{ on
interval } \left[ \frac{i-1}{k},\frac{i}{k} \right).$$ Then
\begin{eqnarray*}
\left| f^*(t) - f^{*}_{k}(t) \right| &=& \left| f^*(t) - f^{*}
\left( \frac{i}{n} \right) \right|\\
& \leq & C_{\alpha} \left|t - \frac{i}{n} \right|^{\alpha}\\
& \leq & C_\alpha k^{-\alpha}. \end{eqnarray*} Repeating the same
reasoning as in the $ 1 < \alpha \leq 2$ case, we arrive at
$$ \frac{1}{n} \sum_{i=1}^n E \left[ \left(
\widehat{f}_n\left(\frac{i}{n}\right) - f^*\left(\frac{i}{n}\right)
\right)^2 \right] = O \left( n^{-\frac{2 \alpha}{2 \alpha + 1}}
\log n \right)$$ for $ 0 < \alpha \leq 1.$ In particular, for
$\alpha=1$ we get $$ \frac{1}{n} \sum_{i=1}^n E \left[ \left(
\widehat{f}_n\left(\frac{i}{n}\right) - f^*\left(\frac{i}{n}\right)
\right)^2 \right] = O \left( n^{-\frac{2}{3}} \log n \right)$$
within a logarithmic factor of the rate we had before (in Lecture
4) for that case!

\section{Summary}
\begin{enumerate}
\item $\widehat{f}_n$ can be computed by finding least-square line
fits to the data on partitions of the form $
\left[0,\frac{1}{k}\right),\left[\frac{1}{k},\frac{2}{k}\right),\ldots
\left[\frac{k-1}{k},1\right)$ for $k=1,\ldots,n,$ and then
selecting the best fit by the $\hat{k}$ that gives the minimum of
the complexity regularization criterion. \item If $f^* \in
H^{\alpha} \left( C_{\alpha} \right)$ for some $ 0 < \alpha \leq
2,$ then $$ MSE \left( \widehat{f}_n \right) = \frac{1}{n}
\sum_{i=1}^n E \left[ \left( \widehat{f}_n\left(\frac{i}{n}\right) -
f^*\left(\frac{i}{n}\right) \right)^2 \right] = O \left(
n^{-\frac{2 \alpha}{2 \alpha + 1}} \log n \right).$$ \item
$\widehat{f}_n$ \textbf{\textit{automatically}} picks the optimal
number of bins. Essentially $\widehat{f}_n$ (indirectly) estimates the
smoothness of $f^*$ and produces a rate which is near minimax
optimal ! ($n^{-\frac{2 \alpha}{2 \alpha + 1}}$ is the best
possible). \item The larger $\alpha$ is the faster the convergence
and the better the denoising !
\end{enumerate}


\end{document}
