\documentclass[12pt]{article}
\usepackage{latexsym}
\usepackage{amssymb,amsmath, amsthm}
\usepackage{amsfonts}
\usepackage{graphicx}

%\numberwithin{equation}{section}
\numberwithin{table}{section} \numberwithin{figure}{section}

%\oddsidemargin -0.1in \textwidth 6.5in \textheight 8.5in
\usepackage[left=1in,top=1in,right=1in]{geometry}
\usepackage{fancyhdr}
\pagestyle{fancy}
\renewcommand{\headrulewidth}{0pt}
\rhead{Jimin Ding}
\lhead{Math 5062 Spring 2018}


\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\vb}[1]{\mbox{\boldmath{$#1$}}}

\newcommand{\indep}{\rotatebox[origin=c]{90}{$\models$}}
\newcommand{\iid}{\overset{\text{iid}}{\sim}}

\newcommand{\toas}{\overset{a.s}{\longrightarrow}}
\newcommand{\toprob}{\overset{p}{\longrightarrow}}
\newcommand{\toDist}{\overset{D}{\longrightarrow}}

\newtheorem{theorem}{Theorem}[section]
%\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}


% for specifying a name

\newtheoremstyle{named}{}{}{\itshape}{}{\bfseries}{.}{.5em}{\thmnote{#3 }#1}
\theoremstyle{named}
\newtheorem*{namedthm}{Theorem}



%\newenvironment{proof}[1][Proof]{\begin{trivlist}
%\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{definition}[1][Definition]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{example}[1][Examples:]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{remark}[1][Remarks:]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}



\newcommand{\Remark}{\noindent \textbf{Remarks:} }
\newcommand{\Example}{\noindent \textbf{Examples:}}
%\newcommand{\qed}{\nobreak \ifvmode \relax \else
%      \ifdim\lastskip<1.5em \hskip-\lastskip
%      \hskip1.5em plus0em minus0.5em \fi \nobreak
%      \vrule height0.75em width0.5em depth0.25em\fi}



\begin{document}
\title{Math 5062: Bayesian Models}
\author{Jimin Ding}
\date{}
\maketitle

\newpage

\section{Lecture 1: Decision Theory Framework}

Goals of statistic analysis: estimation, prediction, testing hypothesis, 
ranking. Examples on page 16 of Bickel and Docksum (2001). These examples 
motivate the 
decision theoretic framework: we need to
\begin{enumerate}
 \item clarify the objectives of a study,
 \item point to what the different possible actions are,
 \item provide assessments of risk, accuracy, and reliability of statistical 
procedures,
\item provide guidance in the choice of procedures for analyzing outcomes of 
experiments.
\end{enumerate}

We begin with a statistical model with observations $X$ whose distribution 
$P_{\theta} \in \mc{P}$. Now we define the four components of the decision 
theory framework. 

\medskip
\noindent \underline{\textbf{Action space}}: the space of 
actions or decisions or claims that we can contemplate making, often denoted 
by $\mathbb{A}$.

\begin{example}
 \item 1. Estimation:  $\mathbb{A}=\mathbb{R}$
 \item 2. Testing: $\mathbb{A}={0,1}$
 \item 3. Ranking: $\mathbb{A}=\{\text{permutations } (i_1,\cdots, i_k) \text{ 
of } 
(1,\cdots,k) \}$
 \item 4. Prediction: Here $\mathbb{A}$ is usually much larger. If the 
response variable $Y$ is real, and the covariate $X \in \mc{X}$, 
then $\mathbb{A}=\{a: a \text{ is a function from } \mc{X} \text{ to } 
\mathbb{R}\}$ with $a(x)$ representing the prediction we would make if the new 
unobserved $Y$ had covariate value $x$.  
\end{example}

\medskip
\noindent \underline{\textbf{Loss function}}: a function that quantifies the 
loss for a given target and an action. $$L: \mc{P} \times 
\mathbb{A} \to \mathbb{R}^+. $$
\vspace{-0.5in}
\begin{example}
(In parametric models, $\mc{P}$ can be indexed by $\theta, \theta\in \Theta$)
 \item $L(\theta,a)=(g(\theta)-a)^2$ Square/quadratic loss function.
 \item $L(\theta,a)=|g(\theta)-a|$ Absolute loss function.
 \item $L(\theta,a)=\min\{ (g(\theta)-a)^2, d^2\}$ Truncated quadratic loss 
function.
\item $L(\theta,a)=1, \text{ if } |g(\theta)-a| > d$, 0 otherwise. 
Confidence interval loss function.
\item $L(\theta,a)=\mathbf{1}(g(\theta)<a)$, Asymmetric loss function.\\

 For $\mathbb{A} \in \mathbb{R}^k, k>1$: 
\item $L(\theta,a)={1\over k} \sum_{j}(g_j(\theta) -a_j)^2$ Squared 
Euclidean distance.
\item $L(\theta,a)= {1\over k} \sum_{j}|g_j(\theta) -a_j|$ Absolute distance
\item  $L(\theta,a)= \max \{|g_j(\theta) -a_j|, j=1,\cdots, k\}$ Supremum 
distance
 \end{example}
\medskip
\noindent \underline{\textbf{Decision procedures}}: any function from the 
sample space taking its values in $\mathbb{A}$, often denoted by $\delta$. If 
data 
$X=x$ is observed, the statistican takes action $\delta(x)$.\\ Question: How to 
choose an ``optimal'' decision/action/procedure? 


\medskip
\noindent \underline{\textbf{Risk function}}: the average loss of the sample 
space. If $\delta$ action is used, $L$ 
is the loss function, $\theta$ is the true value of the parameter, and $X=x$ is 
the outcome of the experiment. Then $L(\theta,\delta(x))$ depends on the 
particular outcome of the experiment. We may want to evaluate the properties of 
the action more based on overall or averaged loss.
\[
 R(\theta,\delta)=E[L(\theta,\delta(X))],
\]
where the expectation is w.r.t $X$ and may involve unknown parameter $\theta$. 
\\
Example: The risk function defined by squared error loss 
function: MSE=Bias$^2+$Variance. \\

The optimal action will be the action that minimizes the risk function. 
However, the risk function depends on the unknown parameter $\theta$, and it is 
usually very hard to find an action that uniformaly minimizes the risk 
function. One way to solve the problem is to restrict our attention to 
``unbiased'' actions and then find the action uniformaly minimizes the risk 
function among the unbiased actions. For example, UMVUE. Another way is to 
aggregate the risk function over all possible values of $\theta \in \Theta$ 
with some weight function $\Pi(\theta)$ and 
then minimizes the aggregated risk. That is, 
\[
 r(\Pi, \delta)=\int_{\Theta} R(\theta, \delta) d\Pi,
\]
where $\Pi$ is a known probability measure that gives the weight of $\theta$. 
This $r(\Pi,\delta)$ is called the 
\textbf{Bayes risk}, and the optimal action that minimizes the Bayes risk is 
called \textbf{Bayes action/rule}, denoted by $\delta^*$.  A third method is to 
consider the worst situation, i.e. $\sup_{\theta \in \Theta} R(\theta, 
\delta)$. 
The resulting action that minimizes  $\sup_{\theta \in \Theta} R(\theta, 
\delta)$ is called the \textbf{minimax rule}.  

\begin{example}{\bf{Normal-Normal model}} 
\\
Let $\vb{X}=(X_1,\cdots, X_n)^T$. Assume
 \[
  X_1, \cdots, X_n |\mu \iid N(\mu,\sigma^2),
 \]
 with a known $\sigma^2$ and 
\[
 \mu \sim N(\mu_0,\sigma_0^2) .
\]
Here $\mu_0$ and $\sigma_0^2$ are 
hyperparameters. Then given the observed data, 
$$
 \mu |\vb{X}=\vb{x} \sim N(\mu_*(\vb{x}),c^2),
$$
where 
\[
 \mu_*(\vb{x})=w \mu_0 + 
(1-w)\bar{x}, \quad c^2= w \sigma_0^2, \quad 
 w=\frac{\sigma^2}{n\sigma_0^2+\sigma^2}.
\]
If we consider squared error loss function and estimation of $g(\theta)=\mu$, 
then
\[
 r(\Pi, \delta)=\int_{\Theta} R(\theta, \delta) 
d\Pi=E\left(E\{[\mu-\delta(\vb{X})]^2|\mu\}\right)
=E\left([\mu-\delta(\vb { X}) ] ^2\right).
\]
The problem of minimizing the Bayes risk can be hence viewed as the prediction 
of $\mu$ using $\vb{X}$. The best predictor is $E(\mu|\vb{X})$. So the Bayes 
decision is
\[
 \delta^*(\vb{x})=\arg\min_{\delta \in \mathbb{A}}
r(\Pi,\delta)=E(\mu|\vb{X}=\vb{x}),
\]
where the 
expectation is w.r.t 
the conditional distribution of $\mu$ given $\vb{X}=\vb{x}$.
In this case, 
$\delta^*(\vb{x})$ is the Bayes estimator of $\mu$.
\medskip

Now let's find the minimax rule in this example. For any decision rule 
$\delta$, 
\[
 \sup_{\theta \in \Theta} R(\theta, \delta) \ge  \int_{\Theta} R(\theta, 
\delta) 
d\Pi \ge 
r(\Pi, \delta^*) = E\left([\mu-\delta^*(\vb{X}) ] ^2\right)= 
E\{E\left([\mu-\delta^*(\vb{X}) ] ^2 |X \right)\} =E(c^2)=c^2.
\]
Since this result is true for any $\sigma_0^2>0$, let $\sigma_0^2 \to \infty$,
\[
  \sup_{\theta \in \Theta} R(\theta, \delta) \ge \sigma^2/n = \sup_{\theta \in 
\Theta} R(\theta, \bar{X}).  
\]
Hence $\bar{X}$ is the minimax estimator of $\mu$.

\end{example}

\subsection*{Related Reading}
\begin{enumerate}
 \item Sh P113-116  %\cite{shao03}
  \item BD chapter 1.2-1.3 %\cite{bd01}
  
\end{enumerate}

\newpage
\section{Lecture 2: Bayesian Model and Estimation}
\underline{Key}: view model parameter $\theta$ as a realization of a random 
variable 
from 
some prior distribution $\Pi$. Two important distributions:
\begin{itemize}
 

\item Prior distribution: based on history/prior information/subjective 
belief. $\Pi(\theta,\xi)$, where $\xi$ are \textbf{hyperparameters} which might 
be further modeled in a hierarchical model. 

\item  Posterior distribution: conditional distribution of the model parameters 
given observed data. $P(\theta|\vb{X})$.

\end{itemize}
\begin{theorem}[\textbf{Bayes Formula}]
Assume $\mc{P}=\{P(x|\theta):\theta\in \Theta\}$ is dominated by a 
$\sigma$-finite measure $\mu$ with pdf $f(x|\theta)=\frac{dP(x|\theta)}{d\mu}$. 
Let $\Pi$ be a prior distribution on $\Theta$. Then the marginal pdf of $X$ 
w.r.t. $\mu$ is $m(x)=\int_{\Theta} f(x|\theta) d\Pi$. Suppose $m(x)>0$. Then
\begin{enumerate}
 \item The posterior $P(\theta|x) \ll \Pi$ with pdf 
$\frac{dP(\theta|X)}{d\Pi}=\frac{f(x|\theta)}{m(x)}$.
\item If $\Pi \ll \lambda$ with pdf $\frac{d\Pi}{d\lambda}=\pi(\theta)$, where 
$\lambda$ is a $\sigma$-finite measure, then the posterior $P(\theta|x) \ll
\lambda$ with pdf 
$\frac{dP(\theta|x)}{d\lambda}=\frac{f(x|\theta)\pi(\theta)}{m(x)}$.
\end{enumerate}

\end{theorem}
\begin{proof}
 Use Fubini's theorem. Detail see page 232 Shao (2003)%\cite{shao03}.
\end{proof}

If both $X$ and $\theta$ are discrete and $\mu$ and $\lambda$ are the counting 
measures, then above theorem becomes the Bayes formula in elementary statistics:
\[
P(\theta=i|X=j)=\frac{P(X=j|\theta=i)P(\theta=i)}{
\sum_i P(X=j|\theta=i)P(\theta=i) } . 
\]

\begin{definition}[Bayes risk:] $$r(\Pi,\delta)=\int_{\Theta}R(\theta,\delta)d 
\Pi(\theta), \quad \text{ 
where } R(\theta,\delta)=E(L(\theta,\delta)).$$
\end{definition}

\begin{definition} [Bayes rule:] $$\delta^*(x)=\arg\min_{\delta \in \mathbb{A}} 
r(\Pi,\delta) 
=\arg\min_{\delta \in \mathbb{A}}  E(L(\theta,\delta(x))|X=x),$$ where the 
expectation is w.r.t. $P(\theta|x)$. We call $\delta^*$ the \textbf{Bayes 
estimator}, if the action is an estimation. 
\end{definition}




\begin{theorem}[\textbf{Existence and Uniqueness of Bayes Estimator}]
 Assume the conditions in the previous theorem hold. Assume $L(\theta,a)$ is 
convex in $a$ for each given $x\in\mc{X}$ and $E(L(\theta,\delta(x))|X=x) 
<\infty $ for some $\delta$ (there exists an estimator with finite risk.) 
Furthermore, assume $\mathbb{A}$ is compact or $L(\theta,a) \to +\infty$ as 
$\|a\| 
\to +\infty$ uniformaly in $\theta$. Then
\begin{enumerate}
 \item Bayes estimator 
 $$\delta^*=\arg\min_{\delta \in \mathbb{A}}  E(L(\theta,\delta(x))|X=x)$$
 exists.
 \item If $L$ is strictly convex, then $\delta^*$ is unique. 
\end{enumerate}
\end{theorem}

\begin{proof}
See page 228 in Lehmann and Casella (1998). %\cite{lc98}. 
\end{proof}

\begin{example}
 \item 1. $L(\theta,a)=[g(\theta)-a]^2$, then 
 \[
  \delta^*(x)=E(g(\theta)|X=x)=\frac{\int_{\Theta}g(\theta)f(x|\theta)d 
\Pi}{\int_{\Theta}f(x|\theta)d \Pi}
 \]
 \item 2. $L(\theta,a)=w(\theta)[g(\theta)-a]^2$ with some known weight 
function $w(\theta)$, then 
 \[
  \delta^*(x)=\frac{\int_{\Theta}w(\theta)g(\theta)f(x|\theta)d 
\Pi}{\int_{\Theta}w(\theta)f(x|\theta)d \Pi}
\]
 \item 3. $L(\theta,a)=|g(\theta)-a|^2$, then 
 $
  \delta^*(x)$ is any median of the posterior. 
  
 \item 4. $L(\theta,a)=\mathbf{1}(|\theta-a|>c)$ with some known constant $c$, 
then 
$
  \delta^*(x)$ is the interval $I$ of length $2c$ which maximizes $P(\theta 
\in I |X=x)$.

\end{example}

\begin{definition}[Conjugate prior:]
the distributions of the posterior and 
prior belong to 
the same family. Depends on both the prior and the model distribution. For 
example,
\begin{enumerate}
 \item Normal-Normal model (midterm of math 5061)
\item  Poisson-Gamma model (Ex.)
\item  Multinomial-Dirichlet model (final of math 5061)
\end{enumerate}

\end{definition}




\begin{definition}[Improper prior:] note that Bayes risk $r(\Pi, \delta)$ is 
well defined 
even if $\Pi$ is not a probability measure but a $\sigma$-finite measure. (Then 
$m(x)$ may not be finite.) 
 \begin{itemize}
 \item $\Pi(\Theta)=1$: proper prior
 \item $\Pi(\Theta)\ne 1$: improper prior $\Rightarrow \delta^*$ is 
referred to as {\bf{a generalized 
Bayes estimator}}.\\
Noninformative prior is usually an improper prior. \\
The resulting generalized 
Bayes estimator is usally the limit of some Bayes estimators from a proper 
prior. \\
Example 4.3 on page 236 Shao 2003. %\cite{shao03}.
\end{itemize}
\end{definition}

\newpage


\begin{example}[More Bayesian Models:]
 \item 1. \textbf{Normal-Gamma model.} Let 
 \[
 X_i \iid N(0,\sigma^2), \quad
  \tau=\frac{1}{2\sigma^2} \sim Gamma(\alpha, \nu).
 \]
($\sigma^2$ follows inverse Gamma distribution.) Then 
\[
 \tau|\vb{X} \sim Gamma(\alpha^*,\nu^*), \quad \alpha^*=n/2+\alpha,\nu^*=(\sum 
X_i^2 +\nu^{-1})^{-1}.
\]
Under square loss, the Bayes esimator for $\sigma^2=1/(2\tau^2)$ is 
\[
E[1/(2\tau^2)|\vb{X}]=\frac{1}{2}  
E[1/(\tau^2)|\vb{X}]=\frac{1}{2}\frac{1}{(\alpha^*-1)\nu^*}   =\frac{\sum 
X_i^2+\nu^{-1}}{n/2-1+\alpha}.           
\]  
\item 2. If we consider {\bf{scale-invariant loss function}}
\[
 L(\theta,a)=(\frac{\theta-a}{\theta})^2 =(\frac{\sigma^2-a}{\sigma^2})^2, 
\]
then the Bayes estimator for $\sigma^2$ is
\[
 \frac{E[w(\theta)g(\theta)]}{E[w(\theta)]}=\frac{E[\frac{1}{\sigma^4} 
\sigma^2]}{E\frac{1}{\sigma^4} } =\frac{E[\tau]}{2E[\tau^2]}=\frac{\sum 
X_i^2 +\nu^{-1}}{2\alpha+n+2}, \quad \alpha^*>1
\]
\item 3. \textbf{Normal-Gamma model with unknown mean and unknown variance.} 
Let 
\begin{eqnarray*}
  X_i &\iid& N(\mu,\sigma^2),\\
  \mu&\sim& N(\mu_0,\sigma_0^/(2\sigma^2))\\
  \tau=\frac{1}{2\sigma^2}&\sim & Gamma(\alpha,\nu).
\end{eqnarray*}
Consider $\theta=(\mu,\sigma^2)$. Posterior of $\theta|\vb{X}$ is proportional 
to
\[
 \tau^{n/2} 
\exp\{-\sum(x_i-\mu)^2\tau\}(\sigma_0^2\tau)^{1/2}\exp\{
-(\mu-\mu_0)^2 \tau /(2\sigma_0^2) \}\tau^{\alpha-1} \exp\{-\tau/\nu\}
\]
\begin{eqnarray*}
  \Rightarrow \mu|\vb{X},\tau &\sim& N(\mu*,c^2), \quad 
\mu*=\frac{n\bar{x}+\mu_0/(2\sigma_0^2)}{n+1/(2\sigma_0^2)}, \quad 
c^2=[(2n+\sigma_0)^2\tau]^{-1},\\
\tau|\vb{X} &\sim & Gamma(\alpha^*,\nu^*), \quad 
\alpha^*=n/2+\alpha,\quad \nu^*=(\sum 
X_i^2 +\nu^{-1} -(n+\frac{1}{2\sigma_0^2})\mu*^2)^{-1}.
\end{eqnarray*}
Hence Bayes estimator for $\theta$ under square loss is 
\[
 \hat{\mu}=\mu*, \quad \hat{\sigma}^2=\frac{1}{2(\alpha^*-1)\nu^*}, \text{ if } 
\alpha^*>1.
\]
Note these are biased estimators. 

\item 4. \textbf{ANOVA with hetergeneous variance. } (Example 4.9 on page 244 
in Shao (2003)%\cite{shao03}
.)
% not a very clearly motivated example, 
%maybe change to example 7.8 on \cite{lc98} p280-282 next time. 

\end{example}


\newpage
\section{Lecture 3: More on Baysian Estimation}

\subsection{Empirical and Hierarchical Bayes Method for Priors}

If hyperparameters, $\xi$, are estimated by historical or observed data, the 
resulting estimation is called \textbf{empirical Bayes estimation}. If the 
hyperparameters are viewed as random variables and modeled by a second-stage 
prior (\textbf{hyper-prior}), the resulting estimation is called 
\textbf{hierarchical Bayes estimation}. 

\begin{example}
 \textbf{Normal-Normal Model}\\
 Let $X_1, \cdots, X_n \iid N(\mu,\sigma^2)$ with unknown $\mu\in \mathbb{R}$ 
and a known $\sigma^2$. Assume $\mu \sim N(\mu_0,\sigma_0^2)$.
 
 \noindent\underline{Empirical Bayes Estimation:} the simplest empirical Bayes 
method is to view the observed data as a ``sample'' from the marginal 
distribution $P(x|\xi)=\int P(x|\theta) d \Pi(\theta|\xi)$. \\
In this case, the pdf of $\vb{X}|\xi$ is $$m(\vb{x}|\xi)=\int_{\mathbb{R}} 
(\sqrt{2 \pi \sigma^2})^{-n/2} \exp\{-\frac{\sum_{i}(x_i-\mu)^2}{2 \sigma^2}\} 
(\sqrt{2 \pi 
\sigma_0^2})^{-n/2} \exp\{-\frac{\sum_{i}(\mu-\mu_0)^2}{2 \sigma_0^2}\} d\mu,$$
where $\xi=(\mu_0,\sigma_0^2)$. One can find the conditional expectation and 
variance of $X|\xi$ without calculating $m(\vb{x}|\xi)$:
\[
 E(X|\xi)=E[E(X|\theta)|\xi]=E[\mu|\xi]=\mu_0,
\]
\[ 
Var(X|\xi)=Var[E(X|\theta)|\xi]+E[Var(X|\theta)|\xi]=Var[\mu|\xi]+E[\sigma^2|\xi
]=\sigma_0^2+\sigma^2.
\]
Then we may estimate prior parameters using the moment method
\[
 \hat{\mu_0}=\bar{x}, \quad \hat{\sigma_0^2}=\frac{1}{n}\sum_{i}x_i^2 
-\bar{x}-\sigma^2=\frac{1}{n}\sum_{i}(x_i 
-\bar{x})^2-\sigma^2.
\]
With respect to squared error loss function, the Bayes estimator for $\mu$ is 
hence  
the sample mean $\bar{x}$.
\smallskip

\noindent\underline{Hierarchical Bayes Estimation:} instead of estimating 
hyperparameters, in hierarchical Bayes approach we put a second-stage prior on 
hyperparameters, $\Lambda(\theta|\xi)$. If the second-stage prior also depends 
on some unknown parameters, one may go on  to consider a third-stage prior. In 
most applications, however, two-stage priors are sufficient, since 
misspecifying a second-stage prior is much less serious than misspecifying a 
first-stage prior. In addition, the second-stage prior can be chosen to be 
noninformative. The hierarchical Bayes estimation is 
\[
 \delta(x)=\int \delta(x,\xi)d P(\xi|x),
\]
where $\delta(x,\xi)$ is the Bayes estimation when $\xi$ is known. \\
In this case, we consider $\Lambda(\xi)$ be the Lebesgue measure. From the 
previous example, we see
\[
\delta(x,\xi)=\frac{\sigma^2}{n\sigma_0^2+\sigma^2}\xi+\frac{n\sigma_0^2}{
n\sigma_0^2+\sigma^2}\bar{x}.
\]
Note the pdf of $\xi|\vb{X}$ is
\[
 f(\xi|\vb{x})=\frac{\int f(\vb{x}|\theta)f(\theta|\xi)f(\xi) d\theta}{\int 
\int f(\vb{x}|\theta)f(\theta|\xi)f(\xi) d\theta d\vb{x}} \propto \int 
f(\vb{x}|\theta)f(\theta|\xi)f(\xi) d\theta. 
\]
Hence, in this example, 
\begin{eqnarray*}
  f(\xi|\vb{x}) &\propto & \int_\mathbb{R} \exp 
\{-\frac{\sum_{i}(x_i-\mu)^2}{2\sigma^2}-\frac{(\mu-\mu_0)^2}{2\sigma_0^2}\} 
d\mu \\
& \propto & \int_\mathbb{R} \exp 
\{-\frac{n(\bar{x}-\mu)^2}{2\sigma^2}-\frac{(\mu-\mu_0)^2}{2\sigma_0^2}\} 
d\mu \\
& = & \int_{\mathbb{R}} \exp\{ - [a \mu^2 -2b \mu +c] \} d\mu \\
&\propto & \exp{c-(2b/a)^2},
\end{eqnarray*}
where $a=n/(2\sigma^2)+1/(2\sigma_0^2)$,$b=n\bar{x}/\sigma^2+\mu_0/\sigma_0^2$, 
and $c=n \bar{x}^2/(2\sigma^2)+\mu_0^2/(2\sigma_0^2)$.
Hence $$ f(\xi|\vb{x}) \propto 
\exp{-\frac{n(\xi-\bar{x})^2}{2(n\sigma_0^2+\sigma^2)}}.$$
So $\xi|\vb{x}$ follows normal distribution with mean $\bar{x}$. We may estimate
\[
 \hat{\mu}_0=\bar{x}.
\]
The hierarchical generalized Bayes rule is then $\bar{x}$. 

\end{example}

\subsection{Properties on Bayesian Estiamtors}
\begin{namedthm}[Biasness of Bayes estimators]
 Let $\delta(X)$ be a Bayes estimator of $g(\theta)$ under squared error loss. 
Then 
$\delta(X)$ is biased unless the Bayes risk $r(\Pi,\delta)=0$, i.e. 
$E[(g(\theta)-\delta(X))^2]=0$, where $E$ is w.r.t both $X$ and $\theta$.
\end{namedthm}

\begin{proof}
 If $\delta(X)$ is unbiased, then $E[\delta(X)|\theta]=g(\theta), \forall 
\theta$. Then 
\begin{eqnarray*}
  E[\delta(X)g(\theta)]&=& E [E(\delta(X)g(\theta)|\theta)]=E[g^2(\theta)],\\
  E[\delta(X)g(\theta)]&=& E [E(\delta(X)g(\theta)|X)]=E[\delta^2(X)].\\
  \therefore 
E[(g(\theta)-\delta(X))^2]&=&E[g^2(\theta)]+E[\delta^2(X)]-2E[\delta(X)g(\theta)
]=0.
\end{eqnarray*}

\end{proof}

\begin{example}[Examples: Sample mean is usually not Bayes estimator.] 
 \item 1.\textbf{Normal-Noraml model.}
 
 Bias$=\frac{\sigma^2}{n\sigma_0^2+\sigma^2}(\mu_0-\mu)$, which is not 0. 
 For large $n$, the bias is the order of $1/n$. 
 
 \item 2.\textbf{Bin-Beta model.} \\
Let $X_i \iid Ber(p), p\sim Beta(a,b)$. Then $p|\vb{X} \sim Beta(a^*,b^*)$,
where 
\[
 a^*=a+\bar{x}, b^*=b+n(1-\bar{X}).
\]
Here $Var(\bar{X})=p(1-p)/n$, which is not a constant as above, but depends on 
the mean $p$. But for $p \in (0,1)$, $p(1-p)>0, \Rightarrow 
r(\Pi,\bar{X})=\int p(1-p)/n d \Pi >0$ unless $\Pi$ only have point mass on 0 
and/or 1. Hence $\bar{X}$ cannot be Bayes estimator. 
 
 \item 3.\textbf{Sample mean in general.}  \\
 Let $X_i \iid E(X_i)=\theta, 
Var(X_i)=\sigma^2,$ which is independent of $\theta$. Note that
\[
 R(\theta,\bar{X})=Var(\bar{X})=\sigma^2/n \neq 0.
\]
Then $r(\Pi,\bar{X}) \neq 0$ w.r.t. any prior $\Pi$, hence $\bar{X}$ cannot be 
Bayes estimator. 



\end{example}

\subsection{Related Reading}
\begin{enumerate}
 \item Sh  P231-245 %\cite{shao03}
  \item LC chapter 4 %\cite{bd01}
  
\end{enumerate}

\end{document}


