\documentclass[12pt]{article}
\usepackage{latexsym}
\usepackage{amssymb,amsmath, amsthm}
\usepackage{amsfonts}
\usepackage{graphicx} 

%\numberwithin{equation}{section}
\numberwithin{table}{section} \numberwithin{figure}{section}

%\oddsidemargin -0.1in \textwidth 6.5in \textheight 8.5in
\usepackage[left=1in,top=1in,right=1in]{geometry}
\usepackage{fancyhdr}
\pagestyle{fancy}
\renewcommand{\headrulewidth}{0pt}
\rhead{Jimin Ding\\Scribe:Bei Wu}
\lhead{Math 5062 Spring 2018}


\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\vb}[1]{\mbox{\boldmath{$#1$}}}

\newcommand{\indep}{\rotatebox[origin=c]{90}{$\models$}}
\newcommand{\iid}{\overset{\text{iid}}{\sim}}

\newcommand{\toas}{\overset{a.s}{\longrightarrow}}
\newcommand{\toprob}{\overset{p}{\longrightarrow}}
\newcommand{\toDist}{\overset{D}{\longrightarrow}}

\newtheorem{theorem}{Theorem}[section]
%\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}


% for specifying a name

\newtheoremstyle{named}{}{}{\itshape}{}{\bfseries}{.}{.5em}{\thmnote{#3 }#1}
\theoremstyle{named}
\newtheorem*{namedthm}{Theorem}



%\newenvironment{proof}[1][Proof]{\begin{trivlist}
%\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{definition}[1][Definition]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{example}[1][Examples:]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{remark}[1][Remarks:]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}



\newcommand{\Remark}{\noindent \textbf{Remarks:} }
\newcommand{\Example}{\noindent \textbf{Examples:}}
%\newcommand{\qed}{\nobreak \ifvmode \relax \else
%      \ifdim\lastskip<1.5em \hskip-\lastskip
%      \hskip1.5em plus0em minus0.5em \fi \nobreak
%      \vrule height0.75em width0.5em depth0.25em\fi}



\begin{document}
\setcounter{section}{3}

\section{Lecture 4: Invariance/Equivariance}
\underline{Motivating example}: Let $X_1, \cdots, X_n \iid Ber(p)$, $p \in (0,1)$. Let \(\hat{p} = \delta(\vb{x})=\delta(x_1, \cdots, x_n)\) be a decision rule.
Consider $X_1^*, \cdots, X_n^* = 1-X_1, \cdots, 1-X_n \iid Ber(p^*)$, where $p^* = 1-p $. it is natural to use the same decision rule to estimate $p^*$, then\[\hat{p^*} = \delta(\vb{x^*}) = \delta(x_1^*, \cdots, x_n^*) = \delta(1-x_1, \cdots, 1-x_n)\]  Also,we have already estimate $\hat{p}$, then, under the same transformation, we can estimate $p^*$ as:
\[\hat{p^*} = 1-\hat{p} = 1-\delta(x_1, \cdots, x_n) \] It is natural to request a decision rule $\delta$ such that $ 1-\delta(x_1, \cdots, x_n) = \delta(1-x_1, \cdots, 1-x_n) $, which is a decision rule invariant under transformation.
For example, \(\delta(\vb{x}) = 1/n\sum_{i=1}^{n} x_i \) is invariant.
\medskip

\noindent Recall: a \underline{\textbf{Group}} $\mc{G} = \{g:g \in \mc{G}\}$ is a class of transformation s.t.
\begin{enumerate}
 \item $\forall g_1,g_2 \in \mc{G}, g_1\circ g_2 \in \mc{G} $
\item $\forall g \in \mc{G}$,  \(g^{-1} \in \mc{G}\) and \(g\circ g^{-1} =g^{-1}\circ g = I  \)
\end{enumerate}
note: Location transformation is a group.
\medskip

\subsection{Location invariant}
\noindent \underline{\textbf{Location family}}: is invariant under location transformations



\medskip

\begin{definition}
\begin{enumerate}
\item $\mc{P} = \{f(x,\theta); \theta \in \Theta \}$ is location invariant if $f(x^*;theta^*) = f(x;theta) $, where $x^* = x+c, \theta^* = \theta+c, \forall c \in \mathbb{R} $. c is location shift.\\
($f(\vb{x},\theta) = f(\vb{x}-\theta) $ )
\begin{example}
 \item 1.$\mc{P} = \{N(\mu, \sigma^2): \mu \in \mathbb{R}, \sigma^2 $ is known$\}$
 \item 2.$\mc{P} = \{E(\mu, \theta): \mu \in \mathbb{R}, \theta $ is known$\}$
\end{example}
\item Loss function $L(\theta,a)$ is location invariant if $L(\theta^*,a^*) = L(\theta,a)$, where $\theta^* = \theta + c, a^* = a + c, \forall c \in \mathbb{R}$.\\
($L(\theta, a) = L(\theta-a)$ )
\begin{example}
 \item 1.$L(\theta,a) = (g(\theta) - a)^2$ when $g(\theta) = \theta $
 \item 2.(counter example) for motivating example, define $L(p,\delta) = \frac{(\delta-p)^2}{p(1-p)}$, then $L(p,\delta) $ is invariant under transformation $1-\delta,1-p$, but not location invariant. 
\end{example}
 \item Estimation problem: $\hat{g}(\theta)$ is location invariant if the distribution family and loss functions are both location invariant.
 \item An estimator $\delta$ is location invariant if $ \delta(x^*) = \delta(x+c)=\delta(x) + c, \forall c \in \mathbb{R}$
\end{enumerate}
\end{definition}

\begin{definition}[MRIE]
minimum risk invariant estimator.
\end{definition}


\subsection{Properties of location invariant estimators}

\begin{theorem}
The bias, variance and risk of location invariance estimators are constant.(independent of $\theta$)
\end{theorem}
\begin{proof}[proof of bias is constant]
The location invariant family has p.d.f. $f(\vb{x}, \theta) = f(\vb{x}- \theta) = f(x_1 - \theta , \cdots, x_n - \theta) $
\begin{equation}
\begin{split}
 bias = E(\delta(\vb{x})) - \theta &= \int \delta(\vb{x})f(\vb{x}- \theta)dx - \theta\\
& =  \int_{\mathbb{R}^n} \delta(x_1,\cdots,x_n)f(x_1-\theta,\cdots,x_n - \theta)dx_1\cdots dx_n - \theta\\
& = \int_{\mathbb{R}^n} \delta(s_1+\theta,\cdots,s_n+\theta)f(s_1,\cdots,s_n )ds_1\cdots ds_n - \theta\\
& = \int_{\mathbb{R}^n} [\delta(s_1,\cdots,s_n)+\theta]f(s_1,\cdots,s_n )ds_1\cdots ds_n - \theta\\
& = \int_{\mathbb{R}^n} [\delta(s_1,\cdots,s_n)]f(s_1,\cdots,s_n )ds_1\cdots ds_n \\
\end{split}
\end{equation}
Bias of $\delta$ is independent of $\theta$, thus it is a constant. 
\end{proof}

To find MRIE, we only need to compare the constant risks and find the $\delta^*$ which has the smallest constant risk. 

\begin{lemma}Let $\delta_0$ be a given location invariant estimator. Then any location invariant estimator $\delta$ satisfies  $$\delta(x) = \delta_0(x) + u(x)$$ where $u(x+c) = u(x), \forall c \in \mathbb{R}, \forall x \in \mathbb{R}^n$
\end{lemma}

\begin{proof}
\begin{enumerate} 
\item $\delta_0(x) + u(x)$ is location invariant since $\delta_0(x) +u(x+c)= \delta_0(x)+c+u(x) $
\item Let $\delta(x)$ be any location invariant. Set $u(x) = \delta(x) - \delta_0(x)$, then $$u(x+c) = \delta(x+c) - \delta_0(x+c) = \delta(x) +c -\delta_0(x) - c = u(x)$$
\end{enumerate}
\end{proof}


$u(x+c) =u(x), u(x_1+c,\cdots, x_n+c) = u(x_1,\cdots, x_n) , \forall c \in \mathbb{R}$
set $c = -x_n$, then 
$$u(x_1-x_n, x_2-x_n,\cdots, x_{n-1}-x_n,0) = u(x_1,\cdots, x_n) $$
So $u$ is a function in $\mathbb{R}^{n-1}$ and is a function of ancillary statistic $x_1-x_n, x_2-x_n,\cdots, x_{n-1}-x_n$

\begin{theorem}
Let $\delta_0$ be a location invariant estimator. Let $d_i = x_i - x_n, i= 1, \cdots, n-1 $, and $ d= (d_i, \cdots, d_n). $Then a necessary and sufficient condition for $\delta$ is also location invariant is that $\exists$ a function $u$ of n-1 arguments for which $$ \delta(x)=\delta_0(x)+u(d),\forall x \in \mathbb{R}$$
\end{theorem}

\begin{theorem}
Let $D= (x_1-x_n,\cdots, x_{n-1}-x_n)^T$. Suppose $\exists$ a location invariant estimator $\delta_0$ such that  $\delta_0$ has a finite risk. Assume $\forall y, \exists$ a $u^*(y)$ which minimizes $E_0[L(\delta_0(x)-u(y))|D=y] $. Then the MRIE exists and is $\delta^*(x) = \delta_0(x)-u^*(y).$
\end{theorem}

\begin{proof}[idea of proof]
If a loss function is location invariant, it can be written as $L(\theta, \delta) = L(\theta- \delta)$.
Then $R(\delta, \theta) = E(L(\theta, \delta)] = E(L(\theta-\delta)] = E_0[L(\delta)] $, since risk is independent of $\theta$, we can set $\theta=0$. And from theorem 4.3
$$E_0[L(\delta(x)] =E_0[L(\delta_0(x)-u(y))]= E[E_0[L(\delta_0(x)-u(y))|D=y]]$$
Thus if $u^*(y)$ minimize $E_0[L(\delta_0(x)-u(y))|D=y]$, $\delta^*(x) = \delta_0(x)-u^*(y)$ minimize the risk function $R(\delta, \theta)$
\end{proof}

\begin{corollary}
Suppose L is convex and not monotone, then MRIE exists. Furthermore, if L is strictly convex, then MRIE is unique. 
\end{corollary}
\begin{example}
\item1. If $L(\theta,a) = (\theta-a)^2$, then $u^*(y) = E_0[\delta_0(x)|D=y]$
\item2. If $ L(\theta,a) = |\theta-a|$, then $ u^*(y) $ is the conditional median of $\delta_0(x)$ given D. 
\end{example}

\subsection{Related Reading}
\begin{enumerate}
 \item Sh  P251-255 %\cite{shao03}
 \item LC Chapter 3.1 %\cite{bd01}
  
\end{enumerate}

\newpage
\rhead{Jimin Ding\\Scribe: Yuchen Han}
\section{Lecture 5: More on Invariance/Equivariance}
\subsection{Properties of location invariant estimators}
\begin{example}[Examples]
\noindent
\begin{enumerate}
\item Let $X_1, X_2, \ldots, X_n$ be i.i.d. from $N(\mu, \sigma^2)$ with an unknown $\mu \in \mathbb{R}$ and a known $\sigma^2$. Consider squared error loss function, $\delta_0(x) =\bar{X}$ is location invariant. \\
By Basu's theorem, $D= (x_1-x_n,\cdots, x_{n-1}-x_n)^T$ and $\bar{X}$ are independent, so
$$u^*(d)=E_0(\bar{X}|x_1-x_n, x_2-x_n, \ldots, x_{n-1}-x_n)=E_0(\bar{X})=0.$$
Furthermore, $\bar{X}$ is MRIE for all convex and even loss functions, since
\begin{align*}
E_0\left[L\left(\delta_0(x)-u(d)\right)|D=d\right] = E_0\left[L\left(\delta_0(x)-u(d)\right)\right] = \int_{\mathbb{R}^n} L\left(\delta_0(x)-u(d)\right)f_0(x)dx
\end{align*}
is minimized if and only if $u(d)=0$, since $f_0(x)$ is also even.

\item Let $X_1, X_2, \ldots, X_n$ be i.i.d. from the exponential distribution $E(\mu, 1)$ with an unknown $\mu \in \mathbb{R}$. $\delta_0(x) =X_{(1)}$ is location invariant. By Basu's theorem, $X_{(1)}$ is independent of $D$. We want to minimize $E_0\left[L\left (X_{(1)}-u(d)\right)|D=d\right] = E_0\left[L\left(X_{(1)}-u(d)\right)\right]$.\\
If consider squared error loss function, then $u^*=E_0(X_{(1)})=\frac{1}{n}$ since $X_{(1)} \sim E(0, n)$. And the MRIE is $\delta^*(x)=X_{(1)}-\frac{1}{n}$.\\
If consider absolute loss function, then $u^*=median_0(X_{(1)})=\frac{\log 2}{n}$ since $$\frac{1}{2}=F_1(x)=1-e^{-nx}, \quad F_1(x) \mbox{ is c.d.f. of } X_{(1)} \sim E(0, n).$$ And the MRIE is $\delta^*(x)=X_{(1)}-\frac{\log 2}{n}$.\\
\end{enumerate}
\end{example}

\begin{theorem}[\textbf{Pitman Estimator}]
If  we have location invariant estimation problem with squared error loss function $L(\theta-a)=(\theta-a)^2$, then
$$\delta^*(x)= \frac{\int_{-\infty}^\infty uf(X_1-u, X_2-u, \ldots, X_n-u) du}{\int_{-\infty}^\infty f(X_1-u, X_2-u, \ldots, X_n-u) du}$$
is the MRIE of $\theta$ which is known as the Pitman estimator and is unbiased.
\end{theorem}
\begin{proof}
Consider $\delta_0(x)=X_n$, $u^*(d)=E_0(X_n|D=d)$.\\
Consider $1-1$ transformation: $X_1, \ldots,X_n \rightarrow Y_1, \ldots, Y_n$, where 
$$Y_1=X_1-X_n, \ldots, Y_{n-1}=X_{n-1}-X_n, Y_n=X_n.$$
\begin{align*}
u^*(d) & =E_0(X_n|D=d)=E_0(Y_n|Y_1=d_1, \ldots, Y_{n-1}=d_{n-1})\\
&=\int y_n f_{Y_n|Y_1,\ldots, Y_{n-1}}dy_n =\int y_n\frac{f_{Y_1,\ldots, Y_{n-1}, Y_n}}{f_{Y_1,\ldots, Y_{n-1}}}dy_n \\
&=\int y_n \frac{f(y_1+y_n,\ldots, y_{n-1}+y_n,y_n)}{\int f(y_1+y_n,\ldots, y_{n-1}+y_n,y_n)dy_n} dy_n \\
&=\frac{\int y_n f(y_1+y_n,\ldots, y_{n-1}+y_n,y_n)dy_n}{\int f(y_1+y_n,\ldots, y_{n-1}+y_n,y_n)dy_n}, \qquad \mbox{let } y_n=x_n-u \\
&= \frac{\int (x_n-u) f(y_1+x_n-u,\ldots, y_{n-1}+x_n-u, x_n-u)d(-u)}{\int f(y_1+x_n-u,\ldots, y_{n-1}+x_n-u, x_n-u)d(-u)}, \qquad \mbox{let } y_i=x_i-x_n, y_n=x_n \\
&= \frac{\int (x_n-u) f(x_1-x_n+x_n-u,\ldots, x_{n-1}-x_n+x_n-u, x_n-u)d(-u)}{\int f(x_1-x_n+x_n-u,\ldots, x_{n-1}-x_n+x_n-u, x_n-u)d(-u)} \\
&= \frac{\int (x_n-u) f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du}{\int f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du} \\
&= \frac{x_n \int  f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du}{\int f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du} -\frac{\int u f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du}{\int f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du} \\
&= x_n -\frac{\int u f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du}{\int f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du} \\
&=\delta_0(x)-\delta^*(x)
\end{align*}
Thus $$\delta^*(x)=\frac{\int u f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du}{\int f(x_1-u,\ldots, x_{n-1}-u, x_n-u)du} .$$
Let $b$ be the constant bias of $\delta^*$, then $\delta_1(x)=\delta^*(x)-b$ is a location invariant estimator of $\theta$ and
$$R(\delta_1, \theta)=E\left[(\delta^*(x)-b-\theta)^2\right] = Var(\delta^*) \leq Var(\delta^*)+b^2=R(\delta^*, \theta).$$
since $\delta^*$ is the MRIE, $b=0$, then $\delta^*$ is unbiased.
\end{proof}

\noindent \underline{\textbf{Risk-unbiasness}}: An estimator $\delta(x)$ for $g(\theta)$ is risk-unbiased if $$E_{\theta}\left[L\left(\theta, \delta(x)\right)\right] \leq E_{\theta}\left[L\left(\theta', \delta(x)\right)\right], \quad \forall \theta' \neq \theta.$$
Interpretation: $\delta(x)$ has the smallest risk at $\theta$.

\begin{theorem}
The MRIE of $\theta$ (location parameter) in a location invariant estimation problem (or decision problem) is risk-unbiased.
\end{theorem}

\subsection{Other non-convex loss functions}

\begin{theorem}
Suppose $0\leq L(t) \leq M$ for all values of $t$ and $L(t) \rightarrow M$ as $t\rightarrow +\infty$, and is risk-unbiased. The density of $X$ is continuous a.e.. Then an MRIE of $\mu$ exists.
\end{theorem}

\begin{example}
\textbf{MRIE under 0-1 loss}\\
$L(\theta,a)=\mathbf{1}(|\theta-a|>k)$ with some known constant $k>0$, $u^*$ maximizes $P_0(|X-U| \leq k)$. Suppose symmetric $f$, if it has unique mode, then $u^*=0$.\\
For example, $N(\mu, \sigma^2)$ with $f(x; \mu)=\frac{1}{\sqrt{2\pi \sigma^2}}\exp\{-\frac{1}{2\sigma^2(x-\mu)^2}\}$, $\delta_0=\bar{X}$, $u^*=0$, $\delta^*(x)=\bar{X}$. 
\end{example}

\noindent\textbf{Remarks}: an MRIE (comparing with UMVUE)
\begin{enumerate}
\item When loss function is non-convex, MRIE typically still exists.
\item MRIE depends on the loss function even for convex loss function.
\item MRIE is often admissible (unlike UMVUE).
\item MRIE is often considered in location-scale families. \\
(UMVUE is more for exponential families and UMVUE for location families usually does not exist.)
\end{enumerate}

\subsection{Related Reading}
\begin{enumerate}
 \item Sh  P253-255 %\cite{shao03}
 \item LC Chapter 3.1 %\cite{bd01}
\end{enumerate}

\end{document}


