\documentclass[12pt]{article}
\usepackage{latexsym}
\usepackage{amssymb,amsmath, amsthm}
\usepackage{amsfonts}
\usepackage{graphicx}

%\numberwithin{equation}{section}
\numberwithin{table}{section} \numberwithin{figure}{section}

%\oddsidemargin -0.1in \textwidth 6.5in \textheight 8.5in
\usepackage[left=1in,top=1in,right=1in]{geometry}
\usepackage{fancyhdr}
\pagestyle{fancy}
\renewcommand{\headrulewidth}{0pt}
\rhead{Jimin Ding \\ Scribe: Jiahui Lyu; Hiro Ono}
\lhead{Math 5062 Spring 2018}


\newcommand{\mc}[1]{\mathcal{#1}}
\newcommand{\vb}[1]{\mbox{\boldmath{$#1$}}}

\newcommand{\indep}{\rotatebox[origin=c]{90}{$\models$}}
\newcommand{\iid}{\overset{\text{iid}}{\sim}}

\newcommand{\toas}{\overset{a.s}{\longrightarrow}}
\newcommand{\toprob}{\overset{p}{\longrightarrow}}
\newcommand{\toDist}{\overset{D}{\longrightarrow}}

\newtheorem{theorem}{Theorem}[section]
%\newtheorem{theorem}{Theorem}
\newtheorem{lemma}[theorem]{Lemma}
\newtheorem{proposition}[theorem]{Proposition}
\newtheorem{corollary}[theorem]{Corollary}
\theoremstyle{definition}


% for specifying a name

\newtheoremstyle{named}{}{}{\itshape}{}{\bfseries}{.}{.5em}{\thmnote{#3 }#1}
\theoremstyle{named}
\newtheorem*{namedthm}{Theorem}
\newtheorem{mydef}{Definition}[subsection]



%\newenvironment{proof}[1][Proof]{\begin{trivlist}
%\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{definition}[1][Definition]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{example}[1][Examples:]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}
\newenvironment{remark}[1][Remarks:]{\begin{trivlist}
\item[\hskip \labelsep {\bfseries #1}]}{\end{trivlist}}



\newcommand{\Remark}{\noindent \textbf{Remarks:} }
\newcommand{\Example}{\noindent \textbf{Examples:}}
%\newcommand{\qed}{\nobreak \ifvmode \relax \else
%      \ifdim\lastskip<1.5em \hskip-\lastskip
%      \hskip1.5em plus0em minus0.5em \fi \nobreak
%      \vrule height0.75em width0.5em depth0.25em\fi}



\begin{document}
\setcounter{section}{5}
\section{Lecture 6: Invariant Estimation under Group Transformations}

\subsection{Invariant Problem under a Transformation}
Let $g$ be a bijection (1-1 and onto) of a sample space $\mathcal{X}$; \\ Let $\mathcal{P}$ be a family of distributions on $\mathcal{X}$, then
\begin{definition}[Definition 1: Invariance Model under a transformation.]
$\mathcal{P}$ is invariant to g if 
$$
 X \sim P  \in \mathcal{P}, \Rightarrow gX \sim P^{\ast} \in 
\mathcal{P}$$
\text{and } 
$$\forall P^{\ast} \in \mathcal{P}, \exists P \text{ s.t. } X \sim P \rightarrow 
gX \sim P^{\ast} $$
\end{definition}

\noindent The second condition ensures that $g$ does not reduce the model.

\medskip
\begin{example}[Example: Normal with unknown variance under scale 
transformation.]
 Let $g(x) = 2x$, and
 $$ \mathcal{P}_A = \{ \mathcal{N} (\mu, \sigma^2): \mu \in \mathbb{R}, \sigma^2 > 0 \} $$
 $$ \mathcal{P}_B = \{ \mathcal{N} (\mu, \sigma^2): \mu \in \mathbb{R}, \sigma^2 > 1 \} $$
 Then $\mathcal{P}_A$ is invariant to $g$ but $\mathcal{P}_B$ is not, as $g$ reduces $\mathcal{P}_B$. 
\end{example}

\medskip
\noindent \underline{\textbf{Induced Transformation}}: Let $\mathcal{P} = \{ 
P_{\theta}: \theta \in \Theta \}$ be a model that is invariant under $g$. 
When $X \sim P_{\theta}, \theta \in \Theta$, then $\exists \theta^* \in 
\Theta s.t. gX \sim P_{\theta^{\ast}}$. Define $\bar 
g$ to be the transformation on the parameter space:
\begin{align*}
 \bar{g}: \quad  \Theta & \to \Theta\\
  \theta & \mapsto  \theta^*.
\end{align*}
The function $\bar g$ is well-defined \underline{\textbf{iff}} the model 
(parametrization) is identifiable. In other words, $\forall \theta_1 \neq 
\theta_2 \Rightarrow P_{\theta_1} \neq P_{\theta_2}$. In particular, the 
identifiability makes $g$ a bijection:
\begin{lemma}
If $\mathcal{P} = \{ P_{\theta}: \theta \in \Theta \}$ is invariant under a bijection (one-to-one and onto) g, and the model is identifiable, then $\bar g$ is a bijection. 
\end{lemma}
\begin{proof}
\item \textbf{1. onto}: That $g$ is onto follows from the second condition in the definition of model invariance under a transformation ($\forall P^{\ast} \in \mathcal{P}, \exists P \text{ s.t. } X \sim P \rightarrow gX \sim P^{\ast} $);
\item \textbf{2. 1-to-1}: To show that $g$ is 1-to-1, if $\forall \theta^{\ast}$ s.t. $\bar g \theta_1 = \bar g \theta_2 = \theta^{\ast}$, we want to show that $\theta_1 = \theta_2$. By the identifiability, we only need to show that $P_{\theta_1} = P_{\theta_2}$. Then $\forall A$  (measurable sets) $\in \sigma(\mathcal{X})$
$$ P_{\theta_1} (X \in A) = P_{\theta^{\ast}} (gX \in gA) = P_{\theta_2} (X \in A), $$
since $\{X \in A \} \equiv \{ gX \in gA \}$. Thus, $P_{\theta_1} = P_{\theta_2} 
$, and by the identifiability of the model, $\Rightarrow \theta_1 = \theta_2$. 
\end{proof}
\medskip
\begin{example}[Example: Recall Normal with unknown variance under scale 
transformation.]
\item  $\mathcal{P}_A = \{ \mathcal{N} (\mu, \sigma^2): \mu \in \mathbb{R}, \sigma^2 > 0 \} $, 
$ g(X) = a+bX $, 
$ \bar g(\mu, \sigma^2) = (a+b\mu, b^2\sigma^2) $. \\
Here, $\Theta = \mathbb{R} \times \mathbb{R}^+, \bar g \Theta = \mathbb{R} \times \mathbb{R}^+$. 
\end{example}
\medskip
\begin{definition}[Definition 2: Invariant loss under a transformation.]
  Let $\mathcal{P}$ be invariant under g (so $\bar g \Theta = \Theta$). A loss 
function $L(\theta, a): \Theta \times \mathbb{A}$ $\rightarrow \mathbb{R}^+$ is 
invariant if 
$$ \forall a \in \mathbb{A}, \exists! \text{ } a^{\ast} \in \mathbb{A} \text{ 
s.t. } L(\theta, a) = L(\bar g \theta, a^{\ast} ) \forall \theta \in \Theta $$

\end{definition}
 

\noindent \underline{\textbf{Induced Transformation}}: Denote $\tilde g a := 
a^{\ast}$, then $\tilde g$ is a bijection on $\mathbb{A}$:
\begin{align*}
 \tilde{g}: \quad  \mathbb{A} & \to \mathbb{A}\\
  a & \mapsto  a^*.
\end{align*}
$\tilde{g}$ is referred to as the induced transformation for decisions. 

\begin{example}[Examples: Normal under linear transformation. ] 
\item $\mathcal{P}_A = \{ 
\mathcal{N} (\mu, \sigma^2): \mu \in \mathbb{R}, \sigma^2 > 0 \} $
\item 
$g(X) = c+bX, \bar g(\mu, \sigma^2) = (c+b\mu, b^2\sigma^2)$
\item $\bar{g}(\mu, \sigma^2) = (c+b\mu, b^2\sigma^2)$
\item 1. Squared error loss: $L_1(\theta,a)=(\mu-a)^2$. 
$$\Rightarrow L_1(\bar g \theta, a^{\ast}) = (c+b\mu 
- a^{\ast})^2.$$
If $L_1$ is invariant under $g$, then 
$$L(\theta, a) = L(\bar g \theta, a^{\ast}), \quad \forall \mu 
\in \mathbb{R}.$$
\[
 \Leftrightarrow (\mu-a)^2=(c+b\mu-a^*)^2, \quad \forall \mu 
\in \mathbb{R}
\]
However, in this case, this is not possible unless $b=1$ since the solution of 
above equation $a^* =a+ c +(b-1) \mu$ depends on $\mu$. Hence squared error 
loss is not invariant under $g$.
\item 2. Scaled squared error loss: $L_2(\theta, a) = \frac{(\mu - 
a)^2}{\sigma^2}$ (standardized by the variance).
$$ \Rightarrow L_2(\bar g \theta, a^{\ast}) = \frac{(c+b\mu - 
a^{\ast})^2}{b^2\sigma^2} = (\mu - \frac{a^{\ast}-c}{b})^2 / \sigma^2. $$
Setting $L(\theta, a) = L(\bar g \theta, a^{\ast})$, then
$$ 
\frac{a^{\ast}-c}{b} = a \quad \Rightarrow \quad a^{\ast} = c + ba,$$  
So $\tilde{g} a=c+ba$ is the induced transformation for decisions, and $L_2$ is 
invariant under $g$.
\end{example}
\medskip
\begin{definition}[Definition 3: Invariant decision problem.]
A decision problem $(\Theta, \mathbb{A}, L)$ is invariant under g if
\begin{itemize}
\item invariant parameter space under the induced transformation $\bar g$ 
(defnition 1)
\item invariant loss function under the induced transformation $\tilde g $ 
(definition 2)
\end{itemize}
\end{definition}






\subsection{Invariance Under a Group}
Typically, if a problem in invariant under a transformation $g$, it is also invariant under a class of related transformations. We always take such a class to be a group. 
\vspace{2mm}

\noindent Recall: a \underline{\textbf{Group}} $\mathcal{G} = \{g: g \in \mathcal{G} \}$ is a class of transformations s.t.:
\begin{enumerate}
\item $\forall g_1, g_2 \in \mathcal{G}, g_1 \circ g_2 \in \mathcal{G}$
\item $\forall g \in \mathcal{G}, g^{-1} \in \mathcal{G}$ and $g \circ g^{-1} = g^{-1} \circ g = I$
\end{enumerate}

\begin{example}
\item 1. Location shift: Additive group.
\item 2. Scale transformations: multiplicative group.
\item 3. Linear transformations. Let $\mathcal{X} = \mathbb{R}$ then
$ \mathcal{G} = \{ g: g(x) = c+ bx, c \in \mathbb{R}, b \in \mathbb{R} \ \{0\} 
\}$
is a group on $\mc{X}$. 
\end{example}



\bigskip
Recall normal model under linear transformation. \\
$\mathcal{P}_A = \{ \mathcal{N} (\mu, \sigma^2): \mu \in \mathbb{R}, 
\sigma^2 > 0 \} $ \\
$\mathcal{G} = \{ g: 
g(x) = c+ bx, c \in \mathbb{R}, b \in \mathbb{R} \ \{0\} \}$,\\
$L(\theta, 
a) = (\mu-a)^2/\sigma^2. $ \\
For each single $g(x) = c+ bx \in \mathcal{G}$, the induced transformations on 
$\Theta$ an $\mathbb{A}$ are $\bar g (\mu,\sigma^2) = (c+b\mu, b^2 \sigma^2)$ 
and $\tilde g a = c+ba$ respectively.\\
We have known this decision problem $(\Theta,\mathbb{A}, L)$ is invariant under 
each $g \in \mc{G}$. Therefore, it is invariant under the group $\mc{G}$. Now 
consider the class of $\bar{g}$ and $\tilde{g}$: $$\bar{\mathcal{G}}= \{\bar g: 
g \in \mathcal{G}\}, \tilde{\mathcal{G}}= \{\tilde g:g \in \mathcal{G}\},$$ 
are referred to as \textbf{Induced Groups}. 


\begin{lemma}
If $\mathcal{P} = \{ P_\theta: \theta \in \Theta \}$ is invariant under a group $\mathcal{G}$, then
 $\bar{\mathcal{G}}= \{\bar g: g \in \mathcal{G}\}$ and $\tilde{\mathcal{G}}= \{\tilde g:g \in \mathcal{G}\}$ are also groups. 
\end{lemma}

\begin{example}
\item 1. Scale transformation \\
$\mathcal{P} = \{ E(0,\theta): \theta \in \Theta = \mathbb{R} \}, f(x; \theta) = \frac{1}{\theta} \mathrm{e}^{-\frac{x}{\theta}}$ \\
$\mathcal{G} = \{g: g(x) = cx, c\in \mathbb{R}^+\}$\\
$L(\theta, a) = (1- \frac{a}{\theta})^2 = \frac{(\theta-a)^2}{\theta^2}$. Then \\
\begin{itemize}
\vspace{-6mm}
\item Invariant model under $\mc{G}$ (definition 1): for each $g\in \mc{G}$,
	\begin{itemize}
	\item If $X \sim P_{\theta}, \theta \sim \Theta$, then $gX = X^{\ast} \sim P_{\theta^{\ast}}, \theta^{\ast} = c\theta $
	\item If $P_{\theta^{\ast}} \in \mathcal{P}, \forall \theta^{\ast} \in \Theta, \exists \theta$ s.t. $\theta^{\ast} = c\theta$
	\end{itemize}
\item Induced group on $\Theta$ : $\bar{\mathcal{G}} = \{ \bar g: 
\bar g \theta = c\theta, c \in \mathbb{R}^+ \}$
\item Invariant loss under $\mc{G}$ (definition 2): $\forall g \in \mc{G}, 
L(\theta, a)=L(\bar{g}\theta, a^*)$. 
\item Induced group on $\mathbb{A}$: 
	\begin{eqnarray*}
	L(\bar g \theta, a^{\ast}) &=& \frac{(c\theta - a^{\ast})^2}{(c\theta)^2} = \frac{(c\theta - ca)^2}{(c\theta)^2} \\
	&=& \frac{(\theta-a)^2}{\theta^2} = L(\theta, a) 
	\end{eqnarray*}
$\Rightarrow \tilde g a = ca$, then we have the group $\tilde{\mathcal{G}} = 
\{\tilde g : \tilde g a = ca, c\in \mathbb{R}^+ \} $. 
\item Invariant decision problem under $\mc{G}$ (definition 3).
\end{itemize}
Note that $\mathcal{G}, \bar{\mathcal{G}}$ and $\tilde{\mathcal{G}}$ are all the 
same group (all scale transformations). But this is not always the case, shown 
as follows.

\item 2. Location and scale transformation \\
$\mathcal{P}_A = \{ \mathcal{N} (\mu, \sigma^2): \mu \in \mathbb{R}, \sigma^2 > 0 \} $ \\
$g(X) = c+bX, \tilde g a = c + ba, \bar g(\mu, \sigma^2) = (c+b\mu, b^2\sigma^2) \Rightarrow \mathcal{G} = \tilde{\mathcal{G}} \neq \bar{\mathcal{G}}$.
\end{example}

\subsection{Invariant Decision Rule}
\underline{\textbf{Invariant principles}}: $X \sim P  \in \mathcal{P}, X^{\ast} \sim P_{\theta^{\ast}} \in \mathcal{P}, L(\theta, a) = L(g\theta, \tilde g a)$; a good decision rule $\delta$ to estimate $\theta$ using $X$
\begin{itemize}
\item Formal invariance:
\[
 \hat{\theta^*}=\widehat{ \bar{g} \theta} \quad  \Leftrightarrow \quad 
\delta(X^*)=\delta(gX)
\]
\item Functional invariance
\[
 \hat{\theta^*}= \tilde{g} \hat{\theta} \quad  \Leftrightarrow \quad 
\delta(X^*)=\tilde{g}\delta(X)
\]
\end{itemize}
Combining the above two types of invariance we have an invariant decision 
rule.
\medskip
\begin{definition}[Definition 4: Invariant Decision Rule.]
 For a decision problem invariant under a group $\mathcal{G}$, an estimator is 
invariant if $$\delta (gX) = \tilde g (\delta (X)), \forall g \in \mathcal{G}$$
\end{definition}


\begin{theorem}
The risk of an invariant estimation satisfies the following condition:
$$R(\theta,\delta) = R(\bar g \theta, \delta), \forall \theta \in \Theta, \bar g 
\in \mathcal{G} .$$
\begin{proof}
\begin{eqnarray*}
R(\theta, \delta) &=& E_\theta[L(\theta, \delta(X))] \\
&=& E_\theta[L(\bar g \theta, \tilde g \delta(X))] \text{ (invariant loss function)} \\
&=& E_\theta[L(\bar \theta, \delta(g X))] \text{ (equivalence of $\delta : \delta (gX)  = \tilde g \delta(X)$) } \\
&=& E_{\theta^{\ast}}[L(\theta^{\ast}, \delta(X^{\ast}))] \\
&=& R(\theta^{\ast}, \delta) \\
&=& R(\bar g \theta, \delta)
\end{eqnarray*}
\end{proof}
\end{theorem}
\begin{mydef}
Two points $\theta_0, \theta_1 \in \Theta$ are said to be equivalent if $\exists 
\bar g$ s.t. $\bar g \theta_1 = \theta_2, \bar g \in \bar{\mathcal{G}}$. The set 
of all such equivalent points is called an orbit:
$$ \Theta(\theta_0) = \{ \bar g (\theta_0): \bar g \in \bar{\mathcal{G}} \} $$
If all points in $\Theta$ are equivalent (a single orbit defined by the group), 
then we say $\bar{\mathcal{G}}$ is \textbf{transitive}.
\end{mydef}
If $\bar G$ is transitive, for any $\theta,\theta_0$ we can choose $\bar g\in 
\bar G$ such that $\bar g(\theta)=\theta_0$. Combined this and the previous 
theorem, we obtain the next Corollary.  

\begin{corollary}
	If $\bar G$ is transitive, then the risk of all invariant estimators is 
constant over parameter space.
\end{corollary}
\begin{theorem}
 MRIE is risk-unbiased if $\bar G$ is transitive and $\tilde G$ is commutative.	
\end{theorem}

\begin{example}
 The followings are examples of commutative and non-commutative 
groups $\tilde G$.
\begin{enumerate}
	\item Location transformation $\tilde G$ is commutative.
	\item Scale transformation $\tilde G$ is commutative.
	\item Scale and location transformation $\tilde G$ is not commutative.
\end{enumerate}	
\end{example}


\subsection*{Related Reading}
\begin{enumerate}
 \item Sh P256-267  
  \item LC chapter 3.1  
  
\end{enumerate}
  
\end{document}


