%%%%%%%%%%%%%%%%%%%%
%I) PRELIMINARIES
%%%%%%%%%%%%%%%%%%%%
\documentclass[11pt,reqno]{amsart}
% Packages
%%%%%%%%%%%%%%%%%%
\usepackage{graphicx}%preferred package for inclusion of graphics
\usepackage{comment}
\usepackage{setspace}
\usepackage{enumerate}%for easy choice of enumerator symbol
\usepackage{enumitem}
\usepackage{tabularx}%for tables width user-defined width
\usepackage{ctable} %for toprule, midrule etc. in tables
\usepackage{multirow} %for more flexibility with tables
\usepackage{textcomp}%for cent-symbol
\usepackage[colorlinks=true, urlcolor=blue]{hyperref}%for inserting hyperlinks
\usepackage{caption} %flexibility with tables
\usepackage{subcaption} %flexibility with tables
\usepackage{html}%to get harvard to work, insert immediately before \usepackage{harvard}
\usepackage{url}%to get harvard to work, insert immediately before \title{...}
\usepackage[dcucite]{harvard} %bibliography style, dcu gives commas before year, semicolon between references, and "and" between authors
\usepackage{amssymb} %for the more esotheric math expressions, such as \approxeq
\usepackage{lineno}%for line numbers
\usepackage{lscape}%for inserting landscape format pages
\usepackage{float} % for more flexibility with tables
\usepackage{appendix}%allows for turning appendices on and off
\usepackage{epstopdf} %to allow import of .eps graphics
% Page formatting
%%%%%%%%%%%%%%%%%%%%
\pagestyle{plain} %puts page number center bottom
\setlength{\topmargin}{0in}
\setlength{\textheight}{8.5in}
\setlength{\oddsidemargin}{.0in}
\setlength{\evensidemargin}{.0in}
\setlength{\textwidth}{6.5in}
\setlength{\footskip}{.5in}
% Customized commands
%%%%%%%%%%%%%%%%%%%%%
% Math
\newcommand{\mlt}[1]{\mathbf{#1}} %matrix bold for Latin symbols
\newcommand{\mgr}[1]{\boldsymbol{#1}}%matrix bold for Greek symbols
\newcommand{\kl}{\left(}
\newcommand{\kr}{\right)}
\newcommand{\kll}{\left\{}
\newcommand{\krr}{\right\}}
\newcommand{\kmu}{\mgr{\mu}}
\newcommand{\kpsi}{\mgr{\psi}}
\newcommand{\kphi}{\mgr{\phi}}
\newcommand{\kgam}{\mgr{\gamma}}
\newcommand{\ktheta}{\mgr{\theta}}
\newcommand{\kbeta}{\mgr{\beta}}
\newcommand{\kdelta}{\mgr{\delta}}
\newcommand{\kt}{^{\prime}}
\newcommand{\kdel}{\partial}
\newcommand{\kdot}{\kl . \kr}
\newcommand{\keps}{\epsilon}
\newcommand{\kx}{\mlt{x}}
\newcommand{\kX}{\mlt{X}}
\newcommand{\kV}{\mlt{V}}
\newcommand{\ky}{\mlt{y}}
\newcommand{\kb}{\mlt{b}}
\newcommand{\ki}{\mlt{i}}
\newcommand{\klam}{\lambda}
\newcommand{\kp}{\mlt{p}}
\newcommand{\kprob}{\text{prob}}
\newcommand{\kz}{\mlt{z}}
\newcommand{\ksig}{\sigma^2}
\newcommand{\kSig}{\mgr{\Sigma}}
\newcommand{\klog}{\text{log}}
\newcommand{\kols}{\kl \kX\kt\kX\kr^{-1}\kX\kt\ky}
\newcommand{\kSSE}{\kl \ky-\kX\kb\kr\kt\kl\ky-\kX\kb\kr}
%Special font within regular document
\newcommand{\chp}[1]{\textbf{\textsl{#1}}}
\newcommand{\km}[1]{\textsf{\small{#1}}} %special font for my own comments
\newcommand{\mlab}{\textbf{\texttt{Matlab }}}
\newcommand{\ktt}[1]{\textbf{\texttt{#1}}}
%Tables
\newcolumntype{C}{>{\centering\arraybackslash}X} %for centered columns within tabularx,instead of justified (the default)
%Others
\newcommand{\kpm}{PM_{2.5}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}
%%%%%%%%%%%%%%%%%%%%%%%%
\citationmode{abbr} %use only "et al" citations
% the following add space before and after equations
\abovedisplayskip=0.7cm
\abovedisplayshortskip=-0.3cm
\belowdisplayskip=0.7cm
\belowdisplayshortskip=0.4cm
%III) TOP MATTER INFORMATION
\renewcommand{\harvardurl}{URL: \url}%to get harvard to work, insert immediately before \title{...}
\title{Common Bayesian Estimation ``Tricks''}
\author{AAEC 6564 \\ Instructor: Klaus Moeltner}
\maketitle %this comes at the end of the top matter to set it.
Assume throughout that $\theta_1$, $\theta_2$, and $z$ are random elements (scalars or vectors), and $y$ symbolizes observed data. The letter $p$ will denote a generic distribution or probability.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{breaking a joint density into marginals and conditionals}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Example:
%
\begin{equation}
\label{equ1}
p\kl \theta_1, \theta_2, z\kr = p\kl \theta_1 \kr p\kl \theta_2, z | \theta_1 \kr = p\kl \theta_1 \kr p\kl \theta_2 | \theta_1 \kr p\kl z | \theta_1,\theta_2\kr
\end{equation}
%
Of course, other split-ups are possible as well. The split-up strategy is usually chosen to be left with as many known densities as possible on the right hand side.
If the original joint density is already conditioned on some other variable, that conditioning is carried through \emph{all} subsequent components.\\
Example:
%
\begin{equation}
\label{equ2}
p\kl \theta_1, \theta_2, z | y\kr = p\kl \theta_1 |y\kr p\kl \theta_2, z | \theta_1, y \kr =
p\kl \theta_1 |y \kr p\kl \theta_2 | \theta_1, y \kr p\kl z | \theta_1,\theta_2, y\kr
\end{equation}
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Turning a marginal into an integrated joint density}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Example:
%
\begin{equation}
\label{equ3}
\begin{split}
p\kl \theta_1\kr = &\int_{\theta_2,z} p\kl \theta_1, \theta_2, z\kr d z \medspace d\theta_2=\\
&\int_{\theta_2,z} p\kl \theta_1| \theta_2, z\kr p \kl \theta_2, z\kr d z \medspace d \theta_2
\end{split}
\end{equation}
%
Same example with pre-existing conditioning:
%
\begin{equation}
\label{equ4}
\begin{split}
p\kl \theta_1|y\kr = &\int_{\theta_2,z} p\kl \theta_1, \theta_2, z|y\kr d z \medspace d\theta_2=\\
&\int_{\theta_2,z} p\kl \theta_1| \theta_2, z, y\kr p \kl \theta_2, z|y\kr d z \medspace d \theta_2
\end{split}
\end{equation}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Obtaining draws from an unknown marginal by drawing from a known conditional}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Continuing with the above example, if $p\kl \theta_1| \theta_2, z, y\kr$ is known, and if \emph{draws} of $\theta_2$ and $z$ from $p \kl \theta_2, z| y\kr$ are available (or, in a rare cases, $p \kl \theta_2, z|y\kr$ is known), draws of $p\kl \theta_1 |y\kr$ can be obtained by drawing from $p\kl \theta_1| \theta_{2,r}, z_r, y\kr$ for many different draws of $\theta_{2,r},z_r$ from $p \kl \theta_2, z|y\kr$.\\
The \emph{Gibbs Sampler} is a special case of this strategy with a built-in reciprocity condition. Dropping $z$ for convenience and without loss in generality, assume we need draws from $p \kl \theta_1 | y \kr$, but we only know the form of $p \kl \theta_1 | \theta_2, y \kr$. Using the integration trick and the ``breaking up a joint''-trick, we obtain:
%
\begin{equation}
\label{equ5}
\begin{split}
p\kl \theta_1|y\kr = &\int_{\theta_2} p\kl \theta_1, \theta_2|y\kr d\theta_2=\\
&\int_{\theta_2} p\kl \theta_1| \theta_2, y\kr p \kl \theta_2|y\kr d \theta_2
\end{split}
\end{equation}
%
The problem here is that we don't know the other marginal either, i.e. we don't know $p \kl \theta_2|y\kr$. However, if we have 1 draw of $\theta_2$ (our staring value for the GS), we can take a single draw of $\theta_1$ from $p\kl \theta_1| \theta_2, y\kr$. By the reasoning above, this will also be a draw from the marginal $p\kl \theta_1|y\kr$. We can then set up the reverse integration problem for $\theta_2$, i.e.
%
\begin{equation}
\label{equ6}
\begin{split}
p\kl \theta_2|y\kr = &\int_{\theta_1} p\kl \theta_1, \theta_2|y\kr d\theta_1=\\
&\int_{\theta_1} p\kl \theta_2| \theta_1, y\kr p \kl \theta_1|y\kr d \theta_1
\end{split}
\end{equation}
%
If $p\kl \theta_2| \theta_1, y\kr$ is known, we can draw $\theta_2$ from it (conditioning on the draw of $\theta_1$ we just obtained from the first step). This will also be a draw from $p\kl \theta_2|y\kr$. This process is then repeated many times to yield draws from the entire support of $p\kl \theta_1|y\kr$ and $p\kl \theta_2|y\kr$.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section*{Monte Carlo integration}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
Another flavor of the integration trick is when we wish to \emph{evaluate} the marginal (or any other unknown density) at a specific point (say $\bar{\theta}_1|y$), or evaluate a specific single-valued function of $\theta_1$ (say its expectation, $E\kl \theta_1|y\kr$). This works well when we already have draws of the remaining model parameters from their respective marginal densities.\\
Example:
%
\begin{equation}
\label{equ7}
\begin{split}
p\kl \bar{\theta}_1|y\kr = &\int_{\theta_2} p\kl \bar{\theta}_1, \theta_2|y\kr d\theta_2=\\
&\int_{\theta_2} p\kl \bar{\theta}_1| \theta_2, y\kr p \kl \theta_2|y\kr d \theta_2
\end{split}
\end{equation}
%
If we know $p\kl \theta_1| \theta_2, y\kr$, and we have draws of $\theta_2$ from $p \kl \theta_2|y\kr$, we can approximate $p\kl \bar{\theta}_1|y\kr$ via:
%
\begin{equation}
\label{equ8}
p\kl \bar{\theta}_1|y\kr \approx \tfrac{1}{R} \sum_{r=1}^R p\kl \bar{\theta}_1 |\theta_{2,r},y\kr
\end{equation}
%
using $r=1\ldots R$ draws of $\theta_2$ from $p \kl \theta_2|y\kr$. \\
Similarly, for $E\kl \theta_1|y\kr$:
%
\begin{equation}
\label{equ9}
\begin{split}
E\kl \theta_1|y\kr = &\int_{\theta_2} \theta_1 \ast p\kl \theta_1, \theta_2|y\kr d\theta_2=\\
&\int_{\theta_2} \theta_1 \ast p\kl \theta_1| \theta_2, y\kr p \kl \theta_2|y\kr d \theta_2
\end{split}
\end{equation}
%
which can be approximated via:
%
\begin{equation}
\label{equ10}
E\kl \theta_1|y\kr \approx \tfrac{1}{R} \sum_{r=1}^R \theta_{1,r}
\end{equation}
%
using $r=1 \ldots R$ draws of $\theta_1$ from $p\kl \theta_1| \theta_{2,r}, y\kr$, which themselves are based on $r=1 \ldots R$ draws of $\theta_2$ from $p \kl \theta_2|y\kr$.\\
The same logic holds for any other (smooth, continuous) function $g\kl \theta_1|y\kr$, which is exploited when generating posterior predictive distributions (PPDs).
%\bibliography{AAEC6984bib}
%\bibliographystyle{dcu}
\end{document}