\documentclass{beamer}

\usepackage{beamerthemesplit}
\usetheme{Boadilla}
\usecolortheme{albatross}

\title[Moments of Multiple Random Variables]{Moments of More that One Random Variable: Lecture IX}
\author{Charles B. Moss}
\date{\today}

\begin{document}

\frame{\titlepage}

\section[Outline]{}
\frame{\tableofcontents}

\section{Covariance and Correlation}
\frame
{
   \frametitle{Covariance and Correlation}
   \begin{itemize}
   \item {\bf Definition 4.3.1} The covariance between two random variables $X$ and $Y$ can be defined as

\begin{equation}
\begin{array}{c}
\displaystyle {\rm Cov}\left( X,Y \right) = {\rm E} \left[ \left( X - {\rm E} \left[ X \right] \right) \left( Y - {\rm E} \left[ Y \right] \right) \right] \\
\displaystyle \; = {\rm E} \left[ XY - X {\rm E} \left[ Y \right] - {\rm E} \left[ x \right] Y + {\rm E} \left[ X \right] \left[ Y \right] \right] \\
\displaystyle \; = {\rm E} \left[ XY \right] - {\rm E} \left[ X \right] {\rm E} \left[Y \right] - {\rm E} \left[ X \right] {\rm E} \left[ Y \right] + {\rm E} \left[ X \right] {\rm E} \left[ Y \right] \\
\displaystyle \; = {\rm E} \left[ XY \right] - {\rm E} \left[ X \right] {\rm E }\left[ Y \right]
\end{array}
\label{eqn:lect09-001}
\end{equation}

      \begin{itemize}
      \item Note that this is simply a generalization of the standard variance formulation.  Specifically, letting $Y \rightarrow X$  yields

\begin{equation}
\begin{array}{c}
\displaystyle {\rm Cov}\left( XX \right) = {\rm E} \left[ XX \right] - {\rm E} \left[ X \right] {\rm E} \left[ X \right] \\
\displaystyle \; = {\rm E} \left[ X^2 \right] - \left( {\rm E} \left[ X \right] \right)^2
\end{array}
\label{eqn:lect09-002}
\end{equation}

     \item From a sample perspective, we have

\begin{equation}
\begin{array}{c}
V \left( X \right) = \frac{1}{N} \sum_{i=1}^N x_i^2 - \bar x ^2 \\ \\
{\rm Cov}\left( X, Y \right) = \frac{1}{N} \sum_{i=1}^N x_i y_i - \bar x \bar y
\end{array}
\label{eqn:lect09-003}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Together the variance and covariance matrices are typically written as a variance matrix

\begin{equation}
\Sigma = \left[ \begin{array}{cc} V\left( X \right) & {\rm Cov} \left( X,Y \right) \\ {\rm Cov} \left( Y,X \right) & V \left( Y \right) \end{array} \right] = \left[ \begin{array}{cc} \sigma_{xx} & \sigma_{xy} \\ \sigma_{yx} & \sigma_{yy} \end{array} \right]
\label{eqn:lect09-004}
\end{equation}

      \noindent Note that ${\rm Cov}\left( X,Y \right) = \sigma_{xy} = \sigma_{yx} = {\rm Cov}\left( Y,X \right)$.
      \item Substituting the sample measures into the variance matrix yields

\begin{equation}
\begin{array}{c}
S = \left[ \begin{array}{cc} s_{xx} & s_{xy} \\ s_{yx} & s_{yy} \end{array} \right] = \left[ \begin{array}{cc} \frac{1}{N} \sum_{i=1}^N x_i x_i - \bar x \bar x & \frac{1}{N} \sum_{i=1}^N x_i y_i \bar x \bar y \\ \frac{1}{N} \sum_{i=1}^N y_i x_i - \bar y \bar x & \frac{1}{N} \sum_{i=1}^N y_i y_i - \bar y \bar y \end{array} \right] \\ \\
\,\,\, = \frac{1}{N} \left[ \begin{array}{cc} \sum_{i=1} x_i x_i & \sum_{i=1}^N x_i y_i \\ \sum_{i=1}^N y_i x_i & \sum_{i=1}^N y_i y_i \end{array} \right] - \left[ \begin{array}{cc} \bar x \bar x & \bar x \bar y \\ \bar y \bar x & \bar y \bar y \end{array} \right]
\end{array}
\label{eqn:lect09-005}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item The sample covariance matrix can then be written as
      
\begin{equation}
S = \frac{1}{N} \left[ \begin{array}{ccc} x_1 & \cdots & x_N \\ y_1 & \cdots & y_N \end{array} \right] \left[ \begin{array}{cc} x_1 & y_1 \\ \vdots & \vdots \\ x_N & y_N \end{array} \right] - \left[ \begin{array}{c} \bar x \\ \bar y \end{array} \right] \left[ \begin{array}{cc} \bar x & \bar y \end{array} \right].
\label{eqn:lect09-006}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item In terms of the theoretical distribution, the variance matrix can be written as

\begin{equation}
\begin{array}{c}
\displaystyle \Sigma = \left[ \begin{array}{c} \int_{-\infty}^\infty \int_{-\infty}^\infty \left(x - \mu_x \right)^2 f\left(x,y\right) dx\,dy \\ \int_{-\infty}^\infty \int_{-\infty}^\infty \left( x - \mu_x \right) \left( y - \mu_y \right) f \left(x,y\right) dx,dy \end{array} \right. \\
\displaystyle \left. \begin{array}{c} \int_{-\infty}^\infty \int_{\infty}^\infty \left( x - \mu_x\right) \left( y - \mu_y\right) f\left(x,y\right) dx\,dy \\ \int_{-\infty}^\infty \int_{-\infty}^\infty \left( y - \mu_y \right)^2 dx \, dy \end{array} \right]
\end{array}
\label{eqn:lect09-007}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
\begin{table}
\caption{Discrete Sample}
\begin{center}
\begin{tabular}{cllll}
\hline
        &    &   &   & Marginal \\
$X \ Y$ & -1 & 0 & 1 & Probability \\
\hline
1  & 0.167 & 0.083 & 0.167 &  0.417 \\
0  & 0.083 & 0.000 & 0.083 &  0.167 \\
-1 & 0.167 & 0.083 & 0.417 &  0.417 \\
Marginal & & & & \\
Probability & 0.417 & 0.167 & 0.417 & \\
\hline
\end{tabular}
\end{center}
\label{tab:lect09-001}
\end{table}
}
\frame
{
   \begin{itemize}
   \item Example 4.3.2. Compute the covariance for the data presented in Table ~\ref{tab:lect09-001}. We start by computing the means from Table ~\ref{tab:lect09-001}

\begin{equation}
\begin{array}{c}
\displaystyle \left[ \begin{array}{c} \bar x \\ \bar y \end{array} \right] = \left[ \begin{array}{c} \begin{array}{c}\left( 0.167 + 0.083 + 0.167\right) \left( 1 \right) \\ + \left(0.083 + 0.000 + 0.083\right) \left( 0 \right) \\ + \left( 0.417 + 0.083 + 0.167 \right) \left(-1\right) \end{array} \\ \\ \begin{array}{c} \left( 0.167 + 0.083 + 0.167\right) \left( 1 \right) \\ + \left(0.083 + 0.000 + 0.083\right) \left( 0 \right)\\ + \left( 0.417 + 0.083 + 0.167 \right) \left(-1\right) \end{array} \end{array} \right] \\ \\
\displaystyle \,\,\, = \left[ \begin{array}{c} \left( 0.417\right)\left(1\right) + \left(0.167\right) \left( 0 \right) + \left(0.417\right)\left(-1\right) \\ \left(0.417\right)\left(1\right) + \left(0.167 \right) \left(0\right) + \left(0.417\right) \left(-1\right) \end{array} \right] = \left[ \begin{array}{c} 0 \\ 0 \end{array} \right].
\end{array}
\label{eqn:lect09-008}
\end{equation}

   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Given that the means for both variables are zero, we can compute the covariance as

\begin{equation}
\begin{array}{c}
\displaystyle {\rm Cov}\left[ X,Y \right] = \frac{1}{9} \sum_{i\in\left\{\-1,0,1\right\}\ } x_i y_i P\left(x_i,y_i\right) \\
\displaystyle \,\, = \left(1\right) \left(-1\right) 0.167 + \left(0\right) \left(-1\right) 0.083 + \cdots \left(-1\right) \left(1\right) 0.167 = 0.
\end{array}
\label{eqn:lect09-009}
\end{equation}

   \item {\bf Theorem 4.3.2.} $V\left(X \pm Y \right) = V\left(X\right) + V\left( Y \right) \pm {\rm Cov}\left( X,Y \right)$

\begin{equation}
\begin{array}{c}
V\left[ X \pm Y \right] = \rm E \left[ \left( X \pm Y \right)\left(X \pm Y \right) \right] \\
\,\,\, = \rm E \left[ XX \pm 2 XY + YY \right] \\
\,\,\, = \rm E \left[ XX \right] + \rm E \left[ YY \right] \pm \rm E \left[XY\right] \\
\,\,\, = V\left(X\right) + V\left( Y\right) \pm 2 {\rm Cov}\left(X,Y\right)
\end{array}
\label{eqn:lect09-010}
\end{equation}

   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Note that this result can be obtained from the variance matrix.  Specifically, $X + Y$ can be written as a vector operation

\begin{equation}
\left[ \begin{array}{cc} X & Y \end{array} \right] \left[ \begin{array}{c} 1 \\ 1 \end{array} \right] = X + Y
\label{eqn:lect09-011}
\end{equation}

   \noindent Given this vectorization of the problem we can define the variance of the sum as

\begin{equation}
\begin{array}{c}
\displaystyle \left[ \begin{array}{cc} 1 & 1 \end{array} \right] \left[ \begin{array}{cc} \sigma_{xx} & \sigma_{xy} \\ \sigma_{xy} & \sigma_{yy} \end{array} \right] \left[ \begin{array}{c} 1 \\ 1 \end{array} \right] = \left[ \begin{array}{cc} \sigma_{xx} + \sigma_{xy} & \sigma_{xy} + \sigma_{yy} \end{array} \right] \left[ \begin{array}{c} 1 \\ 1 \end{array} \right] \\ \\
\displaystyle \,\,\, = \sigma_{xx} + 2 \sigma_{xy} + \sigma_{yy}
\end{array}
\label{eqn:lect09-012}
\end{equation}

   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item {\bf Theorem 4.3.3.} Let $X_i$ , $i=1,2,\cdots$ be pairwise independent.  Then

\begin{equation}
V\left( \sum_{i=1}^N X_i \right) = \sum_{i=1}^N V\left( X_i \right)
\label{eqn:lect09-013}
\end{equation}

   \noindent The simplest proof to this theorem is to use the variance matrix.  Note in the preceding example, if X and Y are independent, we have
   
\begin{equation}
\begin{array}{c}
\displaystyle \left[ \begin{array}{cc} 1 & 1 \end{array} \right] \left[ \begin{array}{cc} \sigma_{xx} & \sigma_{xy} \\ \sigma_{xy} & \sigma_{yy} \end{array} \right] \left[ \begin{array}{c} 1 \\ 1 \end{array} \right] =  \sigma_{xx} + \sigma_{yy}
\end{array}
\label{eqn:lect09-014}
\end{equation}

   \noindent if $\sigma_{xy}=0$. Extending this result to three variables implies

\begin{equation}
\left[ \begin{array}{c} 1 \\ 1 \\ 1 \end{array} \right]' \left[ \begin{array}{ccc} \sigma_{11} & \sigma_{12} & \sigma_{13} \\ \sigma_{21} & \sigma_{22} & \sigma_{23} \\ \sigma_{31} & \sigma_{32} & \sigma_{33} \end{array} \right] \left[ \begin{array}{c} 1 \\ 1 \\ 1 \end{array} \right] = \sigma_{11} + 2 \sigma_{12} + 2 \sigma_{13} + \sigma_{22} + 2 \sigma_{23} + \sigma_{33}
\label{eqn:lect09-015}
\end{equation}

   \noindent if the $x$s are independent, the covariance terms are zero and this expression simply becomes the sum of the variances.
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item {\bf Definition 4.3.2.} The correlation coefficient for two variables is defined as

\begin{equation}
{\rm Corr}\left( X,Y \right) = \frac{\displaystyle {\rm Cov}\left( X,Y \right)}{\displaystyle \sqrt{\sigma_{xx}}\sqrt{\sigma_{yy}}}
\label{eqn:lect09-016}
\end{equation}

      \begin{itemize}
      \item Note that the covariance between any random variable and a constant is equal to zero.  Letting $Y$ equal to zero we have

\begin{equation}
{\rm E} \left[ \left( X - {\rm E} \left[X\right] \right) \left( Y - {\rm E} \left[Y\right]\right) \right] = {\rm E} \left[ \left( X - {\rm E} \left[ X \right] \right) \left( 0 \right) \right] = 0
\label{eqn:lect09-017}
\end{equation}

      \item It stands to reason the correlation coefficient between a random variable and a constant is also zero.
      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item It now possible to derive the ordinary least squares estimator for a linear regression equation.
      \begin{itemize}
      \item We define the ordinary least squares estimator as that set of parameters that minimizes the squared error of the estimate

\begin{equation}
\min_{\alpha,\beta} = {\rm E} \left[ \left( Y - \alpha - \beta X \right)^2 \right] = \min_{\alpha,\beta} {\rm E} \left[ Y^2 - 2 \alpha Y - 2 \beta XY + \alpha^2 + 2\alpha \beta X + \beta^2 X^2 \right]
\label{eqn:lect09-018}
\end{equation}

      \noindent The first order conditions for this minimization problem then becomes

\begin{equation}
\begin{array}{c}
\displaystyle \frac{\displaystyle \partial S}{\displaystyle \partial \alpha} = -2 {\rm E} \left[Y\right] + 2\alpha + 2 \beta \rm E\left[X\right] = 0 \\
\displaystyle \frac{\displaystyle \partial S}{\displaystyle \partial \beta} = -2 {\rm E} \left[ XY \right] + 2 \alpha \rm E \left[ X \right] + 2 \beta \rm E \left[X^2\right] = 0
\end{array}
\label{eqn:lect09-019}
\end{equation}

      \noindent Solving the first equation for $\alpha$ yields

\begin{equation}
\alpha = \rm E \left[ Y \right] - \beta \rm \left[ X \right]
\label{eqn:lect09-020}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Substituting this expression into the second first order condition yields

\begin{equation}
\begin{array}{c}
\displaystyle - {\rm E} \left[ XY \right] + \left( {\rm E} \left[ Y \right] - \beta {\rm E} \left[ X\right] \right) {\rm E} \left[ X \right] + \beta {\rm E} \left[ X^2\right] = 0 \\
\displaystyle - {\rm E} \left[ XY \right] + {\rm E} \left[Y\right] {\rm E} \left[X\right] + \beta \left( {\rm E} \left[X^2\right] - \left( {\rm E } \left[ X \right] \right)^2 \right) = 0 \\
\displaystyle - {\rm Cov}\left( X,Y \right) + \beta V\left( X \right) = 0 \\
\displaystyle \Rightarrow \beta = \frac{\displaystyle {\rm Cov}\left( X,Y \right)}{\displaystyle V \left( X \right)}
\end{array}
\label{eqn:lect09-021}
\end{equation}

      \end{itemize}
   \item {\bf Theorem 4.3.6.} The best linear predictor (or more exactly, the minimum mean-squared-error linear predictor) of $Y$  based on $X$ is given by $\alpha^* + \beta^* X$ , where $\alpha^*$  and $\beta^*$ are the least square estimates.
   \end{itemize}
}
\section{Conditional Mean and Variance}
\frame
{
   \frametitle{Conditional Mean and Variance}
   \begin{itemize}
   \item {\bf Definition 4.4.1} Let $\left(X,Y\right)$ be a bivariate discrete random variable taking values $\left(x_i,y_j\right)$ $i,j = 1,2,\cdots$.  Let $P\left( y_j | X \right)$  be the conditional probability of $Y = y_j$ given $X$.  Let $\phi\left(x_i,y_j\right)$ be an arbitrary function.  Then the conditional mean of $\phi\left(X,Y\right)$ given $X$, denoted ${\rm E} \left[ \phi\left(X,Y\right) | X\right]$  or by ${\rm E}_{Y|X} \left[ \phi \left(X,Y\right)\right]$ , is defined by
   
\begin{equation}
\rm E_{Y|X} \left[\phi\left(X,Y\right)\right] = \sum_{i=1}^\infty \phi\left(X,y_i\right) P\left( y_i | X \right)
\label{eqn:lect09-022}
\end{equation}

   \item {\bf Definition 4.4.2.} Let $\left(X,Y\right)$ be a bivarite continuous random variable with conditional density $f\left(y|x\right)$.  Let $\phi\left(x,y\right)$ be an arbitrary function.  Then the conditional mean of $\phi\left(X,Y\right)$ given $X$ is defined by

\begin{equation}
{\rm E}_{Y|X} \left[ \phi \left(X,Y\right) \right] = \int_{-\infty}^{\infty} \phi\left(X,y\right) f\left(y | X\right) dy
\label{eqn:lect09-023}
\end{equation}

   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item {\bf Theorem 4.4.1} (Law of Iterated Means) ${\rm E} \left[ \phi \left(X,Y \right) \right] = {\rm E}_x {\rm E}_{Y|X} \left[ \phi\left( X,Y \right) \right]$.  (Where the symbol ${\rm E}_x$  denotes the expectation with respect to $X$).

   \item {\bf Theorem 4.4.2} 

\begin{equation}
V\left(\phi\left(X,Y\right) \right) = {\rm E}_X \left[ V_{Y|X} \left[ \phi \left(X,Y\right)\right] \right] + V_x \left[ {\rm E}_{Y|X} \left[ \phi\left(X,Y\right) \right] \right]
\label{eqn:lect09-024}
\end{equation}

   \item {\it Proof}:

\begin{equation}
V_{Y|X} \left[ \phi \right] = {\rm E}_{Y|X} \left[ \phi^2\right] - \left( {\rm E}_{Y|X} \left[ \phi\right] \right)^2
\label{eqn:lect09-025}
\end{equation}

   \noindent Implies
   
\begin{equation}
{\rm E}_X \left[V_{Y|X} \left( \phi \right) \right] = {\rm E} \left[\phi^2\right] - {\rm E}_X \left[ {\rm E}_{Y|X} \left[ \phi \right] \right]
\label{eqn:lect09-026}
\end{equation}

   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item By definition of conditional variance
   
\begin{equation}
V_X\left( {\rm E}_{Y|X} \left[ \phi \right] \right) = {\rm E}_X \left[ {\rm E}_{Y|X} \left[ \phi \right] \right]^2 - \left( {\rm E} \left[ \phi \right] \right)^2
\label{eqn:lect09-027}
\end{equation}


   \begin{itemize}
   \item Adding these expressions yields

\begin{equation}
{\rm E}_X \left[ V_{Y|X} \left(\phi\right) \right] + V_X \left( {\rm E}_{Y|X} \left[ \phi \right] \right) = {\rm E} \left[ \phi^2\right] - \left( {\rm E} \left[ \phi \right] \right) = V \left( \phi \right)
\label{eqn:lect09-028}
\end{equation}

   \item {\bf Theorem 4.4.3} The best predictor (or the minimum mean-squared-error predictor) of $Y$ based on $X$ is given by ${\rm E}\left[ Y | X \right]$.
  
   \end{itemize}
\end{itemize}
}

\end{document}