\documentclass{beamer}

\usepackage{beamerthemesplit}
\usetheme{Boadilla}
\usecolortheme{albatross}

\title[Moment Generating Functions]{Moment Generating Functions: Lecture X}
\author{Charles B. Moss}
\date{\today}

\begin{document}

\frame{\titlepage}

\section[Outline]{}
\frame{\tableofcontents}

\section{Moment Generating Function}
\frame
{
   \frametitle{Moment Generating Function}
   \begin{itemize}
   \item {\bf Definition 2.3.3.} Let $X$ be a random variable with cumulative distribution function $F\left(X\right)$. The moment generating function (mgf) of $X$ (or $F\left(X\right)$ ), denoted $M_X\left(t\right)$, is

\begin{equation}
M_X\left(t\right) = {\rm E} \left[ e^{tX} \right]
\label{eqn:lect10-001}
\end{equation}

   \noindent provided that the expectation exists for $t$ in some neighborhood of 0.  That is, there is an $h>0$ such that, for all $t$ in $-h < t < h$ , ${\rm E} \left[ e^{tX} \right]$ exists.
      \begin{itemize}
      \item If the expectation does not exist in a neighborhood of 0, we say that the moment generating function does not exist.
      \item More explicitly, the moment generating function can be defined as

\begin{equation}
\begin{array}{c}
\displaystyle M_X \left(t\right) = \int_{-\infty}^\infty e^{tx} f\left(x\right) dx \,{\rm for \, continuous \, random \, variables, \, and} \\ \\
\displaystyle M_x \left(t\right) = \sum_x e^{tx} P\left[X = x\right] \, {\rm for \, discrete \, random \, variables}
\end{array}
\label{eqn:lect10-002}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item {\bf Theorem 2.3.2} If $X$ has {\it mgf} $M_X\left(t\right)$ , then

\begin{equation}
{\rm E} \left[ X^n\right] = M_x^{\left(n\right)}\left(0\right)
\label{eqn:lect10-003}
\end{equation}

   \noindent where we define
   
\begin{equation}
M_X^{\left(n\right)} \left( 0 \right) = \frac{\displaystyle d^n}{\displaystyle d t^n} \left. M_X \left(t\right) \right|_{t\rightarrow 0}
\label{eqn:lect10-004}
\end{equation}

      \begin{itemize}
      \item[1.] First note that $e^{tX}$ can be approximated around zero using a Taylor series expansion
      
\begin{equation}
\begin{array}{c}
\displaystyle M_X \left(t\right) = {\rm E} \left[ e^{tx}\right] = {\rm E}\left[ e^0 + t e^{t0} \left(x - 0\right) + \right. \\
\displaystyle \left. \frac{1}{2} t^2 e^{t0} \left(x-0\right)^2 + \frac{1}{6} t^3 e^{t0} \left(x-0\right)^3 + \cdots \right] \\ \\
\displaystyle \,\,\, = 1 + {\rm E}\left[x\right]t+{\rm E}\left[x^2\right]\frac{\displaystyle t^2}{\displaystyle 2} + {\rm E}\left[ x^3 \right] \frac{\displaystyle t^3}{\displaystyle 6} + \cdots
\end{array}
\label{eqn:lect10-005}
\end{equation}

      \noindent Note for any moment $n$

\begin{equation}
M_x^{\left(n\right)}\left(t\right) = \frac{\displaystyle d^n}{\displaystyle dt^n} M_X\left(t\right) = {\rm E} \left[ x^n\right] + {\rm E}\left[ x^{n+1}\right] + {\rm E} \left[ x^{n+2}\right] + \cdots
\label{eqn:lect10-006}
\end{equation}

      \noindent Thus, as $t \rightarrow 0$

\begin{equation}
M_x^{\left(n\right)} \left(0\right) = {\rm E}\left[x^n\right]
\label{eqn:lect10-007}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Leibnitz’s Rule: If $f\left(x,\theta\right)$ , $a\left(\theta\right)$ , and $b\left(\theta\right)$ are differentiable with respect to $\theta$ , then

\begin{equation}
\begin{array}{c}
\displaystyle \frac{\displaystyle d}{\displaystyle d \theta} \int_{a\left(\theta\right)}^{b\left(\theta\right)} f\left(x,\theta\right)dx = - f\left(b\left(\theta\right),\theta\right) \frac{\displaystyle d}{\displaystyle d\theta} a\left(\theta\right) + f\left(a\left(\theta\right)\right) \frac{\displaystyle d}{\displaystyle d \theta} b\left(\theta\right) \\
\displaystyle \int_{a\left(\theta\right)}^{b\left(\theta\right)} \frac{\displaystyle \partial}{\displaystyle \partial \theta} f\left(x,\theta\right) dx
\end{array}
\label{eqn:lect10-008}
\end{equation}

      \item Berger and Casella proof:  Assume that we can differentiate under the integral using Leibnitz’s rule, we have

\begin{equation}
\begin{array}{c}
\displaystyle \frac{\displaystyle d}{\displaystyle dt} M_X \left(t\right) = \frac{\displaystyle d}{\displaystyle dt} \int_{-\infty}^\infty e^{tx} f\left(x\right) dx \\ \\
\,\,\, = \int_{-\infty}^\infty \left( \frac{\displaystyle d}{\displaystyle dt}e^{tx} \right) f\left(x\right) dx \\ \\
\,\,\, = \int_{-\infty}^\infty x e^{tx} f\left(x\right) dx
\end{array}
\label{eqn:lect10-009}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Letting $t \rightarrow 0$, this integral simply bgecomes

\begin{equation}
\int_{-\infty}^\infty x f \left(x \right) dx = {\rm E} \left[ x \right]
\label{eqn:lect10-010}
\end{equation}

      \item This proof can be extended for any moment of the distribution function.
      \end{itemize}
   \item Moment Generating Functions for Specific Distributions
      \begin{itemize}
      \item Application to the Uniform Distribution

\begin{equation}
M_X\left(t\right) = \int_a^b \frac{\displaystyle e^{tx}}{\displaystyle b - a} dx = \frac{\displaystyle 1}{\displaystyle b-a} \frac{\displaystyle 1}{\displaystyle t} \left( e^{tx} \right|_a^b = \frac{\displaystyle e^{bt}-e^{at}}{\displaystyle t \left( b - a \right)}
\label{eqn:lect10-011}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Following the expansion developed earlier, we have
      
\begin{equation}
\begin{array}{c}
\displaystyle M_X\left(t\right) = \frac{\displaystyle \left( 1 - 1 \right) + \left( b - a\right)t + \frac{1}{2}\left(b^2 - a^2\right)t^2 + \frac{1}{6} \left(b^3 - a^3\right)t^3 + \cdots}{\displaystyle \left(b-a\right)t} \\
\displaystyle \,\,\, = 1+\frac{\displaystyle \left(b^2 - a^2\right)t^2}{\displaystyle 2\left(b-a\right)t} + \frac{\displaystyle \left(b^3 - a^3\right)t^3}{\displaystyle 6 \left(b-a\right)t} + \cdots \\
\displaystyle \,\,\, = 1 + \frac{1}{2} \frac{\displaystyle \left(b-a\right)\left(b+a\right)}{\displaystyle \left(b-a\right)} \frac{\displaystyle t^2}{\displaystyle t} + \frac{1}{6}\frac{\displaystyle \left(b-a\right) \left(b^2+ab+a^2\right)}{\displaystyle \left(b-a\right)} \frac{\displaystyle t^3}{\displaystyle t} + \cdots \\
\,\,\, = 1 + \frac{1}{2} \left(a+b\right)t + \frac{1}{6} \left(a^2 + ab + b^2 \right)t^2 + \cdots
\end{array}
\label{eqn:lect10-012}
\end{equation}

      \noindent Letting $b=1$ and $a=0$, the last expression becomes
      
\begin{equation}
M_X\left(t\right) = 1 + \frac{1}{2}t+\frac{1}{6}t^2 + \frac{1}{24}t^3 + \cdots
\label{eqn:lect10-013}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item The first three moments of the uniform distribution are then

\begin{equation}
\begin{array}{c}
M_X^{\left(1\right)} = \frac{1}{2} \\ \\
M_X^{\left(2\right)} = \frac{1}{6} 2 = \frac{1}{3} \\ \\
M_X^{\left(3\right)} = \frac{1}{24} 6 = \frac{1}{4}
\end{array}
\label{eqn:lect10-014}
\end{equation}

      \item Application to the Univariate Normal Distribution

\begin{equation}
\begin{array}{c}
\displaystyle M_X\left(t\right) = \frac{\displaystyle 1}{\displaystyle \sigma \sqrt{2 \pi}} \int_{-\infty}^\infty e^{tx} e^{\displaystyle - \frac{1}{2}\frac{\displaystyle \left(x - \mu\right)^2}{\displaystyle \sigma^2}} dx \\ \\
\displaystyle \,\,\, = \frac{\displaystyle 1}{\displaystyle \sigma \sqrt{2 \pi}} \int_{-\infty}^\infty \exp\left[ tx - \frac{1}{2} \frac{\displaystyle \left(x-\mu\right)^2}{\displaystyle \sigma^2} \right] dx
\end{array}
\label{eqn:lect10-015}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Focusing on the term in the exponent, we have

\begin{equation}
\begin{array}{c}
\displaystyle tx - \frac{1}{2} \frac{\displaystyle \left( x - \mu \right)^2}{\displaystyle \sigma^2} = - \frac{1}{2} \frac{\displaystyle \left(x - \mu \right)^2 - 2 t x \sigma^2}{\displaystyle \sigma^2} \\
\displaystyle \,\,\, = - \frac{1}{2} \frac{\displaystyle x^2 - 2 x \mu + \mu^2 - 2t x\sigma^2}{\displaystyle \sigma^2} \\
\displaystyle \,\,\, = - \frac{1}{2} \frac{\displaystyle x^2 - 2 \left( x \mu + t x \sigma^2\right) + \mu^2 }{\displaystyle \sigma^2} \\
\displaystyle \,\,\, = - \frac{1}{2} \frac{\displaystyle x^2 - 2 x \left( \mu + t \sigma^2\right) + \mu^2}{\displaystyle \sigma^2}
\end{array}
\label{eqn:lect10-016}
\end{equation}

      \noindent The next state is to complete the square in the numerator

\begin{equation}
\begin{array}{c}
\displaystyle x^2 - 2 x \left(\mu + t \sigma^2\right) + \mu^2 + c = 0 \\ \\
\displaystyle \left( x - \left(\mu + t \sigma^2\right) \right)^2 = 0  \\ \\
\displaystyle x^2 - 2 x \left(\mu + t \sigma\right) + \mu^2 + 2 t \sigma^2 \mu + t^2 \sigma^4 = 0 \\ \\
\displaystyle c = 2 t \sigma^2 \mu + t^2 \sigma^4
\end{array}
\label{eqn:lect10-017}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item The complete expression then becomes

\begin{equation}
\begin{array}{c}
\displaystyle tx - \frac{1}{2} \frac{\displaystyle \left( x - \mu \right)^2}{\displaystyle \sigma^2} = -\frac{1}{2} \frac{\displaystyle \left(x - \mu - t \sigma^2\right) - 2 \mu \sigma^2 t - \sigma^4 t^2}{\displaystyle \sigma^2} \\ \\
\,\,\, = - \frac{1}{2} \frac{\displaystyle \left( x - \mu - t \sigma^2\right)}{\displaystyle \sigma^2} + \mu t + \frac{1}{2} \sigma^2 t^2
\end{array}
\label{eqn:lect10-018}
\end{equation}

      \noindent The moment generating function then becomes

\begin{equation}
\begin{array}{c}
\displaystyle M_X \left( t \right) = \exp\left( \mu t + \frac{1}{2} \sigma^2 t^2 \right) \frac{\displaystyle 1}{\displaystyle \sigma \sqrt{2 \pi}} \int_{-\infty}^\infty \exp\left( - \frac{1}{2} \frac{\displaystyle \left(x - \mu - t \sigma^2\right)}{\displaystyle \sigma^2} \right) dx \\ \\
\displaystyle \,\,\, = \exp\left(\mu t + \frac{1}{2}\sigma^2 t^2\right)
\end{array}
\label{eqn:lect10-019}
\end{equation}

      \noindent Taking the first derivative with respect to $t$, we get

\begin{equation}
M_X^{\left(1\right)} \left( t \right) = \left( \mu + \sigma^2 t\right) \exp\left(\mu t + \frac{1}{2} \sigma^2 t^2\right).
\label{eqn:lect10-020}
\end{equation}
      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Letting $t\rightarrow 0$, this becomes
      
\begin{equation}
M_X^{\left(0\right)} = \mu
\label{eqn:lect10-021}
\end{equation}

      \noindent The second derivative of the moment generating function with respect to $t$ yields

\begin{equation}
\begin{array}{c}
\displaystyle M_X^{\left(2\right)} \left(t\right) = \sigma^2 \exp\left(\mu t + \frac{1}{2}\sigma^2 t^2\right) + \\ \\
\displaystyle \left(\mu + \sigma^2 t \right) \left(\mu + \sigma^2 t \right) \exp\left(\mu t + \frac{1}{2} \sigma^2 t^2\right)
\end{array}
\label{eqn:lect10-022}
\end{equation}

      \noindent Again, letting $t\rightarrow 0$ yields

\begin{equation}
M_X^{\left(2\right)}\left(0\right) = \sigma^2 + \mu^2
\label{eqn:lect10-023}
\end{equation}

      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Let $X$ and $Y$ be independent random variables with moment generating functions $M_X\left(t\right)$ and $M_Y\left(t\right)$.  Consider their sum $Z = X + Y$ and its moment generating function

\begin{equation}
\begin{array}{c}
\displaystyle M_Z\left(t\right) = {\rm E}\left[ e^{tz}\right] = {\rm E}\left[ e^{t\left(x+y\right)}\right] = {\rm E}\left[ e^{tx} e^{ty} \right] = \\ \\
\displaystyle {\rm E}\left[ e^{tx}\right] {\rm E}\left[ e^{ty}\right] = \ M_X\left(t\right) M_Y\left(t\right)
\end{array}
\label{eqn:lect10-024}
\end{equation}

         \begin{itemize}
         \item We conclude that the moment generating function for two independent random variables is equal to the product of the moment generating functions of each variable.
         \item Skipping ahead slightly, the multivariate normal distribution function can be written as
         
\begin{equation}
f\left(x\right) = \frac{\displaystyle 1}{\displaystyle \sqrt{2 \pi}} |\Sigma| \exp\left(-\frac{1}{2} \left(x - \mu\right)' \Sigma^{-1} \left(x - \mu \right) \right)
\label{eqn:lect10-025}
\end{equation}

         \end{itemize}
      \end{itemize}
   \end{itemize}
}
\frame
{
   \begin{itemize}
   \item Continued
      \begin{itemize}
      \item Continued
         \begin{itemize}
         \item In order to derive the moment generating function, we now need a vector $\tilde t$.  The moment generating function can then be defined as

\begin{equation}
M_{\tilde X} \left(\tilde t\right) = \exp\left( \mu'{\tilde t} + \frac{1}{2}{\tilde t}' \Sigma {\tilde t} \right)
\label{eqn:lect10-026}
\end{equation}

         \item Normal variables are independent if the variance matrix is a diagonal matrix.
         \item Note that if the variance matrix is diagonal, the moment generating function for the normal can be written as

\begin{equation}
\begin{array}{c}
\displaystyle M_{\tilde X}\left(\tilde x\right) = \exp\left( \mu'{\tilde t} + \frac{1}{2} {\tilde t}' \left( \begin{array}{ccc} \sigma_1^2 & 0 & 0 \\ 0 & \sigma_2^2 & 0 \\ 0 & 0 & \sigma_3^3 \end{array} \right) {\tilde t} \right) \\ \\
\displaystyle \,\,\, = \exp\left(\mu_1 t_1 + \mu_2 t_1 + \mu_3 t_3 +\frac{1}{2} \left(t_1^2\sigma_1^2 + t_2^2 \sigma_2^2 + t_3^2 \sigma_3^2\right) \right) \\ \\
\displaystyle \,\,\, = \exp\left( \left( \mu_1 t_1 + \frac{1}{2}t_1^2 \sigma_1^2 t_1^2 \right) + \left(\mu_2 t_2 + \frac{1}{2} \sigma_2^2 t_2^2 \right) + \left( \mu_3 t_3 + \frac{1}{2} \sigma_3^2 t_3^2 \right) \right) \\ \\
\displaystyle \,\,\, = M_{X_1}\left(t\right) M_{X_2}\left(t\right) M_{X_3}\left(t\right)
\end{array}
\label{eqn:lect10-027}
\end{equation}

         \end{itemize}
      \end{itemize}
   \end{itemize}
}

\end{document}