%                               ANCOVA for STA305 (Experimental Design)
% Notes and comments at the end

% \documentclass[serif]{beamer} % Serif for Computer Modern math font.
\documentclass[serif, handout]{beamer} % Handout mode to ignore pause statements
\hypersetup{colorlinks,linkcolor=,urlcolor=red}


%  To create handout using article mode: Comment above and uncomment below (2 places)
%\documentclass[12pt]{article}
%\usepackage{beamerarticle}
%\usepackage[colorlinks=true, pdfstartview=FitV, linkcolor=blue, citecolor=blue, urlcolor=red]{hyperref}   
     % For live Web links with href in article mode
%\usepackage{fullpage}
 

\usefonttheme{serif} % Looks like Computer Modern for non-math text -- nice!
\setbeamertemplate{navigation symbols}{} % Supress navigation symbols
\usetheme{Berlin} % Displays sections on top
\usepackage[english]{babel}
% \definecolor{links}{HTML}{2A1B81}
% \definecolor{links}{red}

\setbeamertemplate{footline}[frame number] 

\mode<presentation>
% \mode<handout>{\setbeamercolor{background canvas}{bg=black!5}}



\title{The mysterious beauty of the analysis of covariance\footnote{See last slide for copyright information.}}
\subtitle{STA305 Winter 2014}
\date{} % To suppress date


\begin{document}


\begin{frame}
  \titlepage
\end{frame}

\begin{frame}
\frametitle{Background Reading}
\framesubtitle{Optional} 
  \begin{itemize}
    \item Chapter 5 in \emph{Data analysis with SAS} presents some important parts of this material as a special case of regression.
  \end{itemize}
\end{frame}

\begin{frame}
\frametitle{Basic idea}
\begin{itemize}
     \item Lots of things influence the response other than the treatment.
     \item Because of random assignment, they are independent of the treatment.
     \item They all go into the error (background noise) term $\epsilon_{ij}$.
     \item $\sigma^2 = Var(\epsilon_{ij})$ is the loudness of the background noise.
     \item Reduce loudness of background noise by measuring important influences and including them in the model.
     \item Make sure that the treatment is not influencing the covariate.

\end{itemize}
\end{frame}

\begin{frame}
\frametitle{It's just another regression model}
\framesubtitle{The $d_{i,j}$ are dummy variables for the treatments} 
\begin{eqnarray*}
    Y_i &=&  \beta_0 + \beta_1 d_{i,1} + \cdots + \beta_{p-1} d_{i,p-1} + \epsilon_i \\
        &=&  \beta_0^\prime + \beta_1 d_{i,1} + \cdots + \beta_{p-1} d_{i,p-1} + 
             (\alpha_1 X_{i1} + \cdots +  \alpha_k X_{ik} + e_i )\\
        &=&  \mathbf{X}^\prime_i\boldsymbol{\alpha} + \mathbf{d}^\prime_i\boldsymbol{\beta} + e_i
\end{eqnarray*}
\begin{itemize}
    \item $Var(e_i)<Var(\epsilon_i)$.
    \item The $X_{i,j}$ are called \emph{covariates}.
    \item They are random variables, but treat them as fixed.
    \item This is the usual conditional regression model.
    \item The assumption of unit-treatment additivity implies parallel regression planes. 
\end{itemize}

\end{frame}

\begin{frame}
\frametitle{Technical issues with the model $Y_i = \mathbf{x}^\prime_i\boldsymbol{\alpha} + \mathbf{d}^\prime_i\boldsymbol{\beta} + e_i$}
  \begin{itemize}
    \item Assume this model is conditional on $\mathbf{X}_i = \mathbf{x}_i$.
    \item Error terms $e_i$ are identically distributed given $\mathbf{X}_i = \mathbf{x}_i$.
    \item So the model assumes $e_i$ and $\mathbf{X}_i$ are independent.
    \item Thus any other omitted variables that influence $Y_i$ must be \emph{independent of the covariates}.
    \item Impossible to believe, and a well-known recipe for trouble.
    \item Also, covariates are surely measured with error, another recipe for trouble.
  \end{itemize}
\vspace{3mm}
{\LARGE Does it still work?}
\end{frame}

\begin{frame}
\frametitle{A simple example}
\framesubtitle{The true model ($e_i$ is different now)} 
\begin{itemize}
    \item Binary dummy variable for experimental treatment.
    \item One covariate measured with error.
    \item One omitted variable, correlated with the (true) covariate.
\end{itemize}
{\LARGE
\begin{eqnarray*}
    Y_i &=& \beta_0 + \beta_1d_i + \alpha_1X_{i1} + \alpha_2X_{i2} + \epsilon_i \\
    W_i &=& \lambda_0 + \lambda_1 X_{i1} + e_i
\end{eqnarray*} 
} % End size
\begin{itemize}
    \item Observe $(d_i,W_i,Y_i)$.
    \item Fit $Y_i = \beta_0^* + \beta_1^*d_i + \beta_2^*w_i + \delta_i $
    \item Interest is in $\beta_1=\Delta$.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{A simulation study}
%\framesubtitle{} 
\begin{eqnarray*}
    Y_i &=& \beta_0 + \beta_1d_i + \alpha_1X_{i1} + \alpha_2X_{i2} + \epsilon_i \\
    W_i &=& \lambda_0 + \lambda_1 X_{i1} + e_i
\end{eqnarray*} 
\begin{itemize}
    \item $X_1$ and $X_2$ are both strongly related to $Y$.
    \item $X_1$ and $X_2$ are strongly correlated.
    \item Lots of measurement error.
    \item $n_1=n_2=64$
    \item Fit $Y_i = \beta_0^* + \beta_1^*d_i + \beta_2^*w_i + \delta_i $
    \item Test $H_0: \beta_1^*=0$ ten thousand times when $\beta_1=0$ is true, and there is no treatment effect. 
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{No inflation of Type I error probability}
%\framesubtitle{} 
\begin{itemize}
    \item Did it both ways, with and without the (corrupted) covariate $W_i$.
    \item Without covariate: $p \approx 0.0464$
    \item With covariate: $p \approx 0.0537$
    \item These are typical results.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Sampling distribution of $\widehat{\Delta}$}
\framesubtitle{Based on ten thousand simulated data sets} 
\begin{center}
\includegraphics[width=2.25in]{WithoutCovariate}
\includegraphics[width=2.25in]{WithCovariate}
\end{center}
\end{frame}

\begin{frame}
\frametitle{$Var(\widehat{\Delta})$ is smaller with the covariate}
%\framesubtitle{} 
\begin{itemize}
    \item Without covariate, exactly $\sigma^{2\prime}\left( \frac{1}{n_1} + \frac{1}{n_2}  \right) = 0.48125$
    \item With covariate, approximately $0.2769367$ based on the sample variance of 10,000 estimates.
    \item Had $n_1=n_2=64$. Keeping equal sample sizes, what sample size is needed to achieve this precision without the covariate?
\begin{eqnarray*}
     && 15.4 \left( \frac{1}{n_1} + \frac{1}{n_1}  \right) = 0.2769367 \\
    & \Leftrightarrow & n_1 = 111.2
\end{eqnarray*} 
    \item Need about 111+111=222 experimental units to get the same precision without the  covariate. 
    \item The covariate is worth about 222-128=94 experimental units.
    \item An estimator with lower variance is said to be more \emph{efficient}.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Why does the analysis of covariance work so well?}
\framesubtitle{When the model is so wrong} 
After a lot of work,
\begin{eqnarray*}
    \widehat{\Delta} &=& \frac{\hat{\sigma}^2_w (\overline{Y}_1-\overline{Y}_2)
    - \hat{\sigma}_{wy}(\overline{W}_1-\overline{W}_2)}
    {\hat{\sigma}^2_w + q(1-q)(\overline{W}_1-\overline{W}_2)^2} \\ 
    && \\
     &=&  \left(\frac{\hat{\sigma}^2_w}{\hat{\sigma}^2_w + q(1-q)(\overline{W}_1-\overline{W}_2)^2} \right) 
          (\overline{Y}_1-\overline{Y}_2) \\
    && \\
    &&  - ~ \frac{\hat{\sigma}_{wy}(\overline{W}_1-\overline{W}_2)}
    {\hat{\sigma}^2_w + q(1-q)(\overline{W}_1-\overline{W}_2)^2}
\end{eqnarray*} 

And $\overline{W}_1-\overline{W}_2 \rightarrow 0$ as $n \rightarrow \infty$.
\end{frame}

\begin{frame}
\frametitle{The real reason it works (Details omitted)}
\begin{itemize}
    \item If covariates were unrelated to omitted variables and measured without error, everything would be fine.
    \item Call this the ``pretend model."
    \item But actually, covariates are related to omitted variables and measured \emph{with} error.
    \item Call this the ``true model."
    \item Data from the pretend model are indistinguishable from data from the true model.
    \item This does not always happen.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Parameters}
\begin{itemize}
    \item The true model has more parameters (13 versus 6 in the example).
    \item Parameters of the pretend model are \emph{functions} of parameters of the true model.
    \item Regression coefficients of the dummy variables are the same under both models. This is the key.
    \item It happens only because of random assignment.
    \item Other parameters of the pretend model are crazy functions of the parameters of the true model.
    \item But estimation and inference about the \emph{treatment effects} are excellent (as usual) under the pretend model.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{For the little example}
%\framesubtitle{} 
\begin{eqnarray*}
    Y_i &=& \beta_0 + \beta_1d_i + \alpha_1X_{i1} + \alpha_2X_{i2} + \epsilon_i \\
    W_i &=& \lambda_0 + \lambda_1 X_{i1} + e_i \\
    && \\
    Y_i &=& \beta_0^* + \beta_1^*d_i + \beta_2^*W_i + \delta_i
\end{eqnarray*} 
\begin{itemize}
    \item $\beta_1^*=\beta_1$
    \item $Var(W_i) = \lambda_1^2\phi_{11}+\omega$
    \item $\beta_2^* = \frac{\lambda_1(\alpha_1\phi_{11}+\alpha_2\phi_{12})} {\lambda_1^2\phi_{11}+\omega}$
    \item $Var(\delta_i)$ is breathtaking.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Moral of the story}
%\framesubtitle{} 
\begin{itemize}
     \item Analysis of covariance can greatly increase the precision of an analysis by reducing background noise.
     \item Precision of estimation translates directly into time and money.
     \item The covariates may be measured with error and related to other important but unknown variables that influence the dependent variable.
     \item As long as there is random assignment, it still works beautifully even though the model is wrong.
     \item Technically, the analysis of covariance model is ``equivalent to a re-parameterization."
     \item Of course you must be sure that the treatment is not influencing the covariate.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Assumption of Unit-treatment additivity}
%\framesubtitle{} 
\begin{itemize}
    \item Without any treatment, the response is $Y_i = \beta_0 + \beta_1 x_i + \epsilon_i$.
    \item Treatment $j$ just adds $\Delta_j$ to the response, moving all the responses of the units in condition $j$ up (or down) by $\Delta_j$.
    \item Write it as a multiple regression model with dummy variables:
\begin{displaymath}
    Y_i = \beta_0 + \beta_1 x_i + \beta_2 d_{i,1} +  \beta_3 d_{i,2}  + \epsilon_i
\end{displaymath} 
    \item Make a table.
\end{itemize}
\end{frame}

\begin{frame}
\frametitle{Equal slopes model}
% \framesubtitle{} 
\begin{displaymath}
    Y = \beta_0 + \beta_1 x + \beta_2 d_1 + \beta_3 d_2  + \epsilon
\end{displaymath} 
\vspace{5mm}
\begin{center}
\begin{tabular}{|c|c|c|c|} \hline
Treatment & $d_1$ & $d_2$ &  $E(Y|\mathbf{x})$ \\ \hline
1         & 1 & 0  & $(\beta_0+\beta_2) + \beta_1 x$ \\ \hline
2         & 0 & 1  & $(\beta_0+\beta_3) + \beta_1 x$ \\ \hline
3         & 0 & 0  & $~~~~~\beta_0 ~~~~+ \beta_1 x$ \\ \hline
\end{tabular}
\end{center}
\end{frame}

\begin{frame}
\frametitle{Look at the least squares means}
\begin{displaymath}
    \hat{Y} = \hat{\beta}_0 + \hat{\beta_1} x + \hat{\beta}_2 d_1 + \hat{\beta}_3 d_2  
\end{displaymath} 
\vspace{5mm}
\begin{center}
\begin{tabular}{|c|c|c|c|} \hline
Treatment & $d_1$ & $d_2$ &  Estimated Response \\ \hline
1         & 1 & 0  & $(\hat{\beta}_0+\hat{\beta}_2) + \hat{\beta}_1 \overline{x}$ \\ \hline
2         & 0 & 1  & $(\hat{\beta}_0+\hat{\beta}_3) + \hat{\beta}_1 \overline{x}$ \\ \hline
3         & 0 & 0  & $~~~~~\hat{\beta}_0 ~~~~+ \hat{\beta}_1 \overline{x}$ \\ \hline
\end{tabular}
\end{center}
\begin{itemize}
    \item The least squares means are actually $\hat{Y}$ values.
    \item In plain language, call them ``corrected means," or something like ``average teaching evaluation, corrected for teacher's age."
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{Equal slopes assumption is testable}
%\framesubtitle{} 
\begin{itemize}
    \item Interaction means slopes are not equal: ``It depends."
    \item Form a product of quantitative variable by each dummy variable for the categorical variable.

\begin{displaymath}
    Y = \beta_0 + \beta_1 x + \beta_2 d_1 + \beta_3 d_2 
                + \beta_4x\,d_1 + \beta_5 x\,d_2 + \epsilon
\end{displaymath}
    \item Make a table.
\end{itemize}
\end{frame}


\begin{frame}
\frametitle{Unequal slopes model}
% \framesubtitle{Effect of the treatment \emph{depends} on covariate value(s).} 
\begin{displaymath}
    Y = \beta_0 + \beta_1 x + \beta_2 d_1 + \beta_3 d_2 
                + \beta_4x\,d_1 + \beta_5 x\,d_2 + \epsilon
\end{displaymath} 
\vspace{5mm}
\begin{center}
\begin{tabular}{|c|c|c|c|} \hline
Treatment & $d_1$ & $d_2$ &  $E(Y|\mathbf{x})$ \\ \hline
1         & 1 & 0  & $(\beta_0+\beta_2) + (\beta_1+\beta_4) x$ \\ \hline
2         & 0 & 1  & $(\beta_0+\beta_3) + (\beta_1+\beta_5) x$ \\ \hline
3         & 0 & 0  & $~~~~~\beta_0 ~~~~+ ~~~~~\beta_1 ~~~~x$ \\ \hline
\end{tabular}
\end{center}
\end{frame}

\begin{frame}
\frametitle{Sample questions}
%\framesubtitle{} 
\begin{tabular}{|c|c|c|c|} \hline
Group   & $d_1$ & $d_2$ &  $E(Y|\mathbf{x})$ \\ \hline
1       & 1 & 0  & $(\beta_0+\beta_2) + (\beta_1+\beta_4) x$ \\ \hline
2       & 0 & 1  & $(\beta_0+\beta_3) + (\beta_1+\beta_5) x$ \\ \hline
3       & 0 & 0  & $~~~~~\beta_0 ~~~~+ ~~~~~\beta_1 ~~~~x$ \\ \hline
\end{tabular}

\vspace{5mm}

What null hypothesis would you test?
\begin{itemize}
    \item Are all the slopes equal?
    \item Compare slopes for group one vs three.
    \item Compare slopes for group one vs two.
    \item Is there an interaction between treatment and covariate?
    \item Test the null hypothesis of equal regressions.
\end{itemize}
\end{frame}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%






%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\begin{frame}
\frametitle{Copyright Information}

This slide show was prepared by  \href{http://www.utstat.toronto.edu/~brunner}{Jerry Brunner},
Department of Statistics, University of Toronto. It is licensed under a 
\href{http://creativecommons.org/licenses/by-sa/3.0/deed.en_US}
     {Creative Commons Attribution - ShareAlike 3.0 Unported License}. Use any part of it as you like and share the result freely. The \LaTeX~source code is available from the course website:
\href{http://www.utstat.toronto.edu/~brunner/oldclass/305s14} {\footnotesize \texttt{http://www.utstat.toronto.edu/$^\sim$brunner/oldclass/305s14}}

\end{frame}


\end{document}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%















\begin{frame}
\frametitle{Permutation distribution of $\overline{Y}_2-\overline{Y}_1$}
%\framesubtitle{} 
\begin{center}
\includegraphics[width=3in]{PermutationDist}
\end{center}
\end{frame}


\begin{frame}
\frametitle{Basic idea}
\begin{displaymath}
\begin{array}{ccc}
L = \sum_{j=1}^p a_j\mu_j & 
\widehat{L} = \sum_{j=1}^p a_j\overline{Y}_j &
Var(\widehat{L}) = \sigma^2 \sum_{j=1}^p \frac{a_j^2}{n_j}
\end{array}
\end{displaymath}

\begin{itemize}
     \item Lots of things influence the response other than the treatment.
     \item Because of random assignment, they are independent of the treatment.
     \item They all go into the error (background noise) term $\epsilon_{ij}$.
%     \item $Var(\epsilon_{ij}) = \sigma^2$ is loudness of the background noise.
     \item Reduce background noise by measuring important influences and including them in the model.
\end{itemize}
\end{frame}





\begin{frame}
\frametitle{The real reason it works (Proof omitted)}
% \framesubtitle{Hairy technical details omitted} 
\begin{itemize}
    \item If covariates were unrelated to omitted variables and measured without error, everything would be fine.
    \item Call this the ``pretend model."
    \item But actually, covariates are related to omitted variables and measured \emph{with} error.
    \item Call this the ``true model."
    \item Because treatment is independent of both covariates and omitted variables, data from the pretend model are indistinguishable from data from the true model.
    \item Parameters of the pretend model are \emph{functions} of parameters of the true model.
    \item Regression coefficients of the dummy variables are the same under both models.
    \item So estimation and inference for the pretend model are excellent.
%    \item  Everything depends on lack of relationship between $d_i$ and all omitted variables. This comes from random assignment.
\end{itemize}
\end{frame}


For a professional audience:

In practice, covariates are usually random variables that are measured with error and have non-zero covariance with explanatory variables that are not part of the data set. In this case, the usual analysis of covariance model may be viewed as a re-parameterization.