\documentclass[12pt]{article} 
%\usepackage{amsbsy} % for \boldsymbol and \pmb 
\usepackage{graphicx} % To include pdf files!
\usepackage{amsmath}
\usepackage{amsbsy}
\usepackage{amsfonts} % for \mathbb{R} The set of reals
\usepackage[colorlinks=true, pdfstartview=FitV, linkcolor=blue, citecolor=blue, urlcolor=blue]{hyperref} % For links
\usepackage{fullpage}
%\pagestyle{empty} % No page numbers


\begin{document}
%\enlargethispage*{1000 pt} 

\begin{center}   
{\Large \textbf{STA 302f14 Assignment Four}}\footnote{Copyright information is at the end of the last page.}
\vspace{1 mm}
\end{center}

\noindent
Except for Question~\ref{R}, these problems are preparation for the quiz in tutorial on Friday October 10th, and are not to be handed in. Please bring your printout from Question~\ref{R} to the quiz. Please look at the current formula sheet while you do these problems.

\begin{enumerate} 

\item \label{glm} The general linear model is $\mathbf{Y} = \mathbf{X}\boldsymbol{\beta} + \boldsymbol{\epsilon}$, where $\mathbf{X}$ is an $n \times (k+1)$ matrix of observable constants,  $\boldsymbol{\beta}$ is a $(k+1) \times 1$ vector of unknown constants (parameters), and $\boldsymbol{\epsilon}$ is an $n \times 1$ vector of unobservable random variables with $E(\boldsymbol{\epsilon})=\mathbf{0}$ and $cov(\boldsymbol{\epsilon})=\sigma^2\mathbf{I}_n$, where $\sigma^2>0$ is an unknown constant parameter.
    \begin{enumerate}
        \item Show that the matrix $\mathbf{X^\prime X}$ is symmetric.

        \item Recall that the $p \times p$ matrix $\mathbf{A}$ is said to be \emph{non-negative definite} if
           $\mathbf{v}^\prime \mathbf{Av} \geq 0$ for all constant vectors $\mathbf{v} \in \mathbb{R}^p$. Show that $\mathbf{X}^\prime\mathbf{X}$ is non-negative definite.
        \item Show that if the columns of $\mathbf{X}$ are linearly independent, then $\mathbf{X^\prime X}$ is positive definite.
        \item Show that if $\mathbf{X^\prime X}$ is positive definite, then $(\mathbf{X^\prime X})^{-1}$ exists. 
        \item Show that if $(\mathbf{X^\prime X})^{-1}$ exists, then the columns of $\mathbf{X}$ are linearly independent. 
    \end{enumerate}

This is a good problem because it establishes that the least squares estimator $\widehat{\boldsymbol{\beta}} = (\mathbf{X}^\prime\mathbf{X})^{-1}\mathbf{X}^\prime\mathbf{Y}$ exists if and only if the columns of $\mathbf{X}$ are linearly independent, meaning that no independent variable is a linear combination of the other ones.
 

\item \label{simple} This question is an example of \emph{simple regression}. ``Simple" means one independent variable. Chapter 6 in the text is about simple regression. It covers testing as well as estimation. We'll get to testing later.

Here is the model. Let $Y_i = \beta_0 + \beta_1 x_i + \epsilon_i$
for $i=1, \ldots, n$, where  $\epsilon_1, \ldots, \epsilon_n$ are a random sample from a distribution with expected value zero and variance $\sigma^2$. The numbers $x_1, \ldots, x_n$ are known, observed constants, while the parameters $\beta_0$  $\beta_1$ and $\sigma^2$ are unknown constants (parameters). 
    \begin{enumerate}
    \item What is $E(Y_i)$?
    \item What is $Var(Y_i)$?
    \item Find the Least Squares estimates of $\beta_0$ and $\beta_1$ by minimizing the function
        \begin{displaymath}  
            Q(\boldsymbol{\beta})=\sum_{i=1}^n(Y_i-\beta_0 - \beta_1 x_i)^2
        \end{displaymath}                  
over all values of $(\beta_0,\beta_1)$. Let $(\widehat{\beta}_0, \widehat{\beta}_1)$ denote the point at which $Q(\boldsymbol{\beta})$ is minimal. Your answer is a pair of formulas, one for $\widehat{\beta}_0$ and one for $\widehat{\beta}_1$.
    \item Give the equation of the least-squares line. 
    \item Recall that a statistic is an \emph{unbiased estimator} of a parameter if the expected value of the statistic is equal to the parameter.  
        \begin{enumerate}
            \item Is $\widehat{\beta}_0$ an unbiased estimator of $\beta_0$? Answer Yes or No and show your work.
            \item Is $\widehat{\beta}_1$ an unbiased estimator of $\beta_1$? Answer Yes or No and show your work.
        \end{enumerate}
    \item Fitting this simple regression problem into the matrix framework of Question~\ref{glm},
        \begin{enumerate}
            \item What is $\mathbf{X^\prime X}$?
            \item What is $\mathbf{X^\prime Y}$?
            \item What is $(\mathbf{X^\prime X})^{-1}$?
            \item Verify that your expression for  $\widehat{\beta}_1$ agrees with $\widehat{\boldsymbol{\beta}} = (\mathbf{X}^\prime\mathbf{X})^{-1}\mathbf{X}^\prime\mathbf{Y}$. The formula for $\widehat{\beta}_0$ agrees also, but it's messy so don't bother.
        \end{enumerate}

    \item Please use this small data set for the following questions:
\begin{tabular}{crrrrrr} \hline
$x$ &  1  & 8  &  3  &  6   & 4 & 7  \\
$y$ & 14  & 2  & 14  &  10  & 9 & 9 \\ \hline
\end{tabular}
        \begin{enumerate}
            \item What is $\widehat{\beta}_0$? Your answer is a number. Two decimal places of accuracy will be fine.
            \item What is $\widehat{\beta}_1$? Your answer is a number. Two decimal places of accuracy will be fine.
            \item What is $\widehat{Y}_3$? Your answer is a number. Again, two decimal places of accuracy will be fine.
            \item What is $\widehat{\epsilon}_3$? Your answer is a number. 
            \item Based on these data, what value of $y$ would you predict for $x=5$?  Your answer is a number. 
            \item Plot the least-squares line. You can do it freehand; it does not need to be perfect. 
            \item \label{R}Use R to estimate $\beta_0$ and $\beta_1$. Bring your printout (one page maximum) to the quiz. You may be asked to hand it in. You may write your name and student number on the printout (or put them in a comment statement), but don't write anything else on the printout.
        \end{enumerate}
    \end{enumerate}

\item In Question \ref{simple}, the model had both an intercept and one independent variable. But suppose the model has no intercept. This is called simple \emph{regression through the origin}. The model would be $Y_i = \beta_1 x_i + \epsilon_i$.
        \begin{enumerate}
            \item Find the least squares estimator $\widehat{\beta}_1$ with calculus.
            \item What is $\mathbf{X^\prime X}$?
            \item What is $\mathbf{X^\prime Y}$?
            \item What is $(\mathbf{X^\prime X})^{-1}$?
            \item Verify that your expression for  $\widehat{\beta}_1$ agrees with $\widehat{\boldsymbol{\beta}} = (\mathbf{X}^\prime\mathbf{X})^{-1}\mathbf{X}^\prime\mathbf{Y}$. 
        \end{enumerate}

\item There can even be a regression model with an intercept but no independent variable. In this case the model would be $Y_i = \beta_0 + \epsilon_i$.
        \begin{enumerate}
            \item Find the least squares estimator $\widehat{\beta}_0$ with calculus.
            \item What is the $\mathbf{X}$ matrix?
            \item What is $\mathbf{X^\prime X}$?
            \item What is $\mathbf{X^\prime Y}$?
            \item What is $(\mathbf{X^\prime X})^{-1}$?
            \item Verify that your expression for  $\widehat{\beta}_0$ agrees with $\widehat{\boldsymbol{\beta}} = (\mathbf{X}^\prime\mathbf{X})^{-1}\mathbf{X}^\prime\mathbf{Y}$. 
        \end{enumerate}


\item In scalar form, the model of Question~\ref{glm} is 
\begin{displaymath}
    Y_i = \beta_0 + \beta_1 x_{i1} + \cdots + \beta_k x_{ik} + \epsilon_i,
\end{displaymath}
and we obtain least-squares estimates of the $\beta$ values by minimizing the sum of squared differences between  observed $Y_i$ and $E(Y_i)$. That is, we choose $\beta_0, \ldots, \beta_k$ to make 
\begin{displaymath}  
    Q(\boldsymbol{\beta})=\sum_{i=1}^n(Y_i-\beta_0 - \beta_1 x_{i1} - \cdots - \beta_k x_{ik})^2
\end{displaymath}                  
as small as possible. 
    \begin{enumerate}
        \item Differentiate $Q(\boldsymbol{\beta})$ with respect to $\beta_0$ and set the derivative to zero, obtaining the first \emph{normal equation}.
        \item Noting that the quantities $\widehat{\beta}_0, \ldots, \widehat{\beta}_k$ must satisfy the first normal equation, show that the least squares plane must pass through the point $(\overline{x}_1, \overline{x}_2, \ldots, \overline{x}_k, \overline{Y})$. 
        \item Defining ``predicted" $Y_i$ as $\widehat{Y}_i = \widehat{\beta}_0 + \widehat{\beta}_1 x_{i1} + \cdots + \widehat{\beta}_k x_{ik}$, show that $\sum_{i=1}^n \widehat{Y}_i = \sum_{i=1}^n Y_i$. 
        \item The \emph{residual} for observation $i$ is defined by $\widehat{\epsilon}_i = Y_i - \widehat{Y}_i$.  Show that the sum of residuals equals exactly zero. 
    \end{enumerate}

\item Referring to the matrix version of the linear model (see Question~\ref{glm}) and letting  $\widehat{\boldsymbol{\beta}} = (\mathbf{X}^\prime\mathbf{X})^{-1}\mathbf{X}^\prime\mathbf{Y}$ (which implies that the columns of $\mathbf{X}$ must be linearly independent), show that 
$(\mathbf{Y}-\mathbf{X}\widehat{\boldsymbol{\beta}})^\prime (\mathbf{X}\widehat{\boldsymbol{\beta}} - \mathbf{X}\boldsymbol{\beta}) = \mathbf{0}$.

\item Using the result of the preceding question and writing $Q(\boldsymbol{\beta})$ as 
$Q(\boldsymbol{\beta}) = (\mathbf{Y}-\mathbf{X}\boldsymbol{\beta})^\prime (\mathbf{Y}-\mathbf{X}\boldsymbol{\beta})$, show that $Q(\boldsymbol{\beta}) = (\mathbf{Y}-\mathbf{X}\widehat{\boldsymbol{\beta}})^\prime (\mathbf{Y}-\mathbf{X}\widehat{\boldsymbol{\beta}}) + (\widehat{\boldsymbol{\beta}}-\boldsymbol{\beta})^\prime  (\mathbf{X^\prime X}) (\widehat{\boldsymbol{\beta}}-\boldsymbol{\beta})$. Why does this imply that the minimum of $Q(\boldsymbol{\beta})$ occurs at $\boldsymbol{\beta} = \widehat{\boldsymbol{\beta}}$? How do you know that the minimum is unique? 

\item The set of vectors $\mathcal{V} = \{\mathbf{v} = \mathbf{Xb}: \mathbf{b} \in \mathbb{R}^{k+1}\}$ is the subset of $\mathbb{R}^n$ consisting of linear combinations of the columns of $\mathbf{X}$. That is, $\mathcal{V}$ is the space \emph{spanned} by the columns of $\mathbf{X}$. The least squares estimator $\widehat{\boldsymbol{\beta}} = (\mathbf{X}^\prime\mathbf{X})^{-1}\mathbf{X}^\prime\mathbf{Y}$ was obtained by minimizing $(\mathbf{Y}-\mathbf{Xb})^\prime(\mathbf{Y}-\mathbf{Xb})$ over all $\mathbf{b} \in \mathbb{R}^{k+1}$. Thus, $\widehat{\mathbf{Y}} = \mathbf{X}\widehat{\boldsymbol{\beta}}$ is the point in $\mathcal{V}$ that is \emph{closest} to the data vector $\mathbf{Y}$. Geometrically, $\widehat{\mathbf{Y}}$ is the \emph{projection} (shadow) of $\mathbf{Y}$ onto $\mathcal{V}$.

This means the vector of differences $\widehat{\boldsymbol{\epsilon}} = \mathbf{Y} - \widehat{\mathbf{Y}}$ should be perpendicular (at right angles) to each and every vector in $\mathcal{V}$. Prove it, by calculating the inner product $(\mathbf{Xb})^\prime \,\widehat{\boldsymbol{\epsilon}}$ for general $\mathbf{b}$.

\item Is $\widehat{\boldsymbol{\beta}}$ an unbiased estimator of $\boldsymbol{\beta}$? Answer Yes or No and show your work. 
\item Calculate $cov(\widehat{\boldsymbol{\beta}})$ and simplify. Show your work. 

% Gauss-Markov next time, including regression through the origin and simple independent random sample. 


\end{enumerate}

\vspace{110mm}

\noindent
\begin{center}\begin{tabular}{l}
\hspace{6in} \\ \hline
\end{tabular}\end{center}
This assignment was prepared by  \href{http://www.utstat.toronto.edu/~brunner}{Jerry Brunner},
Department of Statistical Sciences, University of Toronto. It is licensed under a 
\href{http://creativecommons.org/licenses/by-sa/3.0/deed.en_US}
     {Creative Commons Attribution - ShareAlike 3.0 Unported License}. Use any part of it as you like and share the result freely. The \LaTeX~source code is available from the course website:
\href{http://www.utstat.toronto.edu/~brunner/oldclass/302f14} {\small\texttt{http://www.utstat.toronto.edu/$^\sim$brunner/oldclass/302f14}}

\end{document}

R work for simple regression
set.seed(444)
x = x = c(1,8,3,6,4,7)
y = 10 -2*x  + rpois(6,10)
plot(x,y)
cbind(x,y)

x; y

> x; y
[1] 1 8 3 6 4 7
[1] 14  2 14 10  9  9


\begin{tabular}{crrrrrr} \hline
$x$ &  1  & 8  &  3  &  6   & 4 & 7  \\
$y$ & 14  & 2  & 14  &  10  & 9 & 9 \\ \hline
\end{tabular}