%                                   302f17Assignment3.tex
\documentclass[12pt]{article} 
%\usepackage{amsbsy} % for \boldsymbol and \pmb 
\usepackage{graphicx} % To include pdf files!
\usepackage{amsmath}
\usepackage{amsbsy}
\usepackage{amsfonts}
\usepackage[colorlinks=true, pdfstartview=FitV, linkcolor=blue, citecolor=blue, urlcolor=blue]{hyperref} % For links
\usepackage{fullpage}
%\pagestyle{empty} % No page numbers


\begin{document}
%\enlargethispage*{1000 pt} 

\begin{center}   
{\Large \textbf{STA 302f17 Assignment Three}}\footnote{Copyright information is at the end of the last page.}
\vspace{1 mm}
\end{center}


\noindent
Please use the formula sheet to do these questions. A copy will be provided with the quiz. As usual, the homework problems are practice for the quiz, ad are not to be handed in.

This exercise set has an unusual feature. \textbf{Some of the questions ask you to prove things that are false}. That is, they are not true in general. In such cases, just write ``The statement is false," and give a brief explanation to make it clear that you are not just guessing. % The explanation is essential for full marks. Sometimes a small counter-example is desirable. 
\vspace{3mm}


\begin{enumerate} 


%%%%%%%%%%%%%%%%%%%%% Linear Algebra %%%%%%%%%%%%%%%%%
% I have rubbed out the boldface for matrices. Bold draft puts them back.
        
   \item \label{firstmat} Which statement is true? (Capital letters are matrices of constants)
        \begin{enumerate}
            \item ${A(B+C) = AB+AC}$
            \item ${A(B+C) = BA+CA}$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}
    \item Which statement is true? ($a$ is a scalar.)
        \begin{enumerate}
            \item $a{(B+C)}=a{B} + a{C}$
            \item $a{(B+C)}={B}a + {C}a$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}

    \item Which statement is true?
        \begin{enumerate}
            \item ${(B+C)A = AB+AC}$
            \item ${(B+C)A = BA+CA}$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}

    \item Which statement is true?
        \begin{enumerate}
            \item ${(AB)^\prime = A^\prime B^\prime}$
            \item ${(AB)^\prime = B^\prime A^\prime}$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}

\pagebreak %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

    \item Which statement is true?
        \begin{enumerate}
            \item ${A^{\prime\prime} = A }$
            \item ${A^{\prime\prime\prime} = A^\prime }$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}

    \item Suppose that the square matrices ${A}$ and ${B}$ both have inverses and are the same size. Which statement is true?
        \begin{enumerate}
            \item ${(AB)}^{-1} = {A}^{-1}{B}^{-1}$
            \item ${(AB)}^{-1} = {B}^{-1}{A}^{-1}$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}


    \item Which statement is true?
        \begin{enumerate}
            \item ${(A+B)^\prime = A^\prime + B^\prime}$
            \item ${(A+B)^\prime = B^\prime + A^\prime }$
            \item ${(A+B)^\prime = (B+A)^\prime}$
            \item All of the above
            \item None of the above
        \end{enumerate}

    \item Which statement is true? ($a$ and $b$ are scalars.)
        \begin{enumerate}
            \item $(a+b){C} = a{C}+ b{C}$
            \item $(a+b){C} = {C}a+ {C}b$
            \item $(a+b){C} = {C}(a+b)$
            \item All of the above
            \item None of the above
        \end{enumerate}

% Don't forget tr(AB)=tr(BA) and prove the one-direction fact for inverses. 

\item Recall that the \emph{trace} of a square matrix is the sum of diagonal elements. So if $C=[c_{ij}]$ is a $p \times p$ matrix, $tr(C) = \sum_{j=1}^p c_{jj}$. Let $A$ be a $p \times q$ constant matrix, and let $B$ be a $q \times p$ constant matrix, so that $AB$ and $BA$ are both defined. Prove $tr(AB) = tr(BA)$.

\item Let $A$ and $B$ be matrices of constants, with $AB=I$. Using $|AB| = |A| \, |B|$, prove $BA=I$. Thus when you are showing that a matrix is the inverse of another matrix, you only need to multiply them in one direction and get the identity. 

\item Prove that inverses are unique, as follows. Let ${B}$ and ${C}$ both be inverses of ${A}$. Show that ${B=C}$. 

\item Suppose that the square matrices ${A}$ and ${B}$ both have inverses. Prove that ${(AB)}^{-1} = {B}^{-1}{A}^{-1}$.

    \item  \label{ss} Let $\mathbf{a}$ be an $n \times 1$ matrix of real constants. How do you know $\mathbf{a}^\prime\mathbf{a}\geq 0$?

    \item Recall the definition of linear independence. The columns of 
${X}$ are said to be \emph{linearly dependent} if there exists a $p \times 1$ vector $\mathbf{v} \neq \mathbf{0}$ with $X\mathbf{v} = \mathbf{0}$. We will say that the columns of ${X}$ are linearly \emph{independent} if $X\mathbf{v} = \mathbf{0}$ implies $\mathbf{v} = \mathbf{0}$. Let ${A}$ be a square matrix. Show that if the columns of ${A}$ are linearly dependent, ${A}^{-1}$ cannot exist. Hint: $\mathbf{v}$ cannot be both zero and not zero at the same time.

    \item \label{ivt} Let ${A}$ be a non-singular square matrix. Prove $({A}^\prime)^{-1} = ({A}^{-1})^\prime$. Start like this. Let ${B} = {A}^{-1}$. Seek to show \ldots
    
    \item Using Question~\ref{ivt}, prove that the if the inverse of a symmetric matrix exists, it is also symmetric. 

    \item The $p \times p$ matrix ${\Sigma}$ is said to be \emph{positive definite} if
           $\mathbf{a}^\prime {\Sigma} \mathbf{a} > 0$ for all $p \times 1$ vectors $\mathbf{a} \neq \mathbf{0}$. Show that the eigenvalues of a positive definite matrix are all strictly positive. Hint: start with the definition of an eigenvalue and the corresponding eigenvalue:  ${\Sigma}\mathbf{v} = \lambda \mathbf{v}$. Eigenvectors are typically scaled to have length one, so you may assume $\mathbf{v}^\prime \mathbf{v} = 1$.


% \pagebreak


    \item  Recall the \emph{spectral decomposition} of a square symmetric matrix (For example, a variance-covariance matrix).  Any such matrix ${\Sigma}$ can be written as 
            ${\Sigma} = {CD} {C}^\prime$,
where ${C}$ is a matrix whose columns are the (orthonormal) eigenvectors of ${\Sigma}$, ${D}$ is a diagonal matrix of the corresponding  eigenvalues, and ${C}^\prime{C} =~{C}{C}^\prime =~{I}$. 

        \begin{enumerate}
           \item Let ${\Sigma}$ be a square symmetric matrix with eigenvalues that are all strictly positive. 
                \begin{enumerate}
                    \item What is ${D}^{-1}$?
                    \item Show 
            ${\Sigma}^{-1} = {C} {D}^{-1} {C}^\prime$
                \end{enumerate}

% \pagebreak

           \item Let ${\Sigma}$ be a square symmetric matrix with non-negative eigenvalues. 
                \begin{enumerate}
                    \item What do you think ${D}^{1/2}$ might be?
                    \item Define ${\Sigma}^{1/2}$ as 
                    ${CD}^{1/2} {C}^\prime$.
                    Show ${\Sigma}^{1/2}$ is symmetric.
                    \item Show ${\Sigma}^{1/2}{\Sigma}^{1/2} = {\Sigma}$.
                \end{enumerate}

            \item Now return to the situation where the eigenvalues of the square symmetric matrix ${\Sigma}$ are all strictly positive. Define ${\Sigma}^{-1/2}$ as ${CD}^{-1/2} {C}^\prime$, where the elements of the diagonal matrix ${D}^{-1/2}$ are the reciprocals of the corresponding elements of ${D}^{1/2}$. 
                \begin{enumerate}
                    \item Show that the inverse of ${\Sigma}^{1/2}$ is ${\Sigma}^{-1/2}$, justifying the notation.
                    \item Show ${\Sigma}^{-1/2} {\Sigma}^{-1/2} = {\Sigma}^{-1}$.
                \end{enumerate}

           \item Let ${\Sigma}$ be a symmetric, positive definite matrix. How do you know that ${\Sigma}^{-1}$ exists?
        \end{enumerate}

    \item Prove that the diagonal elements of a positive definite matrix must be positive. Hint: Can you describe a vector $\mathbf{v}$ such that $\mathbf{v}^\prime A\mathbf{v}$ picks out the $j$th diagonal element?

    \item Using the Spectral Decomposition Theorem and $tr({AB})=tr({BA})$, prove that the trace is the sum of the eigenvalues for a symmetric matrix.

    \item Using the Spectral Decomposition Theorem and $|{AB}| = |{BA}|$, prove that the determinant of a symmetric matrix is the product of its eigenvalues.

%%%%%%%%%%%%%%%%%%%%% Random Vectors %%%%%%%%%%%%%%%%%

\vspace{3mm}
\hrule
\vspace{2mm}

    \item Let the $p \times 1$ random vector $\mathbf{x}$ have expected value $\boldsymbol{\mu}$ and variance-covariance matrix ${\Sigma}$, and let ${A}$ be an $m \times p$ matrix of constants. Prove that the variance-covariance matrix of ${Ax}$ is either 
    \begin{itemize}
        \item ${A} {\Sigma} {A}^\prime$, or
        \item ${A}^2 {\Sigma}$..
    \end{itemize}
Pick one and prove it. Start with the definition of a variance-covariance matrix on the formula sheet.  

    \item Let the $p \times 1$ random vector $\mathbf{y}$ have expected value $\boldsymbol{\mu}$ and variance-covariance matrix ${\Sigma}$. Find $cov(A\mathbf{y},B\mathbf{y})$, where $A$ and $B$ are matrices of constants.

    \item  If the $p \times 1$ random vector $\mathbf{x}$ has mean $\boldsymbol{\mu}$ and variance-covariance matrix ${\Sigma}$, show ${\Sigma} = E(\mathbf{xx}^\prime) - \boldsymbol{\mu \mu}^\prime$.

    \item Let $\mathbf{x}$ be a $p \times 1$ random vector. Starting with the definition on the formula sheet, prove $cov(\mathbf{x})=\mathbf{0}$.. % FALSE

    \item Let the $p \times 1$ random vector $\mathbf{x}$ have mean $\boldsymbol{\mu}$ and variance-covariance matrix ${\Sigma}$, let ${A}$ be an $r \times p$ matrix of constants, and let $\mathbf{c}$ be an $r \times 1$ vector of constants. Find $cov(A\mathbf{x}+\mathbf{c})$. Show your work. 

    \item Let the scalar random variable $y = \mathbf{v}^\prime \mathbf{x}$, where $\mathbf{x}$ is a $p \times 1$ random vector with $cov(\mathbf{x})=\Sigma$ and $\mathbf{v}$ is a constant vector in $\mathbb{R}^p$. Since a variance-covariance matrix reduces to an ordinary variance for the $1 \times 1$ case, $Var(y) = cov(\mathbf{v}^\prime \mathbf{x})$. Use this to prove that $\Sigma$ is positive semi-definite. You have shown that \emph{any} variance-covariance matrix must be positive semi-definite.
% \pagebreak

    \item The square matrix ${A}$ has an eigenvalue equal to $\lambda$ with corresponding eigenvector  $\mathbf{v} \neq \mathbf{0}$ if $A\mathbf{v} = \lambda\mathbf{v}$. Eigenvectors are scaled so that $\mathbf{v}^\prime \mathbf{v} = 1$.
        \begin{enumerate}
            \item Show that the eigenvalues of a variance-covariance matrix cannot be negative.
            \item How do you know that the determinant of a variance-covariance matrix must be greater than or equal to zero? The answer is one short sentence. 
            \item  Let $x$ and $y$ be scalar random variables. Recall $Corr(x,y) = \frac{Cov(x,y)}{\sqrt{Var(x)Var(y)}}$. Using what you have shown about the determinant, show $-1 \leq Corr(x,y) \leq 1$.  You have just proved the Cauchy-Schwarz inequality using probability tools.
        \end{enumerate}

    \item Let $\mathbf{x}$ be a $p \times 1$ random vector with mean $\boldsymbol{\mu}_x$ and variance-covariance matrix ${\Sigma}_x$, and let $\mathbf{y}$ be a $q \times 1$ random vector with mean $\boldsymbol{\mu}_y$ and variance-covariance matrix ${\Sigma}_y$. 
            \begin{enumerate}
                \item What is the $(i,j)$ element of $cov(\mathbf{x},\mathbf{y})$? See the definition on the formula sheet. 
                \item Assuming $p=q$, find an expression for  $cov(\mathbf{x}+\mathbf{y})$ in terms of 
                ${\Sigma}_x$, ${\Sigma}_y$ and $cov(\mathbf{x},\mathbf{y})$. Show your work.
                \item Simplify further for the special case where $Cov(x_i,y_j)=0$ for all $i$ and $j$.
                \item What are the dimensions of these four matrices?
                \item Let $\mathbf{c}$ be a $p \times 1$ vector of constants and $\mathbf{d}$ be a $q \times 1$ vector of constants. Find $ cov(\mathbf{x}+\mathbf{c}, \mathbf{y}+\mathbf{d})$. Show your work.
                \item Starting with the definition on the formula sheet, show $cov(\mathbf{x,y})=cov(\mathbf{y,x})$.. % FALSE
                \item Starting with the definition on the formula sheet, show $cov(\mathbf{x,y})=\mathbf{0}$.. % FALSE
            \end{enumerate}

\end{enumerate}

\vspace{100mm}

\noindent
\begin{center}\begin{tabular}{l}
\hspace{6in} \\ \hline
\end{tabular}\end{center}
This assignment was prepared by  \href{http://www.utstat.toronto.edu/~brunner}{Jerry Brunner},
Department of Statistical Sciences, University of Toronto. It is licensed under a 
\href{http://creativecommons.org/licenses/by-sa/3.0/deed.en_US}
     {Creative Commons Attribution - ShareAlike 3.0 Unported License}. Use any part of it as you like and share the result freely. The \LaTeX~source code is available from the course website:
\href{http://www.utstat.toronto.edu/~brunner/oldclass/302f17} {\small\texttt{http://www.utstat.toronto.edu/$^\sim$brunner/oldclass/302f17}}

\end{document}

% %%%%%%%%%%%%%%%%%%%%%%%%%% Unused questions 

\item Please read pages 11-19 in the textbook. In Section 1.7, the measure of model fit $R^2$ is presented without much explanation. The idea is that $\sum_{i=1}^n(y_i-\overline{y})^2$ represents the sum of squared vertical distances of the points on a scatterplot from a horizontal line with slope zero and intercept $\overline{y}$. Mathematically this could be the best fitting line, but in practice the line $y = b_0 + b_1 x$ is going to do better. That is, $\sum_{i=1}^n(y_i-\widehat{y}_i)^2 =  \sum_{i=1}^ne_i^2  \leq \sum_{i=1}^n(y_i-\overline{y})^2$, so that $0 \leq \frac{\sum_{i=1}^ne_i^2}{\sum_{i=1}^n(y_i-\overline{y})^2} \leq 1$. Small values of this ratio represent good performance of the least squares line relative to the horizontal line. It could be described as an index of ``lack of fit," because big values indicate relatively poor performance. Thus, $R^2 = 1 - \frac{\sum_{i=1}^ne_i^2}{\sum_{i=1}^n(y_i-\overline{y})^2}$ is a measure of good fit. Why it's something \emph{squared} will be taken up later. 
    \begin{enumerate}
        \item Prove that the least squares line must always pass through the point $(\overline{x},\overline{y})$, regardless of the data.
        \item Show the work leading to (1.28). Use the formula sheet. 
        \item Do Exercise 1.2.%   Prove (1.29) on p. 15.
        \item Do Exercise 1.4, but only the expected value, not the variance. The variance is much easier with matrices.
        \item Do Exercise 1.5. Answer all three parts of the question.
        \item Do both parts of Exercise 1.8. You don't have to do all the work of differentiating and solving again. Start with (1.11) and (1.14), and keep simplifying.
    \end{enumerate}


% This is a useful but flawed question. Make A positive definite, move to after proofs, A = B'B hint, possibly set-up question.
    \item \label{rmat} Make up a your own $4 \times 4$ symmetric matrix that is not singular (that is, the inverse exists), and is \emph{not a diagonal matrix}. If your first try is singular, try again. Call it $\mathbf{A}$. Enter it into R using \texttt{rbind} (see lecture slides). Make sure to display the input. Then, 
        \begin{enumerate}
            \item Calculate $|\mathbf{A}^{-1}|$ and $1/|\mathbf{A}|$, verifying that they are equal.
            \item Calculate $|\mathbf{A}^2|$ and $|\mathbf{A}|^2$, verifying that they are equal.
            \item Calculate the eigenvalues and eigenvectors of $\mathbf{A}$.
            \item Calculate $\mathbf{A}^{1/2}$.
            \item  Calculate $\mathbf{A}^{-1/2}$.
        \end{enumerate} 
Display both input and output for each part. Label the output with comments. Bring the printout to the quiz.  


    \item Let $\mathbf{X} = [X_{ij}]$ be a random matrix. Show $E(\mathbf{X}^\prime) = E(\mathbf{X})^\prime$. 

    \item Let $\mathbf{X}$ be a random matrix, and $\mathbf{B}$ be a matrix of constants. Show 
    $E(\mathbf{XB})=E(\mathbf{X})\mathbf{B}$. Recall the definition 
    $\mathbf{AB}=[\sum_{k}a_{i,k}b_{k,j}]$.

    %Let $\mathbf{X}$ and $\mathbf{Y}$ be random matrices of the same dimensions. Show 
    %$E(\mathbf{X} + \mathbf{Y})=E(\mathbf{X})+E(\mathbf{Y})$. Recall the definition 
    %$E(\mathbf{Z})=[E(Z_{i,j})]$.