% 302f20Assignment3.tex   Linear algebra and random vectors
\documentclass[11pt]{article} 
%\usepackage{amsbsy} % for \boldsymbol and \pmb 
\usepackage{graphicx} % To include pdf files!
\usepackage{amsmath}
\usepackage{amsbsy}
\usepackage{amsfonts}
\usepackage{comment}
\usepackage[colorlinks=true, pdfstartview=FitV, linkcolor=blue, citecolor=blue, urlcolor=blue]{hyperref} % For links
\usepackage{fullpage}
%\pagestyle{empty} % No page numbers

\begin{document}
%\enlargethispage*{1000 pt} 

\begin{center}   
{\Large \textbf{STA 302f20 Assignment Three}}\footnote{This assignment was prepared by  \href{http://www.utstat.toronto.edu/~brunner}{Jerry Brunner},
Department of Statistical Sciences, University of Toronto. It is licensed under a 
\href{http://creativecommons.org/licenses/by-sa/3.0/deed.en_US}
     {Creative Commons Attribution - ShareAlike 3.0 Unported License}. Use any part of it as you like and share the result freely. The \LaTeX~source code is available from the course website:
\href{http://www.utstat.toronto.edu/~brunner/oldclass/302f20} {\small\texttt{http://www.utstat.toronto.edu/$^\sim$brunner/oldclass/302f20}}}
\vspace{1 mm}
\end{center}

\noindent 
Please do these review questions in preparation for Quiz Three; they are not to be handed in.  Use the current formula sheet on the course website. Questions 1-12 are based on material in Chapter 2. Questions 13-25 are based on material in Chapter 3.  

\vspace{3mm}

\begin{enumerate} 


\item Recall that the \emph{trace} of a square matrix is the sum of diagonal elements. So if $\mathbf{C}=(c_{ij})$ is a $p \times p$ matrix, $tr(\mathbf{C}) = \sum_{j=1}^p c_{jj}$. Let $\mathbf{A}$ be a $p \times q$ constant matrix, and let $\mathbf{B}$ be a $q \times p$ constant matrix, so that $\mathbf{AB}$ and $\mathbf{BA}$ are both defined. Prove $tr(\mathbf{AB}) = tr(\mathbf{BA})$.

\item Let $\mathbf{A}$ and $\mathbf{B}$ be square matrices of constants, with $\mathbf{AB}=\mathbf{I}$. Using $|\mathbf{AB}| = |\mathbf{A}| \, |\mathbf{B}|$, prove $\mathbf{BA}=\mathbf{I}$. Thus when you are showing that a matrix is the inverse of another matrix, you only need to multiply them in one direction and get the identity. 

    \item In the textbook, do Problems 2.28, 2.35 and 2.36. % For 2.36 see p. 26
    \item In the textbook, do Problem 2.38. It is asking you to show that if the symmetric matrix $\mathbf{A}$ is positive definite, then $\mathbf{A} = \mathbf{P}^\prime\mathbf{P}$ for some non-singular $\mathbf{P}$.  
    \item In the textbook, do Problems 2.53 and 2.76.
% For 2.53 see p. 40; for 2.76 p. 53 but it's big in the notes.

    \item \label{r} Let 
$\mathbf{A} = \left(\begin{array}{cccc}
1.00 & 0.75 & 0.75 & 0.75 \\
0.75 & 1.00 & 0.75 & 0.75 \\
0.75 & 0.75 & 1.00 & 0.75 \\
0.75 & 0.75 & 0.75 & 1.00
\end{array}\right)$. Enter $\mathbf{A}$ into R using \texttt{rbind}.    
        \begin{enumerate}
            \item Calculate $|\mathbf{A}^{-1}|$ and $1/|\mathbf{A}|$, verifying that they are equal.
            \item Calculate $|\mathbf{A}^2|$ and $|\mathbf{A}|^2$, verifying that they are equal.
            \item Calculate the eigenvalues and eigenvectors of $\mathbf{A}$. 
            \item Calculate $\mathbf{A}^{1/2}$. Multiply the matrix by itself to get $\mathbf{A}$.
            \item  Calculate $\mathbf{A}^{-1/2}$. Multiply the matrix by $\mathbf{A}^{1/2}$ to get the identity.
        \end{enumerate} 
Display the creation of $\mathbf{A}$, and also the input and output for each part. Label the output with comments. Be ready to hand in a pdf with the quiz if requested.

\begin{comment}
# Q6

A = rbind(c(1.00, 0.75, 0.75, 0.75),
          c(0.75, 1.00, 0.75, 0.75),
          c(0.75, 0.75, 1.00, 0.75),
          c(0.75, 0.75, 0.75, 1.00) ); A
          
# a)
c(det(solve(A)), 1/det(A))

# b) 
c(det(A%*%A), det(A)^2)

# c) 
eigen(A)

# d) A^{1/2}
C = eigen(A)$vectors; D = diag(eigen(A)$values)
Ahalf = C %*% sqrt(D) %*% t(C); Ahalf
Ahalf %*% Ahalf # Should be A

# e) A^{-1/2}
Aminushalf = C %*% solve(sqrt(D)) %*% t(C); Aminushalf
Aminushalf %*% Ahalf # Should be I
\end{comment}

    \item Let $\mathbf{A}$ be a square matrix. Show that if the columns of $\mathbf{A}$ are linearly dependent, $\mathbf{A}^{-1}$ cannot exist. Hint: $\mathbf{v}$ cannot be both zero and not zero at the same time.
    
    \item  \label{ss} Let $\mathbf{a}$ be an $n \times 1$ matrix of real constants. How do you know $\mathbf{a}^\prime\mathbf{a}\geq 0$?

\pagebreak

    \item  Recall the \emph{spectral decomposition} of a square symmetric matrix (For example, a variance-covariance matrix).  Any such matrix $\boldsymbol{\Sigma}$ can be written as 
            $\boldsymbol{\Sigma} = \mathbf{CD} \mathbf{C}^\prime$,
where $\mathbf{C}$ is a matrix whose columns are the (orthonormal) eigenvectors of $\boldsymbol{\Sigma}$, $\mathbf{D}$ is a diagonal matrix of the corresponding  eigenvalues, and $\mathbf{C}^\prime\mathbf{C} =~\mathbf{C}\mathbf{C}^\prime =~\mathbf{I}$. 

        \begin{enumerate}
           \item Let $\boldsymbol{\Sigma}$ be a square symmetric matrix with eigenvalues that are all strictly positive. 
                \begin{enumerate}
                    \item What is $\mathbf{D}^{-1}$?
                    \item Show 
            $\boldsymbol{\Sigma}^{-1} = \mathbf{C} \mathbf{D}^{-1} \mathbf{C}^\prime$
                \end{enumerate}

           \item Let $\boldsymbol{\Sigma}$ be a square symmetric matrix, and this time some of the eigenvalues might be zero. 
                \begin{enumerate}
                    \item What do you think $\mathbf{D}^{1/2}$ might be?
                    \item Define $\boldsymbol{\Sigma}^{1/2}$ as 
                    $\mathbf{CD}^{1/2} \mathbf{C}^\prime$.
                    Show $\boldsymbol{\Sigma}^{1/2}$ is symmetric.
                    \item Show $\boldsymbol{\Sigma}^{1/2}\boldsymbol{\Sigma}^{1/2} = \boldsymbol{\Sigma}$.
                \end{enumerate}

            \item Now return to the situation where the eigenvalues of the square symmetric matrix $\boldsymbol{\Sigma}$ are all strictly positive. Define $\boldsymbol{\Sigma}^{-1/2}$ as $\mathbf{CD}^{-1/2} \mathbf{C}^\prime$, where the elements of the diagonal matrix $\mathbf{D}^{-1/2}$ are the reciprocals of the corresponding elements of $\mathbf{D}^{1/2}$. 
                \begin{enumerate}
                    \item Show that the inverse of $\boldsymbol{\Sigma}^{1/2}$ is $\boldsymbol{\Sigma}^{-1/2}$, justifying the notation.
                    \item Show $\boldsymbol{\Sigma}^{-1/2} \boldsymbol{\Sigma}^{-1/2} = \boldsymbol{\Sigma}^{-1}$.
                \end{enumerate}

           \item The (square) matrix $\boldsymbol{\Sigma}$ is said to be \emph{positive definite} if
           $\mathbf{v}^\prime \boldsymbol{\Sigma} \mathbf{v} > 0$ for all vectors $\mathbf{v} \neq \mathbf{0}$. Show that the eigenvalues of a positive definite matrix are all strictly positive. 
           \item Let $\boldsymbol{\Sigma}$ be a symmetric, positive definite matrix. Putting together a couple of results you have proved above, establish that $\boldsymbol{\Sigma}^{-1}$ exists.
        \end{enumerate}

\item Using the Spectral Decomposition Theorem and $tr(\mathbf{AB})=tr(\mathbf{BA})$, prove that the trace is the sum of the eigenvalues for a symmetric matrix.

\item Using the Spectral Decomposition Theorem and $|\mathbf{AB}| = |\mathbf{BA}|$, prove that the determinant of a symmetric matrix is the product of its eigenvalues.

    \item Prove that the diagonal elements of a positive definite matrix must be positive. Hint: Can you describe a vector $\mathbf{v}$ such that $\mathbf{v}^\prime \mathbf{Av}$ picks out the $j$th diagonal element?


\vspace{3mm}\hrule\vspace{3mm} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Random vectors and matrices

    \item Let $\mathbf{X}$ be a random matrix, and $\mathbf{B}$ be a matrix of constants. Show 
    $E(\mathbf{XB})=E(\mathbf{X})\mathbf{B}$. 

    \item Do Problem 3.10 in the text.
 
    \item Let the $p \times 1$ random vector $\mathbf{x}$ have expected value $\boldsymbol{\mu}$ and variance-covariance matrix $\mathbf{\Sigma}$, and let $\mathbf{A}$ be an $m \times p$ matrix of constants. Prove that the variance-covariance matrix of $\mathbf{Ax}$ is either 
    \begin{itemize}
        \item $\mathbf{A} \boldsymbol{\Sigma} \mathbf{A}^\prime$, or
        \item $\mathbf{A}^2 \boldsymbol{\Sigma}$..
    \end{itemize}
Pick one and prove it. Start with the definition of a variance-covariance matrix on the formula sheet. If the two expressions are equal, say so.

    \item Let the $p \times 1$ random vector $\mathbf{y}$ have expected value $\boldsymbol{\mu}$ and variance-covariance matrix $\boldsymbol{\Sigma}$. Find $cov(\mathbf{Ay},\mathbf{By})$, where $A$ and $B$ are matrices of constants.

    \item Let $\mathbf{x}$ be a $p \times 1$ random vector. Starting with the definition on the formula sheet, prove $cov(\mathbf{x})=\mathbf{0}$.. % FALSE

    \item Let the $p \times 1$ random vector $\mathbf{x}$ have mean $\boldsymbol{\mu}$ and variance-covariance matrix $\boldsymbol{\Sigma}$, let $\mathbf{A}$ be an $r \times p$ matrix of constants, and let $\mathbf{c}$ be an $r \times 1$ vector of constants. Find $cov(\mathbf{Ax}+\mathbf{c})$. Show your work. 

    \item Comparing the definitions, one can see that viewing a scalar random variable as a $1 \times 1$ random vector, the variance-covariance matrix is just the ordinary variance. Accordingly, let the scalar random variable $Y = \mathbf{v}^\prime \mathbf{x}$, where $\mathbf{x}$ is a $p \times 1$ random vector with covariance matrix $\boldsymbol{\Sigma}$, and $\mathbf{v}$ is a $p \times 1$ vector of constants. What is $Var(Y)$? Why does this tell you  that \emph{any} variance-covariance matrix must be positive semi-definite? 

    \item Using definitions on the formula sheet and other material from this assignment, 
        \begin{enumerate}
            \item Show that the eigenvalues of a variance-covariance matrix cannot be negative.
            \item How do you know that the determinant of a variance-covariance matrix must be greater than or equal to zero? The answer is one short sentence. 
            \item  Let $X$ and $Y$ be scalar random variables. Recall $Corr(X,Y) = \frac{Cov(X,Y)}{\sqrt{Var(X)Var(Y)}}$. Using what you have shown about the determinant, show $-1 \leq Corr(X,Y) \leq 1$. You have just proved the Cauchy-Schwarz inequality.
        \end{enumerate}

    \item Let $\mathbf{x}$ be a $p \times 1$ random vector with mean $\boldsymbol{\mu}_x$ and variance-covariance matrix $\mathbf{\Sigma}_x$, and let $\mathbf{y}$ be a $q \times 1$ random vector with mean $\boldsymbol{\mu}_y$ and variance-covariance matrix $\mathbf{\Sigma}_y$. 
            \begin{enumerate}
                \item What is the $(i,j)$ element of $\boldsymbol{\Sigma}_{xy} = cov(\mathbf{x},\mathbf{y})$? 
                \item Find an expression for  $cov(\mathbf{x}+\mathbf{y})$ in terms of 
                $\mathbf{\Sigma}_x$, $\mathbf{\Sigma}_y$ and $\boldsymbol{\Sigma}_{xy}$. Show your work.
                \item Simplify further for the special case where $Cov(X_i,Y_j)=0$ for all $i$ and $j$.

                \item Let $\mathbf{c}$ be a $p \times 1$ vector of constants and $\mathbf{d}$ be a $q \times 1$ vector of constants. Find $ cov(\mathbf{x}+\mathbf{c}, \mathbf{y}+\mathbf{d})$. Show your work.
            \end{enumerate}

    \item Let the random vectors $\mathbf{x}_1$ and $\mathbf{x}_2$ be $p \times 1$, and the random vectors $\mathbf{y}_1$ and $\mathbf{y}_2$ be $p \times 1$, with
$E(\mathbf{x}_1)=\boldsymbol{\mu}_1$, $E(\mathbf{x}_2)=\boldsymbol{\mu}_2$, $E(\mathbf{y}_1)=\boldsymbol{\mu}_3$, $E(\mathbf{y}_2)=\boldsymbol{\mu}_4$.    
Show that the $p \times q$ matrix of covariances $cov(\mathbf{x}_1+\mathbf{x}_2, \mathbf{y}_1+\mathbf{y}_2) = 
cov(\mathbf{x}_1,\mathbf{y}_1) + cov(\mathbf{x}_1,\mathbf{y}_2) +cov(\mathbf{x}_2,\mathbf{y}_1) +cov(\mathbf{x}_2,\mathbf{y}_2)$. 

    \item Starting with the definition on the formula sheet, show $cov(\mathbf{x,y})=cov(\mathbf{y,x})$.. % FALSE


    \item Starting with the definition on the formula sheet, show $cov(\mathbf{x,y})=\mathbf{0}$.. % FALSE


    \item  Do problem 3.20 in the text. The answer is in the back of the book. 



\end{enumerate} % End all the questions

% \vspace{130mm}


% How about distribution of y-hat?

\end{document}


\vspace{3mm}\hrule\vspace{3mm} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


# False start on Q6
set.seed(9999)
B = rpois(16,5)*2; dim(B) = c(4,4); A = 0.5 * (B + t(B)); A
# Pretend I made it up
A = rbind(c(14, 12, 20,  6),
          c(12, 14,  9, 14),
          c(20,  9, 14,  5),
          c( 6, 14,  5, 18) ); A
# a)
c(det(solve(A)), 1/det(A))
# b) 
c(det(A%*%A), det(A)^2)
# c) 
eigen(A)
# d) A^{1/2}
C = eigen(A)$vectors; D = diag(eigen(A)$values)
Ahalf = C %*% sqrt(D) %*% t(C)
Ahalf %*% Ahalf