\documentclass[12pt]{article} 
%\usepackage{amsbsy} % for \boldsymbol and \pmb 
\usepackage{graphicx} % To include pdf files!
\usepackage{amsmath}
\usepackage{amsbsy}
\usepackage{amsfonts}
\usepackage[colorlinks=true, pdfstartview=FitV, linkcolor=blue, citecolor=blue, urlcolor=blue]{hyperref} % For links
\usepackage{fullpage}
%\pagestyle{empty} % No page numbers


\begin{document}
%\enlargethispage*{1000 pt} 

\begin{center}   
{\Large \textbf{STA 2101/442 Assignment 1 (Mostly Review)}}\footnote{This assignment was prepared by  \href{http://www.utstat.toronto.edu/~brunner}{Jerry Brunner},
Department of Statistics, University of Toronto. It is licensed under a 
\href{http://creativecommons.org/licenses/by-sa/3.0/deed.en_US}
     {Creative Commons Attribution - ShareAlike 3.0 Unported License}. Use any part of it as you like and share the result freely. The \LaTeX~source code is available from the course website:
\href{http://www.utstat.toronto.edu/~brunner/oldclass/appliedf13} {\texttt{http://www.utstat.toronto.edu/$^\sim$brunner/oldclass/appliedf13}}}
\vspace{1 mm}
\end{center}

\noindent
Except for Question~\ref{sat}, the questions are practice for the quiz on Friday Sept. 20th, and are not to be handed in. For the linear algebra part starting with Question~\ref{firstmat}, there is an excellent review in Chapter Two of Renscher and Schaalje' \emph{Linear models in statistics}. The chapter has more material than you need for this course.

\begin{enumerate} 

     \item Let $Y_1, \ldots, Y_n$ be numbers, and $\overline{Y}=\frac{1}{n}\sum_{i=1}^nY_i$. Show that the sum of squares $Q_m = \sum_{i=1}^n(Y_i-m)^2$ is minimized when $m = \overline{Y}$.

    \item Let $Y_1, \ldots, Y_n$ be a random sample from a distribution with mean $\mu$ and standard deviation $\sigma$. 
    \begin{enumerate}
        \item Show that the sample variance $S^2=\frac{\sum_{i=1}^n(Y_i-\overline{Y})^2}{n-1}$ is an unbiased estimator of $\sigma^2$.
        \item Denote the sample standard deviation by $S = \sqrt{S^2}$. Assume that the data come from a continuous distribution, so it's easy to see that $Var(S) \neq 0$. Using this fact, show that $S$ is a \emph{biased} estimator of $\sigma$.
    \end{enumerate}

        \item Let $Y_1, \ldots, Y_n$ be a random sample from a normal distribution with mean $\mu$ and variance $\sigma^2$, so that $T = \frac{\sqrt{n}(\overline{Y}-\mu)}{S} \sim t(n-1)$. This is something you don't need to prove, for now. 
    \begin{enumerate}
        \item Derive a $(1-\alpha)100\%$ confidence interval for $\mu$. ``Derive" means show all the high school algebra. Use the symbol $t_{\alpha/2}$ for the number satisfying $Pr(T>t_{\alpha/2})= \alpha/2$.
        \item \label{ci} A random sample with $n=23$ yields $\overline{Y} = 2.57$ and a sample variance of $S^2=5.85$. Using the critical value $t_{0.025}=2.07$, give a 95\% confidence interval for $\mu$. The answer is a pair of numbers.
        \item Test $H_0: \mu=3$ at $\alpha=0.05$. 
            \begin{enumerate}
                \item Give the value of the $T$ statistic. The answer is a number.
                \item State whether you reject $H_0$, Yes or No.
                \item Can you conclude that $\mu$ is different from 3? Answer Yes or No.
                \item If the answer is Yes, state whether $\mu>3$ or $\mu<3$. Pick one.
            \end{enumerate}
        \item Show that using a $t$-test, $H_0:\mu=\mu_0$ is rejected at significance level $\alpha$ if and only the $(1-\alpha)100\%$ confidence interval for $\mu$ does not include $\mu_0$. The problem is easier if you start by writing the set of $T$ values for which $H_0$ is \emph{not} rejected.  
        \item In Question~\ref{ci}, does this mean $Pr\{1.53<\mu<3.61\}=0.95$? Answer Yes or No and briefly explain.
    \end{enumerate}


\item For each of the following distributions, derive a general expression for the Maximum Likelihood Estimator (MLE); don't bother with the second derivative test. Then use the data to calculate a numerical estimate; you should bring a calculator to the quiz in case you have to do something like this.
    \begin{enumerate}
        \item $p(x)=\theta(1-\theta)^x$ for $x=0,1,\ldots$, where $0<\theta<1$. Data: \texttt{4, 0, 1, 0, 1, 3, 2, 16, 3, 0, 4, 3, 6, 16, 0, 0, 1, 1, 6, 10}. Answer: 0.2061856   
                % Geometric .25, thetahat = 1/xbar 
        \item $f(x) = \frac{\alpha}{x^{\alpha+1}}$ for $x>1$, where $\alpha>0$. Data: \texttt{1.37, 2.89, 1.52, 1.77, 1.04, 2.71, 1.19, 1.13, 15.66, 1.43} Answer: 1.469102
                % Pareto alpha = 1 (one over uniform) alphahat = 1/mean(log(x))
        \item $f(x) = \frac{\tau}{\sqrt{2\pi}} e^{-\frac{\tau^2 x^2}{2}}$, for $x$ real, where $\tau>0$. Data: \texttt{1.45, 0.47, -3.33, 0.82, -1.59, -0.37, -1.56, -0.20 } Answer: 0.6451059 % Normal mean zero   tauhat = sqrt(1/mean(x^2))

        \item $f(x) = \frac{1}{\theta} e^{-x/\theta}$ for $x>0$, where $\theta>0$. Data: \texttt{0.28, 1.72, 0.08, 1.22, 1.86, 0.62, 2.44, 2.48, 2.96} Answer: 1.517778  % Exponential, true theta=2, thetahat = xbar
    \end{enumerate}


        \item The random variable $X$ has density $f(x) = \frac{\Gamma(\alpha+\beta)}{\Gamma(\alpha) \Gamma(\beta)} x^{\alpha-1}(1-x)^{\beta-1}$ for $0<x<1$, where $\alpha>0$ and $\beta>0$. Find $E(X)$; show your work. Hint: $f(x)$ is a density for \emph{any} $\alpha>0$ and $\beta>0$.


\item \label{sat} In the United States, admission to university is based partly on high school marks and recommendations, and partly on applicants' performance on a standardized multiple choice test called the Scholastic Aptitude Test (SAT). The SAT has two sub-tests, Verbal and Math. A university administrator selected a random sample of 200 applicants, and recorded the Verbal SAT, the Math SAT and first-year university Grade Point Average (GPA) for each student. The data are given in the file
\href{http://www.utstat.toronto.edu/~brunner/appliedf13/code_n_data/hw/sat.data}
     {\texttt{sat.data}}. There is a link on the course web page in case the one in this document does not work.

The university administrator knows that the Verbal and Math SAT tests have the same number of questions, and the maximum score on both is 800. But are they equally difficult on average for this population of students?

Using R, do a reasonable analysis to answer the question.  Bring your printout to the quiz; you may be asked to hand it in. Be ready to 
    \begin{itemize}
        \item State your model.
        \item Justify your choice of model. Would you expect Verbal and Math scores from the same student to be independent?         
        \item State your null and alternative hypotheses, in symbols.
        \item Express your conclusion (if any) in plain, non-statistical language that could be understood by someone who never had a Statistics course. Your answer is something about which test is more difficult for these students. Marks will be deducted for use of technical terms like null hypothesis, significance level, critical value, $p$-value, and so on even if what you say is correct.
    \end{itemize}

Remember, the computer assignments in this course are \emph{not group projects}. You are expected to do the work yourself. There is more than one correct answer. I did the analysis several different ways, and I consider all of them correct. I can think of about five more acceptable ways that I did not try. The number of bad ways to analyze the data is virtually unlimited.



    \item \label{firstmat} Which statement is true? (Quantities in boldface are matrices of constants.)
        \begin{enumerate}
            \item $\mathbf{A(B+C) = AB+AC}$
            \item $\mathbf{A(B+C) = BA+CA}$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}
    \item Which statement is true?
        \begin{enumerate}
            \item $a\mathbf{(B+C)}=a\mathbf{B} + a\mathbf{C}$
            \item $a\mathbf{(B+C)}=\mathbf{B}a + \mathbf{C}a$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}


    \item Which statement is true?
        \begin{enumerate}
            \item $\mathbf{(B+C)A = AB+AC}$
            \item $\mathbf{(B+C)A = BA+CA}$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}

    \item Which statement is true?
        \begin{enumerate}
            \item $\mathbf{(AB)^\prime = A^\prime B^\prime}$
            \item $\mathbf{(AB)^\prime = B^\prime A^\prime}$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}


    \item Which statement is true?
        \begin{enumerate}
            \item $\mathbf{A^{\prime\prime} = A }$
            \item $\mathbf{A^{\prime\prime\prime} = A^\prime }$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}

\pagebreak 

    \item Suppose that the square matrices $\mathbf{A}$ and $\mathbf{B}$ both have inverses and are the same size. Which statement is true?
        \begin{enumerate}
            \item $\mathbf{(AB)}^{-1} = \mathbf{A}^{-1}\mathbf{B}^{-1}$
            \item $\mathbf{(AB)}^{-1} = \mathbf{B}^{-1}\mathbf{A}^{-1}$
            \item Both a and b
            \item Neither a nor b
        \end{enumerate}
    \item Which statement is true?
        \begin{enumerate}
            \item $\mathbf{(A+B)^\prime = A^\prime + B^\prime}$
            \item $\mathbf{(A+B)^\prime = B^\prime + A^\prime }$
            \item $\mathbf{(A+B)^\prime = (B+A)^\prime}$
            \item All of the above
            \item None of the above
        \end{enumerate}


    \item Which statement is true?
        \begin{enumerate}
            \item $(a+b)\mathbf{C} = a\mathbf{C}+ b\mathbf{C}$
            \item $(a+b)\mathbf{C} = \mathbf{C}a+ \mathbf{C}b$
            \item $(a+b)\mathbf{C} = \mathbf{C}(a+b)$
            \item All of the above
            \item None of the above
        \end{enumerate}

    \item Let $\mathbf{A}$ be a square matrix with the determinant of $\mathbf{A}$ (denoted $|\mathbf{A}|$) equal to zero. What does this tell you about $\mathbf{A}^{-1}$? No proof is required here.
        
    \item Recall that an inverse of the matrix $\mathbf{A}$ (denoted $\mathbf{A}^{-1}$) is defined by two properties: $\mathbf{A}^{-1}\mathbf{A=I}$ and $\mathbf{AA}^{-1}=\mathbf{I}$. Prove that inverses are unique, as follows. Let $\mathbf{B}$ and $\mathbf{C}$ both be inverses of $\mathbf{A}$. Show that $\mathbf{B=C}$. 
    \item Let $\mathbf{X}$ be an $n$ by $p$ matrix with $n \neq p$. Why is it incorrect to say that $(\mathbf{X^\prime X})^{-1}= \mathbf{X}^{-1}\mathbf{X}^{\prime -1}$?

    \item Suppose that the square matrices $\mathbf{A}$ and $\mathbf{B}$ both have inverses. Prove that $\mathbf{(AB)}^{-1} = \mathbf{B}^{-1}\mathbf{A}^{-1}$. You have two things to show. 

    \item \label{ivt} Let $\mathbf{A}$ be a non-singular square matrix. Prove $(\mathbf{A}^{-1})^\prime=(\mathbf{A}^\prime)^{-1}$. 
    
    \item Using Question~\ref{ivt}, prove that the if the inverse of a symmetric matrix exists, it is also symmetric. 

    \item  \label{ss} Let $\mathbf{a}$ be an $n \times 1$ matrix of real constants. How do you know $\mathbf{a}^\prime\mathbf{a}\geq 0$?

    \item  Recall the \emph{spectral decomposition} of a square symmetric matrix (for example, a variance-covariance matrix).  Any such matrix $\boldsymbol{\Sigma}$ can be written as 
            $\boldsymbol{\Sigma} = \mathbf{P} \boldsymbol{\Lambda} \mathbf{P}^\prime$,
where $\mathbf{P}$ is a matrix whose columns are the (orthonormal) eigenvectors of $\boldsymbol{\Sigma}$, $\boldsymbol{\Lambda}$ is a diagonal matrix of the corresponding (non-negative) eigenvalues, and $\mathbf{P}^\prime\mathbf{P} =~\mathbf{P}\mathbf{P}^\prime =~\mathbf{I}$. 

        \begin{enumerate}
           \item Let $\boldsymbol{\Sigma}$ be a square symmetric matrix with eigenvalues that are all strictly positive. 
                \begin{enumerate}
                    \item What is $\boldsymbol{\Lambda}^{-1}$?
                    \item Show 
            $\boldsymbol{\Sigma}^{-1} = \mathbf{P} \boldsymbol{\Lambda}^{-1} \mathbf{P}^\prime$
                \end{enumerate}
           \item Let $\boldsymbol{\Sigma}$ be a square symmetric matrix, and this time some of the eigenvalues might be zero. 
                \begin{enumerate}
                    \item What do you think $\boldsymbol{\Lambda}^{1/2}$ might be?
                    \item Define $\boldsymbol{\Sigma}^{1/2}$ as 
                    $\mathbf{P} \boldsymbol{\Lambda}^{1/2} \mathbf{P}^\prime$.
                    Show $\boldsymbol{\Sigma}^{1/2}$ is symmetric.
                    \item Show $\boldsymbol{\Sigma}^{1/2}\boldsymbol{\Sigma}^{1/2} = \boldsymbol{\Sigma}$.
                \end{enumerate}


           \item Now return to the situation where the eigenvalues of the square symmetric matrix $\boldsymbol{\Sigma}$ are all strictly positive. Define $\boldsymbol{\Sigma}^{-1/2}$ as $\mathbf{P} \boldsymbol{\Lambda}^{-1/2} \mathbf{P}^\prime$, where the elements of the diagonal matrix $\boldsymbol{\Lambda}^{-1/2}$ are the reciprocals of the corresponding elements of $\boldsymbol{\Lambda}^{1/2}$. 
                \begin{enumerate}
                    \item Show that the inverse of $\boldsymbol{\Sigma}^{1/2}$ is $\boldsymbol{\Sigma}^{-1/2}$, justifying the notation.
                    \item Show $\boldsymbol{\Sigma}^{-1/2} \boldsymbol{\Sigma}^{-1/2} = \boldsymbol{\Sigma}^{-1}$.
                \end{enumerate}

           \item The (square) matrix $\boldsymbol{\Sigma}$ is said to be \emph{positive definite} if
           $\mathbf{v}^\prime \boldsymbol{\Sigma} \mathbf{v} > 0$ for all vectors $\mathbf{v} \neq \mathbf{0}$. Show that the eigenvalues of a positive definite matrix are all strictly positive. Hint: start with the definition of an eigenvalue and the corresponding eigenvalue:  $\boldsymbol{\Sigma}\mathbf{v} = \lambda \mathbf{v}$. 
           \item Let $\boldsymbol{\Sigma}$ be a symmetric, positive definite matrix. Putting together a couple of results you have proved above, establish that $\boldsymbol{\Sigma}^{-1}$ exists.
        \end{enumerate}

    \item Let $\mathbf{X}$ be an $n \times p$ matrix of constants. The idea is that $\mathbf{X}$ is the ``design matrix" in the linear model 
$\mathbf{Y} = \mathbf{X}\boldsymbol{\beta} + \boldsymbol{\epsilon}$, so this problem is really about linear regression.

        \begin{enumerate}

            \item Recall that $\mathbf{A}$ symmetric means $\mathbf{A=A^\prime}$. Let $\mathbf{X}$ be an $n$ by $p$ matrix. Show that $\mathbf{X^\prime X}$ is symmetric.
    
            \item Recall the definition of linear independence. The columns of 
$\mathbf{A}$ are said to be \emph{linearly dependent} if there exists a column vector $\mathbf{v} \neq \mathbf{0}$ with $\mathbf{Av} = \mathbf{0}$. We will say that the columns of $\mathbf{A}$ are linearly \emph{independent} if $\mathbf{Av} = \mathbf{0}$ implies $\mathbf{v} = \mathbf{0}$.

 Show that if the columns of $\mathbf{X}$ are linearly independent, then the columns of $\mathbf{X}^\prime\mathbf{X}$ are also linearly independent. Use  Problem~\ref{ss} and the definition of linear independence.
% Proof: Given that if Xv=0 then v=0. Let (X'X)v=0 => v'X'Xv = v'0 = 0
%        But v'X'Xv = (Xv)'(Xv) = 0, so  Xv=0. 
%        Hence by def of linear independence, v=0. Done.

            \item Show that if the columns of  $\mathbf{X}^\prime\mathbf{X}$ are linearly independent, then the columns of $\mathbf{X}$ are linearly independent.
% Proof: Given (X'X)v=0 => v=0. Let Xv=0. Then X'Xv = X'0 = 0 
%        Hence by def of linear independence, v=0. Done.

            \item Show that if the columns of  $\mathbf{X}^\prime\mathbf{X}$ are linearly independent, then $(\mathbf{X}^\prime\mathbf{X})^{-1}$ exists.
% Proof: By an earlier problem, the linear independence of the cols of X'X implies linear independence of the columns of X. Now, a'X'Xa = (Xa)'Xa >= 0 because it's a sum of squares. Suppose it equals zero. Then Xa = 0 => a = 0 because the cols of X are linearly independent. Thus X'X is positive definite as well as symmetric., and by an earlier problem, its inverse exists. Done.

            \item Show that if $(\mathbf{X}^\prime\mathbf{X})^{-1}$ exists, then the columns of  $\mathbf{X}^\prime\mathbf{X}$ are linearly independent.
        \end{enumerate}
        
%This is a good problem because it establishes that the least squares estimator $\widehat{\boldsymbol{\beta}} = (\mathbf{X}^\prime\mathbf{X})^{-1}\mathbf{X}^\prime\mathbf{Y}$ exists if and only if the columns of $\mathbf{X}$ are linearly independent.
 
\end{enumerate}


\end{document}

R work for the sat data

sat = read.table("http://www.utstat.utoronto.ca/~brunner/appliedf13/code_n_data/hw/sat.data")
attach(sat)
D = VERBAL-MATH
t.test(D)

	One Sample t-test

data:  D
t = -9.047, df = 199, p-value < 2.2e-16
alternative hypothesis: true mean is not equal to 0
95 percent confidence interval:
 -65.62406 -42.13594
sample estimates:
mean of x 
   -53.88 

library(help = "stats")
help(wilcox.test)

> wilcox.test(D)

	Wilcoxon signed rank test with continuity correction

data:  D
V = 3632.5, p-value = 4.881e-15
alternative hypothesis: true location is not equal to 0

# Sign test?
length(VERBAL[VERBAL>MATH])
length(VERBAL[VERBAL<MATH])

> length(VERBAL[VERBAL>MATH])
[1] 52
> 
> length(VERBAL[VERBAL<MATH])
[1] 148