% 431Assignment3.tex  Parameter space, More MLE and MOM, Consistency
\documentclass[10pt]{article} 
%\usepackage{amsbsy} % for \boldsymbol and \pmb 
\usepackage{graphicx} % To include pdf files!
\usepackage{amsmath}
\usepackage{amsbsy}
\usepackage{amsfonts}
\usepackage[colorlinks=true, pdfstartview=FitV, linkcolor=blue, citecolor=blue, urlcolor=blue]{hyperref} % For links
\usepackage{fullpage}
%\pagestyle{empty} % No page numbers


\begin{document}

%\enlargethispage*{1000 pt} 


\begin{center}   
{\Large \textbf{STA 431s17 Assignment Three}}\footnote{This assignment was prepared by  \href{http://www.utstat.toronto.edu/~brunner}{Jerry Brunner},
Department of Statistical Sciences, University of Toronto. It is licensed under a 
\href{http://creativecommons.org/licenses/by-sa/3.0/deed.en_US}
     {Creative Commons Attribution - ShareAlike 3.0 Unported License}. Use any part of it as you like and share the result freely. The \LaTeX~source code is available from the course website:
\href{http://www.utstat.toronto.edu/~brunner/oldclass/431s17} {\small\texttt{http://www.utstat.toronto.edu/$^\sim$brunner/oldclass/431s17}}}
\vspace{1 mm}
\end{center}

% This assignment is confined to the Estimation lecture. 
% Basic MLE and MOM appeared on Assignment 2. Here, MLE for normal and MOM for regression. See 2017A3. First take some problems directly from the Estimation lecture. See 2017A3 for LR test questions. Take consistency from 2101. 
% Put normal and MVN on the formula sheet. 

\noindent
This assignment is on material from Lecture Slide Set 5: Statistical models and estimation. There is also some background in Section A.6 of Appendix A, but that section is dominated by numerical maximum likelihood with R, which is nice but not needed for this course.

\begin{enumerate} 

%%%%%%%%%%%%%%%%%%%%%%%% Parameter space %%%%%%%%%%%%%%%%%%%%%%%%

\item Let $X_1, \ldots, X_n$ be a random sample from a Poisson distribution with expected value $\lambda > 0$.
    \begin{enumerate}
        \item What is the parameter of this model?
        \item What is the parameter space? See the lecture slides for how to write it.
    \end{enumerate}

\item Let $Y_1, \ldots, Y_n$ be a random sample from a normal distribution with expected value $\mu$ and variance $\sigma^2$.
    \begin{enumerate}
        \item What are the parameters of this model?
        \item What is the parameter space? 
    \end{enumerate}
    
\item Independently for $i=1, \ldots, n$, let $Y_i = \beta X_i + \epsilon_i$, where $X_i \sim N(\mu_x,\sigma^2_x)$, $\epsilon_i \sim N(0,\sigma^2_\epsilon)$, and  $X_i$ and $\epsilon_i$ are independent.
    \begin{enumerate}
        \item What are the parameters of this model?
        \item What is the parameter space? 
    \end{enumerate}


\item For $i=1, \ldots, n$, let $Y_i = \beta_0 + \beta_1 x_{i1} + \cdots + \beta_k x_{ik} + \epsilon_i$, where
    \begin{itemize}
        \item[] $\beta_0, \ldots, \beta_k$ are unknown constants.
        \item[] $x_{ij}$ are known constants.
        \item[] $\epsilon_1, \ldots, \epsilon_n$ are independent $N(0,\sigma^2)$ random variables.
        \item[] $\sigma^2$ is an unknown constant.
        \item[] $Y_1, \ldots, Y_n$ are observable random variables.
    \end{itemize}
    \begin{enumerate}
        \item What are the parameters of this model?
        \item What is the parameter space? 
    \end{enumerate}

% \pagebreak

%%%%%%%%%%%%%%%%%%%% More MLE and MOM %%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\item Let \label{uvn} $X_1, \ldots, X_n$ be a random sample from a normal distribution with expected value $\mu$ and variance $\sigma^2$. 
        \begin{enumerate}
            \item What is the parameter space for this model?
            \item Derive the Maximum Likelihood Estimator of the pair $\theta = (\mu,\sigma^2)$. Show your work.  
            \item Find a Method of Moments estimator of $\theta$. Use the fact that $E(X_i)=\mu$ and $Var(X_i)=\sigma^2$. This is very quick. Don't waste time and effort doing unnecessary things. 
            \item \label{numbers} In the following R output, data are in the vector $x$. Based on this, give $\widehat{\theta}$. Your answer is a pair of numbers. I needed a calculator because R's \texttt{var} function uses $n-1$ in the denominator. 
\begin{verbatim}
> c(length(x),mean(x),var(x))
[1]  20.0000  94.3800 155.1554
\end{verbatim} % MLE = MOM = (94.38, 147.3976)
            \item Give the maximum likelihood estimator of the standard deviation $\sigma$. The answer is a number. Do it the easy way. How do you know that this is okay?
        \end{enumerate}

\pagebreak

\item The formula sheet has a useful expression for the multivariate normal likelihood. 
    \begin{enumerate}
        \item Show that you understand the notation by giving the univariate version, in which $X_1, \ldots, X_n \stackrel{i.i.d}{\sim} N(\mu,\sigma^2)$. Your answer will have no matrix notation for the trace, transpose or inverse.
        \item Now starting with the univariate normal density (also on the formula sheet), show that the univariate normal likelihood is the same as your answer to the previous question. Hint: Add and subtract $\overline{X}$.
        \item How does this expression allow you to see \emph{without differentiating} that the MLE of $\mu$ is $\overline{X}$? % A.5.2 in the text.
    \end{enumerate}

\item Starting with the multivariate normal density on the formula sheet, derive the multivariate normal likelihood, also on the formula sheet. You will use $tr(\mathbf{AB}) = tr(\mathbf{AB}) $ and other properties of the trace.

\item Independently for $i=1, \ldots, n$, let $\mathbf{Y}_i = \boldsymbol{\beta}_0 + \boldsymbol{\beta}_1 \mathbf{X}_i +   \boldsymbol{\epsilon}_i$, where
\begin{itemize}
    \item $\mathbf{Y}_i$ is an $q \times 1$ random vector of observable response variables; there are $q$ response variables. 
    \item $\mathbf{X}_i$ is a $p \times 1$ observable random vector; there are $p$ explanatory variables. $E(\mathbf{X}_i) = \boldsymbol{\mu}_x$ and $V(\mathbf{X}_i) = \boldsymbol{\Phi}_{p \times p}$. The positive definite matrix $\boldsymbol{\Phi}$ is unknown. 
    \item $\boldsymbol{\beta}_0$ is a  $q \times 1$ matrix of unknown constants.
    \item $\boldsymbol{\beta}_1$ is a  $q \times p$ matrix of unknown constants.  
    \item $\boldsymbol{\epsilon}_i$ is a $q \times 1$ random vector with expected value zero and unknown positive definite variance-covariance matrix $V(\boldsymbol{\epsilon}_i) = \boldsymbol{\Psi}_{q \times q}$.  
    \item $\boldsymbol{\epsilon}_i$ is independent of $\mathbf{X}_i$.
\end{itemize}
Letting $\mathbf{D}_i = \left(\begin{array}{c} \mathbf{X}_i  \\ \hline  \mathbf{Y}_i \end{array} \right)$, we have 
$V(\mathbf{D}_i) = \boldsymbol{\Sigma} = \left( \begin{array}{c|c}
                \boldsymbol{\Sigma}_x & \boldsymbol{\Sigma}_{xy} \\ \hline
                \boldsymbol{\Sigma}_{yx} & \boldsymbol{\Sigma}_y    
                                                 \end{array} \right)$, and 
$\widehat{\boldsymbol{\Sigma}} = \left( \begin{array}{c|c}
        \widehat{\boldsymbol{\Sigma}}_x & \widehat{\boldsymbol{\Sigma}}_{xy} \\ \hline
        \widehat{\boldsymbol{\Sigma}}_{yx} & \widehat{\boldsymbol{\Sigma}}_y    
                                                 \end{array} \right)$.
    \begin{enumerate}
        \item Start by writing $\boldsymbol{\Sigma}$ in terms of the unknown parameter matrices.
        \item Give a Method of Moments Estimator for $\boldsymbol{\Phi}$. Just write it down. 
        \item Obtain formulas for the Method of Moments Estimators of $\boldsymbol{\beta}_1$, $\boldsymbol{\beta}_0$ and $\boldsymbol{\Psi}$. Show your work. You may give $\widehat{\boldsymbol{\beta}}_0$   in terms of $\widehat{\boldsymbol{\beta}}_1$, but simplify $\widehat{\boldsymbol{\Psi}}$.
        \item If the distributions of $\mathbf{X}_i$ and $\boldsymbol{\epsilon}_i$ are multivariate normal, how do you know that your Method of Moments estimates are also the MLEs?
    \end{enumerate}


%%%%%%%%%%%%%%%%%%%%%%%%% Consistency, lifted from Exercises A.5 %%%%%%%%%%%%%%%%%%%%%%%

\item  Let $X_1 , \ldots, X_n$ be a random sample from a continuous distribution with density
\begin{displaymath}
    f(x;\theta) = \frac{1}{\theta^{1/2}\sqrt{2\pi}} \, e^{-\frac{x^2}{2\theta}},
\end{displaymath}
where the parameter $\theta>0$.  Propose a reasonable estimator for the parameter $\theta$, and use the Law of Large Numbers to show that your estimator is consistent.

\item  Let $X_1 , \ldots, X_n$ be a random sample from a Gamma distribution with $\alpha=\beta=\theta>0$. That is, the density is
\begin{displaymath}
    f(x;\theta) = \frac{1}{\theta^\theta \Gamma(\theta)} e^{-x/\theta} x^{\theta-1}, 
\end{displaymath}
for $x>0$.  Let $\widehat{\theta} = \overline{X}_n$. Is $ \widehat{\theta}$ consistent for $\theta$? Answer Yes or No and prove your answer. Hint: The expected value of a Gamma random variable is $\alpha\beta$.

\item \label{thruorigin} Independently for $i = 1 , \ldots, n$, let 
\begin{displaymath}
    Y_i = \beta X _i + \epsilon_i,
\end{displaymath}
where $E(X_i)=\mu_x$, $Var(X_i)=\sigma^2_x$, $E(\epsilon_i)=0$, $Var(\epsilon_i)=\sigma^2_\epsilon$, and $\epsilon_i$ is independent of $X_i$.  Let
\begin{displaymath}
    \widehat{\beta} = \frac{\sum_{i=1}^n X_i Y_i}{\sum_{i=1}^n X_i^2}.
\end{displaymath}
Is $\widehat{\beta}$ consistent for $\beta$? Answer Yes or No and prove your answer. 

\item Another Method of Moments estimator for Problem~\ref{thruorigin} is $\widehat{\beta}_2 = \frac{\overline{Y}_n}{\overline{X}_n}$.
    \begin{enumerate}
        \item Show that $\widehat{\beta}_2 \stackrel{p}{\rightarrow} \beta$ in most of the parameter space.
        \item However, consistency means that the estimator converges to the parameter in probability \emph{everywhere} in the parameter space. Where does  $\widehat{\beta}_2$ fail, and why?
    \end{enumerate}

\item Let $X_1, \ldots, X_n$ be a random sample from a distribution with expected value $\mu$ and variance $\sigma^2_x$. Independently of $X_1, \ldots, X_n$, let $Y_1, \ldots, Y_n$ be a random sample from a distribution with the same expected value $\mu$ and variance $\sigma^2_y$. Let Let $T_n= \alpha \overline{X}_n + (1-\alpha) \overline{Y}_n$, where  $0 \leq \alpha \leq 1$. Is $T_n$ always a consistent estimator of $\mu$? Answer Yes or No and show your work. % Always is deliberately misleading. Have confidence!

\item Let $X_1, \ldots, X_n$ be a random sample from a distribution with mean $\mu$. Show that $T_n = \frac{1}{n+400}\sum_{i=1}^n X_i$ is consistent for $\mu$. Hint: If a sequence of constants $a_n \rightarrow a$ as an ordinary limit, you can view the constants as degenerate random variables and write $a_n \stackrel{p}{\rightarrow} a$. Then you can use continuous mapping and so on with confidence.

\item \label{varconsistent} Let $X_1, \ldots, X_n$ be a random sample from a distribution with mean $\mu$ and variance $\sigma^2$. Prove that the sample variance $S^2=\frac{\sum_{i=1}^n(X_i-\overline{X})^2}{n-1}$ is consistent for $\sigma^2$. 

\item \label{covconsistent} Let $(X_1, Y_1), \ldots, (X_n,Y_n)$ be a random sample from a bivariate distribution with $E(X_i)=\mu_x$, $E(Y_i)=\mu_y$, $Var(X_i)=\sigma^2_x$, $Var(Y_i)=\sigma^2_y$, and $Cov(X_i,Y_i)=\sigma_{xy}$. Show that the sample covariance 
$S_{xy} = \frac{\sum_{i=1}^n(X_i-\overline{X})(Y_i-\overline{Y})}{n-1}$ is a consistent estimator of $\sigma_{xy}$.

\item Let $X_1 , \ldots, X_n$ be a random sample from a Poisson distribution with parameter $\lambda$. You know that $E(X_i)=Var(X_i)=\lambda$; there is no need to prove it.

From the Law of Large Numbers, it follows immediately that $\overline{X}_n$ is consistent for $\lambda$. Let
\begin{displaymath}
    \widehat{\lambda} = \frac{\sum_{i=1}^n (X_i-\overline{X}_n)^2}{n-4}.
\end{displaymath}
Is $\widehat{\lambda}$ also consistent for $\lambda$? Answer Yes or No and prove your answer.



\end{enumerate}


\end{document}