% LaTEX source code
% Steve Miller

\documentclass[12pt,letterpaper]{article}

\usepackage{times}
\usepackage[T1]{fontenc}
\usepackage{mathrsfs}
\usepackage{latexsym}
\usepackage[dvips]{graphics}
\usepackage{epsfig}
\usepackage{makeidx}

\makeindex

%\usepackage{hyperref, amsmath, amsthm, amsfonts, amscd, flafter,epsf}
\usepackage{amsmath,amsfonts,amsthm,amssymb,amscd}
\input amssym.def
\input amssym.tex

\addtolength{\textwidth}{2cm} \addtolength{\hoffset}{-1cm}
\addtolength{\marginparwidth}{-1cm} \addtolength{\textheight}{2cm}
\addtolength{\voffset}{-1cm}

\newcommand{\mindex}[1]{#1\index{#1}}
\newcommand{\sindex}[2]{#2\index{#1!#2}}
\newcommand{\cfindex}[1]{\index{continued fraction!#1}}
\newcommand{\prindex}[1]{\index{probability!#1}}
\newcommand{\faindex}[1]{\index{Fourier analysis!#1}}
\newcommand{\rmtindex}[1]{\index{Random Matrix Theory!#1}}
\newcommand{\rzfindex}[1]{\index{Riemann zeta function!#1}}
\newcommand{\matindex}[1]{\index{matrices!#1}}
\newcommand{\vindex}[1]{\index{vector!#1}}
\newcommand{\cmindex}[1]{\index{circle method!#1}}
\newcommand{\roindex}[1]{\index{Roth's Theorem!#1}}
\newcommand{\nindex}{\index{NEEDS ATTENTION}}


\newcommand{\un}{\text{U}}
\newcommand{\sy}{\text{USp}}
\newcommand{\usp}{\text{USp}}
\newcommand{\soe}{\text{SO(even)}}
\newcommand{\soo}{\text{SO(odd)}}
\newcommand{\so}{\text{O}}
\newcommand{\chiint}{\chi_{[-\foh,\foh]}}
\newcommand{\kot}[1]{ \frac{\sin \pi({#1}) }{\pi ({#1})} }
\newcommand{\kkot}[1]{ \frac{\sin \pi {#1} }{\pi {#1} } }


\renewcommand{\d}{{\mathrm{d}}} % differential, used in integrals
\newcommand{\bs}[2]{b_{i_{#1}-i_{#2}}}
\newcommand{\zeev}{Ze\'{e}v}
\newcommand{\oa}[1]{\vec{#1}}

\newcommand{\hf}{\hat{f}}
\newcommand{\hg}{\hat{g}}
\newcommand{\sumnz}{\sum_{n \in \Z}}
\newcommand{\summz}{\sum_{m \in \Z}}
\newcommand{\sumkz}{\sum_{k \in \Z}}
\newcommand{\sumooo}{\sum_{n=1}^\infty}
\newcommand{\intr}{\int_{\R}}
\newcommand{\intzo}{\int_0^1}


\newcommand{\ol}[1]{\overline{#1}}
\newcommand{\lp}{\left(}
\newcommand{\rp}{\right)}

%%%%%%%%%%%%%% Dirichlet characters

\newcommand{\logpN}{ \frac{\log p}{\log(N/\pi)} }      %logp / log(m/pi)
\newcommand{\logptN}{ 2\frac{\log p}{\log(N/\pi)} }    %2logp / log(m/pi)
\newcommand{\fologN}{O\left(\frac{1}{\log N}\right)}                       %O(1/logm)


\newcommand{\logpm}{ \frac{\log p}{\log(m/\pi)} }      %logp / log(m/pi)
\newcommand{\logptm}{ 2\frac{\log p}{\log(m/\pi)} }    %2logp / log(m/pi)
\newcommand{\logpnp}{ \frac{\log p}{\log(\frac{N}{\pi})} }     %logp / log(N/pi)
\newcommand{\logptnp}{ 2\frac{\log p}{\log(\frac{N}{\pi})} }    %2logp / log(N/pi)
\newcommand{\bchi}{ \overline{\chi} }                               %Chi-Bar
\newcommand{\fologm}{O\left(\frac{1}{\log m}\right)}                       %O(1/logm)
\newcommand{\fommt}{\frac{1}{m-2}}                             %1/m-2
\newcommand{\hphi}{\widehat{\phi}}  %phi^
\newcommand{\phiint}{\int_{-\infty}^{\infty} \phi(y) dy}
\newcommand{\FD}{\mathcal{F}} %fundamental domain

\newcommand{\pnddisc}{p \ {\notdiv} \ \triangle}  %p not dividing disc
\newcommand{\pddisc}{p | \triangle}          %p dividing disc
\newcommand{\epxxi}{e^{2 \pi i x \xi}}  %exp(2 Pi i x xi)
\newcommand{\enxxi}{e^{-2 \pi i x \xi}} %exp(-2Pi i x xi)
\newcommand{\epzxi}{e^{2 \pi i (x+iy) \xi}}
\newcommand{\enzxi}{e^{-2 \pi i (x+iy) \xi}}
\newcommand{\fologn}{O\left(\frac{1}{\log N}\right)}
\newcommand{\fllnln}{O\left(\frac{\log \log N}{\log N}\right)}
\newcommand{\flogpn}{\frac{\log p}{\log N}}
\newcommand{\plogx}{ \frac{\log p}{\log X}}
\newcommand{\plogm}{\frac{\log p}{\log M}}
%\newcommand{\plognt}{\frac{\log p}{\log N_t}}
% change in notation: use C(t) for conductor, not C(t)
\newcommand{\plogCt}{\frac{\log p}{\log C(t)}}
\newcommand{\pilogm}{\frac{\log p_i}{\log M}}
\newcommand{\pjlogm}{\frac{\log p_j}{\log M}}
\newcommand{\pologm}{\frac{\log p_1}{\log M}}
\newcommand{\ptlogm}{\frac{\log p_2}{\log M}}
%%%%%%%%%%%%%% Dirichlet characters


\newcommand{\Norm}[1]{\frac{#1}{\sqrt{N}}}

\newcommand{\sumii}[1]{\sum_{#1 = -\infty}^\infty}
\newcommand{\sumzi}[1]{\sum_{#1 = 0}^\infty}
\newcommand{\sumoi}[1]{\sum_{#1 = 1}^\infty}

\newcommand{\eprod}[1]{\prod_p \left(#1\right)^{-1}}


\newcommand\st{\text{s.t.\ }}
\newcommand\be{\begin{equation}}
\newcommand\ee{\end{equation}}
\newcommand\bea{\begin{eqnarray}}
\newcommand\eea{\end{eqnarray}}
\newcommand\bi{\begin{itemize}}
\newcommand\ei{\end{itemize}}
\newcommand\ben{\begin{enumerate}}
\newcommand\een{\end{enumerate}}
\newcommand\bc{\begin{center}}
\newcommand\ec{\end{center}}
\newcommand\ba{\begin{array}}
\newcommand\ea{\end{array}}
%\newcommand\mod{\text{mod\ }}
\newcommand{\mo}{\text{mod}\ }

\newcommand\ie{{i.e.,\ }}
\newcommand{\tbf}[1]{\textbf{#1}}
\newcommand\CF{{Continued Fraction}}
\newcommand\cf{{continued fraction}}
\newcommand\cfs{{continued fractions}}

% General Symbols

\def\notdiv{\ \mathbin{\mkern-8mu|\!\!\!\smallsetminus}}
\newcommand{\done}{\Box} %use in linux

%Blackboard Letters

\newcommand{\R}{\ensuremath{\mathbb{R}}}
\newcommand{\C}{\ensuremath{\mathbb{C}}}
\newcommand{\Z}{\ensuremath{\mathbb{Z}}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\F}{\mathbb{F}}
\newcommand{\W}{\mathbb{W}}
\newcommand{\Qoft}{\mathbb{Q}(t)}  %use in linux

% Finite Fields and Groups

\newcommand{\Fp}{ \F_p }
\newcommand{\Fpf}{ \Fp^{*} }
\newcommand{\ZnZ}{ \Z / n\Z}
\newcommand{\ZnZf}{ (\Z / n\Z)^{*}}
\newcommand{\ZqZ}{ \Z / q\Z}
\newcommand{\ZqZf}{ (\Z / q\Z)^{*}}

% Fractions

\newcommand{\fof}{\frac{1}{4}}  %oneforth
\newcommand{\foh}{\frac{1}{2}}  %onehalf
\newcommand{\fot}{\frac{1}{3}}  %onethird
\newcommand{\fop}{\frac{1}{\pi}}    %1/pi
\newcommand{\ftp}{\frac{2}{\pi}}    %2/pi
\newcommand{\fotp}{\frac{1}{2 \pi}} %1/2pi
\newcommand{\fotpi}{\frac{1}{2 \pi i}}

% Legendre Symbols

\newcommand{\jsi}[1]{{\underline{#1}\choose p_i}}
\newcommand{\jso}[1]{{\underline{#1}\choose p_1}}
\newcommand{\jst}[1]{{\underline{#1}\choose p_2}}
\newcommand{\jsthree}[1]{ {\underline{#1}\choose 3}}
\newcommand\lag[2]{\ensuremath{\left(\frac{#1}{#2}\right)}}
\newcommand{\jsq}[1]{{\underline{#1}\choose q}}            %(*/q)
\newcommand{\jsn}[1]{{\underline{#1}\choose n}}            %(*/q)
\newcommand{\js}[1]{{\underline{#1}\choose p}}

% Continued Fraction Shortcuts

\newcommand{\cffour}[4]{#1+\cfrac{1}{#2+\cfrac{1}{#3+\cfrac{1}{\ \ddots\ +\cfrac{1}{#4}}}}}
\newcommand{\cfthree}[3]{#1+\cfrac{1}{#2+\cfrac{1}{\ \ddots \+ \cfrac{1}{#3}}}}
\newcommand{\cftwo}[2]{#1+\cfrac{1}{\ \ddots \ + \cfrac{1}{#2}}    }

\newcommand{\alphampq }{ \Bigg| \alpha - \frac{p}{q} \Bigg|  }

\newcommand{\gi}[2]{ \left[\frac{#1}{#2}\right]  }
\newcommand{\gismall}[2]{ \Big[\frac{#1}{#2}\Big]  }


% Subscript Shortcuts

\newcommand{\pnpo}{p_{n+1}}
\newcommand{\pmpo}{p_{m+1}}
%\newcommand{\pm}{p_m}
\newcommand{\pn}{p_n}
\newcommand{\pmmo}{p_{m-1}}
\newcommand{\pnmo}{p_{n-1}}
\newcommand{\pmmt}{p_{m-2}}
\newcommand{\pnmt}{p_{n-2}}

\newcommand{\qmpo}{q_{m+1}}
\newcommand{\qnpo}{q_{n+1}}
\newcommand{\qm}{q_m}
\newcommand{\qn}{q_n}
\newcommand{\qmmo}{q_{m-1}}
\newcommand{\qnmo}{q_{n-1}}
\newcommand{\qmmt}{q_{m-2}}
\newcommand{\qnmt}{q_{n-2}}

\newcommand{\ampo}{a_{m+1}}
\newcommand{\anpo}{a_{n+1}}
\newcommand{\am}{a_m}
\newcommand{\an}{a_n}
\newcommand{\ammo}{a_{m-1}}
\newcommand{\anmo}{a_{n-1}}
\newcommand{\ammt}{a_{m-2}}
\newcommand{\anmt}{a_{n-2}}

% Theorem / Lemmas et cetera

\newtheorem{thm}{Theorem}[section]
\newtheorem{conj}[thm]{Conjecture}
\newtheorem{cor}[thm]{Corollary}
\newtheorem{lem}[thm]{Lemma}
\newtheorem{prop}[thm]{Proposition}
\newtheorem{exa}[thm]{Example}
\newtheorem{defi}[thm]{Definition}
\newtheorem{exe}[thm]{Exercise}
\newtheorem{exeh}[thm]{Exercise\textsuperscript{(h)}}
\newtheorem{exehr}[thm]{Exercise\textsuperscript{(hr)}}

\newtheorem{que}[thm]{Question}
\newtheorem{prob}[thm]{Problem}
\newtheorem{cla}[thm]{Claim}
\newtheorem{proj}[thm]{Research Project}

%\theoremstyle{definition}
\newtheorem{rek}[thm]{Remark}

\newcommand{\vars}[2]{ #1_1, \dots, #1_{#2} }
\newcommand{\ncr}[2]{{#1 \choose #2}}
\newcommand{\twocase}[5]{#1 \begin{cases} #2 & \text{{\rm #3}}\\ #4
&\text{{\rm #5}} \end{cases}   }
\newcommand{\twocaseother}[3]{#1 \begin{cases} #2 & \text{#3}\\ 0
&\text{otherwise} \end{cases}   }

\newcommand{\vectwo}[2]
{\left(\begin{array}{cc}
                        #1   \\
                        #2
                          \end{array}\right)}

\newcommand{\mattwo}[4]
{\left(\begin{array}{cc}
                        #1  & #2   \\
                        #3 &  #4
                          \end{array}\right) }

\newcommand{\matthree}[9]
{\left(\begin{array}{ccc}
                        #1  & #2 & #3  \\
                        #4 &  #5 & #6 \\
                        #7 &  #8 & #9
                          \end{array}\right) }

%\newcommand{\matfour}[16]
%{\left(\begin{array}{cccc}
%                        #1  & #2 & #3 & #4 \\
%                        #5  & #6 & #7 & #8 \\
%                        #9  & #10 & #11 & #12 \\
%                        #13 & #14 & #15 & #16
%                          \end{array}\right) }


%\newcommand{\matfive}[25]
%{\left(\begin{array}{ccccc}
%                        #1  & #2 & #3 & #4  & #5 \\
%                        #6 & #7 &  #8 & #9  & #10 \\
%                        #11 & #12 & #13 & #14 & #15 \\
%                        #16 & #17 & #18 & #19 & #20 \\
%                        #21 & #22 & #23 & #24 & #25
%                         \end{array}\right) }

\newcommand{\dettwo}[4]
{\left|\begin{array}{cc}
                        #1  & #2   \\
                        #3 &  #4
                          \end{array}\right| }

\newcommand{\detthree}[9]
{\left|\begin{array}{ccc}
                        #1  & #2 & #3  \\
                        #4 &  #5 & #6 \\
                        #7 &  #8 & #9
                          \end{array}\right| }


% arrows

\newcommand{\ua}{\uparrow}
\newcommand{\da}{\downarrow}
\newcommand{\uda}{\updownarrow}
\newcommand{\ra}{\rightarrow}
\newcommand{\la}{\leftarrow}
\newcommand{\hookra}{\hookrightarrow}
\newcommand{\hookla}{\hookleftarrow}
\newcommand{\circlela}{\circlearrowleft}
\newcommand{\circlera}{\circlearrowright}
%\newcommand{\squigglela}{\leftrightsquigarrow}
\newcommand{\squigglera}{\rightsquigarrow}
\newcommand{\mto}{\mapsto}  %maps to

\newcommand{\hqone}{\frac{h_1}{q_1}}
\newcommand{\hqm}{\frac{h_m}{q_m}}
\newcommand{\hqi}{\frac{h_i}{q_i}}

\newcommand{\gep}{\epsilon}
\newcommand{\gl}{\lambda}
\newcommand{\ga}{\alpha}
\newcommand{\gb}{\beta}
\newcommand{\gd}{\delta}
\newcommand{\gG}{\Gamma}
\newcommand{\gL}{\Lambda}
\newcommand{\GG}{\Gamma}
\newcommand{\w}{\omega}

\newcommand{\bra}[1]{\left\langle #1\right|}
\newcommand{\ket}[1]{\left|#1\right\rangle}
\newcommand{\braket}[2]{\left\langle #1 \left|#2\right.\right\rangle}
\newcommand{\braOket}[3]{\left\langle #1\left|#2\right|#3\right\rangle}
% matrices
% 2x2 symm
\newcommand{\sym}[3]{\ensuremath{\left(\begin{array}{ll}#1 &#2 \\ #2 &#3%
\end{array}\right)}}
\newcommand{\mat}[4]{\ensuremath{\left(\begin{array}{ll}#1 &#2 \\ #3 &#4%
\end{array}\right)}}
\newcommand{\half}{\text{\small $\frac{1}{2}$}}
\newcommand{\sfrac}[1]{\text{\small $\frac{1}{#1}$}}
\newcommand{\mbf}[1]{{\mathbf #1}}
\newcommand{\pd}[2]{\frac{\partial #1}{\partial #2}}
\newcommand{\infint}{\int_{-\infty}^{\infty} \!\!}  % infinite integral
\newcommand{\diag}{diag}
\newcommand{\OO}{O}

% Random Matrices

\newcommand{\Poo}{P_{11}}
\newcommand{\Pot}{P_{12}}
\newcommand{\Ptt}{P_{22}}
\newcommand{\aoo}{a_{11}}
\newcommand{\aot}{a_{12}}
\newcommand{\att}{a_{22}}
\newcommand{\oof}{\frac{1}{|\mathcal{F}|}} % 1/|F|
\newcommand{\sumef}{\sum_{E \in \mathcal{F}}}
\newcommand{\plogne}{\frac{\log p}{\log N_E}}
\newcommand{\hfi}{\widehat{f_i}}
\newcommand{\hfo}{\widehat{f_1}}
\newcommand{\hft}{\widehat{f_2}}
\newcommand{\NF}{|\mathcal{F}|}   %|F|
\newcommand{\sumtinF}{\sum_{t \in \mathcal{F}}}
\newcommand{\ooNF}{\frac{1}{|\mathcal{F}|}}   %1/|F|
\newcommand{\oop}{\frac{1}{p}}


    %=======================================================
    %   THIS IS WHERE YOU PUT SHORTCUT DEFINITIONS
    %========================================================


    %=======================================================
    %   THIS IS USED TO LABEL THE EQUATIONS 1.1, 1.2, ..., 2.1
    %========================================================
\renewcommand{\theequation}{\thesection.\arabic{equation}}



\title{The Method of Least Squares}

\author{Steven J. Miller\thanks{E-mail: \texttt{sjm1@williams.edu}}}

%\date{\today}
\date{}

\begin{document}

\maketitle \centerline{Department of Mathematics and Statistics}
 \centerline{Williams College}
 \centerline{Williamstown, MA 01267}

\begin{abstract} The Method of Least Squares is a procedure to
determine the best fit line to data; the proof uses calculus
and linear algebra. The basic problem is to find the best fit
straight line $y = a x + b$ given that, for $n \in \{1, \dots, N\}$,
the pairs $(x_n, y_n)$ are observed. The method easily generalizes
to finding the best fit of the form \be y \ = \ a_1 f_1(x) + \cdots
+ c_K f_K(x); \ee it is not necessary for the functions $f_k$ to be
linearly in $x$ -- all that is needed is that $y$ is to be a linear
combination of these functions.
\end{abstract}

%\pagebreak

\tableofcontents

\setcounter{equation}{0}
    %=======================================================
    %   You need the setcounter command above to reset the
    %   equation counting in each section. without it, you'd
    %   have 1.1, 1.2, 1.3, 2.4, 2.5 -- with it (you put it
    %   after each new section) you have 1.1, 1.2, 1.3, 2.1,
    %========================================================



            %**********************************
            %**********************************
            %
            %       N E W      S E C T I O N
            %
            %**********************************
            %**********************************


%\setcounter{theorem}{0} \setcounter{thm}{0}

\section{Description of the Problem}

Often in the real world one expects to find linear relationships
between variables. For example, the force of a spring linearly
depends on the displacement of the spring: $y = k x$ (here $y$ is
the force, $x$ is the displacement of the spring from rest, and $k$
is the spring constant). To test the proposed relationship,
researchers go to the lab and measure what the force is for various
displacements. Thus they assemble data of the form $(x_n,y_n)$ for
$n \in \{1, \dots, N\}$; here $y_n$ is the observed force in Newtons
when the spring is displaced $x_n$ meters.

Unfortunately, it is extremely unlikely that we will observe a
perfect linear relationship. There are two reasons for this. The
first is experimental error; the second is that the underlying
relationship may not be exactly linear, but rather only
approximately linear. (A standard example is the force felt on a falling body. We initially approximate the force as $F = mg$ with $g$ the acceleration due to gravity; however, this is not quite right as there is a resistive force which depends on the velocity.) See Figure \ref{fig:springdataplot} for a
simulated data set of displacements and forces for a spring with
spring constant equal to $5$.

\begin{figure}
\begin{center}
\scalebox{1}{\includegraphics{springdataplot.eps}}
\caption{\label{fig:springdataplot} 100 ``simulated'' observations
of displacement and force ($k = 5$).}
\end{center}\end{figure}

The Method of Least Squares is a procedure, requiring just some
calculus and linear algebra, to determine what the ``best fit'' line
is to the data. Of course, we need to quantify what we mean by
``best fit'', which will require a brief review of some probability
and statistics.

A careful analysis of the proof will show that the method is capable
of great generalizations. Instead of finding the best fit line, we
could find the best fit given by \emph{any} finite linear
combinations of specified functions. Thus the general problem is
given functions $f_1, \dots, f_K$, find values of coefficients $a_1,
\dots, a_K$ such that the \emph{linear} combination \be y \ = \ a_1
f_1(x) + \cdots + a_K f_K(x) \ee is the best approximation to the
data.

\section{Probability and Statistics Review\index{probability}}\label{chap:introprob}

We give a quick introduction to the basic elements of probability
and statistics which we need for the Method of Least Squares; for
more details see \cite{BD,CaBe,Du,Fe,Kel,LF,MoMc}.

Given a sequence of data $x_1, \dots, x_N$, we define the
\textbf{mean} (or the \textbf{expected value}) to be $(x_1+\cdots +
x_N)/N$. We denote this by writing a line above $x$: thus \be
\overline{x} \ = \ \frac1N \sum_{n=1}^N x_n. \ee The mean is the
average value of the data.

Consider the following two sequences of data: $\{10, 20, 30, 40,
50\}$ and $\{30, 30, 30, 30, 30\}$. Both sets have the same mean;
however, the first data set has greater variation about the mean.
This leads to the concept of variance, which is a useful tool to
quantify how much a set of data fluctuates about its mean. The
\textbf{variance}\footnote{For those who know more advanced statistics, for technical reasons the correct definition of the sample variance is to divide by $N-1$ and not $N$.} of $\{x_1, \dots, x_N\}$, denoted by $\sigma^2_x$, is \be
\sigma_x^2 \ = \ \frac1N \sum_{n=1}^N (x_i - \overline{x})^2; \ee
the \textbf{standard deviation} $\sigma_x$ is the square root of the
variance: \be \sigma_x \ = \ \sqrt{ \frac1N \sum_{n=1}^N (x_i -
\overline{x})^2}. \ee

Note that if the $x$'s have units of meters then the variance
$\sigma^2_x$ has units of ${\rm meters}^2$, and the standard
deviation $\sigma_x$ and the mean $\overline{x}$ have units of
meters. Thus it is the standard deviation that gives a good measure
of the deviations of the $x$'s around their mean, as it has the same units as our quantity of interest.

There are, of course, alternate measures one can use. For example,
one could consider \be \frac1N\sum_{n=1}^N (x_n - \overline{x}). \ee
Unfortunately this is a signed quantity, and large positive
deviations can cancel with large negatives. In fact, the definition
of the mean immediately implies the above is zero! This, then, would
be a terrible measure of the variability in data, as it is zero
regardless of what the values of the data are.

We can rectify this problem by using absolute values. This leads us
to consider \be\label{eq:absvalues} \frac1N\sum_{n=1}^N |x_n -
\overline{x}|. \ee While this has the advantage of avoiding
cancellation of errors (as well as having the same units as the
$x$'s), the absolute value function is not a good function
analytically. It is not differentiable. This is primarily why we
consider the standard deviation (the square root of the variance) --
this will allow us to use the tools from calculus.

We can now quantify what we mean by ``best fit''. If we believe $y =
ax + b$, then $y - (ax+b)$ should be zero. Thus given observations
\be \{(x_1,y_1), \ \dots, \ (x_N,y_N)\}, \ee we look at \be \{y_1 -
(ax_1+b), \ \dots, \ y_N - (ax_N+b)\}. \ee The mean should be small
(if it is a good fit), and the sum of squares of the terms will measure how good of a
fit we have.

We define \be E(a,b) \ :=\  \sum_{n=1}^N \left(y_n - (ax_n+b)\right)^2. \ee Large
errors are given a higher weight than smaller errors (due to the
squaring). Thus our procedure favors many medium sized errors over a
few large errors. If we used absolute values to measure the error
(see equation \eqref{eq:absvalues}), then all errors are weighted
equally; however, the absolute value function is not differentiable,
and thus the tools of calculus become inaccessible.

\begin{rek}[Choice of how to measure errors] As the point is so important, it is worth looking at one more time. There are three natural candidates to use in measuring the error between theory and observation: \be\label{eq:errornoabs} E_1(a,b) \ = \ \sum_{n=1}^N \left(y_i - (ax_i+b)\right), \ee \be\label{eq:errorabs} E_2(a,b) \ = \ \sum_{n=1}^N \left|y_i - (ax_i+b)\right| \ee and \be\label{eq:errorsq} E_3(a,b) \ = \ \sum_{n=1}^N \left(y_i - (ax_i+b)\right)^2. \ee The problem with \eqref{eq:errornoabs} is that the errors are signed quantities, and positive errors can cancel with negative errors. The problem with \eqref{eq:errorabs} is that the absolute value function is not differentiable, and thus the tools and results of calculus are unavailable. The problem with \eqref{eq:errorsq} is that errors are not weighted equally: large errors are given significantly more weight than smaller errors. There are thus problems with all three. That said, the problems with \eqref{eq:errorsq} is not so bad when compared to its advantages, namely that errors cannot cancel and that calculus is available. Thus, most people typically use \eqref{eq:errorsq} and measure errors by sums of squares. \end{rek}

\section{The Method of Least Squares}

Given data $\{(x_1,y_1), \dots, (x_N,y_N)\}$, we defined the
error associated to saying $y = ax+b$ by \be\label{eq:error} E(a,b)
\ := \ \sum_{n=1}^N \left(y_n - (ax_n+b)\right)^2. \ee
%This is just
%$N$ times the variance of the data set $\{y_1 - (ax_1+b), \dots, y_n
%- (ax_N+b)\}$. It makes no difference whether or not we study the
%variance or $N$ times the variance as our error, and
Note that the
error is a function of two variables, the unknown parameters $a$ and $b$.

The goal is to find values of $a$ and $b$ that minimize the error.
In multivariable calculus we learn that this requires us to find the
values of $(a,b)$ such that the gradient of $E$ with respect to our variables (which are $a$ and $b$) vanishes; thus we require \be \nabla E \ = \ \left(\frac{\partial E}{\partial a}, \frac{\partial E}{\partial b}\right) \ = \ (0,0), \ee or
\be \frac{\partial E}{\partial a} \ = \
0, \ \ \ \frac{\partial E}{\partial b} \ = \ 0. \ee Note we do not
have to worry about boundary points: as $|a|$ and $|b|$ become
large, the fit will clearly get worse and worse. Thus we do not need
to check on the boundary.

Differentiating $E(a,b)$ yields \bea \frac{\partial E}{\partial a} &
\ = \ & \sum_{n=1}^N 2 \left(y_n - (ax_n+b)\right) \cdot (-x_n)
\nonumber\\ \frac{\partial E}{\partial b} & \ = \ & \sum_{n=1}^N 2
\left(y_n - (ax_n+b)\right) \cdot (-1). \eea

Setting $\partial E/\partial a = \partial E/\partial b = 0$ (and
dividing by -2) yields \bea  \sum_{n=1}^N \left(y_n - (ax_n+b)\right)
\cdot x_n & \ = \ & 0 \nonumber\\  \sum_{n=1}^N \left(y_n -
(ax_n+b)\right) & \ = \ & 0. \eea Note we can divide both sides by -2 as it is just a constant; we cannot divide by $x_i$ as that varies with $i$.

We may rewrite these equations as \bea \left(\sum_{n=1}^N x_n^2
\right)a + \left(\sum_{n=1}^N x_n\right) b & \ = \ & \sum_{n=1}^N
x_n y_n \nonumber\\ \left(\sum_{n=1}^N x_n\right) a +
\left(\sum_{n=1}^N 1\right) b & \ = \ & \sum_{n=1}^N y_n. \eea

We have obtained that the values of $a$ and $b$ which minimize the
error (defined in \eqref{eq:error}) satisfy the following matrix
equation: \be\label{eq:keyeqinvolvingmatrixvector} \mattwo{\sum_{n=1}^N x_n^2}{\sum_{n=1}^N x_n\\
}{\sum_{n=1}^N x_n}{\sum_{n=1}^N 1} \vectwo{a\\ }{b} \ = \
\vectwo{\sum_{n=1}^N x_n y_n\\ }{\sum_{n=1}^N y_n}. \ee

We need a fact from linear algebra. Recall the inverse of a matrix $A$ is a matrix $B$ such that $AB = BA = I$, where $I$ is the identity matrix. If $A = \mattwo{\alpha}{\beta}{\gamma}{\delta}$ is a $2\times 2$ matrix where $\det A = \alpha \delta - \beta \gamma \neq 0$, then $A$ is invertible and \be A^{-1} \ = \ \frac{1}{\alpha \delta - \beta \gamma} \mattwo{\delta}{-\gamma}{-\beta}{\alpha}. \ee In other words, $A A^{-1} = \mattwo{1}{0}{0}{1}$ here. For example, if $A = \mattwo{1}{2}{3}{7}$ then $\det A = 1$ and $A^{-1} = \mattwo{7}{-2}{-3}{1}$; we can check this by noting (through matrix multiplication) that \be \mattwo{1}{2}{3}{7} \mattwo{7}{-2}{-3}{1} \ = \ \mattwo{1}{0}{0}{1}. \ee


We can show
the matrix in \eqref{eq:keyeqinvolvingmatrixvector} is invertible (so long as at least two of the $x_n$'s are distinct), which implies \be\label{eq:mls} \vectwo{a\\
}{b} \ = \ \mattwo{\sum_{n=1}^N x_n^2}{\sum_{n=1}^N x_n\\
}{\sum_{n=1}^N
x_n}{\sum_{n=1}^N 1}^{-1} \ \vectwo{\sum_{n=1}^N x_n y_n\\
}{\sum_{n=1}^N y_n}. \ee

Denote the matrix from \eqref{eq:keyeqinvolvingmatrixvector} by $M$. The determinant of $M$ is \be \det M \ = \
\sum_{n=1}^N x_n^2 \cdot \sum_{n=1}^N 1 - \sum_{n=1}^N x_n \cdot
\sum_{n=1}^N x_n. \ee As \be \overline{x} \ = \ \frac1N \sum_{n=1}^N
x_n, \ee we find that \bea \det M & \ = \ & N \sum_{n=1}^N x_n^2 -
(N\overline{x})^2 \nonumber\\ & = & N^2 \left( \frac1N \sum_{n=1}^N
x_n^2 - \overline{x}^2\right) \nonumber\\ & = & N^2 \cdot \frac1N
\sum_{n=1}^N(x_n - \overline{x})^2, \eea where the last equality
follows from simple algebra. Thus, as long as all the $x_n$ are not
equal, $\det M$ will be non-zero and $M$ will be invertible. Using the definition of variance, we notice the above could also be written as \be \det M \ = \ N^2 \sigma_x^2. \ee \\

\textbf{Thus we find that, so long as the $x$'s are not all equal,
the best fit values of $a$ and $b$ are obtained by solving a linear
system of equations; the solution is given in \eqref{eq:mls}.}\\

We rewrite \eqref{eq:mls} in a simpler form. Using the inverse of the matrix and the definition of the mean and variance, we find  \be\label{eq:mls2} \vectwo{a\\
}{b} \ = \ \frac1{N^2 \sigma_x^2} \mattwo{N}{-N\overline{x}}{-N\overline{x}}{\sum_{n=1}^N x_n^2} \ \vectwo{\sum_{n=1}^N x_n y_n\\
}{\sum_{n=1}^N y_n}. \ee

Expanding gives

\Large
\bea\label{eq:formulasab} a & \ = \ & \frac{N \sum_{n=1}^N x_n y_n - N\overline{x} \sum_{n=1}^N y_n}{N^2 \sigma_X^2} \nonumber\\ b &=& \frac{-N\overline{x}\sum_{n=1}^N x_n y_n + \sum_{n=1}^N x_n^2 \sum_{n=1}^N y_n}{N^2 \sigma_X^2} \nonumber\\ & & \overline{x} \ = \ \frac1N \sum_{n=1}^N x_i \nonumber\\ & & \sigma_x^2 \ = \ \frac1N \sum_{n=1}^N (x_i-\overline{x})^2.\eea
\normalsize

\ \\

As the formulas for $a$ and $b$ are so important, it is worth giving another expression for them. We also have \Large \bea\label{eq:formulasabtaketwo} a & \ = \ & \frac{\sum_{n=1}^N 1 \sum_{n=1}^N x_n y_n - \sum_{n=1}^N x_n \sum_{n=1}^N y_n}{\sum_{n=1}^N 1 \sum_{n=1}^N x_n^2 -  \sum_{n=1}^N x_n \sum_{n=1}^N x_n} \nonumber\\ \  & & \ \nonumber\\ \ & & \ \nonumber\\  b &=&
\frac{\sum_{n=1}^N x_n \sum_{n=1}^N x_n y_n - \sum_{n=1}^N x_n^2 \sum_{n=1}^N y_n}{\sum_{n=1}^N x_n \sum_{n=1}^N x_n -  \sum_{n=1}^N x_n^2 \sum_{n=1}^N 1}.
\eea

\ \\
\normalsize

\begin{rek} The formulas above for $a$ and $b$ are reasonable, as can be seen by a unit analysis. For example, imagine $x$ is in meters and $y$ is in seconds. Then if $y = ax + b$ we would need $b$ and $y$ to have the same units (namely seconds), and $a$ to have units seconds per meter. If we substitute in the units for the various quantities on the right hand side of \eqref{eq:formulasab}, we do see $a$ and $b$ have the correct units. While this is not a proof that we have not made a mistake, it is a great reassurance. No matter what you are studying, you should always try unit calculations such as this. \end{rek}

There are other, equivalent formulas for $a$ and $b$; these give the same answer, but arrange the algebra in a slightly different sequence of steps. Essentially what we are doing is the following: image we are given \bea 4 &\ = \ & 3a + 2b \nonumber\\ 5 &=& 2a + 5b.\nonumber\ \eea If we want to solve, we can proceed in two ways. We can use the first equation to solve for $b$ in terms of $a$ and substitute in, or we can multiply the first equation by 5 and the second equation by 2 and subtract; the $b$ terms cancel and we obtain the value of $a$. Explicitly, \bea 20 & \ = \ & 15a + 10b \nonumber\\ 10 &=& 4a + 10b, \nonumber\ \eea  which yields $$10 \ = \ 11a,$$ or $$a \ = \ 10/11.$$

\begin{rek} The data plotted in Figure \ref{fig:springdataplot} was
obtained by letting $x_n = 5 + .2n$ and then letting $y_n = 5x_n$
plus an error randomly drawn from a normal distribution with mean
zero and standard deviation $4$ ($n \in \{1, \dots, 100\})$. Using
these values, we find a best fit line of \be y \ = \ 4.99x + .48;
\ee thus $a = 4.99$ and $b = .48$. As the expected relation is $y =
5x$, we expected a best fit value of $a$ of $5$ and $b$ of $0$.

While our value for $a$ is very close to the true value, our value
of $b$ is significantly off. We deliberately chose data of this
nature to indicate the dangers in using the Method of Least Squares.
Just because we know $4.99$ is the best value for the slope and
$.48$ is the best value for the $y$-intercept does \emph{not} mean
that these are good estimates of the true values. The theory needs
to be supplemented with techniques which provide error estimates.
Thus we want to know something like, given this data, there is a
$99\%$ chance that the true value of $a$ is in $(4.96, 5.02)$ and
the true value of $b$ is in $(-.22, 1.18)$; this is far more useful
than just knowing the best fit values.

If instead we used \be\label{eq:Eabserror} E_{{\rm abs}}(a,b) \ = \ \sum_{n=1}^N
\left|y_n - (ax_n+b)\right|, \ee then numerical techniques yield
that the best fit value of $a$ is $5.03$ and the best fit value of
$b$ is less than $10^{-10}$ in absolute value. The difference
between these values and those from the Method of Least Squares is
in the best fit value of $b$ (the least important of the two
parameters), and is due to the different ways of weighting the
errors.
\end{rek}

\begin{exe} Consider the observed data $(0,0), (1,1), (2,2)$. It should be clear that the best fit line is $y=x$; this leads to zero error in all three systems of measuring error, namely \eqref{eq:errornoabs}, \eqref{eq:errorabs} and \eqref{eq:errorsq}; however, show that if we use \eqref{eq:errornoabs} to measure the error then line $y=1$ also yields zero error, and clearly this should not be the best fit line! \end{exe}


\begin{exe} Generalize the method of least squares to find the best
fit quadratic to $y = ax^2 + bx + c$ (or more generally the best fit
degree $m$ polynomial to $y=a_mx^m + a_{m-1}x^{m-1} + \cdots +
a_0)$.
\end{exe}

While for any real world problem, direct computation determines
whether or not the resulting matrix is invertible, it is nice to be
able to prove the determinant is always non-zero for the best fit
line (if all the $x$'s are not equal).

\begin{exe} If the $x$'s are not all equal, must the determinant be
non-zero for the best fit quadratic or the best fit cubic?
\end{exe}

Looking at our proof of the Method of Least Squares, we note that it
was not essential that we have $y = ax + b$; we could have had $y =
af(x) + bg(x)$, and the arguments would have proceeded similarly.
The difference would be that we would now obtain \be\label{eq:genfg}
\mattwo{\sum_{n=1}^N f(x_n)^2}{\sum_{n=1}^N f(x_n)g(x_n)\\
}{\sum_{n=1}^N f(x_n)g(x_n)}{\sum_{n=1}^N g(x_n)^2} \vectwo{a\\ }{b}
\ = \ \vectwo{\sum_{n=1}^N f(x_n) y_n\\ }{\sum_{n=1}^N g(x_n) y_n}.
\ee

Finally, we comment briefly on a very important change of variable that allows us to use the Method of Least Squares in many more situations than one might expect. Consider the case of a researcher trying to prove Newton's Law of Universal Gravity, which says the force felt by two masses $m_1$ and $m_2$ has magnitude $Gm_1m_2/r^2$, where $r$ is the distance between the objects. If we fix the masses, then we expect the magnitude of the force to be inversely proportional to the distance. We may write this as $F = k/r^n$, where we believe $n=2$ (the value for $k$ depends on $G$ and the product of the masses). Clearly it is $n$ that is the more important parameter here. Unfortunately, as written, we cannot use the Method of Least Squares, as one of the unknown parameters arises non-linearly (as the exponent of the separation).

We can surmount this problem by taking a logarithmic transform of the data. Setting $\mathcal{K} = \log k$, $\mathcal{F} = \log F$ and $\mathcal{R} = \log r$, the relation $F = k/r^n$ becomes $\mathcal{F} = n \mathcal{R} + \mathcal{K}$. We are now in a situation where we can apply the Method of Least Squares. The only difference from the original problem is how we collect and process the data; now our data is not the separation between the two masses, but rather the logarithm of the separation. Arguing along these lines, many power relations can be converted to instances where we can use the Method of Least Squares. We thus (finally) fulfill a promise made by many high school math teachers years ago: logarithms can be useful!

\begin{exe} Consider the generalization of the Method of Least
Squares given in \eqref{eq:genfg}. Under what conditions is the
matrix invertible?
\end{exe}

\begin{exe} The method of proof generalizes further to the case when one expects $y$ is
a \emph{linear} combination of $K$ fixed functions. The functions
need not be linear; all that is required is that we have a linear
combination, say $a_1 f_1(x) + \cdots + a_K f_K(x)$. One then
determines the $a_1, \dots, a_K$ that minimize the variance (the sum
of squares of the errors) by calculus and linear algebra. Find the
matrix equation that the best fit coefficients $(a_1,\dots,a_K)$
must satisfy.
\end{exe}

\begin{exe}
Consider the best fit line from the Method of Least Squares, so the
best fit values are given by \eqref{eq:mls}. Is the point
$(\overline{x}, \overline{y})$, where $\overline{x} = \frac1n
\sum_{n=1}^N x_n$ and $\overline{y} = \sum_{n=1}^N y_n$, on the best
fit line? In other words, does the best fit line go through the
``average'' point?
\end{exe}

\begin{exe}[Kepler's Third Law] Kepler's third law states that if $T$ is the orbital period of a planet traveling in an elliptical orbit about the sun (and no other objects exist), then $T^2 = C L^3$, where $L$ is the length of the semi-major axis. I always found this the hardest of the three laws; how would one be led to the right values of the exponents from observational data? One way is through the Method of Least Squares. Set $\mathcal{T} = \log T$, $\mathcal{L} = \log L$  and $c = \log \mathcal{C}$. Then a relationship of the form $T^a = C L^b$ becomes $a \mathcal{T} = b  \mathcal{L} + c$, which is amenable to the Method of Least Squares. The semi-major axis of the the 8 planets (sadly, Pluto is no longer considered a planet) are Mercury 0.387, Venus 0.723, Earth 1.000, Mars 1.524, Jupiter 5.203, Saturn
9.539, Uranus 19.182, Neptune 30.06 (the units are astronomical units, where one astronomical unit is 1.496 $\cdot 10^8$ km); the orbital periods (in years) are 0.2408467, 	0.61519726, 	1.0000174, 	1.8808476, 	 11.862615, 	29.447498, 	84.016846 and 	164.79132. Using this data, apply the Method of Least Squares to find the best fit values of $a$ and $b$ in $T^a = C L^b$ (note, of course, you need to use the equation $a\mathcal{T} = b\mathcal{L} + \mathcal{C}$).

Actually, as phrased above, the problem is a little indeterminate for the following reason. Imagine we have $T^2 = 5 L^3$ or $T^4 = 25 L^6$ or $T = \sqrt{5} L^{1.5}$ or even $T^{4} = 625 L^{12}$. \textbf{All of these are the same equation!} In other words, we might as well make our lives easy by taking $a=1$; there really is no loss in generality in doing this. This is yet another example of how changing our point of view can really help us. At first it looks like this is a problem involving \textbf{three} unknown parameters, $a$, $b$ and $C$; however, \textbf{there is absolutely no loss in generality in taking $a=1$}; thus let us make our lives easier and just look at this special case.

For your convenience, here are the natural logarithms of the data: the lengths of the semi-major axes are
$$\{-0.949331,\ -0.324346,\ 0,\ 0.421338,\ 1.64924,\ 2.25539,\ 2.95397,\ 3.4032\}$$ and the natural logarithms of the periods (in years) are $$\{-1.42359,\ -0.485812,\ 0.0000173998,\ 0.631723,\ 2.47339,\ 3.38261,\ 4.43102,\ 5.10468\}.$$

The problem asks you to find the best fit values of $a$ and $b$. In some sense this is a bit misleading, as there are infinitely many possible values for the pair $(a,b)$; however, all of these pairs will have the same \textbf{ratio $b/a$} (which Kepler says should be close to 3/2 or 1.50). It is this ratio that is truly important. The content of Kepler's Third Law is that the square of the period is proportional to the cube of the semi-major axis. The key numbers are the powers of the period and the length (the $a$ and the $b$), not the proportionality constant. This is why I only ask you to find the best fit values of $a$ and $b$ and not $C$ (or $\mathcal{C}$), as $C$ (or $\mathcal{C})$ is not as important. If we take $a=1$ then the best fit value of $\mathcal{C}$ is 0.000148796, and the best fit value of $b$ is almost $1.50$.

Our notes above have many different formulas to find the best fit values $a$ and $b$ for a relation $y = ax + b$. For us, we have $\mathcal{T} = \frac{b}{a} \mathcal{L} + \frac{\mathcal{C}}{a}$. Thus, for this problem, the role of $a$ from before is being played by $\frac{b}{a}$ and the role of $b$ from before is being played by $\frac{\mathcal{C}}{a}$. Therefore if we want to find the best fit value for the ratio $\frac{b}{a}$ for this problem, we just use the first of the two formulas from \eqref{eq:formulasabtaketwo}.
\end{exe}

\begin{thebibliography}{PTW02} % '2nd argument contains the widest acronym'

\bibitem[BD]{BD}
P. Bickel and K. Doksum, \emph{Mathematical Statistics: Basic Ideas
and Selected Topics}, Holden-Day, San Francisco, 1977.

\bibitem[CaBe]{CaBe}
G. Casella and R. Berger, \emph{Statistical Inference}, 2nd edition,
Duxbury Advanced Series, Pacific Grove, CA, 2002.

\bibitem[Du]{Du}
R. Durrett, \emph{Probability: Theory and Examples}, 2nd edition,
Duxbury Press, 1996.

\bibitem[Fe]{Fe}
W. Feller, \emph{An Introduction to Probability Theory and Its
Applications}, 2nd edition, Vol. II, John Wiley \& Sons, New York,
1971.

\bibitem[Kel]{Kel}
D. Kelley, \emph{Introduction to Probability}, Macmillan Publishing
Company, London, 1994.

\bibitem[LF]{LF}
R. Larson and B. Farber, \emph{Elementary Statistics: Picturing the
World}, Prentice-Hall, Englewood Cliffs, NJ, 2003.

\bibitem[MoMc]{MoMc}
D. Moore and G. McCabe, \emph{Introduction to the Practice of
Statistics}, W. H. Freeman and Co., London, 2003.


\end{thebibliography}


\end{document}
