%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%                                                                 %
%  GEANT manual in LaTeX form                                     %
%                                                                 %
%  Version 1.00                                                   %
%                                                                 %
%  Last Mod.  9 June 1993  19:30  MG                              %
%                                                                 %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\Origin{L.Urb\'{a}n}
\Submitted{26.10.84} \Revised{16.12.93}
\Version{Geant 3.16}\Routid{PHYS330}
\Makehead{Ionisation processes induced by e+/e-}

\section{Subroutines}
\Shubr{GDRELA}{}
\Rind{GDRELA} initialises the ionisation energy loss tables for different
materials for muons, electrons and positrons and other particles.
The energy binning is set within the array
{\tt ELOW} (common \FCind{/GCMULO/}) in the routine \Rind{GPHYSI}.
The tables are filled with the quantity $dE/dx$ in GeV/cm (formula
(\ref{eq:phys330-3}) below)
which is valid for an element as well as a mixture or a compound.
In the tables the dE/dx due to ionisation is summed with the
energy loss coming from bremsstrahlung.
For energy loss of electrons and positrons, it calls \Rind{GDRELE}
using the following pointers:

\begin{tabular}{llr}
 \mbox{\tt JMA = LQ(JMATE-I)} &  pointer to the I$^{th}$ material \\
 \mbox{\tt JEL1 = LQ(JMA-1) } &  pointer for dE/dx for electrons  \\
 \mbox{\tt JEL1+90          } &  pointer for dE/dx for positron \\
\end{tabular}

\Rind{GDRELA} is called at initialisation time by \Rind{GPHYSI}.
\Shubr{GDRELE}{(EEL,CHARGE,JMA,DEDX)}
\Rind{GDRELE} computes the ionisation energy loss ({\tt DEDX}) for electrons
({\tt CHARGE = -1}) and positrons ({\tt CHARGE = +1}) with kinetic
energy {\tt EEL} in the material indicated by {\tt I} in
{\tt JMA = LQ(JMATE-I)}. It is called by the routine \Rind{GDRELA}.
\Shubr{GDRSGA}{}
{\tt CDRSGA} calculates the total cross-section in all materials for
delta rays for M\"{o}ller (\Pem\Pem) and Bhabha (\Pem\Pep)
scattering and for muons.
It tabulates the mean free path, $ \lambda = 1/\Sigma $ (in cm)
as a function of the medium and the energy.
The energy binning is set within the array {\tt ELOW} (common
\FCind{/GCMULO/})
in the routine \Rind{GPHYSI}. The following
pointers are used (see {\tt JMATE} data structure):

\begin{tabular}{l@{\hspace{3cm}}l}
{\tt JMA = LQ(JMATE-I)}&pointer to the I$^{th}$ material \\
{\tt JDRAY = LQ(JMA-11)}&pointer to $\delta$-ray cross-sections\\
{\tt JDRAY}  &pointer for electrons                          \\
{\tt JDRAY+90} &pointer for positrons                        \\
{\tt JDRAY+180} &pointer for $\mu^+/\mu^-$.                    \\
\end{tabular}

The routine is called during initialisation by \Rind{GPHYSI}.

\section{Method}
Let: \[\frac{d\sigma(E,T)}{dT}\]
be the differential cross-section
for the ejection of an electron with kinetic energy $T$ by an incident
${e^{\pm}}$ of total energy $E$ moving in a medium of density $\rho$.

The variable {\tt DCUTE} in common \FCind{/GCCUTS/}
is the kinetic energy cut-off for the generated $\delta$-rays.
Below this threshold the soft
electrons ejected are simulated as continuous energy loss by the incident
${\rm e^{\pm}}$, and above it they are explicitly generated.

The mean value of the energy lost by the incident ${\rm e^{\pm}}$ to
the soft $\delta$-rays is:
\begin{equation}
\label{eq:phys330-1}
E_{soft}(E,\mbox{\tt DCUTE})
= \int_{0}^{\mbox{\tt DCUTE}} \frac{d \sigma (E,T)}{dT} T \: dT
\end{equation}
whereas the total cross-section for the ejection of
an electron of energy $ T > \mbox{\tt DCUTE} $ is:
\begin{equation}
\label{eq:phys330-2}
\sigma (E,{\mbox{\tt DCUTE}})
= \int_{\mbox{\tt DCUTE}}^{T_{Max}}\frac{d \sigma (E,T)} {dT} \: dT
\end{equation}
where $T_{Max}$ is the maximum energy transferable to the free electron:
\begin{equation}
T_{Max} = \left\{ \begin{array}{ll}
             E-m & {\rm for \hspace{.2cm} }  \Pep  \\
             (E-m)/2 & {\rm for \hspace{.2cm} }  \Pem , \\
              \end{array} \right .
\end{equation}
where $m$ is the electron mass.
The method of calculation of the continuous energy loss
and the total cross-section are
explained below. {\tt [PHYS331]} deals with the explicit
simulation of the $\delta$-rays.

\subsection{Continuous energy loss}
The integration of (\ref{eq:phys330-1}) leads to the Berger-Seltzer
formulae \latexhtml{%
   \cite{bib-BERG,bib-BETH,bib-BLOC,bib-EGS3,bib-STER,bib-MES1}}%
  {\cite{bib-BERG}, \cite{bib-BETH}, \cite{bib-BLOC},
   \cite{bib-EGS3}, \cite{bib-STER}, \cite{bib-MES1}}:

\begin{equation}
\label{eq:phys330-3}
\frac {dE}{dx} = \frac{2 \pi r_0^ 2 mn }{\beta^2}
       \left [\ln \frac{2(\tau + 2)} {(I/m)^2}+ F^{\pm} (\tau , \Delta )
- \delta \right ],
\end{equation}
 where

\[
\begin{array}{LL}
\gamma           & \frac{E}{m}                           \\
\beta^2          & 1-\frac{1}{\gamma^2}                  \\
\tau             & \gamma-1                              \\
\tau_c           & \frac{\mbox{\tt DCUTE}}{m}           \\
\mbox{\tt DCUTE}   &  \mbox{energy cut for} \: e^{\pm}      \\
\tau_{max}       & \mbox{maximum possible energy transfer in \Pem mass:
  $\tau$ for \Pep, $\tau/2$ for \Pem}  \\
\Delta           & \min(\tau_c,\tau_{max})              \\
n                & \mbox{electron density of the medium}        \\
I                & \mbox{average mean ionisation energy}        \\
\delta           & \mbox{density effect correction}.
\end{array}
\]

The functions $ F^{\pm}$  are given by
\begin{eqnarray}
F^+ (\tau,\Delta)& =& \ln(\tau\Delta ) -
\frac{\Delta^2}{\tau}\left[\tau + 2 \Delta -
\frac{3\Delta^2 y } {2} -\left(\Delta - \frac{\Delta^3 }{3} \right) y^2
- \left (\frac{\Delta^2}{2} - \tau
       \frac{\Delta^3}{3} + \frac{\Delta^4 } {4} \right)
          y^3  \right]  \\
F^- (\tau,\Delta )& =& -1 -\beta^2 +\ln \left [(\tau - \Delta)
\Delta \right ] + \frac{\tau}{\tau -\Delta}+\frac{\left [
\frac{\Delta^2}{2} + ( 2\tau +1) \ln
\left (1- \frac{\Delta}{\tau} \right ) \right ]}{\gamma^2},
\end{eqnarray}
where $ y = 1/(\gamma+1) $.

The density effect correction is calculated as in~\cite{bib-STER}:
\begin{equation}
\delta = \left\{\begin{array}{l@{\hspace{2cm}}l}
0                          & \mbox{if } x<x_0                        \\
2\ln 10 + x+C+a(x_1-x)^m  & \mbox{\phantom{if }}x_0 \leq x\leq x_1  \\
2\ln 10 + x+C             & \mbox{\phantom{if }}x_1<x,
\end{array}\right .
\end{equation}
where $ x = \ln(\gamma^2-1) / (2\ln 10) $
 The quantities n, I and the parameters of the density effect correction
($x_0, x_1, C, a, m$) are computed in the routine \Rind{GPROBI},
and we give the corresponding formulae here.
The electron density of the medium, n, can be written as

\begin{equation}
n = \left\{ \begin{array}{LL}

N_{Av} \rho \frac{Z}{A} & \mbox{for elements} \\ [.2cm]
N_{Av} \rho \frac{\sum_{i} n_i Z_i }{\sum_{i} n_i A_i} &
\mbox{for compounds/mixtures, }
\end{array} \right .
\end{equation}

where

\begin{DLtt}{MMMMM}
\item [$N$]     Avogadro's number;
\item[$Z_i$]   atomic number;
\item[$A_i$]   atomic weight;
\item[$\rho$]    density of the material;
\item[$n_i$]     proportion by number of the $i^{th}$ element in the
        material (for a mixture
       $n_i  = W p_i/A_i$
    where $p_i$ the proportion by weight and $W$ is the molecular
    weight).
\end{DLtt}

The average mean ionisation energy can be calculated as~\cite{bib-BERG}
\cite{bib-BETH} \cite{bib-BLOC} \cite{bib-EGS3}:

\begin{equation}
I (GeV) = \left\{ \begin{array}{LL}
16 \cdot 10^{-9} \; Z^{0.9} & \mbox{for a chemical element} \\
\exp \left [ \sum_{i} n_i Z_i \ln I(Z_i ) / \sum_i n_i Z _i \right ]
& \mbox{for a compound or mixture}
\end{array} \right .
\end{equation}

The density effect correction parameters can be computed (for
condensed medium \cite{bib-STER}) as
\begin{eqnarray*}
C & = & 1 + 2\ln \frac{I}{28.8 \cdot 10^{-9} \sqrt{\rho} \sum n_i Z_i /
              \sum n_i A_i }\\
m       &=& 3                          \\
x_a     &=& \frac{C} {2\ln 10}   \\
a       &=& 2\ln 10 \frac{(x_a -x_0)}{(x_1 - x_0 )}^m \\
\end{eqnarray*}

\begin{center}
$ \begin{array}{|c|c|c|c|}
\hline
I        &         C    &       x_0       &             x_1\\
\hline
<10^{-7} &        <3.681      &        0.2       &            2 \\
 <10^{-7}&        \geq 3.681  &      -0.326C-1   &            2 \\
\hline
\geq10^{-7} &       5.215     &       0.2         &          3 \\
\geq10^{-7} &      \geq 5.215 &       -0.326C-1.5 &         3 \\
\hline
\end{array}$\end{center}

\subsection{Total cross-sections}
The integration of formula (\ref{eq:phys330-2}) gives the total cross-section
\cite{bib-EGS4}, \cite{bib-MESS} for M\"{o}ller scattering ($\Pem\Pem$):
\begin{equation}
\label{eq:phys330-9}
\sigma ( Z,E,\mbox{\tt DCUTE} ) =\frac {2 \pi r_0^2 mZ}{\beta^2(E-m)}
 \left[\frac{(\gamma-1)} {\gamma^2}\left(\frac{1}{x}-1\right)
             +\frac{1}{x}-\frac{1}{1-x}-\frac{2\gamma-1}{\gamma^2}\ln
\frac{1-x}{x}\right]
\end{equation}
and for Bhabha scattering ($\Pep \Pem $):
\begin{equation}
\label{eq:phys330-10}
\sigma (Z,E,\mbox{\tt DCUTE}) =\frac{ 2 \pi r^2_0  mZ }
      {(E-m)}\left [\frac {1 }{\beta^2}  \left(\frac{1}{x}-1\right)
  + B_1 x + B_2 (1-x) -
 \frac {B_3 } {2} ( 1-x^2 ) +\frac{B_4}{3}(1-x^3)\right]
\end{equation}
where

\[
\begin{array}{L@{\hspace{3cm}}L}
\gamma = \frac{E}{m}       & \beta^2 = 1-\frac{1}{\gamma^2} \\ [.2cm]
 x     =\frac {\mbox{\tt DCUTE}}{E-m}  & \gamma=\frac{1}{\gamma + 1} \\ [.2cm]
 B_1=2-y^2                 &  B_2=(1-2y)(3+y^2)              \\ [.2cm]
B_3=(1-2y)^2+(1-2y)^3      &  B_4=(1-2y)^3
\end{array}
\]

The formulae [\ref{eq:phys330-9}] and [\ref{eq:phys330-10}]
give the total cross-section of the scattering
above the threshold energies

\begin{equation}
T_{\rm Moller}^{\rm thr} =2\mbox{\tt DCUTE} \mbox{\hspace{2cm}and\hspace{2cm}}
T_{\rm Bhabha}^{\rm thr} = \mbox{\tt DCUTE}
\end{equation}
The interaction length for the production of $\delta$-rays is calculated
during initialisation by routine \Rind{GDRSGA}.


\documentclass[14pt]{article}

\usepackage{fancyhdr}
\usepackage{extramarks}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{tikz}
\usepackage[plain]{algorithm}
\usepackage{algpseudocode}
\usepackage{enumitem}
\usepackage{relsize}
\usepackage{scrextend}

\usetikzlibrary{automata,positioning}

%
% Basic Document Settings
%

\topmargin=-0.45in
\evensidemargin=0in
\oddsidemargin=0in
\textwidth=6.5in
\textheight=9.0in
\headsep=0.25in

\linespread{1.1}

\pagestyle{fancy}
\lhead{\hmwkAuthorName}
\chead{\hmwkClass\ (\hmwkClassInstructor): \hmwkTitle}
\rhead{\firstxmark}
\lfoot{\lastxmark}
\cfoot{\thepage}

\renewcommand\headrulewidth{0.4pt}
\renewcommand\footrulewidth{0.4pt}

\setlength\parindent{0pt}

%
% Create Problem Sections
%

\newcommand{\enterProblemHeader}[1]{
    \nobreak\extramarks{}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
    \nobreak\extramarks{Problem \arabic{#1} (continued)}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
}

\newcommand{\exitProblemHeader}[1]{
    \nobreak\extramarks{Problem \arabic{#1} (continued)}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
    \stepcounter{#1}
    \nobreak\extramarks{Problem \arabic{#1}}{}\nobreak{}
}

\setcounter{secnumdepth}{0}
\newcounter{partCounter}
\newcounter{homeworkProblemCounter}
\setcounter{homeworkProblemCounter}{1}
\nobreak\extramarks{Problem \arabic{homeworkProblemCounter}}{}\nobreak{}

%
% Homework Problem Environment
%
% This environment takes an optional argument. When given, it will adjust the
% problem counter. This is useful for when the problems given for your
% assignment aren't sequential. See the last 3 problems of this template for an
% example.
%
\newenvironment{homeworkProblem}[1][-1]{
    \ifnum#1>0
        \setcounter{homeworkProblemCounter}{#1}
    \fi
    \section{Problem \arabic{homeworkProblemCounter}}
    \setcounter{partCounter}{1}
    \enterProblemHeader{homeworkProblemCounter}
}{
    \exitProblemHeader{homeworkProblemCounter}
}

%
% Homework Details
%   - Title
%   - Due date
%   - Class
%   - Section/Time
%   - Instructor
%   - Author
%

\newcommand{\hmwkTitle}{Homework\ \#1}
\newcommand{\hmwkDueDate}{September 13, 2016}
\newcommand{\hmwkDueTime}{2:30pm}
\newcommand{\hmwkClass}{CS 477}
\newcommand{\hmwkClassInstructor}{Monica Nicolescu}
\newcommand{\hmwkAuthorName}{Matthew J. Berger}

%
% Title Page
%

\title{
    \vspace{2in}
    \textmd{\textbf{\hmwkClass:\ \hmwkTitle}}\\
    \normalsize\vspace{0.1in}\small{Due\ on\ \hmwkDueDate\ at \hmwkDueTime}\\
    \vspace{0.1in}\large{\textit{\hmwkClassInstructor}}
    \vspace{3in}
}

\author{\textbf{\hmwkAuthorName}}
\date{}

\renewcommand{\part}[1]{\textbf{\large Part \Alph{partCounter}}\stepcounter{partCounter}\\}

%
% Various Helper Commands
%

% Useful for algorithms
\newcommand{\alg}[1]{\textsc{\bfseries \footnotesize #1}}

% For derivatives
\newcommand{\deriv}[1]{\frac{\mathrm{d}}{\mathrm{d}x} (#1)}

% For partial derivatives
\newcommand{\pderiv}[2]{\frac{\partial}{\partial #1} (#2)}

% Integral dx
\newcommand{\dx}{\mathrm{d}x}

% Alias for the Solution section header
\newcommand{\solution}{\textbf{\large Solution}}

% Alias for the Big-O Symbol
\newcommand{\bigo}{\mathcal{O}}

% Probability commands: Expectation, Variance, Covariance, Bias
\newcommand{\E}{\mathrm{E}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\Bias}{\mathrm{Bias}}

\begin{document}

\maketitle

\pagebreak

\begin{homeworkProblem}
    \textbf{(U \& G-required)[30 points]} Arrange the following list of functions in ascending order of growth rate. That is, if function g(n) immediately follows function f(n) in your list.

    \begin{itemize}
        \item[] \(f_1(n) = n^{4.5}\)
        \item[] \(f_2(n) = \sqrt{2n^2 + 1}\)
        \item[] \(f_3(n) = n^2 + 10\)
        \item[] \(f_4(n) = 10^n\)
        \item[] \(f_5(n) = 100^n\)
        \item[] \(f_6(n) = n^2\log{n}\)
    \end{itemize}

    \textbf{Solution:}

    \begin{table}[h]
    \def\arraystretch{1.5}%
    \begin{tabular}{|l|l|}
        \hline
        \multicolumn{1}{|c|}{Formula} & \multicolumn{1}{|c|}{Complexity} \\
        \hline
        \(f_2(n) = \sqrt{2n^2 + 1} \) & \(\bigo(n)\)          \\
        \(f_6(n) = n^2\log{n} \)      & \(\bigo(n^2\log{n})\) \\
        \(f_3(n) = n^2 + 10 \)        & \(\bigo(n^2)\)        \\
        \(f_1(n) = n^{4.5} \)         & \(\bigo(n^{4.5})\)    \\
        \(f_4(n) = 10^n \)            & \(\bigo(10^n)\)       \\
        \(f_5(n) = 100^n \)           & \(\bigo(100^n)\)      \\
        \hline
    \end{tabular}
\end{table}

\end{homeworkProblem}

\pagebreak

\begin{homeworkProblem}
    \textbf{(U \& G-required)[30 points]} Using mathematical induction, show that the following relations are true for every $n \geq 1$:

    \begin{enumerate}[label=\alph*)]
    	\item \(\mathlarger{\sum\limits_{i=1}^n i^3=\left[\frac{n(n+1)}{2}\right]^2}\)
    	\item \(\mathlarger{\sum\limits_{i=1}^ni(i+1)=\frac{n(n+1)(n+2)}{3}}\)
    \end{enumerate}

    \textbf{Solution:}

    \textbf{Part A}
    \begin{addmargin}[2em]{2em}
    \textsl{\textbf{\underline{Proof:}} We will prove by induction that, for all n $\in \mathbb{Z_+}$,}

    \begin{itemize}
    	\item[(1)] \(\hfil\mathlarger{\sum\limits_{i=1}^n i^3=\left[\frac{n(n+1)}{2}\right]^2}\)
    \end{itemize}

        \textsl{\textbf{\underline{Base Case:}} When $n = 1$, the left side of (1) is $(1)^3 = 1$, and the right side is $\left[\frac{1(1+1)}{2}\right]^2 = \left[\frac{2}{2}\right]^2 = 1$, so both sides are equal and (1) is true for $n = 1$.}

        \textsl{\textbf{\underline{Induction Step:}} Let $k \in \mathbb{Z_+}$ be given and suppose (1) is true for $n = k$. Then}
    \[
    	\begin{split}
    		\sum\limits_{i=1}^{k+1} i^3&=\sum\limits_{i=1}^k +\ (k+1)^3 \\
    		\left[\frac{(k+1)(k+2)}{2}\right]^2 &=\ \left[\frac{k(k+1)}{2}\right]^2 + (k+1)^3\\
    		&=\ \frac{k^2(k+1)}{2^2}^2 + (k+1)^3\\
    		&=\ \frac{k^2(k+1)}{4}^2 + \frac{4(k+1)^3}{4}\\
    		&=\ \frac{(k+1)^2(k^2 + 4k+4)}{4}\\
    		&=\ \frac{(k+1)^2(k+2)^2}{2^2}\\
    		&=\ \left[\frac{(k+1)(k+2)}{2}\right]^2\\
    	\end{split}
    \]
    \textsl{Thus, (1) holds for $n = k + 1$, and the proof of the induction step is complete.}

    \textsl{\textbf{\underline{Conclusion:}} By the principle of induction, (1) is true for all $n \in \mathbb{Z_+}$}
    \end{addmargin}

    \pagebreak

    \textbf{Part B}
    \begin{addmargin}[2em]{2em}
    \textsl{\textbf{\underline{Proof:}} We will prove by induction that, for all n $\in \mathbb{Z_+}$,}

    \begin{itemize}
    	\item[(1)] \(\hfil\mathlarger{\sum\limits_{i=1}^ni(i+1)=\frac{n(n+1)(n+2)}{3}}\)
    \end{itemize}

        \textsl{\textbf{\underline{Base Case:}} When $n = 1$, the left side of (1) is $(1)(1+1) = 2$, and the right side is $\frac{(1)(1 + 1)(1 + 2)}{3} = 2$, so both sides are equal and (1) is true for $n = 1$.}

        \textsl{\textbf{\underline{Induction Step:}} Let $k \in \mathbb{Z_+}$ be given and suppose (1) is true for $n = k$. Then}
    \[
    	\begin{split}
    		\sum\limits_{i=1}^{k+1} i(i+1)      &= \sum\limits_{i=1}^k +\ (k+1)(k+2) \\
    		\frac{(k+1)(k+2)(k+3)}{3} &= \frac{k(k+1)(k+2)}{3} + (k+1)(k+2)\\
    		&= \frac{k(k+1)(k+2)}{3} + \frac{3(k+1)(k+2)}{3}\\
    		&= \frac{(k+1)(k+2)(k+3)}{3}\\
    	\end{split}
    \]
    \textsl{Thus, (1) holds for $n = k + 1$, and the proof of the induction step is complete.}

    \textsl{\textbf{\underline{Conclusion:}} By the principle of induction, (1) is true for all $n \in \mathbb{Z_+}$}
    \end{addmargin}
\end{homeworkProblem}

\pagebreak

\begin{homeworkProblem}
	\textbf{(U \& G-required)[40 points]} Indicate whether the first function of each of the following pairs has a smaller, same, or larger order of growth (to within a constant multiple) than the second function:

    \begin{enumerate}[label=\alph*)]
    	\item \(\mathit{(n-1)!}\ \text{and}\ \mathit{n!}\)
    	\item \(\mathit{\log_2n}\ \text{and}\ \mathit{\ln{n}}\)
    	\item \(\mathit{2^\mathit{n-1}}\ \text{and}\ \mathit{2^n}\)
    	\item \(\mathit{\log_2^2n}\ \text{and}\ \mathit{\log_2n^2}\)
    \end{enumerate}

    \textbf{Solution}

    \begin{enumerate}[label=\alph*)]
    	\item \(smaller\)
    	\item \(larger\)
    	\item \(smaller\)
    	\item \(larger\)
    \end{enumerate}
\end{homeworkProblem}

\begin{homeworkProblem}[5]
	\textbf{[Extra credit - 20 points]} Using the formal definition of the asymptotic notations, prove the following statements:

    \begin{enumerate}[label=\alph*)]
    	\item \(5n^2 + 20 \in \bigo(n^2)\)
    	\item \(n + 23 \in \bigo(n^3)\)
    \end{enumerate}

    \textbf{Solution}

    \begin{enumerate}[label=\alph*)]
    	\item \(0 \leq 5n^2 + 20 \leq cn^2  \rightarrow c \geq \frac{20}{n^2} + 5\ \text{and}\ n_0 \geq 1 \\ \therefore c = 25\ \text{and}\ n_0=1\ \text{and}\ 5n + 20 \in \bigo(n^2)\)
    	\item \(0 \leq n + 23 \leq cn^3 \rightarrow c \geq \frac{n+23}{n^3}\ \text{and}\ n_0 \geq 1 \\ \therefore c = 24\ \text{and}\ n_0=1\ \text{and}\ n + 23 \in \bigo(n^3)\)
    \end{enumerate}
\end{homeworkProblem}

\end{document}


\lab{Linked Lists}{Linked Lists}

\objective{One of the fundamental problems in programming is knowing which data structures to use to optimize code.
The type of data structure used determines how quickly data is accessed and modified, which affects the overall speed of a program.
In this lab, we introduce a basic data structure called a linked list and create a class to implement it.}

% \section*{Linked Lists} % ===================================================

A \emph{linked list} is a data structure that chains data together.
Every linked list needs a reference to the first item in the chain, called the \li{head}.
A reference to the last item in the chain, called the \li{tail}, is also often included.
Each item in the list stores a piece of data, plus at least one reference to another item in the list.
The items in the list are called \emph{nodes}.

\subsection*{Nodes} % ---------------------------------------------------------

Think of data as several types of objects that need to be stored in a warehouse.
A \emph{node} is like a standard size box that can hold all the different types of objects.
For example, suppose a particular warehouse stores lamps of various sizes.
Rather than trying to carefully stack lamps of different shapes on top of each other, it is preferable to first put them in boxes of standard size.
Then adding new boxes and retrieving stored ones becomes much easier.
A \emph{data structure} is like the warehouse, which specifies where and how the different boxes are stored.

A node class is usually simple.
The data in the node is stored as an attribute.
Other attributes may be added (or inherited) specific to a particular data structure.

\begin{problem} % Restricting data types of the Node class.
Consider the following generic node class.
\begin{lstlisting}
class Node:
    """A basic node class for storing data."""
    def __init__(self, data):
        """Store the data in the value attribute."""
        self.value = data
\end{lstlisting}

Modify the constructor so that it only accepts data of type \li{int}, \li{float}, or \li{str}.
If another type of data is given, raise a \li{TypeError} with an appropriate error message.
Modify the constructor docstring to document these restrictions.
\end{problem}

\begin{comment}
\begin{info} % Nodes are often dictionary keys.
Often the data stored in a node is actually a \emph{key} value.
The key might be a memory address, a dictionary key, or the index of an array where the true desired information resides.
For simplicity, in this and the following lab we store actual data in node objects, not references to data located elsewhere.
\end{info}
\end{comment}

The nodes of a \emph{singly linked list} have a single reference to the next node in the list (see Figure \ref{fig:singly_linked}), while the nodes of a \emph{doubly linked list} have two references: one for the previous node, and one for the next node (see Figure \ref{fig:doubly_linked}).
This allows for a doubly linked list to be traversed in both directions, whereas a singly linked list can only be traversed in one direction.

\begin{lstlisting}
class LinkedListNode(Node):
    """A node class for doubly linked lists. Inherits from the Node class.
    Contains references to the next and previous nodes in the linked list.
    """
    def __init__(self, data):
        """Store the data in the value attribute and initialize
        attributes for the next and previous nodes in the list.
        """
        Node.__init__(self, data)       # Use inheritance to set self.value.
        self.<<next>> = None                # Reference to the next node.
        self.prev = None                # Reference to the previous node.
\end{lstlisting}

\begin{figure}[H] % Singly linked list.
\centering
\begin{tikzpicture}[->,>=stealth',shorten >=1pt,auto, node distance=1.5cm,thick,main node/.style={rectangle,draw}, minimum size=.5cm]
\tikzset{rect node/.style={rectangle, draw, minimum height = .5cm, minimum width=.2cm}}
    \node[main node] (1) {A};
    \node[main node] (2) [right of=1] {B};
    \node[main node] (3) [right of=2] {C};
    \node[main node] (4) [right of=3] {D};
    \node[draw = none, black, node distance=1.5cm] [above left of=1](H) {\li{head}};
\foreach \r in {1, 2, 3, 4}{
    \node[rect node][right of=\r, node distance = .36cm]{};}
\node[draw = none, node distance = 1.5cm] [right of=4]{};
\foreach \s/\t  in {1/2, 2/3, 3/4}{\path[draw](\s) edge[shorten <=.1cm](\t);}
    \draw[blue] (H) edge (1.north);
\end{tikzpicture}
\caption{A singly linked list.
Each node has a reference to the next node in the list. The head attribute is always assigned to the first node.}
\label{fig:singly_linked}
\end{figure}

\begin{figure}[H] % Doubly linked list.
\centering
\begin{tikzpicture}[->,>=stealth',shorten >=1pt,auto, node distance=1.5cm, thick,main node/.style={rectangle,draw}, minimum size=.5cm]
\tikzset{rect node/.style={rectangle, draw, minimum height = .5cm, minimum width=.9cm}}
    \node[main node] (1) {A};
    \node[main node] (2) [right of=1] {B};
    \node[main node] (3) [right of=2] {C};
    \node[main node] (4) [right of=3] {D};
    \node[draw=none, node distance=.07cm] (1up) [above of=1] {};
    \node[draw=none, node distance=.07cm] (1dn) [below of=1] {};
    \node[draw=none, node distance=.07cm] (2up) [above of=2] {};
    \node[draw=none, node distance=.07cm] (2dn) [below of=2] {};
    \node[draw=none, node distance=.07cm] (3up) [above of=3] {};
    \node[draw=none, node distance=.07cm] (3dn) [below of=3] {};
    \node[draw=none, node distance=.07cm] (4up) [above of=4] {};
    \node[draw=none, node distance=.07cm] (4dn) [below of=4] {};
    \node[draw = none, black, node distance = 1.5cm] [above right of=4] (T) {\li{tail}};
    \node[draw = none, black, node distance = 1.5cm] [above left of=1] (H) {\li{head}};
    \node[rect node](1.5)[]{};
    \node[rect node](2.5)[right of=1.5]{};
    \node[rect node](3.5)[right of=2.5]{};
    \node[rect node](4.5)[right of=3.5]{};
\foreach \s/\t  in {1up/2up, 2dn/1dn, 2up/3up, 3dn/2dn, 3up/4up, 4dn/3dn}{
        \path[draw](\s) edge[shorten <=.1cm, shorten >=.1cm](\t);}
    \draw[blue] (H) edge (1.north);
    \draw[blue] (T) edge (4.north);
\end{tikzpicture}
\caption{A doubly linked list. Each node has a reference to the node before it and a reference to the node after it. In addition to the head attribute, this list has a tail attribute that is always assigned to the last node.}
\label{fig:doubly_linked}
\end{figure}

The following \li{LinkedList} class chains \li{LinkedListNode} instances together by modifying each node's \li{<<next>>} and \li{prev} attributes.
The list is empty initially, so the \li{head} and \li{tail} attributes are assigned the placeholder value \li{None} in the constructor.
The \li{append()} method makes a new node and adds it to the very end of the list (see Figure \ref{fig:linked-list-append}).
There are two cases for appending that must be considered separately in the implementation: either the list is empty, or the list is nonempty.

\begin{figure}[H] % append().
\centering
\begin{tikzpicture}[->,>=stealth',shorten >=1pt,auto, node distance=1.6cm,thick,main node/.style={rectangle,draw}, minimum size=.5cm]
\tikzset{rect node/.style={rectangle, draw, minimum height = .5cm, minimum width=.2cm}}
    \node[main node] (1) {A};
    \node[main node] (2) [right of=1] {B};
    \node[main node] (3) [right of=2, fill=yellow!15] {C};
    \node[main node] (5) [right of=3, node distance=3.5cm] {A};
    \node[main node] (6) [right of=5] {B};
    \node[main node] (7) [right of=6] {C};
    \node[draw=none, node distance=.07cm] (1up) [above of=1] {};
    \node[draw=none, node distance=.07cm] (1dn) [below of=1] {};
    \node[draw=none, node distance=.07cm] (2up) [above of=2] {};
    \node[draw=none, node distance=.07cm] (2dn) [below of=2] {};
    \node[draw=none, node distance=.07cm] (3up) [above of=3] {};
    \node[draw=none, node distance=.07cm] (3dn) [below of=3] {};
    \node[draw=none, node distance=.07cm] (5up) [above of=5] {};
    \node[draw=none, node distance=.07cm] (5dn) [below of=5] {};
    \node[draw=none, node distance=.07cm] (6up) [above of=6] {};
    \node[draw=none, node distance=.07cm] (6dn) [below of=6] {};
    \node[draw=none, node distance=.07cm] (7up) [above of=7] {};
    \node[draw=none, node distance=.07cm] (7dn) [below of=7] {};
    \node[draw=none, black, node distance=1.5cm] [above left of=1](H1) {\li{head}};
    \node[draw = none, black, node distance=1.5cm] [above right of=2](T1) {\li{tail}};
    \node[draw=none, black, node distance=1.5cm] [above left of=5](H2) {\li{head}};
    \node[draw = none, black, node distance=1.5cm] [above right of=7](T2) {\li{tail}};
\foreach \r in {1, 2, 5, 6, 7}{
        \node[rect node][right of=\r, node distance = .36cm]{};
        \node[rect node][left  of=\r, node distance = .36cm]{};}
    \node[rect node][right of=3, node distance = .36cm, fill=yellow!15]{};
    \node[rect node][left  of=3, node distance = .36cm, fill=yellow!15]{};
\foreach \s/\t in {1up/2up, 2dn/1dn, 5up/6up, 6dn/5dn}{
        \path[draw](\s) edge[shorten <=.1cm, shorten >=.1cm](\t);}
\foreach \s/\t in {6up/7up, 7dn/6dn}{
        \path[draw](\s) edge[green, shorten <=.1cm, shorten >=.1cm](\t);}
    \draw[blue, shorten >=.1cm] (H1) edge (1.north);
    \draw[blue, shorten >=.1cm] (T1) edge (2.north);
    \draw[blue, shorten >=.1cm] (H2) edge (5.north);
    \draw[blue, shorten >=.1cm] (T2) edge (7.north);
\end{tikzpicture}
\caption{Appending a new node to the end of a nonempty doubly linked list.
The green arrows are the new connections.
Note that the \li{tail} attribute is reassigned from B to C.}
\label{fig:linked-list-append}
\end{figure}

\begin{lstlisting}
class LinkedList:
    """Doubly linked list data structure class.

    Attributes:
        head (LinkedListNode): the first node in the list.
        tail (LinkedListNode): the last node in the list.
    """
    def __init__(self):
        """Initialize the head and tail attributes by setting
        them to None, since the list is empty initially.
        """
        self.head = None
        self.tail = None

    def append(self, data):
        """Append a new node containing the data to the end of the list."""
        # Create a new node to store the input data.
        new_node = LinkedListNode(data)
        if self.head is None:
            # If the list is empty, assign the head and tail attributes to
            # new_node, since it becomes the first and last node in the list.
            self.head = new_node
            self.tail = new_node
        else:
            # If the list is not empty, place new_node after the tail.
            self.tail.<<next>> = new_node               # tail --> new_node
            new_node.prev = self.tail               # tail <-- new_node
            # Now the last node in the list is new_node, so reassign the tail.
            self.tail = new_node
\end{lstlisting}

\begin{warn} % Warning about 'is' vs '=='.
The \li{is} operator is \textbf{not} the same as the \li{==} operator.
While \li{==} checks for numerical equality, \li{is} evaluates whether or not two objects are the same by checking their location in memory.

\begin{lstlisting}
>>> 7 == 7.0            # True since the numerical values are the same.
<<True>>

# 7 is an int and 7.0 is a float, so they cannot be stored at the same
# location in memory. Therefore 7 "is not" 7.0.
>>> 7 is 7.0
<<False>>
\end{lstlisting}

For numerical comparisons, always use \li{==}.
When comparing to built-in Python constants such as \li{None}, \li{True}, \li{False}, or \li{NotImplemented}, use \li{is} instead.
\end{warn}

\subsection*{Locating Nodes} % ------------------------------------------------

The \li{LinkedList} class only explicitly keeps track of the first and last nodes in the list via the \li{head} and \li{tail} attributes.
To access any other node, use each successive node's \li{<<next>>} and \li{prev} attributes.

\begin{lstlisting}
>>> my_list = LinkedList()
>>> for data in (2, 4, 6):
...     my_list.append(data)
...
# To access each value, use the head attribute of the LinkedList
# and the next and value attributes of each node in the list.
>>> my_list.head.value
2
>>> my_list.head.<<next>>.value                     # 2 --> 4
4
>>> my_list.head.<<next.next>> is my_list.tail      # 2 --> 4 --> 6
<<True>>
\end{lstlisting}

% Problem 2: LinkedList.find(), LinkedList.get()
\begin{problem}
Add the following methods to the \li{LinkedList} class.
\begin{enumerate}
\item \li{find()}: Accept a piece of data and return the first node in the list containing that data (return the actual \li{LinkedListNode} object, not its \li{value}).
If no such node exists, or if the list is empty, raise a \li{ValueError} with an appropriate error message.
\\(Hint: if \li{n} is assigned to one of the nodes the list, what does \li{n = n.<<next>>} do?)

\item \li{get()}: Accept an integer $i$ and return the $i$th node in the list.
If $i$ is negative or greater than or equal to the number of nodes in the list, raise an \li{IndexError}.
\\(Hint: add an attribute that tracks the current size of the list.
Update it every time a node is successfully added or removed, such as at the end of the \li{append()} method.)
\end{enumerate}
\label{prob:linked-list-node-finders}
\end{problem}

\subsection*{Magic Methods} % -------------------------------------------------

Endowing data structures with magic methods makes them much more intuitive to use.
Consider, for example, how a Python list responds to built-in functions like \li{len()} and \li{print()}.
At the bare minimum, the \li{LinkedList} class should have the same functionality.

\begin{problem} % LinkedList.__len__(), LinkedList.__str__().
Add the following magic methods to the \li{LinkedList} class.
\begin{enumerate}
\item Write the \li{__len__()} method so that the length of a \li{LinkedList} instance is equal to the number of nodes in the list.

\item Write the \li{__str__()} method so that when a \li{LinkedList} instance is printed, its output matches that of a Python list.
Entries are separated by a comma and one space; strings are surrounded by single quotes, or by double quotes if the string itself has a single quote.
\\(Hint: use \li{repr()} to deal with quotes easily.)
\end{enumerate}
\end{problem}

\subsection*{Removal} % ------------------------------------------------------

To delete a node, all references to the node must be removed.
Python automatically deletes the object once there is no way for the user to access it.
Na{\"i}vely, this might be done by finding the previous node to the one being removed, and setting its \li{<<next>>} attribute to \li{None}.
However, there is a problem with this approach.

\begin{lstlisting}
class LinkedList:
    # ...
    def remove(self, data):
        """Attempt to remove the first node containing the specified data.
        This method incorrectly removes additional nodes.
        """
        # Find the target node and sever the links pointing to it.
        target = self.find(data)
        target.prev.<<next>> = None                     # -/-> target
        target.<<next>>.prev = None                     # target <-/-
\end{lstlisting}

Removing all references to the target node deletes the node (see Figure \ref{fig:linked-list-remove-bad}).
Unfortunately, the nodes before and after the target node are no longer linked.

\begin{lstlisting}
>>> my_list = LinkedList()
>>> for i in range(10):
...     my_list.append(i)
...
>>> print(my_list)
[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]

>>> my_list.remove(4)               # Removing a node improperly results in
>>> print(my_list)                  # the rest of the chain being lost.
[0, 1, 2, 3]                        # Should be [0, 1, 2, 3, 5, 6, 7, 8, 9].
\end{lstlisting}

\begin{figure}[H] % Incorrect remove().
\centering
\begin{tikzpicture}[->,>=stealth',shorten >=1pt,auto, node distance=1.5cm,thick,main node/.style={rectangle,draw}, minimum size=.5cm]
\tikzset{rect node/.style={rectangle, draw, minimum height = .5cm, minimum width=.2cm}}
    \node[main node] (1) {A};
    \node[main node] (2) [right of=1] {B};
    \node[main node] (3) [right of=2] {C};
    \node[main node] (4) [right of=3] {D};
    \node[main node] (5) [right of=4, node distance=2.6cm] {A};
    \node[main node] (6) [right of=5] {B};
    \node[main node] (7) [right of=6, fill=red!10] {C};
    \node[main node] (8) [right of=7, fill=red!10] {D};
    \node[draw=none, node distance=.07cm] (1up) [above of=1] {};
    \node[draw=none, node distance=.07cm] (1dn) [below of=1] {};
    \node[draw=none, node distance=.07cm] (2up) [above of=2] {};
    \node[draw=none, node distance=.07cm] (2dn) [below of=2] {};
    \node[draw=none, node distance=.07cm] (3up) [above of=3] {};
    \node[draw=none, node distance=.07cm] (3dn) [below of=3] {};
    \node[draw=none, node distance=.07cm] (4up) [above of=4] {};
    \node[draw=none, node distance=.07cm] (4dn) [below of=4] {};
    \node[draw=none, node distance=.07cm] (5up) [above of=5] {};
    \node[draw=none, node distance=.07cm] (5dn) [below of=5] {};
    \node[draw=none, node distance=.07cm] (6up) [above of=6] {};
    \node[draw=none, node distance=.07cm] (6dn) [below of=6] {};
    \node[draw=none, node distance=.07cm] (7up) [above of=7] {};
    \node[draw=none, node distance=.07cm] (7dn) [below of=7] {};
    \node[draw=none, node distance=.07cm] (8up) [above of=8] {};
    \node[draw=none, node distance=.07cm] (8dn) [below of=8] {};
\foreach \r in {1, 2, 3, 4, 5, 6, 8}{
    \node[rect node][right of=\r, node distance = .36cm]{};
    \node[rect node][left of=\r, node distance = .36cm]{};}
\foreach \r in {7, 8}{
    \node[rect node][right of=\r, node distance = .36cm, fill=red!10]{};
    \node[rect node][left of=\r, node distance = .36cm, fill=red!10]{};}
\foreach \s/\t in {1up/2up, 2dn/1dn, 2up/3up, 3dn/2dn, 3up/4up, 4dn/3dn, 5up/6up, 6dn/5dn, 7dn/6dn, 7up/8up}{
        \path[draw](\s) edge[shorten <=.1cm, shorten >=.1cm](\t);}
    \node[draw=none, node distance=1.5cm] [right of=8]{};  % Centralize
\end{tikzpicture}
\caption{Na{\"i}ve removal for doubly linked Lists.
Deleting all references pointing to C deletes the node, but it also separates nodes A and B from node D.}
\label{fig:linked-list-remove-bad}
\end{figure}

This can be remedied by pointing the previous node's \li{<<next>>} attribute to the node after the deleted node, and similarly changing that node's \li{prev} attribute.
Then there will be no reference to the removed node and it will be deleted, but the chain will still be connected.

\begin{figure}[H] % Correct remove().
\centering
\begin{tikzpicture}[->,>=stealth',shorten >=1pt,auto, node distance=1.5cm,thick,main node/.style={rectangle,draw}, minimum size=.5cm]
\tikzset{rect node/.style={rectangle, draw, minimum height = .5cm, minimum width=.2cm}}
    \node[main node] (1) {A};
    \node[main node] (2) [right of=1] {B};
    \node[main node] (3) [right of=2] {C};
    \node[main node] (4) [right of=3] {D};
    \node[main node] (5) [right of=4, node distance=2.6cm] {A};
    \node[main node] (6) [right of=5] {B};
    \node[main node] (7) [right of=6, fill=red!10] {C};
    \node[main node] (8) [right of=7] {D};
    \node[draw=none, node distance=.07cm] (1up) [above of=1] {};
    \node[draw=none, node distance=.07cm] (1dn) [below of=1] {};
    \node[draw=none, node distance=.07cm] (2up) [above of=2] {};
    \node[draw=none, node distance=.07cm] (2dn) [below of=2] {};
    \node[draw=none, node distance=.07cm] (3up) [above of=3] {};
    \node[draw=none, node distance=.07cm] (3dn) [below of=3] {};
    \node[draw=none, node distance=.07cm] (4up) [above of=4] {};
    \node[draw=none, node distance=.07cm] (4dn) [below of=4] {};
    \node[draw=none, node distance=.07cm] (5up) [above of=5] {};
    \node[draw=none, node distance=.07cm] (5dn) [below of=5] {};
    \node[draw=none, node distance=.07cm] (6up) [above of=6] {};
    \node[draw=none, node distance=.07cm] (6dn) [below of=6] {};
    \node[draw=none, node distance=.07cm] (7up) [above of=7] {};
    \node[draw=none, node distance=.07cm] (7dn) [below of=7] {};
    \node[draw=none, node distance=.07cm] (8up) [above of=8] {};
    \node[draw=none, node distance=.07cm] (8dn) [below of=8] {};
\foreach \r in {1, 2, 3, 4, 5, 6, 7, 8}{
    \node[rect node][right of=\r, node distance = .36cm]{};
    \node[rect node][left of=\r, node distance = .36cm]{};}
\node[rect node][right of=7, node distance = .36cm, fill=red!10]{};
\node[rect node][left of=7, node distance = .36cm, fill=red!10]{};
\foreach \s/\t in {1up/2up, 2dn/1dn, 2up/3up, 3dn/2dn, 3up/4up, 4dn/3dn, 5up/6up, 6dn/5dn, 7dn/6dn, 7up/8up}{
        \path[draw](\s) edge[shorten <=.1cm, shorten >=.1cm](\t);}
    \path[draw, shorten <=.1cm, shorten >=.1cm](6) edge[green, bend left] (8);
    \path[draw, shorten <=.1cm, shorten >=.1cm](8) edge[green, bend left] (6);
    \node[draw=none, node distance=1.5cm] [right of=8]{};  % Centralize
\end{tikzpicture}
\caption{Correct removal for doubly linked Lists.
To avoid gaps in the chain, nodes B and D must be linked together.}
\label{fig:linked-list-remove}
\end{figure}

\begin{problem} % LinkedList.remove().
Modify the \li{remove()} method given above so that it correctly removes the first node in the list containing the specified data.
Also account for the special cases of removing the first, last, or only node, in which \li{head} and/or \li{tail} must be reassigned.
Raise a \li{ValueError} if there is no node in the list that contains the data.
\\(Hint: use the \li{find()} method from Problem \ref{prob:linked-list-node-finders} to locate the target node.)
\end{problem}

\begin{warn} % Garbage collection warning.
Python keeps track of the variables in use and automatically deletes a variable (freeing up the memory that stored the object) if there is no access to it.
This feature is called \emph{garbage collection}.
In many other languages, leaving a reference to an object without explicitly deleting it can lead to a serious memory leak.
See \url{https://docs.python.org/3/library/gc.html} for more information on Python's garbage collection system.
\end{warn}

\subsection*{Insertion} % -----------------------------------------------------

The \li{append()} method can add new nodes to the end of the list, but not to the middle.
To do this, get references to the nodes before and after where the new node should be, then adjust their \li{<<next>>} and \li{prev} attributes.
Be careful not to disconnect the nodes in a way that accidentally deletes nodes like in Figure \ref{fig:linked-list-remove-bad}.

\begin{figure}[H] % insert().
\centering
\begin{tikzpicture}[->,>=stealth',shorten >=1pt,auto, node distance=1.6cm,thick,main node/.style={rectangle,draw}, minimum size=.5cm]
\tikzset{rect node/.style={rectangle, draw, minimum height = .5cm, minimum width=.2cm}}
    \node[main node] (1) {A};
    \node[main node] (2) [right of=1] {B};
    \node[main node] (3) [right of=2] {D};
    \node[main node] (4) [below right of=2, fill=yellow!15] {C};
    \node[main node] (5) [right of=3, node distance=3.5cm] {A};
    \node[main node] (6) [right of=5] {B};
    \node[main node] (7) [right of=6, node distance=2.2cm] {D};
    \node[main node] (8) [below right of=6] {C};
    \node[draw=none, node distance=.07cm] (1up) [above of=1] {};
    \node[draw=none, node distance=.07cm] (1dn) [below of=1] {};
    \node[draw=none, node distance=.07cm] (2up) [above of=2] {};
    \node[draw=none, node distance=.07cm] (2dn) [below of=2] {};
    \node[draw=none, node distance=.07cm] (3up) [above of=3] {};
    \node[draw=none, node distance=.07cm] (3dn) [below of=3] {};
    \node[draw=none, node distance=.07cm] (4up) [above of=4] {};
    \node[draw=none, node distance=.07cm] (4dn) [below of=4] {};
    \node[draw=none, node distance=.07cm] (5up) [above of=5] {};
    \node[draw=none, node distance=.07cm] (5dn) [below of=5] {};
    \node[draw=none, node distance=.07cm] (6up) [above of=6] {};
    \node[draw=none, node distance=.07cm] (6dn) [below of=6] {};
    \node[draw=none, node distance=.07cm] (7up) [above of=7] {};
    \node[draw=none, node distance=.07cm] (7dn) [below of=7] {};
    \node[draw=none, node distance=.07cm] (8up) [above of=8] {};
    \node[draw=none, node distance=.07cm] (8dn) [below of=8] {};
    \node[draw=none, black, node distance=1.5cm] [above left of=1](H1) {\li{head}};
    \node[draw = none, black, node distance=1.5cm] [above right of=3](T1) {\li{tail}};
    \node[draw=none, black, node distance=1.5cm] [above left of=5](H2) {\li{head}};
    \node[draw = none, black, node distance=1.5cm] [above right of=7](T2) {\li{tail}};
\foreach \r in {1, 2, 3, 5, 6, 7, 8}{
        \node[rect node][right of=\r, node distance = .36cm]{};
        \node[rect node][left  of=\r, node distance = .36cm]{};}
    \node[rect node][right of=4, node distance = .36cm, fill=yellow!15]{};
    \node[rect node][left  of=4, node distance = .36cm, fill=yellow!15]{};
\foreach \s/\t in {1up/2up, 2dn/1dn, 2up/3up, 3dn/2dn, 5up/6up, 6dn/5dn}{
        \path[draw](\s) edge[shorten <=.1cm, shorten >=.1cm](\t);}
\foreach \s/\t in {6up/8up, 8dn/6dn, 8up/7up, 7dn/8dn}{
        \path[draw](\s) edge[green, shorten <=.1cm](\t);}
    \draw[blue, shorten >=.1cm] (H1) edge (1.north);
    \draw[blue, shorten >=.1cm] (T1) edge (3.north);
    \draw[blue, shorten >=.1cm] (H2) edge (5.north);
    \draw[blue, shorten >=.1cm] (T2) edge (7.north);
    \node[draw = none, node distance = 1.5cm] [right of=8]{};  % Centralize
\end{tikzpicture}
\caption{Insertion for doubly linked lists.}
\label{fig:linked-list-insert}
\end{figure}

\begin{problem} % LinkedList.insert()
Add an \li{insert()} method to the \li{LinkedList} class that accepts an integer \li{index} and data to add to the list.
Insert a new node containing the data immediately \textbf{before} the node in the list currently at position \li{index}.
After the insertion, the new node should be at position \li{index}.
For example, Figure \ref{fig:linked-list-insert} places a new node containing C at index 2.
Carefully account for the special case of inserting before the first node, which requires \li{head} to be reassigned.
\\(Hint: except when inserting before the head, get references to the nodes that should be immediately before and after the new node following the insertion.
Consider using the \li{get()} method from Problem \ref{prob:linked-list-node-finders} to locate one of these nodes.)

If \li{index} is equal to the number of nodes in the list, append the node to the end of the list by calling \li{append()}.
If \li{index} is negative or strictly greater than the number of nodes in the list, raise an \li{IndexError}.
\end{problem}

\begin{info} % Big-O rates for linked lists.
The temporal complexity for inserting to the beginning or end of a linked list is $O(1)$, but inserting anywhere else is $O(n)$, where $n$ is the number of nodes in the list.
This is quite slow compared other data structures.
In the next lab we turn our attention to \emph{trees}, special kinds of linked lists that allow for much quicker sorting and data retrieval.
\end{info}

\section*{Restricted-Access Lists} % ==========================================

It is sometimes wise to restrict the user's access to some of the data within a structure.
In particular, because insertion, removal, and lookup are $O(n)$ for data in the middle of a linked list, cutting off access to the middle of the list forces the user to only use $O(1)$ operations at the front and end of the list.
The three most common and basic restricted-access structures that implement this idea are \emph{stacks}, \emph{queues}, and \emph{deques}.
Each structure restricts the user's access differently, making them ideal for different situations.

\begin{itemize}
\item \textbf{Stack}: \emph{Last In, First Out} (LIFO).
Only the last item that was inserted can be accessed.
A stack is like a pile of plates: the last plate put on the pile is (or should be) the first one to be taken off.
Stacks usually have two main methods: \li{push()}, to insert new data, and \li{pop()}, to remove and return the last piece of data inserted.

\item \textbf{Queue} (pronounced ``cue''): \emph{First In, First Out} (FIFO).
New nodes are added to the end of the queue, but an existing node can only be removed or accessed if it is at the front of the queue.
A queue is like a polite line at the bank: the person at the front of the line is served first, while newcomers add themselves to the back of the line.
Queues also usually have a \li{push()} and a \li{pop()} method, but \li{push()} inserts data to the end of the queue while \li{pop()} removes and returns the data at the front of the queue.
The \li{push()} and \li{pop()} operations are sometimes called \li{enqueue()} and \li{dequeue()}, respectively.

\item \textbf{Deque} (pronounced ``deck''): a double-ended queue.
Data can be inserted or removed from either end, but data in the middle is inaccessible.
A deque is like a deck of cards, where only the top and bottom cards are readily accessible.
A deque has two methods for insertion and two for removal, usually called \li{append()}, \li{appendleft()}, \li{pop()}, and \li{popleft()}.

A deque can act as a queue by using only \li{append()} and \li{popleft()} (or \li{appendleft()} and \li{pop()}), or as a stack by using only \li{append()} and \li{pop()} (or \li{appendleft()} and \li{popleft()}).
\end{itemize}

\begin{problem} % Deque class.
Write a \li{Deque} class that inherits from \li{LinkedList}.
%
\begin{enumerate}
\item Write the following methods.
Since they all involve data at the endpoints, avoid iterating through the list so the resulting operations are $O(1)$.
\begin{itemize}
    \item \li{pop()}: Remove the last node in the list and return its data.
    Account for the special case of removing the only node in the list.
    Raise a \li{ValueError} if the list is empty.
    \item \li{popleft()}: Remove the first node in the list and return its data.
    Raise a \li{ValueError} if the list is empty.
    \\(Hint: use inheritance and the \li{remove()} method of \li{LinkedList}.)
    \item \li{appendleft()}: Insert a new node at the beginning of the list.
    \\(Hint: use inheritance and the \li{insert()} method of \li{LinkedList}.)
\end{itemize}
Note that the \li{LinkedList} class already implements \li{append()}.

\item Override the \li{remove()} method with the following code.

\begin{lstlisting}
def remove(*args, **kwargs):
    raise NotImplementedError("Use pop() or popleft() for removal")
\end{lstlisting}

This effectively disables \li{remove()} for the \li{Deque} class, preventing the user from removing a node from the middle of the list.

\item Disable \li{insert()} as well.
\end{enumerate}
\end{problem}

\begin{info}
The \li{*args} argument allows the \li{remove()} method to receive any number of positional arguments without raising a \li{TypeError}, and the \li{**kwargs} argument allows it to receive any number of keyword arguments.
This is the most general form of a function signature.
\end{info}

Python lists have \li{append()} and \li{pop()} methods, so they can be used as stacks.
However, data access and removal from the front is much slower than from the end, as Python lists are implemented as dynamic arrays and not linked lists.

The \li{collections} module in the standard library has a \li{deque} object that is implemented as a doubly linked list.
This is an excellent object to use in practice instead of a Python list when speed is of the essence and data only needs to be accessed from the ends of the list.
Both lists and deques are slow to modify elements in the middle, but lists can access middle elements quickly.
Table \ref{tab:list-deque-complexity} describes the complexity for common operations on lists v. deques in Python.

\begin{table}[H]
\begin{tabular}{|c|c|c|}
\hline
Operation& List Complexity & Deque Complexity\\
\hline
Append/Remove from the end& $O(1)$ & $O(1)$\\
\hline
Append/Remove from the start & \cellcolor{red!20}$O(n)$ & $O(1)$ \\
\hline
Insert/Delete in the middle   & \cellcolor{red!20}$O(n)$ & \cellcolor{red!20}$O(n)$ \\
\hline
Access element at the start/end  & $O(1)$ & $O(1)$ \\
\hline
Access element in the middle & $O(1)$ & \cellcolor{red!20}$O(n)$ \\
\hline
\end{tabular}
\caption{Complexity of operations on lists and deques.}
\label{tab:list-deque-complexity}
\end{table}

\begin{problem} % Reverse a file using a stack.
Write a function that accepts the name of a file to be read and a file to write to.
Read the first file, adding each line of text to a stack.
After reading the entire file, pop each entry off of the stack one at a time, writing the result to the second file.

For example, if the file to be read has the following list of words on the left, the resulting file should have the list of words on the right.

\begin{lstlisting}
<<My homework is too hard for me.         I am a mathematician.
I do not believe that                   Programming is hard, but
I can solve these problems.             I can solve these problems.
Programming is hard, but                I do not believe that
I am a mathematician.                   My homework is too hard for me.
\end{lstlisting}

You may use a Python list, your \li{Deque} class, or \li{collections.deque} for the stack.
Test your function on the file \texttt{english.txt}, which contains a list of over 58,000 English words in alphabetical order.
% Which data structure is the fastest?
\end{problem}

\newpage

\section*{Additional Material} % ==============================================

\subsection*{Possible Improvements to the LinkedList Class} % -----------------

The following are some ideas for expanding the functionality of the \li{LinkedList} class.

\begin{enumerate}
\item Add a keyword argument to the constructor so that if an iterable is provided, each element of the iterable is immediately added to the list.
This makes it possible to cast an iterable as a \li{LinkedList} the same way that an iterable can be cast as one of Python's standard data structures.

\begin{lstlisting}
>>> my_list = [1, 2, 3, 4, 5]
>>> my_linked_list = LinkedList(my_list)  # Cast my_list as a LinkedList.
>>> print(my_linked_list)
[1, 2, 3, 4, 5]
>>> type(my_linked_list)
<<LinkedList>>
\end{lstlisting}

\item Add the following methods.
\begin{itemize}
\item \li{count()}: return the number of occurrences of a specified value.
\item \li{reverse()}: reverse the ordering of the nodes (in place).
\item \li{roll()}: shift the nodes a given number of steps to the right or left (in place).
\item \li{sort()}: sort the nodes by their data (in place).
\end{itemize}

\item Implement more magic methods.
\begin{itemize}
\item \li{__add__()}: concatenate two lists.
\item \li{__getitem__()} and \li{__setitem__()}: enable standard bracket indexing.
Try to allow for negative indexing as well.
\item \li{__iter__()}: support \li{for} loop iteration, the \li{iter()} built-in function, and the \li{in} statement.
\end{itemize}
\end{enumerate}

\subsection*{Other Kinds of Linked Lists} % -----------------------------------

The \li{LinkedList} class can also be used as the backbone for more specialized data structures.
%
\begin{enumerate}
\item A \emph{sorted list} adds new nodes strategically so that the data is always kept in order.
Therefore, a \li{SortedLinkedList} class should have an \li{add()} method that receives some \li{data} and inserts a new node containing \li{data} before the first node in the list that has a \li{value} that is greater or equal to \li{data} (thereby preserving the ordering).
Other methods for adding nodes should be disabled.
Note however, that a linked list is \textbf{not} an ideal implementation for a sorted list because each insertion is $O(n)$ (try sorting \texttt{english.txt}).

\item In a \emph{circular linked list}, the ``last'' node connects back to the ``first'' node.
Thus a reference to the tail is unnecessary.
The \li{roll()} method mentioned above is used often so the \li{head} attribute is at an ``active'' part of the list where nodes are inserted, removed, or accessed often.
This data structure can therefore decrease the average insertion or removal time for certain data sets. % by constantly adjusting what is considered to be ``in the middle'' of the list.
\end{enumerate}


\input{../header.tex}
\title{\vspace{-2cm}INF3490/INF4490 Exercise Solutions - Advanced Neural Networks}
\author{Eivind Samuelsen\input{../author_footnote.tex}}
\date{}

% Removing paragraph indents is sometimes useful:
% \setlength\parindent{0pt}
% ==============================================================================

% ================================= DOCUMENT ===================================
\begin{document}
    \renewcommand\marginsymbol[1][0pt]{%
  \tabto*{0cm}\makebox[-1cm][c]{$\mathbb{P}$}\tabto*{\TabPrevPos}}

\maketitle
\input{../intro.tex}

\section{Perceptron activation functions}
Last week we used the activation function
\begin{equation}
g(h) =
\begin{cases}
1 & h > 0 \\
0 & h \leq 0
\end{cases}
\end{equation}
Why is this not used with backpropagation?\\

\noindent\textit{Answer:}

\noindent
Backpropagation is entirely dependent on the output of the neurons \emph{not} being hard zeroes and ones, so that it can make gradual changes to them.
From a technical perspective, it is dependent on the activation function \(y_i = f(a_i)\) being differentiable, with a derivative \(f'(a_i)\) that can be written as a function of the neuron output \(y_i\) (you could perhaps write it as \(g'(y_i) = f'(f^{-1}(yi))\)), in order to estimate how much the weighted sum of inputs \(a_i\) has to change in order to achieve a change in \(y_i\) that is proportional to the derivative of the error \(y_i - t_i\).

\section{Hidden layers}
What is the minimum number of hidden neuron layers needed in order to approximate an arbitrary continuous function, and why?\\

\noindent\textit{Answer:}

\noindent
You need three layers (one output layer plus two hidden layers) in order to create any decision boundary.
Take the two-dimensional case: one layer gives you straight lines, and two layers gives you any shape that is in between any number of lines, so any convex shape - which includes all triangles.
By combining enough triangles (remember, we haven’t put any limits on how many neurons we have in each layer) we can approximate any shape.
Luckily, this holds for any number of dimensions, so three layers is enough no matter how many input nodes we have.

\section{Validation}
Why do we use a validation set?
Describe how the three different cross-validation methods presented in the lecture slides work, and what their advantages and disadvantages are.\\

\noindent\textit{Answer:}

\noindent
First off, there has been some confusion about exactly what cross-validation means.
The lecture slides and the book describes it as something involving three sets: a training set, a validation set and a test set, and is described as something you do in order to implement early stopping.
Contrary to this, the cross-validation that you were to do in the second mandatory exercise only involves a training set and a test set, and isn’t used for early stopping at all!

The first definition is actually just an extension to cross-validation that is used in order to do early stopping properly.
In both cases we split up our data and use parts of it for training data while the rest remains “unseen” by the training, and is used for validating the training.
To better avoid bias, this data then has to be split into separate sets for evaluating when to stop training (the “validation set” in the lectures) and for evaluating final performance (the “test set” in the lectures).

In general then, a validation set is used in order to evaluate the performance of
the trained classifier on unseen data.
In accordance with the terminology used in
the book, it would also be valid to answer that the validation set is used to test for
when it is right to stop early.

The three validation methods work as follows:

\begin{itemize}
    \item Simple cross-validation: the data is simply split into two (three) equally large parts, with one part used for training, (one part used for determining early stopping) and one part used for final evaluation.
    \item k-fold cross-validation: the data is separated into k folds. The training is then done k times, each time using a different fold as the test data (or with early stopping: one fold for validation during training and one fold for the final evaluation) and the rest as training data.
    \item Leave one out cross-validation: in turn, leave out each single data sample and train on the rest, testing on the single sample you left out, and calculate the success rate. (kind of an extreme case of k-fold cross-validation).
\end{itemize}

\section{Multi Layer Perceptron}
Implement the MLP shown below, and train it to correctly perform the XOR function.

\begin{figure}[H]
\begin{center}
\includegraphics[width=0.4\textwidth]{mlp.png}
\caption{MLP with one hidden layer and two hidden nodes.}
\label{fig:mlp}
\end{center}
\end{figure}

\noindent\textit{Answer:}

\noindent
No solution given, as this is very similar to the mandatory assignment.

\section{Delta(error) function}
In the lecture slides the backpropagation deltas are first presented as
\[
\delta_k = (y_k - t_k)y_k(1-y_k)
\]
What does this tell us about the activation function in use?\\

\noindent\textit{Answer:}

\noindent
In general the backpropagation deltas are defined as \(\delta_k = (y_k - t_k)g'(y_k)\) where \(g'(y_k)\) is the derivative of the activation function as a function of the activation output as mentioned in the previous exercise.
In this case we have \(\delta_k = (y_k-t_k)y_k(1-y_k)\), so we must have \(g'(y_k) = y_k(1-y_k)\).
Then we would need to be either clever at math or hope that this is a common activation function so that we can find it in some text about neural networks.
In fact, this \(g'(y_k)\) corresponds to the single most common activation function:
\begin{equation}
    f(x) = \frac{1}{1+e^{-x}}
\end{equation}
The derivation goes like this:
\begin{align*}
    f'(x) &= \frac{e^{-x}}{(1+e^{-x})^2}\\
        a &= 1+e^{-x}\\
    f'(x) &= \frac{e^{-x}}{(a)^2}\\
    f'(x) &= \frac{1 + e^{-x} - 1}{(a)(a)}\\
    f'(x) &= \frac{a - 1}{(a)(a)}\\
    f'(x) &= \frac{a - 1}{(a)} \frac{1}{(a)}\\
    f'(x) &= (\frac{a}{a} - \frac{1}{a})\frac{1}{a}\\
    f'(x) &= (1 - \frac{1}{a})\frac{1}{a}\\
    f'(x) &= (1 - f(x))f(x)
\end{align*}

\section{Natural language MLP}
You are to design an MLP that would learn to hyphenate words correctly.
You would have a dictionary that shows correct hyphenation examples for lots of words.
Think about the following:
What should the input to the neural net be?
\begin{itemize}
    \item How should this input be encoded to work well with the classifier?
    \item How is should the output be encoded?
    \item How many layers do you need?
    \item How many neurons should there be in each layer?
\end{itemize}

\noindent\textit{Answer:}

\noindent
There are many ways of solving this, but the simplest way to arrange the input is to take pairs of letters as input, and assume that some external mechanism will feed us with candidate letter-pairs.

The next question is then how to encode the letter-pairs.
Neural networks takes their input as a fixed number of real-valued scalars.
So we need an encoding in that form.
But, there is no clear and easy way to put the relevant set of letters on a one-dimensional scale, or even a multidimensional scale.

What we can do is to use binary inputs: we encode each letter with one input for each possible value.
In the biological analogy of neural networks this would correspond to having a specialized neuron in our brain that detects a single letter.
E.g. in the English alphabet we would get 26 inputs per letter.
Then we want two letters as input, so we would get a total of 52 inputs to our neural net.

Finally, since we take pairs of letters as input, the output can simply be a true or false value that says whether this is a good place to hyphenate or not.
In that case, the output layer only needs to have one single neuron.

The number of hidden layers and their neuron count depends on how difficult we believe the problem to be.
We don’t expect there to be any easy patterns to which letter-pairs that are “hyphenatable”, so the classification problem is unlikely to be linearly separable, or maybe even separable by a convex shape, so it would probably be wise to have two hidden layers, so that the network can classify complex shapes in the input space.
The number of neurons in each layer would have to be up to experimentation, but a wise default value might be to reduce the number of neurons in each layer linearly, so if you have N inputs, you would have \(2N/3\) neurons in the first hidden layer and \(N/3\) neurons in the second.
Again, with the English alphabet, this would give \(104/3 \approx 35\) and \(52/3 \approx 17\) hidden nodes.

\input{../contact.tex}
\end{document}
% ==============================================================================


\documentclass[12pt]{article}
\usepackage[usenames]{color} %used for font color
\usepackage{amsmath, amssymb, amsthm}
\usepackage{wasysym}
\usepackage[utf8]{inputenc} %useful to type directly diacritic characters
\usepackage{graphicx}
\usepackage{caption}
\usepackage{subcaption}
\usepackage{float}
\usepackage{mathtools}
\usepackage [english]{babel}
\usepackage [autostyle, english = american]{csquotes}
\MakeOuterQuote{"}
\graphicspath{ {./} }
\newcommand{\Z}{\mathbb{Z}}
\newcommand{\N}{\mathbb{N}}
\newcommand{\R}{\mathbb{R}}
\newcommand{\Q}{\mathbb{Q}}
\newcommand{\prob}{\mathbb{P}}
\newcommand{\degrees}{^{\circ}}
\DeclarePairedDelimiter\ceil{\lceil}{\rceil}
\DeclarePairedDelimiter\floor{\lfloor}{\rfloor}

\author{Tianshuang (Ethan) Qiu}
\begin{document}
\title{Math 74, Week 12}
\maketitle
\newpage

\section{Mon Lec, 1c}
Assume that there is a point $x_0, x_1 \in [a,b]$ such that $f(x_0) \not = f(x_1)$, then by the Mean Value Theorem there exists $x_2 \in [x_0, x_1]$ such that $f'(x_2)=\frac{f(x_0)-f(x_1)}{x_0-x_1}$. We know this to be non-zero since $f(x_0) \not = f(x_1)$, however this contradicts the precondition that $f'(x)=0 \forall x \in [a,b]$. We have a contradiction and therefore $f(x_0)=f(x_1)$ for all $x_0, x_1 \in [a,b]$
Thus our function must be constant.

\section{Mon Lec, 2b}
Let $A(t)$ be the distance runner A has traveled during the race at any time $t$ in seconds, and let $B(t)$ be the same with runner B. Let $f(t)=A(t)-B(t)$, since they both begin at the same spot, $A(0)=B(0)=0$
\newline
Similarly, since they both finish the same race, and end in a tie. Let them take $t_0$ seconds to finish, so we have $A(t_0)=B(t_0)$. Thus we have $f(t_0)=f(0)=0$, now by Rolle's Theorem there exists $x \in [0,x_0]$ such that $f'(x)=0$. We know that the derivative of $f$ is $A'(t)-B'(t)$
Furthermore, we know that the derivative of the distance with respect to time is the speed of travel, so we know that at $t=x$, the speed of runner A and B are the same.

\section{Mon Lec, 2d}
$f(4)=\frac{1}{1}=1, f(1)=\frac{1}{(-2)^2}=\frac{1}{4}$, and $f(4)-f(1)=\frac{3}{4}$, thus $f'(c)=3 \times \frac{3}{4} = \frac{1}{4}$. Now take the derivative of the function $f'(x)=\frac{-2}{(x-3^3)}$
$$\frac{-2}{(x-3)^3} = \frac{1}{4}$$
$$-8 = (x-3)^3$$
$$x=1$$
This is the only real solution and is not in $(1,4)$, so there is no solution to the problem. This does not contradict the Mean Value Theorem because the function is not continuous and therefore not differentiable at $x=3$
\newpage

\section{Mon Dis, 1d}
Let the farmer be $a$ km away from the river, the cow $b$ km away, and if they both move to the river, they are $c$ km away from each other. Now, let the distance the farmer travels be $F(x)$, where $x$ is the horizontal distance the farmer travels to his point in the river. $F(x)= \sqrt{a^2+x^2}+\sqrt{(c-x)^2+b^2}$ by the Pythagorean Theorem.
We want to find the minimum of this function, so we apply Fermat's Theorem and set
$$F'(x)= \frac{x}{\sqrt{a^2+x^2}} + \frac{x-c}{\sqrt{(c-x)^2+b^2}} = 0$$
Therfore $\frac{x}{\sqrt{a^2+x^2}} = \frac{c-x}{\sqrt{(c-x)^2+b^2}}$, and $x\sqrt{(c-x)^2+b^2} = (c-x)\sqrt{a^2+x^2}$. Finally square both sides $x^2((c-x)^2+b^2) = (c-x)^2(a^2+x^2)$, $(c-x)^2x^2+b^2x^2=(c-x)^2x^2+(c-x)^2a^2$, now take the square root of both sides after cancelling
$$bx=ac-ax$$
$x = \frac{ac}{a+b}$

\section{Mon Dis, 2}
\subsection{d}
Fermat's theorem assumes that $f'(c)$ exists at the maximum or minimum, the existence of the derivative implies that the function is continuous. Therefore FT's assumption is stronger than EVT and CIM which just assumes continuity. We can show that an assumption is necessary by constructing a function that does not follow the assumption, but follows the hypothesis and does not follow the conclusion.

\subsection{e}
FT deals with local maximums and minimums, which already implies a non-finite interval that we are working with, since we are only concerned with values close to $c$, we don't really care about the end points. EVT and CIM needs closed intervals becauseit is in the name and extrema may not exist in an open interval.

\subsection{f}
RT, MVT (CMVT), and IVT are existential theorems, EVT, CIM, and FT show us how to find the extremes.

\subsection{g}
FT Converse: If $f'(c)=0$, then it is the local maximum or minimum of a function. It is not true since the function could be constant, and maximums need to be strictly bigger.
\newline
EVT Converse: If $f$ has a global maximum and a global minimum at some numbers $c,d$, let $c,d \in [a,b]$ then $f$ is continuous on $[a,b]$. This is not true since we can take a constant function and define a point to be arbitrarily big and another to be small, the function is not continuous.
\newline
CIM converse: The smallest/largest values from all critical points in $(a,b)$ and the end points in $[a,b]$ are the global minimum and maximums on the closed interval $[a,b]$. This is true since it is what we use to write the CIM.
\newpage

\section{Wed Lec, 4a}
$f(1)=8$, $f(-1)=-6$, so by IVT there must exist some number $r \in (1,1)$ such that $f(r)=0$, therefore there is at least one root.
\newline
We take the derivative: $2+3x^2+20x^4 = 0$, since each term with $x$ is non-negative, this polynomial has no real roots. Therefore if there are more than one root, by Rolle's theorem its derivative must have at least one root. Since this is not the case the original function can have at most one root.
\newline
Thus it has only one root.

\section{Wed Lec, 4b}
Consider the function $g(x) = f(x)-x$. Since $0 \leq f(x) \leq 1$, $g(0) \geq 0$ and $g(1) \leq 0$. Furthermore, since $f$ is continuous, $g$ is also continuous. Then by the Intermediate Value Theorem there exists a point in $[0,1]$ such that $g(x)=0$, and therefore $f(x)=x$

\section{Wed Lec, 4c}
Our sequence is defined as $a_n = \sqrt{2a_{n-1}}, a_1 = \sqrt{2}$. We first show that $a_n$ is monotonically increasing. In order to show that $a_n < a_{n+1}$ is equivalent to showing $a_n < \sqrt{2a_{n}}$ which simplifies to $a_n < 2$
\newline
We can show this inductively. Our base case is $n=1, a_1 = \sqrt{2}<2$. Then assume that for some $n \in \N$ we have $a_n < 2$, then $a_{n+1} = \sqrt{2a_n} < \sqrt{2 \times 2} = 2$. Thus we have proven our inductive case.
\newline
The sequence is monotonic and bounded, therefore it must converge. Let it converge to $L$, since every subsequence of the same sequence also converge to $L$, $\lim_{n \to \infty}a_n = \lim_{n \to \infty}a_{n+1}=L$, so we must have $L = \sqrt{2L}$
\newline
Solving for $L$ gives $L = 2,0$. Since we know that $a_n$ is increasing and $a_1>0$, $L=2$. Therefore the sequence converges to 2.
\newpage

\section{Wed Dis, 3c}
Since $f$ is continuous on $[a,b]$, let it take its maximum at $x_0$ and minimum at $x_1$. Now by Rolle's Theorem's assumption $f(a)=f(b)$. If $x_0, x_1 \in \{a,b\}$, then $f$ is constant on $[a,b]$ and $f'(x)=0$ for all $x \in (a,b)$. Otherwise by Fermat's Theorem if $x_0 \in (a,b)$ then $f'(x_0)=0$, or if $x_1 \in (a,b)$ then $f'(x_1)=0$. Thus it is proven.

\section{Wed Dis, 4b}
Consider $h(x)=g(x)-f(x)$. $h(a) = g(a)-f(a)=0, h'(x)=g'(x)-f'(x)$. By the specification of the problem we know that $h'(x) > 0 $ when $x \in (a,b)$
\newline
Now consider $h(b)$, assume that $g(b) \leq f(b)$, so $h(b) \leq 0$, then by MVT there must be a point $c \in (a,b)$ such that $h'(c) = \frac{h(b)-h(a)}{a-b}$. Since $h(a)=0$, then $h'(c)\leq 0$, so $g'(c)-f'(c)\leq 0$, but we know that this is not possible since $f'(x)<g'(x)$ for all $x \in (a,b)$. We have reached a contradiction, and our assumption incorrect and $g(b)>f(b)$.
\newline
$\blacksquare$
\newpage

\section{Fri Lec, 2d}
\subsection{a}
Let $\epsilon>0$, $\exists N \in \N$ such that for all $n > N$, $|x_n-7|<\epsilon$

\subsection{b}
Let $\epsilon > 0$, $\exists \delta > 0$ such that $|x-5|<\delta \implies |f(x)-7|<\epsilon$

\subsection{c}
$\exists \delta > 0, u \in \R$ such that $|x-5|< \delta \implies f(x)<u$

\subsection{d}
Let $x_0 \in \R$, $\exists \epsilon > 0$ and $x_1<x_0 \in \R$ such that $|f(x_1)-7| \geq \epsilon$

\section{Fri Lec, 5b}
We claim that $\lim_{x \to 9}\sqrt{x}=3$
\newline
Let $\epsilon > 0$, pick $\delta = \epsilon ^ 2 + 6\epsilon$, for $|x-9|<\delta$, we can use the triangle inequality to show that $|x|<|\delta + 9|$. We have $|f(x)-3| = |\sqrt{x}-3| < |\sqrt{|\delta + 9|-3}| = |\sqrt{(3+\epsilon)^2}-3|= \epsilon$
\newline
Now reorganizing the epxression above: $|f(x)-3| < \epsilon$. Thus we have proven the epsilon delta property, and the limit at $9$ is $3$.

\end{document}


\subsection*{\comp{RKST(100)}}
\begin{enumerate}
\item \comp{ESCF} The calculated heat of formation in kcal.mol$^{-1}$.
\item \comp{ENUCLR} The nuclear energy in eV.
\addtocounter{enumi}{1}
\item \comp{ATHEAT} The reference heat of atomization plus ionization, in eV.
\item \comp{FRACT} The fractional orbital occupancy, in electrons.  Range:
0.0 to 2.0.
\item \comp{EMIN} Within any geometry optimization calculation, the lowest
heat of formation calculated.  Used in deciding when to exit the SCF.
\item \comp{COSINE} The angle between the current and previous gradient
vector.  Range: -1.0 to +1.0.
\item \comp{GNORM} The scalar of the gradient vector. \comp{GNORM=MOD(GRAD)}.
\item \comp{TIME0} The internal zero of time for determining CPU usage.
\item \comp{EZQ} The nuclear - solvent interaction term in the Tomasi model.
\item \comp{EEQ} The electronic - solvent interaction term in the Tomasi model.
\item \comp{AREA} The surface area of a molecule, in \AA $^2$, calculated by COSMO.
\addtocounter{enumi}{1}
\item \comp{GVW} Used by the Tomasi method.
\item \comp{GVWS} Used by the Tomasi method.
\item \comp{ELC1} Used by the Tomasi method.
\item \comp{TLEFT} The allowed amount of CPU time left for the calculation.
Default: one hour.
\item \comp{TDUMP} The interval between checkpoints (dumps of the calculation
that can be used by \comp{RESTART}).  Default: one hour.
\item \comp{ELECT} The electronic energy in eV.
\item \comp{STEP} The step size in a reaction path.
\item \comp{TVEC} A three by three matrix representing the translation vector.
The first vector is \comp{TVEC(1,1):TVEC(3,1)}.
\addtocounter{enumi}{8}
\item \comp{RJKAB1} The two-electron integral over M.O.s used in
calculating the correction to the energy in the half-electron approximation.
\item \comp{WTMOL} The molecular weight in AMU.  Calculated in \comp{MOPAC}.
\item \comp{VERSON} The version number of MOPAC, e.g., \mopacversion .0.
\item \comp{TOTIME} The total time, in seconds, for the calculation,
including time used in previous runs, if \comp{RESTART} is used.
\item \comp{EOUTER} Nuclear energy in eV due to atom pairs not present in
\comp{P}, \comp{H}, or \comp{F}. Calculated in \comp{HCORZ}, and
used by \comp{WRITMN} only.
\item \comp{THRESH} In MOZYME, the threshold value for intensity of a
LMO on an atom.  An atom with intensity below \comp{THRESH} would either
(a) not be added to a LMO (in \comp{DIAGG2}), or (b) would be deleted from the
 LMO (in \comp{TIDY}).
\addtocounter{enumi}{2}
\item \comp{CORHYB} The correction to the MOZYME energy due to the use of the
dipole approximation. (for interactions arising from atoms separated
by more than \comp{CUTOF2} but less than \comp{CUTOF1}, that is, atoms pairs
not present in \comp{P}, \comp{H}, or \comp{F}.
\item \comp{ESCFL} In an intersystem crossing, \comp{ESCFL} is the $\Delta H_f$
 of the lower state.  Note \mbox{\comp{ESCFL} $<$ \comp{ESCF}}.
\item \comp{VOLUME} The volume of the system in \AA $^3$, calculated by COSMO.
\item \comp{FEPSI} Used by COSMO
\item \comp{RDS}  Used by COSMO.
\item \comp{DISEX2} Used by COSMO.  \comp{DISEX2} is needed in both \comp{INITSN}
and \comp{CONSTS}.
\item \comp{CUTOF1} The cutoff distance for dipolar electrostatic terms.  Beyond
\comp{CUTOF1}, only point-charge terms are used.  Default: 30 \AA ngstroms.
\item \comp{CUTOF2} The cutoff distance for NDDO two-electron, two-center terms.
Beyond \comp{CUTOF2}, only dipole and point charge terms are used.
\item \comp{CUTOFS} The distance beyond which overlap integrals are {\em not}
calculated.  By default, \comp{CUTOFS=7\AA }.
\item \comp{CUTOFP} In polymers, all electrostatic terms involving
atoms separated
by more than \comp{CUTOFP} are calculated as if they were separated by
\item \comp{CUTOFP}  Needed in order to ensure that equivalent atoms are correctly
predicted.
\item \comp{FNSQ}
\item \comp{CLOWER}  In a polymer calculation, \comp{CLOWER} marks the onset,
in \AA ngstroms, of the truncation function for electrostatic interactions.
Set in function \comp{TRUNK}
\item \comp{CUPPER}  In a polymer calculation, \comp{CUPPER} marks the upper bound,
in \AA ngstroms, of the truncation function for electrostatic interactions.
Set in function \comp{TRUNK}
\item \comp{DLM}  Used by intersystem crossing.
\item \comp{XNORM}  Used by intersystem crossing.
\item \comp{DIFE}  In an intersystem crossing, \comp{DIFE} is the square
of the difference in energy of the two states involved.
\item \comp{PRESSURE} In a solid-state calculation, \comp{PRESSURE} is the stress
(in kcal/mol) that the system is under.
\end{enumerate}


\documentclass[]{article}
\usepackage{ragged2e}
\usepackage{amsmath}
\usepackage{listings}
\usepackage{graphicx}

\usepackage[utf8]{inputenc}
\usepackage[english]{babel}

\usepackage{hyperref}
\hypersetup{
	colorlinks=true,
	linkcolor=blue,
	filecolor=magenta,
	urlcolor=cyan,
}

\urlstyle{same}

\usepackage{xcolor}

\definecolor{codegreen}{rgb}{0,0.6,0}
\definecolor{codegray}{rgb}{0.5,0.5,0.5}
\definecolor{codepurple}{rgb}{0.58,0,0.82}
\definecolor{backcolour}{rgb}{0.95,0.95,0.92}

\lstdefinestyle{mystyle}{
	backgroundcolor=\color{backcolour},
	commentstyle=\color{codegreen},
	keywordstyle=\color{magenta},
	numberstyle=\tiny\color{codegray},
	stringstyle=\color{codepurple},
	basicstyle=\ttfamily\footnotesize,
	breakatwhitespace=false,
	breaklines=true,
	captionpos=b,
	keepspaces=true,
	numbers=left,
	numbersep=5pt,
	showspaces=false,
	showstringspaces=false,
	showtabs=false,
	tabsize=2
}

\lstset{style=mystyle}

%opening
\title{Microprocessors ETI 2407 \\
Assignment I}
\author{\textbf{Mugo Mwaura}
	ENE221-0280/2016}
\date{August 31, 2020\endgraf\rule{\textwidth}{.9pt}}

\begin{document}

\maketitle

\section{Question One}

\justify
Write an assembly program that displays whole numbers and their squares. A user should
input the last number. The output below shows what would happen if a user entered 5. What
is the highest value you could enter that gave correct results? Explain why this number is the
limit and what can be done to improve this limit:

\begin{align*}
x \qquad x^2 \\
1 \qquad 1 \\
2 \qquad 4 \\
3\qquad 9 \\
4 \hspace*{1.5em} 16 \\
5\hspace*{1.5em} 25\\
\end{align*}

\subsection{Answer Explanation}

\textbf{Highest value} entered that gave the correct result is $181$ i.e., $floor\left(\sqrt{\frac{2^{16}}{2}}\right)$.
Highest number giving result = $2^{16}$.
Registers (AX) store 16 bit values. However, if this program was using a signed
representation, the maximum would be:
\begin{equation*}
\frac{2^{16}}{2} = 32,767
\end{equation*}

For a $32 \ bit$ answer, the result is placed in DX:AX or effectively, if using a single 32 bit register, EAX.
This program does compute the answer upto a max of $65,556$ therefore,
but since we are only printing from AX (using \textbf{emu8086 print\_num} procedure), and haven't manually implemented
printing from DX:AX, the maximum result we get is similar to that of a signed value.

\subparagraph{What can be done to improve the limit?}
Print the entire result from EAX for a $2^{16}$ value. Beyond that, we can't go as the addresses
only access $64 \ KB$ of addressable memory

\subsection{Pseudo-code}
\begin{enumerate}
	\item Get maximum value $x$ from user
	\item initialize loop counter $c$ to $1$
	\item \textbf{while $c $ $<= x$ }
	\begin{itemize}
		\item print $c$
		\item print $c \times c$
		\item \textbf{end loop}
	\end{itemize}
\end{enumerate}

\subsection{Code} \label{script 1}
\begin{flushleft}
	\lstinputlisting[language=python]{Q_one/q_one.asm}
\end{flushleft}

\clearpage
\subsection{Program Output}

\begin{figure}[h]
	\includegraphics[width=15cm]{Q_one/q1_updated_aug26.png}
	\centering
	\caption{A screenshot of the output from running script \ref{script 1}
	}
\end{figure}

\newpage
\clearpage
\section{Question Two}
Write a program that allows you convert from a number to the corresponding ASCII value.

\centering
E.g., if a user enters $35$, the output should be $\#$.

\justify
The program should also have a provision
for displaying the ASCII value of an entered character.

\centering
E.g., if a user enters $*$, the output
should be 42.

\justify
At the start of the program, a user should choose the option between the $2$
modes (either ASCII value to character OR character to ASCII value).
NB: Your code should be able to deal with erroneous input appropriately

\centering
E.g. entering a number above $255$.

\justify

\subsection{Psedo-code}

\begin{enumerate}
	\item Select program mode \\
	\hspace*{3em} a) ASCII to Decimal \\
	\hspace*{3em} b) Decimal to ASCII \\
	\item Get input from user $input$
	\begin{itemize}
		\item If mode $(b)$ and input$ >$ $255$, \textbf{error}
	\end{itemize}

	\item Initialize ASCII counter $c$. Initialize DEC counter $d$

	\item \textbf{Start loop:}
	\begin{itemize}
	\item \textbf{If $input$ is equal} to $ASCII\{c\}$, (where $ASCII$ is the set of all ASCII characters $[0: 255]$) \\
	\hspace*{6em} print $ASCII\{c\}$, print $DEC\{d\}$, (where $DEC$ is an \hspace*{6em} enumeration of the $ASCII$ set $[0_d: 255_d]$) \\
	\textbf{end loop}

	\end{itemize}

\end{enumerate}

\subsection{Code} \label{sketch 2}
\lstinputlisting[language=python]{q_two/q_two.asm}

\clearpage
\subsection{Program Output}

\begin{figure}[h]
	\includegraphics[width=12cm]{q_two/q2_ascii_to_dec.png}
	\centering
	\caption{A screenshot of the output from running script \ref{sketch 2}. \textbf{Converting ASCII to decimal}}
\end{figure}

\begin{figure}[h]
	\includegraphics[width=12cm]{q_two/q2_dec_to_ascii.png}
	\centering
	\caption{A screenshot of the output from running script \ref{sketch 2}. \textbf{Converting decimal to ASCII}}
\end{figure}

\begin{figure}[h]
	\includegraphics[width=12cm]{q_two/q2_input_gthan_255.png}
	\centering
	\caption{A screenshot of the output from running script \ref{sketch 2}. \textbf{Error on Decimal value greater that $255$}}
\end{figure}

\clearpage
\section{Question Three}

Develop a virus that changes the letters being typed randomly into other letters and thus frustrates the user. E.g. if he intends to type \emph{Hello boss}, it could instead display something like \emph{Sdupl dwsp}. (For simplicity, assume that the email will be written into your running program. i.e., you do not need to write code for attacking the browser, \textbf{ though if you do that, come get a cookie}).

After writing your code, describe what would happen if a user typed
1234?

\subsection{Answer Explanation}
\textbf{If the user types "\emph{1234?}}" the program will output "1234?" to stdout(See Figure \ref{fig:5}). The program only changes alphabetical ASCII inputs. i.e., $65 - 90 $ and $97 - 122$. The rest of the inputs are printed as keyed in by the user.

\subsection{Pseudo-code}
\begin{enumerate}
	\item \textbf{Loop:}
	\begin{itemize}
		\item (a) Get character $c$ from user without echo
		\item \textbf{If} $c$ is an alphabet: \\
		\hspace*{6em} Alter the value of $c$ randomly
		\item Print the value of $c$
		\item Go back to \textbf{(a)}

	\end{itemize}
\end{enumerate}

\subsection{Code} \label{sketch 3}
\lstinputlisting[language=python]{q_three/q_three.asm}

\clearpage
\subsection{Program Output}

\begin{figure}[h]
	\includegraphics[width=12cm]{q_three/"q_3 print 123456789?".png}
	\centering
	\caption{A screenshot of the output from running script \ref{sketch 3}. \textbf{An input of "1234?" outputs "1234?"}}
	\label{fig:5}
\end{figure}

\begin{figure}[h]
	\includegraphics[width=12cm]{q_three/"q_3_printing_I'm a noisy machine".png}
	\centering
	\caption{A screenshot of the output from running script \ref{sketch 3}. \textbf{The output of the input \emph{"I'm a noisy machine"}}}
\end{figure}

\begin{figure}[h]
	\includegraphics[width=12cm]{q_three/Q3_printing_make_me_cry!!!.png}
	\centering
	\caption{A screenshot of the output from running script \ref{sketch 3}. \textbf{The output of the input \emph{"make me cry!!!"}}}
\end{figure}

\clearpage
\section{Question Four}
Explain the following code and what it does. What output does it produce?

\begin{lstlisting}[language={[x86masm]Assembler}]
start:
	mov AL, 200
	mov CX, 1
	mov BL,5
	div BL
	mov BL,AL
	shl CX,3
somewhere:
	mov AH, 2
	mov DL, 30h
	test BL, 10000000b
	jz elsewhere
	mov DL, 31h
elsewhere:
	int 21h
	shl BL, 1
	loop somewhere
\end{lstlisting}

\subsection{Answer Explanation}
\emph{Short Answer:}

This sketch \textbf{performs division} on two digits and \textbf{prints the result} to stdout. The exact details are mentioned as comments in sketch \ref{sketch 4}

\subsection{Code explanation} \label{sketch 4}
\lstset{
	showstringspaces=false
}

\lstinputlisting[language=python]{q_four/q_four.asm}

\end{document}


% -----------------------------*- LaTeX -*------------------------------
\documentclass[12pt]{report}
\usepackage{scribe_hgen486}
\def \baselinestretch{1.2}
\def\ni{\noindent}
\def\Be{\text{Be}}

\usepackage{amsmath,url,enumerate,graphicx}

\usepackage{chngcntr}
\counterwithout{table}{section}
\begin{document}

\scribe{Alan Hutchison}		% required
\lecturenumber{5}			% required, must be a number
\lecturedate{January 19}		% required, omit year
\lecturer{Matthew Stephens}

\maketitle

% please leave this comment
\framebox[.95\textwidth]{\parbox{.93\textwidth}{ {{\bf Note:}} These
lecture notes are still rough, and have only have been mildly
proofread.  }}
\vspace*{.1in}

% feel free to delete content below this line
% ----------------------------------------------------------------------

\section{Introduction to Bayesian Inference}

\emph{See github.com/stephens999/stat302 for further notes by M. Stephens on Bayesian Inference (Lects. 1-4)}

From last class:\
Posterior Odds = Prior Odds * Likelihood Ratio (aka Bayes Factor)

\[
\frac{Pr(M_1|x)}{Pr(M_0|x)} = \frac{P(M_1)}{P(M_0)} * \frac{P(x|M_1}{P(x|M_0)}
\]

This is the two model $M_1$ vs. $M_0$ case. We saw two examples of this in the previous lecture, regarding 1) Identifying Elephant Tusks and 2) Protein measurements in the blood. In these cases we were using likelihood ratios (LRs) for imagining a population that we're screening to tell us the conditional probability of a state (elephant type of tusk / disease state of patient) given data.\\

What if we only have 1 tusk and a LR of 1.8? We can think of the LR as a measure of the uncertainty of event (tusk is Savannah, tusk is Forest) based on everything we know. Instead of thinking in terms of a `measure of frequency' we can think in terms of a `measure of uncertainty'.\\

\begin{enumerate}
\item ``50\% chance of snow tomorrow'' This is simplest understood as the uncertainty of snow, not as a prediction of a hypothetical population of days.
\item The answer to ``Did it snow in Denver?'' can be expressed as an uncertainty, even though the event `snowed in Denver' is not a random variable.
\end{enumerate}

One way of thinking about uncertainty is by ``comparing an event to a standard'', such as a flip of a (fair) coin or a roll of a (fair) dice. It is not unreasonable to start with these standards (usually uniformly distributed probabilities). Heuristically, if one's prior odds are off by a factor of 10, then it starts to be more problematic.

\section{Comparison of more than two models}

Now, we will look at $k$ different models instead of 2 models, as we have done previously.\\

In the elephant case, for example, instead of thinking of just Forest vs. Savannah elephants, we could think about further subdividing these categories by genetic differences amongst the different cardinal directions (N E S W). \\

Let's refer to the different models as $Z_i \in \{1,2,...,k\}$, the likelihood of data $P(x_i|Z_i=k)$ as $L_{ik}$ and the prior probability $P(Z_i=k)$ as $\Pi_k$. We are using subscript $i$ to denote a series of tusks, but we could just as easily drop the $i's$.\\

Using the Law of Total Probability (LTP), we can write

\[
P(x_i) = \sum_{k'} P(x_i,Z_i=k') = \sum_{k'} P(x_i | Z_i=k') P(Z_i = k') = \sum_{k'} L_{ik'} \Pi_{k'}
\]
\[
P(Z_i=k|x_i) = \frac{P(x_i|Z_i=k)P(Z_i=k)}{P(x_i)} = \frac{L_{ik}\Pi_k}{\sum_{k'} L_{ik'} \Pi_{k'}}.
\]

Note that the denominator is simply the sum over $k$ of all possible numerators. We use the symbol $k'$ to denote the same set of states $k$, only for the sum across all $k$ of them. This means we can simply compute $L_{ik}\Pi_k$ for each $k$, then divide by the sum of them at the end. If all $k$ models are equally likely, then we end up just normalizing by the sum of the likelihoods.\\

Instead of using equalities, we can use proportions to avoid thinking about the normalizing factor. We can think about this as informing our posterior uncertainty by what we learn from the data (the likelihood) with what we knew or believed beforehand (the prior).
\[
Posterior \propto Likelihood * Prior
\]

Remember that the likelihoods themselves don't really matter, it's the ratios that give you information about the different models. In ratios the normalizing constant ends up canceling out altogether.

\section{Continuum of models case: Parametric model}
\subsection{Estimating an allele frequency}

As previously, sample 100 alleles, see 40 of type `1' and 60 of type `0'. $D$: Data. $M_q$: ``freq. of allele `1' is q''\\

Originally we used,
\[
L(q) = q^{40} (1-q)^{60}
\]

Maximizing this provides a Maximum Likelihood Estimate (MLE) point estimate for $q$, but we may be interested incorporating prior information or belief for $q$. We can compute the posterior distribution of $q$ using Bayes Theorem. Let our prior distribution on $q$ be $P(q)$.

\[
P(q|D) \propto q^{40} (1-q)^{60} * p(q)
\]
Assuming that all values of $q$ are equally probable $P(q)=1$

\[
P(q|D) \propto q^{40} (1-q)^{60}
\]
 now we need to normalize by $\int q'^{40}(1-q')^{60} dq'$. This normalization step can be a complicated integral, and there are some shortcuts that can be used here. Instead of solving out the integral, it is easier if we can recognize it as a standard distribution whose behavior we know.

 Since we are used the assumption that $P(q)=1$, i.e. all values of $q$ are equally probable, we can also write this uniform distribution as a Beta distribution of $q$, $Be(q;\alpha,\beta)=q^{1-\alpha} (1-q)^{1-\beta}$ with parameters $\alpha=1$ and $\beta=1$, $P(q) = Be(q;1,1) = q^0 (1-q)^0 = 1$.

\[
P(q|D) \propto q^{40} (1-q)^{60} * p(q) =  q^{40} (1-q)^{60} q^0 (1-q)^0 = q^{40} (1-q)^{60}
\]

Rewritten as a Beta distribution, we have

\[
P(q|D) \propto q^{40} (1-q)^{60} = q^{\alpha -1 = 40} (1-q)^{\beta -1 = 60} = Be(q;\alpha,\beta) = Be(q;41,61)
\]

Having an expression proportional to the posterior distribution $P(q|D)$, we have the means for summarizing $q$ as either a point estimate or as an interval. For point estimates, common choices include the \emph{posterior mean}, \emph{posterior median}, and \emph{posterior mode}. For an interval estimate, \emph{posterior quantiles} allow us to describe the spread of likely values of $q$. Giving the 2.5\% quantile and 97.5\% quantile provides the interval for which there is a 95\% probability that $q$ lies in it.

There are two takeaways from this section

\begin{enumerate}
\item For computing the posterior distribution in the continuous case, we use the same procedure as the the discrete case but we have the trick of using conjugate prior distributions to make the math simpler.
\item Once we have the posterior distribution, we can summarize it with both point estimates and interval estimates.
\end{enumerate}

\subsection{Conjugacy}

 For many types of likelihood functions there are corresponding prior \emph{conjugate} distributions such that when the two are multiplied together the resulting distribution is also in the same family of conjugate distributions. More formally, \emph{conjugacy means that the posterior distribution comes from the same family as the prior distribution}. These priors may not be realistic interpretations of the underlying mechanism of the prior distribution, but they are powerful and useful. For the Beta distributions in particular, any distribution with support of [0,1] can be approximated as a mixture of Beta distributions.

A question in class asked if there were any properties of likelihood functions that would indicate whether a conjugate prior distribution exists. The short answer is no, though it has been proven that exponential likelihood families have conjugate priors. Many common distributions have been identified as having conjugate priors, and they are summarized in Table \ref{tab:test}.

\begin{table}
\begin{center}
\begin{tabular}{|c|c|}\hline
Observed Likelihood Function & Conjugate Prior \\ \hline \hline
Binomial & Beta \\ \hline
Multinomial & Dirichlet \\ \hline
Mean of Normal & Normal \\ \hline
Variance of Normal & Inverse Normal \\ \hline
Mean of Poisson & Gamma \\ \hline
\end{tabular}
\caption{Observed Likelihood Functions and their Conjugate Priors}
\label{tab:test}
\end{center}
\end{table}

Let's return to the above example. Instead of assuming the uniform prior distribution $P(q) = Be(q;1,1)$, let's leave it as a general Beta distribution.

\[
P(q|D) \propto q^{40} (1-q)^{60} * p(q) =  q^{40} (1-q)^{60} q^{\alpha-1} (1-q)^{\beta-1} = q^{40+\alpha-1} (1-q)^{60+\beta-1} = Be(q;40+\alpha,60+\beta)
\]

Here we can see exactly how the choice of the prior distribution parameters $\alpha$ and $\beta$ can include the resulting posterior distribution. Figure \ref{fig:2015-01-19_beta} shows the Beta distribution for a variety of $\alpha,\beta$ combinations.

\begin{figure}
\includegraphics[width=\textwidth]{Betas.pdf}
\label{fig:2015-01-19_beta}
\caption{Beta distributions for different $\alpha$ and $\beta$ values. Black refers to the density, red to the distribution function, and green the quantile function.}
\end{figure}

 Given that it is reasonable to assume that the prior distribution of allele frequencies in population genetics is U-shaped, we can use a prior distribution of $Be(q;0.5,0.5)$.

 \[
P(q|D) \propto  q^{40} (1-q)^{60} q^{1/2-1} (1-q)^{1/2-1} = q^{40-1/2} (1-q)^{60-1/2} = Be(q;40.5,60.5)
\]
As you can see, this slightly changes our posterior distribution with a uniform prior, from $Be(q;41,61)$ to $Be(q;40.5,60.5)$

More generally, we can think of the observed alleles in our sample as $n_1$ of type ``1'' and $n_0$ of type ``0''. This means our posterior distribution will be

\[
P(q|D) \sim\  Be(q;n_1+\alpha,n_0+\beta)
\]

and the resulting posterior mean estimate for $q$ would be

\[
E[P(q|D)] = \frac{n_1+\alpha}{n_0+n_1+\alpha+\beta}
\]

Using a Maximum Likelihood Estimate (MLE) approach instead, our result would be independent of the prior distribution

\[
E[L(q|D)] = \frac{n_1}{n_0+n_1}
\]

Likewise, estimating $q$ solely off the prior distribution would be independent of observed data $n_0$ or $n_1$.

\[
E[Be(q;\alpha,\beta)] = \frac{\alpha}{\alpha+\beta}
\]

What this comparison of point estimates of $q$ shows is that the posterior mean of $q$ is a combination or tradeoff or compromise of the MLE estimate and the prior estimate: the results is `weighted' by the evidence presented by $n_0$ and $n_1$, but is balanced against the prior information or uncertainty about the population encoded in $\alpha$ and $\beta$. If $\alpha$ and $\beta$ are big relative to $n_0$ and $n_1$, then the posterior mean will be close to the prior mean. If $n_0$ and $n_1$ are big relative to $\alpha$ and $\beta$, then the posterior mean will be close to the MLE. ``In sufficient quantities, data can overwhelm belief.''

Note: in some situations, $\alpha$ and $\beta$ can be thought of as pseudo-counts. This can be useful in a situation where you want a prior that strongly discounts the possibility of an event but you don't want to make it zero. In that case, you could use a prior of $Be(1,100)$ to indicate the prior belief that it is extremely unlikely and then let the data determine whether the posterior probability of the event increases.

\subsection{Example: Estimating Normal Mean}

1 observation: data $X \sim\ N(\theta,\sigma^2)$

assuming $\sigma^2$ is known (e.g. measurement error).

Prior on $\theta$, $\theta \sim\ N(\mu_0,\sigma_0^2)$. We use the subscript `0' to denote that these are prior parameters.

\[
P(\theta|X=x) \propto\ prior * likelihood
\]
\[
P(\theta|X=x) \propto\ \exp{[\frac{-(\theta - \mu_0)^2}{2 \sigma_0^2}]} \exp{[\frac{-(x - \theta)^2}{2 \sigma^2}]}
\]

\[
P(\theta|X=x) \propto\ \exp{[\theta^2 [-\frac{1}{2\sigma_0^2}+\frac{1}{2\sigma^2}] + \theta[\frac{\mu_0}{\sigma_0^2}+\frac{x}{\sigma^2}]]}
\]

Given we are solving for the normal mean using a normal conjugate prior, we know our posterior distribution is going to have the form
\[
P(\theta|X=x) \propto\ N(\mu_1,\sigma_1^2) = \exp{[\frac{-(\theta - \mu_1)^2}{2 \sigma_1^2}]}
\]

which can be re-written without the constants as

\[
P(\theta|X=x) \propto\  \exp{[\theta^2\frac{-1}{2 \sigma_1^2}+\theta \frac{-\mu_1^2}{2 \sigma_1^2}]}
\]

Comparing this with the above equation, we can determine that

\[
\frac{1}{\sigma_1^2} = \frac{1}{\sigma_0^2} + \frac{1}{\sigma^2}
\]
The inverse of the variance $\frac{1}{\sigma^2}$ is referred to as the precision, which is mathematically sometimes more tractable. This equation means that our posterior precision is the sum of our prior precision and our precision from our data. Our precision gets bigger, and our variance gets smaller.

\[
\frac{\mu}{\sigma_1^2} = \frac{\mu_0}{\sigma_0^2} + \frac{x}{\sigma^2}
\]

This means that the product of our posterior mean and our posterior precision is the sum of the product of our prior mean and prior precision with product of our observed $x$ and observed precision.

\end{document}


\documentclass[a4paper,12pt]{article}

\begin{document}

\title{wassopies fififiifif}
\author{mr poops}
\date{\today}
\maketitle

\section{the poopings}
this is a chapter where i will teach you some math. let's go.
\subsection{start}
we must find out that $\frac{1}{2}$ is equivalent to $\frac{4}{8}.$
let's find it out.
$$\frac{1}{2} = \frac{1 \times 4}{2 \times 4} = \frac{4}{8}$$
\section{milk}
$\omega$ is $\Omega$ but lowercase
BUT!!!!!!!! DOES 9 + 77 = NOTHING?!!?!
we can find out by multiplying $77 \times 0$. haha jk. ok goodbye.
REMEBER THIS:
$$9 + 7 = Nothing$$
\section{I LIKE BIG TRUCKS AND I CAN NOT LIE}
\subsection{idk}
\subsubsection{O RELLY}
\end{document}