\documentclass[a4paper]{article}

\usepackage[margin=2cm]{geometry}		% For desired margins
\usepackage{fontspec}					% utf-8 support
\usepackage{enumitem}					% Continuous enumerations
\usepackage{float}						% Floating images
\usepackage[dvipsnames]{xcolor}			% Colors
\usepackage{pgfplots}					% Custom plots
\usepackage{listings}					% Code highlighting
\usepackage[hidelinks]{hyperref}		% Hyper links
\usepackage{graphicx}					% To include images
\usepackage{tikz}						% To draw images
\usepackage{pgfplots}					% To draw graphs
\usepackage[dvipsnames]{xcolor	}
\usepackage{subcaption}

\pgfplotsset{compat=1.13}
\usetikzlibrary{positioning}

\setlength{\parindent}{0pt}
\setlength{\parskip}{1em}

\title{PAR: Laboratory 5 \\
		\texttt{\large par4201}}
\author{Joan Marcè i Igual \and Es6teve Tarragó i Sanchis}

\newenvironment{questionenum}{%
\setlist[enumerate]{resume}
\restartlist{enumerate}
\newcommand{\question}[1]{
\begin{enumerate}
	\item\bfseries ##1
\end{enumerate}
}}{%
}

\lstset{
	basicstyle=\small\ttfamily,
	showspaces=false,
	showstringspaces=false,
	tabsize=4,
	numbers=left,
	numbersep=5pt,
	stringstyle=\color{orange},
	commentstyle=\color{OliveGreen},
	keywordstyle=\color{blue},
	numberstyle=\color{Gray},
	frame=single
}

\begin{document}
\maketitle
\tableofcontents

\section{Analysis with Tareador}

\begin{questionenum}
	\question{Include the relevant parts of the modified \texttt{solver-tareador.c} code and comment where the calls to the Tareador API have been placed. Comment also about the task graph generated and the causes of the dependencies that appear in the two solvers: \emph{Jacobi} and \emph{Gauss-Seidel}. How will you protect them in the parallel \texttt{OpenMP} code.}
\end{questionenum}

\begin{center}
	\begin{minipage}{0.8\textwidth}
\begin{lstlisting}[language=C, title=\texttt{heat-tareador.c}]
switch( param.algorithm ) {
	case 0: // JACOBI
		tareador_start_task("relax_jacobi");
		residual = relax_jacobi(param.u, param.uhelp, np, np);
		tareador_end_task("relax_jacobi");
		// Copy uhelp into u
		tareador_start_task("Copy_mat");
		copy_mat(param.uhelp, param.u, np, np);
		tareador_end_task("Copy_mat");
		break;
	case 1: // GAUSS
		tareador_start_task("relax_gauss");
		residual = relax_gauss(param.u, np, np);
		tareador_end_task("relax_gauss");
		break;
}
		\end{lstlisting}
	\end{minipage}

	\begin{minipage}{0.8\textwidth}
		\begin{lstlisting}[language=C, title=\texttt{solver-tareador.c}]
// Set tareador task
tareador_start_task("jacobi_relax");
// Disable sum object for better parallelization potential
tareador_disable_object(&sum);
utmp[i*sizey+j]= 0.25 * ( u[ i*sizey     + (j-1) ]+  // left
u[ i*sizey     + (j+1) ]+  // right
u[ (i-1)*sizey + j     ]+  // top
u[ (i+1)*sizey + j     ]); // bottom
diff = utmp[i*sizey+j] - u[i*sizey + j];
sum += diff * diff;

// Reenable sum object
tareador_enable_object(&u);

// End of tareador task
tareador_end_task("jacobi_relax");
		\end{lstlisting}
	\end{minipage}
\end{center}

\section{OpenMP parallelization and execution analysis: \emph{Jacobi}}
\begin{questionenum}
	\question{Describe the data decomposition strategy that is applied to solve the problem, including a picture with the part of the data structure that is assigned to each processor.}

	The decomposition strategy used is giving each thread a different part of the matrix by rows. This can be seen in the \autoref{fig:jacobi-data} where each thread has a different block of rows of the matrix to be computed.

	\begin{figure}[H]
		\centering
		\includegraphics[width=0.5\textwidth]{images/jacobi/data}
		\caption{Figure describing the data decomposition, each thread has one color}
		\label{fig:jacobi-data}
	\end{figure}

	\question{Include the relevant portions of the parallel code that you implemented to solve the heat equation using the \emph{Jacobi} solver, commenting whatever necessary. Including captures of Paraver windows to justify your explanations and the differences observed in the execution.}

	The heat computation part can be optimized in the following way with \texttt{OpenMP}.
	\begin{center}

	\begin{minipage}{0.9\textwidth}
		\begin{lstlisting}[language=C, title=\texttt{solver-tareador.c}]
double relax_jacobi (double *u, double *utmp, unsigned sizex, unsigned sizey)
{
	double diff, sum=0.0;

	// In this part max threads is used instead of get_num_threads() beacuse
	// there's only one thread at this point.
	int howmany = omp_get_max_threads();

	// Parallelize by blocks, assigning each block to one different thread
	#pragma omp parallel for reduction(+:sum) private(diff)
	for (int blockid = 0; blockid < howmany; ++blockid) {
		int i_start = lowerb(blockid, howmany, sizex);
		int i_end = upperb(blockid, howmany, sizex);
		for (int i=max(1, i_start); i<= min(sizex-2, i_end); i++) {
			for (int j=1; j<= sizey-2; j++) {
				utmp[i*sizey+j]= 0.25 * ( u[ i*sizey     + (j-1) ]+  // left
				u[ i*sizey     + (j+1) ]+  // right
				u[ (i-1)*sizey + j     ]+  // top
				u[ (i+1)*sizey + j     ]); // bottom
				diff = utmp[i*sizey+j] - u[i*sizey + j];
				sum += diff * diff;
			}
		}
	}

	return sum;
}
		\end{lstlisting}
	\end{minipage}
	\end{center}

	Also there's another complementary way to optimize. The \texttt{copy\_mat()} function can be parallelized to allow a better way of copying the matrix.

	\begin{center}
		\begin{minipage}{0.8\textwidth}
			\begin{lstlisting}[language=C, title=\texttt{solver-tareador.c}]
#pragma omp parallel for collapse(2)
for (int i=1; i<=sizex-2; i++)
	for (int j=1; j<=sizey-2; j++)
		v[ i*sizey+j] = u[ i*sizey+j ];
			\end{lstlisting}
		\end{minipage}
	\end{center}

	Another way to attack the problem is avoiding the matrix copy part and only swapping the pointers of the matrix. This can be done in the \texttt{heat-omp.c} file between the iterations.

	\begin{center}
		\begin{minipage}{0.8\textwidth}
			\begin{lstlisting}[language=C, title=\texttt{heat-omp.c}]
residual = relax_jacobi(param.u, param.uhelp, np, np);
// Swap the pointers to avoid copying the matrix
double *temp = param.u;
param.u = param.uhelp;
param.uhelp = temp;

// Disable the matrix copy
//copy_mat(param.uhelp, param.u, np, np);
			\end{lstlisting}
		\end{minipage}
	\end{center}

	\begin{figure}[H]
		\centering
		\includegraphics[width=\textwidth]{images/jacobi/paraver-8}
		\caption{Capture of \texttt{paraver} using 8 processors}
		\label{fig:jacobi-paraver}
	\end{figure}

	\begin{figure}[H]
		\centering
		\includegraphics[width=\textwidth]{images/jacobi/paraver-8-zoom}
		\caption{Zoom of \texttt{paraver} capture to show that the processor 0 is still doing work}
		\label{fig:jacobi-paraver-zoom}
	\end{figure}

	As it can be seen in the \autoref{fig:jacobi-paraver} all processors are working so the work is properly shared between all the processors. It seems that the processor 0 is only doing synchronizations but actually it's working too as it can be seen in the \autoref{fig:jacobi-paraver-zoom}.

	\question{Include the speed-up (strong scalability) plots that have been obtained for the different numbers of processors. Reason about the performance that is observed.}

	\begin{figure}[H]
		\begin{subfigure}[t]{0.49\textwidth}
			\begin{tikzpicture}
				\begin{axis}[
						width = \textwidth,
						xmin = 1, xmax = 12,
						domain = 0:12,
						xlabel = number of processors,
						ylabel = execution time,
						enlarge x limits = 0.05,
					]
					\addplot table[x = P, y = T]{data/jacobi/elapsed.csv};
					\addplot[dashed, mark = none] {4.5/x};
				\end{axis}
			\end{tikzpicture}
			\caption{Elapsed time for each number of processors}
			\label{fig:jacobi-elapsed}
		\end{subfigure}
		\hfill
		\begin{subfigure}[t]{0.49\textwidth}
			\begin{tikzpicture}
				\begin{axis}[
						width = \textwidth,
						xmin = 1, xmax = 12,
						ymin = 0, ymax = 12,
						domain = 0:12,
						xlabel = number of processors,
						ylabel = speedup,
						enlarge x limits = 0.05,
					]
					\addplot[dashed, mark=none]{x};
					\addplot table[x = P, y = S]{data/jacobi/speedup.csv};
				\end{axis}
			\end{tikzpicture}
			\caption{Speedup obtained for each number of processors}
			\label{fig:jacobi-speedup}
		\end{subfigure}
	\caption{Comparison between speedup and elapsed time by number of processors in the Jacobi solution}
	\end{figure}
\end{questionenum}

As it can bee seen in the \autoref{fig:jacobi-speedup} the speedup increases and then it remains nearly constant. This is due to the overhead caused by \texttt{OpenMP} as the number of processors increases. If the data is divided by many processors there's more time spend during the synchronization than computing the value and that's why the speedup is not increasing anymore.

\section{OpenMP parallelization and the execution analysis: \emph{Gauss-Seidel}}
\begin{questionenum}
	\question{Include the relevant portions of the parallel code that implements the \emph{Gauss-Seidel} solver, commenting how you implemented the synchronization between threads.}

	\begin{center}
		\begin{minipage}{0.8\textwidth}
			\begin{lstlisting}[language=C, title=\texttt{solver-omp-gauss.c}]
double relax_gauss (double *u, unsigned sizex, unsigned sizey)
{
	int howmany = omp_get_max_threads();
	int numblocks = howmany;
	int blocksfinished[numblocks];
	for(int i =0; i < numblocks; ++i) blocksfinished[i]=-1;
	double unew, diff, sum=0.0;

	#pragma omp parallel for reduction(+:sum) private(diff,unew)
	for (int blockid = 0; blockid < numblocks; ++blockid) {
		for (int blockidcol = 0; blockidcol < numblocks; ++blockidcol){
			if (blockid > 0) {
				// Added block waiting
				while( blocksfinished[blockid-1] < blockidcol ) {
				#pragma omp flush(blocksfinished)
			}
		}
		int i_start = lowerb(blockid, numblocks, sizex);
		int i_end = upperb(blockid, numblocks, sizex);
		int j_start = lowerb(blockidcol,numblocks, sizey);
		int j_end = upperb(blockidcol, numblocks,sizey);
		for (int i=max(1, i_start); i<= min(sizex-2, i_end); i++) {
			for (int j=max(1, j_start); j<= min(sizey-2, j_end); j++) {
				unew = 0.25 * (
					u[ i*sizey 		+ (j-1) ]+  // left
					u[ i*sizey		+ (j+1) ]+  // right
					u[ (i-1)*sizey	+ j     ]+  // top
					u[ (i+1)*sizey	+ j     ]); // bottom
				diff = unew - u[i*sizey+ j];
				sum += diff * diff;
				u[i*sizey+j]=unew;
				}
			}

			// Finish block
			blocksfinished[blockid] = blockidcol;
			#pragma omp flush(blocksfinished)
		}
	}

	return sum;
}
			\end{lstlisting}
		\end{minipage}
	\end{center}

	For the synchronization we use the array \texttt{blocksfinished [row]} in which we store the last column achieved by the row. A row can only start a block if the previous row has already been calculated.

	In order to synchronize the variable \texttt{blocksfinished}  we use the \texttt{flush} sentence. We are also using \texttt{reduction} in order to join the result.

	\question{Include the speed-up (strong-scalability) plot that has been obtained for the different numbers of processors. Reason about the performance that is observed, including captures of Paraver windows to justify your explanations.}

	\begin{figure}[H]
		\begin{subfigure}{0.49\textwidth}
			\centering
			\begin{tikzpicture}
				\begin{axis}[
						xmin = 1, xmax = 12,
						domain = 1:12,
						enlarge x limits = 0.05,
						xlabel = number of processors,
						ylabel = execution time,
						width = \textwidth
					]
					\addplot table[x=P, y=T]{data/gauss/elapsed.csv};
					\addplot[dashed, mark = none]{6.5/x};
				\end{axis}
			\end{tikzpicture}
		\end{subfigure}
		\hfill
		\begin{subfigure}{0.49\textwidth}
			\centering
			\begin{tikzpicture}
				\begin{axis}[
						xmin = 1, xmax = 12,
						domain = 1:12,
						enlarge x limits = 0.05,
						xlabel = number of processors,
						ylabel = speedup,
						width = \textwidth
					]
					\addplot table[x=P, y=S] {data/gauss/speedup.csv};
					\addplot[dashed, mark=none]{x};
				\end{axis}
			\end{tikzpicture}
		\end{subfigure}
	\end{figure}

	We can see that the synchronizations costs are more than linear. This is due to the number of blocks which is quadratic in the number of threads and a flush is need in each region.

	\question{Explain how did you obtain the optimum value for the ration computation/synchronization in the parallelization of this solver for 8 threads.}
\end{questionenum}

	\begin{figure}[H]
		\centering
		\begin{tikzpicture}
			\begin{axis}[
					xmin = 4, xmax = 16,
					ymin = 0, ymax = 7,
					xlabel = Number of blocks,
					ylabel = Time,
					enlarge x limits = 0.05,
				]
				\addplot table[x=N, y=T]{data/gauss/optimum.csv};
			\end{axis}
		\end{tikzpicture}
	\end{figure}

We can see that the optimum value is 8, which mean 8 rows with 8 columns each. With a lower number there are threads with no job and with a higher value we are increasing the synchronization cost.

\end{document}


\documentclass{article}

\usepackage{fancyhdr}
\usepackage{extramarks}
\usepackage{amsmath}
\usepackage{amsthm}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{tikz}
\usepackage{physics}
\usepackage[plain]{algorithm}
\usepackage{algpseudocode}

\usetikzlibrary{automata,positioning}

%
% Basic Document Settings
%

\topmargin=-0.45in
\evensidemargin=0in
\oddsidemargin=0in
\textwidth=6.5in
\textheight=9.0in
\headsep=0.25in

\linespread{1.1}

\pagestyle{fancy}
\lhead{\hmwkAuthorName}
\chead{\hmwkClass\ (\hmwkClassInstructor\ \hmwkClassTime): \hmwkTitle}
\rhead{\firstxmark}
\lfoot{\lastxmark}
\cfoot{\thepage}

\renewcommand\headrulewidth{0.4pt}
\renewcommand\footrulewidth{0.4pt}

\setlength\parindent{0pt}

%
% Create Problem Sections
%

\newcommand{\enterProblemHeader}[1]{
    \nobreak\extramarks{}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
    \nobreak\extramarks{Problem \arabic{#1} (continued)}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
}

\newcommand{\exitProblemHeader}[1]{
    \nobreak\extramarks{Problem \arabic{#1} (continued)}{Problem \arabic{#1} continued on next page\ldots}\nobreak{}
    \stepcounter{#1}
    \nobreak\extramarks{Problem \arabic{#1}}{}\nobreak{}
}

\setcounter{secnumdepth}{0}
\newcounter{partCounter}
\newcounter{homeworkProblemCounter}
\setcounter{homeworkProblemCounter}{1}
\nobreak\extramarks{Problem \arabic{homeworkProblemCounter}}{}\nobreak{}

%
% Homework Problem Environment
%
% This environment takes an optional argument. When given, it will adjust the
% problem counter. This is useful for when the problems given for your
% assignment aren't sequential. See the last 3 problems of this template for an
% example.
%
\newenvironment{homeworkProblem}[1][-1]{
    \ifnum#1>0
        \setcounter{homeworkProblemCounter}{#1}
    \fi
    \section{Problem \arabic{homeworkProblemCounter}}
    \setcounter{partCounter}{1}
    \enterProblemHeader{homeworkProblemCounter}
}{
    \exitProblemHeader{homeworkProblemCounter}
}

%
% Homework Details
%   - Title
%   - Due date
%   - Class
%   - Section/Time
%   - Instructor
%   - Author
%

\newcommand{\hmwkTitle}{Homework\ \#2}
\newcommand{\hmwkDueDate}{February 12, 2014}
\newcommand{\hmwkClass}{Calculus}
\newcommand{\hmwkClassTime}{}
\newcommand{\hmwkClassInstructor}{Professor Isaac Newton}
\newcommand{\hmwkAuthorName}{\textbf{Aditya Vijaykumar}}

%
% Title Page
%

\title{
    %\vspace{2in}
    \textmd{\textbf{\hmwkClass:\ \hmwkTitle}}\\
    \normalsize\vspace{0.1in}\small{\hmwkDueDate\ }\\
%    \vspace{3in}
}

\author{\hmwkAuthorName}
\date{}

\renewcommand{\part}[1]{\textbf{\large Part \Alph{partCounter}}\stepcounter{partCounter}\\}

%
% Various Helper Commands
%

% Useful for algorithms
\newcommand{\alg}[1]{\textsc{\bfseries \footnotesize #1}}

% For derivatives
\newcommand{\deriv}[1]{\frac{\mathrm{d}}{\mathrm{d}x} (#1)}

% For partial derivatives
\newcommand{\pderiv}[2]{\frac{\partial}{\partial #1} (#2)}

% Integral dx
\newcommand{\dx}{\mathrm{d}x}

% Alias for the Solution section header
\newcommand{\solution}{\textbf{\large Solution}}

% Probability commands: Expectation, Variance, Covariance, Bias
\newcommand{\E}{\mathrm{E}}
\newcommand{\Var}{\mathrm{Var}}
\newcommand{\Cov}{\mathrm{Cov}}
\newcommand{\Bias}{\mathrm{Bias}}

\begin{document}

\maketitle

%\pagebreak

\begin{homeworkProblem}
    Give an appropriate positive constant \(c\) such that \(f(n) \leq c \cdot
    g(n)\) for all \(n > 1\).

    \begin{enumerate}
        \item \(f(n) = n^2 + n + 1\), \(g(n) = 2n^3\)
        \item \(f(n) = n\sqrt{n} + n^2\), \(g(n) = n^2\)
        \item \(f(n) = n^2 - n + 1\), \(g(n) = n^2 / 2\)
    \end{enumerate}

    \textbf{Solution}

    We solve each solution algebraically to determine a possible constant
    \(c\).
    \\

    \textbf{Part One}

    \[
        \begin{split}
            n^2 + n + 1 &=
            \\
            &\leq n^2 + n^2 + n^2
            \\
            &= 3n^2
            \\
            &\leq c \cdot 2n^3
        \end{split}
    \]

    Thus a valid \(c\) could be when \(c = 2\).
    \\

    \textbf{Part Two}

    \[
        \begin{split}
            n^2 + n\sqrt{n} &=
            \\
            &= n^2 + n^{3/2}
            \\
            &\leq n^2 + n^{4/2}
            \\
            &= n^2 + n^2
            \\
            &= 2n^2
            \\
            &\leq c \cdot n^2
        \end{split}
    \]

    Thus a valid \(c\) is \(c = 2\).
    \\

    \textbf{Part Three}

    \[
        \begin{split}
            n^2 - n + 1 &=
            \\
            &\leq n^2
            \\
            &\leq c \cdot n^2/2
        \end{split}
    \]

    Thus a valid \(c\) is \(c = 2\).

\end{homeworkProblem}

\pagebreak

\begin{homeworkProblem}
    Let \(\Sigma = \{0, 1\}\). Construct a DFA \(A\) that recognizes the
    language that consists of all binary numbers that can be divided by 5.
    \\

    Let the state \(q_k\) indicate the remainder of \(k\) divided by 5. For
    example, the remainder of 2 would correlate to state \(q_2\) because \(7
    \mod 5 = 2\).

    \begin{figure}[h]
        \centering
        \begin{tikzpicture}[shorten >=1pt,node distance=2cm,on grid,auto]
            \node[state, accepting, initial] (q_0)   {$q_0$};
            \node[state] (q_1) [right=of q_0] {$q_1$};
            \node[state] (q_2) [right=of q_1] {$q_2$};
            \node[state] (q_3) [right=of q_2] {$q_3$};
            \node[state] (q_4) [right=of q_3] {$q_4$};
            \path[->]
                (q_0)
                    edge [loop above] node {0} (q_0)
                    edge node {1} (q_1)
                (q_1)
                    edge node {0} (q_2)
                    edge [bend right=-30] node {1} (q_3)
                (q_2)
                    edge [bend left] node {1} (q_0)
                    edge [bend right=-30] node {0} (q_4)
                (q_3)
                    edge node {1} (q_2)
                    edge [bend left] node {0} (q_1)
                (q_4)
                    edge node {0} (q_3)
                    edge [loop below] node {1} (q_4);
        \end{tikzpicture}
        \caption{DFA, \(A\), this is really beautiful, ya know?}
        \label{fig:multiple5}
    \end{figure}

    \textbf{Justification}
    \\

    Take a given binary number, \(x\). Since there are only two inputs to our
    state machine, \(x\) can either become \(x0\) or \(x1\). When a 0 comes
    into the state machine, it is the same as taking the binary number and
    multiplying it by two. When a 1 comes into the machine, it is the same as
    multipying by two and adding one.
    \\

    Using this knowledge, we can construct a transition table that tell us
    where to go:

    \begin{table}[ht]
        \centering
        \begin{tabular}{c || c | c | c | c | c}
            & \(x \mod 5 = 0\)
            & \(x \mod 5 = 1\)
            & \(x \mod 5 = 2\)
            & \(x \mod 5 = 3\)
            & \(x \mod 5 = 4\)
            \\
            \hline
            \(x0\) & 0 & 2 & 4 & 1 & 3 \\
            \(x1\) & 1 & 3 & 0 & 2 & 4 \\
        \end{tabular}
    \end{table}

    Therefore on state \(q_0\) or (\(x \mod 5 = 0\)), a transition line should
    go to state \(q_0\) for the input 0 and a line should go to state \(q_1\)
    for input 1. Continuing this gives us the Figure~\ref{fig:multiple5}.
\end{homeworkProblem}

\begin{homeworkProblem}
    Write part of \alg{Quick-Sort($list, start, end$)}

    \begin{algorithm}[]
        \begin{algorithmic}[1]
            \Function{Quick-Sort}{$list, start, end$}
                \If{$start \geq end$}
                    \State{} \Return{}
                \EndIf{}
                \State{} $mid \gets \Call{Partition}{list, start, end}$
                \State{} \Call{Quick-Sort}{$list, start, mid - 1$}
                \State{} \Call{Quick-Sort}{$list, mid + 1, end$}
            \EndFunction{}
        \end{algorithmic}
        \caption{Start of QuickSort}
    \end{algorithm}
\end{homeworkProblem}

\pagebreak

\begin{homeworkProblem}
    Suppose we would like to fit a straight line through the origin, i.e.,
    \(Y_i = \beta_1 x_i + e_i\) with \(i = 1, \ldots, n\), \(\E [e_i] = 0\),
    and \(\Var [e_i] = \sigma^2_e\) and \(\Cov[e_i, e_j] = 0, \forall i \neq
    j\).
    \\

    \part

    Find the least squares esimator for \(\hat{\beta_1}\) for the slope
    \(\beta_1\).
    \\

    \solution

    To find the least squares estimator, we should minimize our Residual Sum
    of Squares, RSS:

    \[
        \begin{split}
            RSS &= \sum_{i = 1}^{n} {(Y_i - \hat{Y_i})}^2
            \\
            &= \sum_{i = 1}^{n} {(Y_i - \hat{\beta_1} x_i)}^2
        \end{split}
    \]

    By taking the partial derivative in respect to \(\hat{\beta_1}\), we get:

    \[
        \pderiv{
            \hat{\beta_1}
        }{RSS}
        = -2 \sum_{i = 1}^{n} {x_i (Y_i - \hat{\beta_1} x_i)}
        = 0
    \]

    This gives us:

    \[
        \begin{split}
            \sum_{i = 1}^{n} {x_i (Y_i - \hat{\beta_1} x_i)}
            &= \sum_{i = 1}^{n} {x_i Y_i} - \sum_{i = 1}^{n} \hat{\beta_1} x_i^2
            \\
            &= \sum_{i = 1}^{n} {x_i Y_i} - \hat{\beta_1}\sum_{i = 1}^{n} x_i^2
        \end{split}
    \]

    Solving for \(\hat{\beta_1}\) gives the final estimator for \(\beta_1\):

    \[
        \begin{split}
            \hat{\beta_1}
            &= \frac{
                \sum {x_i Y_i}
            }{
                \sum x_i^2
            }
        \end{split}
    \]

    \pagebreak

    \part

    Calculate the bias and the variance for the estimated slope
    \(\hat{\beta_1}\).
    \\

    \solution

    For the bias, we need to calculate the expected value
    \(\E[\hat{\beta_1}]\):

    \[
        \begin{split}
            \E[\hat{\beta_1}]
            &= \E \left[ \frac{
                \sum {x_i Y_i}
            }{
                \sum x_i^2
            }\right]
            \\
            &= \frac{
                \sum {x_i \E[Y_i]}
            }{
                \sum x_i^2
            }
            \\
            &= \frac{
                \sum {x_i (\beta_1 x_i)}
            }{
                \sum x_i^2
            }
            \\
            &= \frac{
                \sum {x_i^2 \beta_1}
            }{
                \sum x_i^2
            }
            \\
            &= \beta_1 \frac{
                \sum {x_i^2 \beta_1}
            }{
                \sum x_i^2
            }
            \\
            &= \beta_1
        \end{split}
    \]

    Thus since our estimator's expected value is \(\beta_1\), we can conclude
    that the bias of our estimator is 0.
    \\

    For the variance:

    \[
        \begin{split}
            \Var[\hat{\beta_1}]
            &= \Var \left[ \frac{
                \sum {x_i Y_i}
            }{
                \sum x_i^2
            }\right]
            \\
            &=
            \frac{
                \sum {x_i^2}
            }{
                \sum x_i^2 \sum x_i^2
            } \Var[Y_i]
            \\
            &=
            \frac{
                \sum {x_i^2}
            }{
                \sum x_i^2 \sum x_i^2
            } \Var[Y_i]
            \\
            &=
            \frac{
                1
            }{
                \sum x_i^2
            } \Var[Y_i]
            \\
            &=
            \frac{
                1
            }{
                \sum x_i^2
            } \sigma^2
            \\
            &=
            \frac{
                \sigma^2
            }{
                \sum x_i^2
            }
        \end{split}
    \]

\end{homeworkProblem}

\pagebreak

\begin{homeworkProblem}
    Prove a polynomial of degree \(k\), \(a_kn^k + a_{k - 1}n^{k - 1} + \hdots
    + a_1n^1 + a_0n^0\) is a member of \(\Theta(n^k)\) where \(a_k \hdots a_0\)
    are nonnegative constants.

    \begin{proof}
        To prove that \(a_kn^k + a_{k - 1}n^{k - 1} + \hdots + a_1n^1 +
        a_0n^0\), we must show the following:

        \[
            \exists c_1 \exists c_2 \forall n \geq n_0,\ {c_1 \cdot g(n) \leq
            f(n) \leq c_2 \cdot g(n)}
        \]

        For the first inequality, it is easy to see that it holds because no
        matter what the constants are, \(n^k \leq a_kn^k + a_{k - 1}n^{k - 1} +
        \hdots + a_1n^1 + a_0n^0\) even if \(c_1 = 1\) and \(n_0 = 1\).  This
        is because \(n^k \leq c_1 \cdot a_kn^k\) for any nonnegative constant,
        \(c_1\) and \(a_k\).
        \\

        Taking the second inequality, we prove it in the following way.
        By summation, \(\sum\limits_{i=0}^k a_i\) will give us a new constant,
        \(A\). By taking this value of \(A\), we can then do the following:

        \[
            \begin{split}
                a_kn^k + a_{k - 1}n^{k - 1} + \hdots + a_1n^1 + a_0n^0 &=
                \\
                &\leq (a_k + a_{k - 1} \hdots a_1 + a_0) \cdot n^k
                \\
                &= A \cdot n^k
                \\
                &\leq c_2 \cdot n^k
            \end{split}
        \]

        where \(n_0 = 1\) and \(c_2 = A\). \(c_2\) is just a constant. Thus the
        proof is complete.
    \end{proof}
\end{homeworkProblem}

\pagebreak

%
% Non sequential homework problems
%

% Jump to problem 18
\begin{homeworkProblem}[18]
    Evaluate \(\sum_{k=1}^{5} k^2\) and \(\sum_{k=1}^{5} (k - 1)^2\).
\end{homeworkProblem}

% Continue counting to 19
\begin{homeworkProblem}
    Find the derivative of \(f(x) = x^4 + 3x^2 - 2\)
\end{homeworkProblem}

% Go back to where we left off
\begin{homeworkProblem}[6]
    Evaluate the integrals
    \(\int_0^1 (1 - x^2) \dx\)
    and
    \(\int_1^{\infty} \frac{1}{x^2} \dx\).
\end{homeworkProblem}

\end{document}


\documentstyle{article}
\begin{document}

\title{API for plane-waves in NWChem}

\author{Arthur Smith}

\date{\today}
\maketitle

\section{General outline}
There are two general features of plane-wave basis sets which justify
a substantially different treatment from, for example, Gaussian basis
sets. First, the basis set is not linked to the actual atomic positions,
other than to the size of the periodic cell in which the atoms are contained,
(in fact, for plane waves each individual basis element extends throughout
this cell). Second, the basis sets tend to be very large while calculations
involving a single basis element (overlaps for example) are essentially
trivial, so the computational challenges differ considerably. For plane-wave
density-functional theory with separable non-local pseudopotentials,
all calculations can be accomplished in a time linear (or $N \log(N)$) in
the basis-set size (there are further dependences on the number of atoms
and orbitals, of course).

Both of these features (independence of atomic positions, and large
size with mostly trivial overlaps) can also be said to apply to
other types of basis sets that we may wish to implement in the future:
pure real-space grid methods (the plane-wave technique already does
much of the computation on a real-space grid), multi-grid methods,
mixed-basis techniques, or even wavelets. I believe the API should
be designed in such a way that programmers will need to make only minimal
modifications to a higher-level application code in order to utilize
a new basis set when/if one becomes available.

\section{Pre-requisites}

Plane-wave basis sets are only directly applicable to periodic systems -
solids in particular, although other systems can be readily treated
by super-cell methods. Thus the application must have previously
specified a periodically repeated cell of some sort.

In addition,
particularly for relative small periodic cells, a collection of
special {\bf k}-points must be specified (selected according to the
cell symmetry) which will be used to approximate the integration of
quantities over the Brillouin zone of the cell with a weighted sum
over these {\bf k}-points.

For large systems or for molecules/clusters treated in the supercell
approximation, it is often appropriate to just use the single point
{\bf k = 0}. In any case, the code should assume the {\bf k}-points
have been previously specified.

\section{Handling the basis sets}

There is then a slightly different plane-wave basis set for each
{\bf k}-point, and another (usually larger one) for the charge density.
These are specified by the dimensions of the periodic
cell of the system, and by some sort of cutoff - generally the requirement
that:
\begin{equation}
\hbar^2|{\bf k + G}|^2/2m < E_{cut}
\end{equation}
for the wavefunction basis sets and:
\begin{equation}
\hbar^2 |{\bf G}|^2/2m < d \cdot E_{cut}
\end{equation}
for the charge density, where the parameter $d$ is between 1 and 4 and
determines the degree to which the charge density completely reproduces
all oscillations in the wavefunctions.

The reciprocal lattice vectors {\bf G} are simply integer linear combinations
\begin{equation}
{\bf G}^{\alpha} = n^{\alpha}_1 {\bf b}_1 + n^{\alpha}_2 {\bf b}_2 +
			n^{\alpha}_3 {\bf b}_3
\end{equation}
of the fundamental reciprocal lattice vectors ${\bf b}_i$, which satisfy
\begin{equation}
{\bf b}_i \cdot {\bf a}_j = 2 \pi \delta_{ij}
\end{equation}
and where the ${\bf a}_j$ are the fundamental lattice vectors specifying
the periodic cell of the system.

The {\bf G} values used (and the corresponding $n^{\alpha}_i$ triplets)
are specified by spherical (ellipsoidal) regions surrounding the origin for
the charge density and {\bf k = 0} wavefunctions, or by similar regions
surrounding points close to the origin for ${\bf k} \neq {\bf 0}$.

In order to apply Fast-Fourier-Transform (FFT) techniques to allow
transformations between the plane-wave basis set and a real-space grid on
which charge density and potential calculations are performed,
the triplets $n^{\alpha}_i$ must be mapped to corresponding points in
a three-dimensional integer rectangular grid $N_1 \times N_2 \times N_3$.
Each of the $N_i$ must be greater than twice the largest absolute value
of all the $n_i$, and for most library FFTs must be a power of two. After
applying the FFT, the resulting data is associated with a real-space grid
at the points:
\begin{equation}
{\bf r}_{ijk} = {i \over N_1} {\bf a}_1 + {j \over N_2} {\bf a}_2 +
		{k \over N_3} {\bf a}_3
\end{equation}

Note that the point i, j, k in the $N_1 \times N_2 \times N_3$ reciprocal
space array is associated with a triplet $n^{\alpha}_l$ close to the
origin as follows:
\begin{equation}
{\rm if\ \ } i < N_1/2 {\rm\ \  then\ \ } n^{\alpha}_1 = i
\end{equation}
\begin{equation}
{\rm if\ \ } i > N_1/2 {\rm\ \  then\ \ } n^{\alpha}_1 = i - N_1
\end{equation}
and similarly for j and k.

\subsection{Data distribution}
Utilizing these FFT methods, the application of the Hamiltonian to
a wavefunction involves diagonal operators or products of diagonal
operators (see the section on applying H to $\psi_i$ below). One
nice aspect of a diagonal operator on a parallel computer is that
however the vectors involved are distributed on the machine, application
of the operator involves no communication (as long as the components
of the operator itself are either distributed in the same way as the
vectors, or are calculated when needed). In other words, calculation of
\begin{equation}
	g_i ({\bf r}) = f({\bf r}) \psi_i({\bf r})
\end{equation}
is a local operation, no matter where the components $\psi_i({\bf r})$
actually reside.

However, communication {\it is}\ required during the FFT operations
if $\psi_i({\bf r})/\psi_i({\bf G})$ is distributed in {\bf r}/{\bf G},
and extensive communication
is also required during orthogonalization and similar inner-product
operations if $\psi_i({\bf r})/\psi_i({\bf G})$ is distributed in the
index $i$. In addition, since we are thinking of data-decomposition and not
task-decomposition here, steps must be taken to ensure that every
process carries roughly the same number of coefficients $\psi_i({\bf G})$,
which is best accomplished by some form of interleaving. Note that
$\psi_i({\bf r})$ will naturally be evenly distributed in {\bf r} under most
decompositions because the {\bf r} lie on a rectangular grid, while
the relevant {\bf G} values lie within a sphere, for which block
decompositions may leave some processors with few or no {\bf G}-vectors.

A further complication in the {\bf k = 0} situation is that the
coefficients for {\bf G} and {\bf -G} are complex conjugates, and therefore
nearly a factor of two in memory and speed can be gained by keeping only
one of the two coefficients. But that means in the data decomposition,
{\bf G} and {\bf -G} should be covered by the same process, requiring
a reversed interleaving for negative values of the $n_i$.

\subsection{The initialization API}
Thus the fundamental information needed to initialize a particular plane-wave
basis set is:
\begin{enumerate}
\item The periodic cell vectors ${\bf a}_i$
\item The {\bf k}-vectors used.
\item The energy cutoff $E_{cut}$
\item The charge density basis parameter $d$.
\item Some information about number of processes, preferred data
decomposition, and possibly process communication topology, in order
to determine the distribution of G and R vectors among nodes.
\item Preferred values of $N_1, N_2, N_3$ (optional)
\item The vectors ${\bf b}_i$ (optional - might be useful
for modularization).
\end{enumerate}

Returned from the initialization will be:
\begin{enumerate}
\item The size of the basis set for each k-point, and for the charge
density, including total size, and number allocated to this process.
\item Some form of listing of the G-vectors: at least the $n^{\alpha}_i$,
preferably also the actual vectors {\bf G} and their (squared?) magnitudes.
\item The output values of $N_1, N_2, N_3$ for the whole space and
for the sub-blocks (or other arrangement) allocated to this process.
\item Optionally, the ${\bf b}_i$ vectors.
\end{enumerate}

\subsection{The allocation API}
Typically, we'll be allocating some number $n_{orbitals}$ of
complex coefficients of the $ng({\bf k})$ plane waves for each of
the {\bf k}. In addition, there will be several auxiliary arrays
individually of approximate size $ng$, including arrays to handle
the charge density and components of the potential. And we also need
a few complex and real arrays on the $N_1\times N_2 \times N_3$
real-space grid.

These would presumably be allocated with GA routines or something
similar - for example in our hybrid decomposition approach, the first
set of coefficient arrays would be distributed across all the processors. The
second set, of auxiliary arrays, would be distributed across the processors
within a group (the spatial decomposition subgroups) but duplicated (sometimes
with identical data) in all such groups, in order to eliminate all
communications except during FFT and orthogonalization (and global operation)
routines.

Inputs then to an allocator would be:
\begin{enumerate}
\item A tag specifying the basis set (previously initialized).
\item The data type of the coefficients (real or complex, or something else?)
\item The number of ($ng$ arrays) of this type to be
allocated ($n_{orbitals}$ for example.
\item Optionally: A flag indicating whether the data should be
initialized with random values or with zeroes.
\end{enumerate}

Output would be a tag specifying the coefficient matrix, similar to
GA matrix allocation, maybe?

\subsection{API for basis set operations}

Once we have the basis set specifications, there are a number of
operations we'll need to do anything with them:
\begin{enumerate}
\item Copy
\item Scale or AXPY
\item application of a diagonal operator (GBMV?)
\item Inner product
\item Send and receive, or get and put to a different process
\item Transformation between basis sets
\item Computation of basis set overlaps for different basis sets
\end{enumerate}

Most of these are straightforward: probably only the last two
will require much thought. We'll want to do more complicated
things with the charge density (at least in calculation of
the exchange-correlation energy), but that is probably a special
case that can be dealt with separately.

There are a couple of transformations between basis sets we
might want to consider implementing initially: within a G-space
representation, changing form one cutoff energy to a different
one is trivial in principle (just chop, or zero-pad) but
not necessarily in practice where reading in a starting configuration
from an old basis set may misplace coefficients if you are not
careful (we have not yet successfully implemented this using
our current code, for example). Transformations between G-space and
real-space are essential to computation of the charge density and
evaluation of the real-space potentials, and this can be viewed
as one kind of general basis-set transformation (implemented
through fast Fourier transforms).

I think computation of overlaps will not be necessary until
mixed Gaussian and plane-wave bases are being considered, but
it might be worth-while thinking about how to do it now.

\section{Applying H to $\psi_i$}

We can use various minimization techniques to isolate the
lowest $n_{occupied}$ eigenstates of H. Because $ng$ is so much
larger, we never want to construct the full $ng\times ng$ matrix,
so most of these techniques start from the residual
\begin{equation}
H \psi_i - \epsilon_i \psi_i.
\end{equation}
and the most computationally intensive part of the code is
going to be evaluation of the Hamiltonian-vector product H$\psi_i$.

\subsection{Kinetic energy}

The kinetic energy operator is evaluated in the reciprocal space
representation:
\begin{equation}
T \psi_i({\bf G}) = {\hbar^2|{\bf k + G}|^2 \over 2m} \psi_i({\bf G})
\end{equation}

The values of $|{\bf k + G}|^2$ should be pre-stored (possibly
returned from the initialization routine above) and so this is
a straightforward operation. However, since this is peculiar to
the plane-wave basis, it might be a good idea to define a kinetic
energy API (for example, for pure real-space grids, the kinetic energy
could be evaluated using finite-difference techniques). For such
an API the input is a basis-set specifier and the coefficient matrix
(with an integer specifying how many orbitals to work on at once)
and output is the matrix of result vectors (placed in a work area provided).

\subsection{The real-space potentials}

These are comprised of the local pseudopotential, the Hartree
(electron-electron Coulomb) potential and  the exchange-correlation potential.
The potentials themselves are evaluated as follows:
\begin{equation}
V_{local}({\bf G}) = \sum_{is} u_{is}({\bf G}) S_{is}({\bf G})
\end{equation}
where $is$ is an index counting the atomic species in the system,
$u_{is}$ is the potential from a single atom of this species, and
$S_{is}$ is a structure factor for the species:
\begin{equation}
S_{is}({\bf G}) = \sum_{ia} \exp(i {\bf G}\cdot {\bf r}_{ia})
\end{equation}
where $ia$ runs over all atoms of species $is$. $u_{is}$ and $S_{is}$
would be distributed in the same fashion as the {\bf G}'s, but with
all species $is$ together (not distributed), and so duplicated
in the same fashion as $V_{local}({\bf G})$ itself: among processes with the
same {\bf G}'s but different collections of orbitals. So this computation
involves no communication, except for the FFT required in the end
to produce a real-space potential.

The electron-electron direct Coulomb interaction is easily evaluated:
\begin{equation}
V_{Hartree}({\bf G}) = 4\pi \rho(G)/G^2
\end{equation}
except for ${\bf G} = {\bf 0}$ where the corresponding term is
a constant that is added to the constant electrostatic energy of the
periodic system usually evaluated by Ewald summation. Similarly, the
only communication required here is the final FFT, although there
will be duplication if the orbitals are distributed.

Both $V_{local}$ and $V_{Hartree}$ are then FFT'd to real space, and
can be added to the exchange correlation potential:
\begin{equation}
V_{xc}({\bf r}) = V_{xc}(\rho({\bf r}), \nabla \rho({\bf r}), ...)
\end{equation}
which requires no communication at all, except in the
evaluation of the charge density and gradients:

The charge density $\rho({\bf r})$ is calculated as the sum
\begin{equation}
\rho({\bf r}) = \sum_{i} |\psi_i({\bf r})|^2
\end{equation}
and the gradient is:
\begin{equation}
\nabla\rho({\bf r}) = FFT \bigl\{ i {\bf G} \rho({\bf G}) \bigr\}
\end{equation}
and similarly for higher derivatives.

The duplication discussed could be eliminated at the expense of
more communication: we should probably study this at some point. In
particular since the gradient-dependent exchange correlation potential
cannot be done by table-lookup (because it depends on more than one
parameter, unlike the LDA) the computation involved may be sufficient
to justify distributing it.

An API for a set of subroutine calls to evaluate these potentials
seems appropriate - what is needed on input is quite a variety of
things: for the charge density calculation we probably want to
pass the basis set information and the G-space coefficient matrix,
and the output would be either the charge density on the real-space
grid, or else the fitted charge-density in G-space (or both).
For the local potential we need to pass in geometry information and
information about the pseudopotential, as well as the basis set - output
is the G-space representation of that potential. For the Hartree potential
we pass in the charge density and basis set info, output is the G-space
potential. For the gradients calculation pass in the G-space charge
density, return the real-space charge density and gradients to be
passed to an exchange-correlation subroutine, or else just return the
real-space exchange-correlation potential.

We also need an initialization API for the pseudopotential to
generate the $u_{is}({\bf G})$, and
for the exchange-correlation potentials (at least to allow
choices between the possibilities, and also possibly to allow
setting some of their parameters). More on the pseudopotential below...

\subsection{The non-local pseudopotential}
Using the Kleinman-Bylander separable decomposition, the non-local
pseudopotential (for each species $is$!) can be evaluated in
reciprocal space as follows (from memory, so bear with me if I
missed a minus sign or something):
\begin{eqnarray}
V_{nl}^{lm} \psi_i ({\bf G}) = C^l
		&\sum_{ia} U_{nl}^l({\bf k+G})Y_{lm}^*({\bf k + G})
		\exp(i {\bf G}\cdot{\bf r}_{ia}) \cdot \cr
		&\sum_{{\bf G}'} U_{nl}^l({\bf k + G}')Y_{lm}({\bf k + G}')
		\exp(-i {\bf G}'\cdot{\bf r}_{ia})
		\psi_i({\bf G}')\cr
\end{eqnarray}
where $ia$ is an index over all the atoms of species $is$, and $l$, $m$ are
standard angular momentum quantum numbers. The $U_{nl}^l$ and $Y_{lm}$
can be computed at initialization and are $O(N)$ in size so not too bad.
This would then be summed over $l$, $m$, and species $is$.

One big potential problem data and
communication-wise here is the $\exp(i {\bf G}\cdot {\bf r}_{ia})$ matrix,
which if precomputed and stored is $O(N^2)$ in size, and yet needs to be present
for {\bf all} the orbitals (note that non-local calculation is an $O(N^3)$
operation and thus one of the major time-consuming parts of the computation
for large systems). What we do is evaluate the matrix only for {\bf G}
vectors along the principal axes ${\bf b}_i$, and then compute the product
of three complex numbers to get the final value when needed - this means
we need to only store a matrix $O(N^{4/3})$ in size, but we need to
ensure the correct sort of data-locality for that matrix.

There are other ways of doing the nonlocal pseudopotentials: real-space only
evaluations for example, or taking advantage of possible redundancy if a part
of the system has a shorter periodicity (for example, the bulk portion of
a surface problem) or other symmetry, or using complicated hybrids like
the ultra-soft Vanderbilt pseudopotentials, and presumably other methods
will come along.

So I think two (or possibly three) subroutine calls would be involved
that could be standardized on: initialization of the pseudopotential
(calculation of the $U_{nl}({\bf k+ G})$ and local $u({\bf G})$ values)
and of other needed quantities ($Y_{lm}({\bf k + G})$ and the
$\exp(i {\bf G}\cdot {\bf r}_{ia})$ matrix); and application of
the pseudopotential to a set of orbitals (either with an ion force
calculation, or with forces calculated separately).

\subsection{Dealing with the pseudopotentials}

There may be several types of pseudopotentials we would want to
handle - for example, it might be worth testing the Gaussian-derived
ECP's with plane waves (maybe not?) Anyway, we must initially
read in some representation of the pseudopotentials about each species
of atom in the system, and the input format we currently use most
often is a simple radial grid. These then need to be converted to
the local $u({\bf G})$ and nonlocal $U_{nl}({\bf k+G})$ forms needed in
the actual application of the pseudopotentials.

Thus there needs to be some specification of the input representation
of the pseudopotentials, and an initialization routine for converting
that representation to one appropriate for the basis set. The associated
API would have as input the input representation and specification
of the (plane-wave) basis set, and as output the converted pseudopotential
values.

\section{Achieving self-consistency}

With plane-wave basis sets the central problem of converging to
a self-consistent charge-density (represented by a collection of
occupied Kohn-Sham eigenfunctions) is fundamentally the same as for
density functional theory with other basis sets. However, the principal
constraint imposed by the large size of the plane-wave basis set is
that we never know the full matrix $H$, only matrix-vector products
$H \psi_i$, meaning that direct diagonalization cannot be performed,
and instead iterative approaches are needed. In addition, of course,
$H$ itself changes with each iteration because the charge density
is changing. Among a wide variety of possibilities there are basically
two iterative schemes that have become adopted in plane-wave basis-set
methods: Car-Parrinello ``molecular-dynamics'' type iterations, and
Teter-Payne-Allen preconditioned gradient iterations (often
misnamed ``conjugate gradient'' iterations because the first implementation
happened to use a multiple conjugate gradients updating scheme that
has since proved inefficient compared to just using a single preconditioned
gradient).

The Car-Parrinello approach is to treat the energy gradient $H\psi$ as
a force on the coefficients of $\psi$, so that in a ``time-step'' $\tau$,
\begin{equation}
\psi_i(t + \tau) = \psi_i(\tau) - {\tau^2 \over 2 m_{eff}} H\psi_i(tau),
\end{equation}
after which the new $\psi_i$ are orthogonalized to one another.
When doing real dynamics a velocity term may also be included - as stated
this is effectively just a steepest-descents algorithm. As such it
is simple but rather inefficient for getting self-consistency, but
on the other hand it is rather robust (always converging to the lowest
occupied subspace, if you wait long enough) and in fact when coupled
with atomic dynamics can be competitive with the Born-Oppenheimer
approach required when using the preconditioned gradient methods.

The preconditioned gradient methods have the following iterative step:
\begin{enumerate}
\item Generate residual vectors: $ R_i = (H - \epsilon_i) \psi_i$
\item Apply a preconditioner: $B_i = P R_i$. As with the
Davidson method, the preconditioner should be close to an inverse
of the Hamiltonian.  In practice, a rational function of the kinetic
energy (approaching 1/K.E. for high energy) works well with a
plane-wave basis set.
\item Orthogonalize the $B$'s to one another and to the $\psi's$.
\item Generate $ H B_i $ and the subspace matrix  $ <V_i|H|V_j>$ where
$V_i$ is $\psi_i$ for $i < n$ and $B_{i - n}$ for $i >= n$.
\item Diagonalize within the subspace, and let the new $\psi$ vectors
be the lowest $n$ eigenvectors.
\item Generate the new charge density (generally by mixing in some
of the old, otherwise instabilities arise) and the resulting new
Hamiltonian, and repeat.
\end{enumerate}
These methods (there are variants based on whether you let $n = 1$ and
have an outer loop over bands, or else let $n =$ all states, or else
with different preconditioners) tend to be much faster than the
steepest-descent approach to obtain a self-consistent set of electronic
states. On the other hand they require more computation per step
(twice as many Hamiltonian-vector operations, for example) and experience
shows they tend to be less robust.

Both methods could be implemented with no knowledge of the
underlying basis sets or data decomposition, as long as the
following subroutine calls were available:
\begin{enumerate}
\item Application of $H$ to $\psi$-type vectors
\item Application of a preconditioner to $\psi$-type vectors
\item AXPY operations on $\psi$-type vectors and $\rho$-type vectors
\item Inner products on $\psi$-type vectors (at least for
generating the $<V|H|V>$ matrix, possibly also for orthogonalization unless
that was a separate subroutine call)
\item Generating $\rho$ from $\psi_i$, and modifying $V_{xc}$ and $V_{Hartree}$
accordingly, to complete the self-consistent loop.
\end{enumerate}
In addition, a diagonalization routine for $2 n\times 2 n$ matrices is
needed, and probably should be done in parallel in accordance with
the orbital decomposition of the data.

\section{Energy and forces}

At the highest level, an API that calls on the electronic
energy minimization to generate the energy and forces for a particular
configuration of atoms might be workable, and would certainly be
useful. This would facilitate rapid comparison of different methods
for a particular problem, in addition to providing a natural interface
for standard molecular dynamics codes.

Input to such a high level API would include:
\begin{enumerate}
\item Geometry - atomic configuration
\item Basis set, pseudopotential specifications (data decomp?)
\item (Optionally) Trial set of orbitals
\item Convergence parameters: iteration method, maximum number of
iterations, tolerance on the final numbers
\end{enumerate}

Output would of course be energy and forces, as well as
(optionally) a final set of orbitals for restarting, and a flag
specifying whether the convergence tolerance was actually reached.

In addition, I think it would be important to have callable
routines that estimate (or at least keep track of) memory usage
and time per iteration, to allow analysis of resource utilization
by the calling program. To do the estimate means having some kind
of performance model of the code - we do already have such a thing
for the memory usage, but have never tried it for timing...

\end{document}
\bye


\chapter{The Theory of Alignment}
\label{sec:alignTheory}

Whenever a new detector is built, people are careful to mount it as close to the correct location as they can. In order to check this, survey measurements are
performed to check the position with a precision of $\SI{100}{\micro\metre}$.
To achieve an even greater precision, \textit{software alignment} is performed.

The reason why alignment is of great importance is that a misaligned detector
yields worse momentum resolutions, low reconstruction efficiencies and biased mass estimations and mass peak resolutions.
The most prominent area of misalignment for a spectrometer are asymmetries.
In general, asymmetries in data, including unexpected trigger inefficiencies, can be reduced when the misalignment is found and corrected.
% references

\section{Clustering of SciFi hits}
\label{sec:clustering}

The signals generated by the SiPMs are read out by a specifically developed integrated circuit, the \textit{PACIFIC}\footnote{(low Power Asic for the sCIntillating Fibres traCker}\cite{readout}.
It performs the shaping, integration and digitization of the signal. An effect called spillover caused by the propagation delay of photons inside the fibres and the recovery time of the SiPMs. This leads to the signal not being completely inside of its 25-ns timeslot for the bunch crossing. The \textit{PACIFIC} shapes the signal from the SiPM to reduce the spillover and suppress the signal tails\cite{techreport}. Afterwards, the signal is integrated over the 25-ns timeslot and a 2-bit analog-digital converter (ADC) digitizes the signal.
Three comparators implement the ADC and can be set individually for each channel, as seen in figure \ref{fig:pac}.

\begin{figure}
    \centering
    \includegraphics[width=0.8\textwidth]{plots/pacific.png}
    \caption{Figure of the PACIFIC chip. From left to right, an amplifier, a shaper and an integrator are build in. The Signal the is digitized and the Clustering is performed to the 2-bit output on the far right.}
    \label{fig:pac}
\end{figure}

To achieve a reduction in data rate and suppress the signal noise, clusterization is performed. A diagram of the clusterization is shown in Figure \ref{fig:tikz_cluster}

\input{plots/clusters.tex}

During the digitization the specific thresholds are set. The second comparator sets the \textit{seeding threshold} which marks channels exceeding said threshold as cluster candidates. Up to three more neighboring channels can be added to the cluster if they pass the threshold set by the first comparator (\textit{neighboring threshold}).
The third comparator sets the \textit{high threshold}. A cluster can be form by just one channel if it exceeds this threshold.
The cluster position is the weighted mean of the contributing channels to this cluster. The cluster resolution can be better than $\SI{100}{\micro\metre}$ \cite{techreport}. In the next section, the way that clusters are combined to reconstruct tracks is described.

\section{Track Reconstruction}
\label{sec:kalman}

In order for LHCb to be used for physics, all of the detector hit information has to be converted into tracks, which is a challenging task.
The track reconstruction algorithm needs to find the correct hits from each sub detector to build the track. This can be problematic just because of the amount of tracks per event (roughly 100).
The aim is to have the track efficiency as high as possible. in order to estimate the track parameters, such as the curvature parameter and track slopes ($t_x$, $t_y$), as accurately as possible.
A good track fit is needed in order to find to best estimates for the track parameters and covariances. The estimates are used in the event reconstruction to find the correct tracks for each particle and the decay products. The information provided is used in the RICH rings, ECAL and HCAL and muon detectors. With these information, particle and track parameters such as the invariant mass can be measured and vertex origins of particle decays can be found.
There are several track models that can be used. In general, a track is built from numerous segments which are either straight or curved because of an active magnetic field. Depending on the environment of the track either model is good.
The track segments are called track states and are defined by a position in $x$ and $y$ at a given distance $z$ where the hit was located, then a slopes $t_{x,y}$ at the hit position and a momentum parameter obtained from the track curvature inside the magnetic field\cite{VanTilburg}.

In order to correctly reconstruct the track it is important to know where the hit is localized and for the upcoming hits, where to particle track came from. From the momentum measurement of the track curvature caused by the magnetic field, the parameter $q/p$ is also added where $q$ stands for the charge of the track that is determined from the direction into which of the track bends.

\begin{align*}
  \vec{r} = \left(\begin{array}{c} x \\ y \\ t_x \\ t_y \\ \frac{q}{p}\end{array}\right) &\,\, t_x = \frac{\partial x}{\partial z} & t_y = \frac{\partial y}{\partial z}
\end{align*}

The uncertainty of the five-component state vector is a $5\times5$ covariance matrix $C$.
A track state can be anywhere on the trajectory but is easier to choose it at real detection points. Combining the track state with a real measurement point is called \textit{node}.
The propagation from node $k-1$ to node $k$ is described by a propagation function

\begin{equation*}
  \vec{r}_k = f_k(\vec{r}_{k_{-1}}) + \vec{w}_k\,.
\end{equation*}

This means node $k$ is acquired by propagating node $k-1$ through the propagation function $f_k$ and shifting it by the \textit{process noise} $\vec{w}_k$.

LHCb uses process noise to model the scattering caused by particle interactions with the detector material.
Depending on the type of propagation, linear or curved, a different propagation function is used.
for a linear extrapolation, $f_k$ results in
\begin{equation*}
  f_k \left(\vec{r}_{k-1}\right) = F_k \vec{r}_{k-1}
\end{equation*}
with the transport matrix $F_k$
\begin{gather*}
  F_K = \begin{pmatrix}
    1 & 0 & \Delta z & 0 & 0 \\
    0 & 1 & 0 & \Delta z & 0 \\
    0 & 0 & 1 & 0 & 0 \\
    0 & 0 & 0 & 1 & 0 \\
    0 & 0 & 0 & 0 & 1 \\
  \end{pmatrix}
\end{gather*}
and $\Delta z$ being the difference in z between the nodes
\begin{equation*}
  \Delta z = z_k - z_{k-1}
\end{equation*}

Trajectory information for each node is provided by the real measurement where the relation between measurement $m_k$ and track state at a given node $k$ is defined as

\begin{equation*}
  m_k = h_k(\vec{r}_k) + \epsilon_k
\end{equation*}

with the projection function $h_k$ and \textit{measurement noise} $\epsilon_k$.
So if the detector only measures the $y$ coordinate of state, the projection function
will be
\begin{equation*}
  h_k(\vec{r}_k) = H_k \vec{r}_k
\end{equation*}
with
\begin{gather*}
  H_k = \begin{pmatrix}
    0 & 1 & 0 & 0 & 0 \\
  \end{pmatrix}\,.
\end{gather*}

When measuring more parameters the measurement matrix $H_k$ and projection matrix have dimension $n\times5$ with $n$ being the numbers of parameters measured.

With this track model, $\epsilon_k$ and $w_k$ are random and unknown and have an expectation value of zero.

\section{The Kalman filter method}
In general a track is an ensemble of measurements and track states and the Kalman filter method\cite{VanTilburg} is used to fit tracks.
The idea of the Kalman filter is, to have a starting node and add measurements one by one. In between the addition of measurements, the local track state is updated with the new information.
The Kalman filter method is a $\chi^2$ minimising problem for the measurement of the track. Because of the iterative nature of the method, it is fast and also used in other fields than physics, for example GPS and meteorology.
The three steps of the Kalman filter will be briefly outlined and later described in further detail.

The first step is the $\symbf{Prediction}$: The next track state of the trajectory is predicted based on the track state at the previous node.
The second step is the $\symbf{Filter}$ procedure: By using filter equations, the prediction is updated with measurement information in this node. The prediction and filter are repeated for each measurement. With more measurements added, the estimate for the best trajectory is the track state after each filter step.
The final step is called $\symbf{Smoother}$: When the trajectory is complete, smoother equations are applied from the last node to the previous node. Therefore the information from all measurements is used in both forward- and back-propagation which results in a more
defined track.

\subsection{First Step: Prediction}
For a given state vector at node \textit{k-1}, the prediction for the $k^{\text{th}}$ state vector and its covariance matrix results from the propagation relations

\begin{align*}
  \vec{r}_p^{k-1} &= f_p\left( \vec{r}_{k-1} \right) \\
  \text{Cov}_k^{k-1} &= F_k C_{k-1} F_k^T + Q_k
\end{align*}

The superscript of the state vector shows the amount of information used in the estimate.
That means $\vec{r}_k^n$ is the smoothed state vector which used all information,
$\vec{r}_k^k-1$ is the predicted state vector and $\vec{r}_k^k \equiv \vec{r}_k$ is the filtered state.

$Q_k$ is the process noise in matrix form and it is part of the predicted
covariance matrix $C_k^{k-1}$.
Because the first state cannot take measurements from the previous state, an initial prediction is taken from the track finding algorithm instead.
The predicted residual between the measurement, $m_k$ and the state vector results in
\begin{equation*}
  \text{res}_k^{k-1} = m_k - h_k\left( \vec{r}_k^{k-1} \right)
\end{equation*}
and the corresponding covariance matrix is defined as
\begin{equation*}
  \text{Cov}_{\text{res},k}^{k-1} = V_k + H_k C_k^{k-1} H_k^T\,.
\end{equation*}

Here, $V_k$ is the measurement variance. With these metrics the minimal $\chi^2$ for the optimal track states can be calculated via
\begin{equation*}
  \left( \chi^2 \right)_k^{k-1} =
  \text{res}_k^{k-1} \left(\text{Cov}_{\text{res},k}^{k-1}\right)^{-1} \text{res}_k^{k-1}
\end{equation*}

\subsection{Second Step: Filter}
During the filter step, the track state is updated with the measurement information.
Iteratively, each measurement is added and the filtered state $\vec{r}_k$ and the corresponding covariance matrix is calculated via
\begin{align*}
  \vec{r}_k &= \vec{r}_k^{k-1} + G_p \text{res}+k^{k-1} \\
  \text{Cov}_k &= \left(\mathbb{1} - G_k H_k\right) \text{Cov}_k^{k-1}\,,
\end{align*}
where $G_k$ is the gain matrix of dimension $5\times1$ and is defined as
\begin{equation*}
  G_k = C_k^{k-1} H_k^T \left( \text{Cov}_{\text{res},k}^{k-1} \right)^{-1}
\end{equation*}

Afterwards the residuals and its covariance matrix are calculated and the filtered total $\chi^2$ is defined as
\begin{equation*}
  \left( \chi^2_{\text{filter}} \right)_k = \text{res}_k \text{Cov}_{\text{res},k}^{-1} \text{res}_k\,.
\end{equation*}

The prediction and filter procedure is continued for all measurements until the track is fully reconstructed.
Because the last node at $k \, = \, n$ has the most information in it, a backward update is performed to infuse the previous nodes with the same information as in last node.
This is called \textit{smoother}-step.

\subsection{Third Step: Smoother}
The smoother function returns the best possible estimate for track states at
the previous nodes. The method used is called \textit{Rauch-Tung-Striebel}-smoother\cite{RTS}.
The idea is to use backward information and construct a smoothed state vector and covariance matrix
\begin{align*}
  \tilde{r}_k^n &= \vec{r}_k + S_k \left( \vec{r}_{k+1}^n - \vec{r}_{k+1}^k \right) \\
  \tilde{C}_k^n &= C_k
\end{align*}
and the Smoother-matrix $S_k$ of dimension $5\times5$
\begin{equation*}
  S_k = C_k F_{k+1}^T \left( C_{k+1}^p \right)^{-1}\,.
\end{equation*}

In order to calculate the smoothed $\chi^2$ the residual and corresponding covariance matrix are
\begin{align*}
  \text{res}_k &= m_k - h_k \vec{h}_k^n \\
  \text{Cov}_{\text{res},k}^n &= V_k - H_k C_k^n H_k^T
\end{align*}

The $\chi^2$ is calculated analogously to the one during the filter step with the difference being the new residuals and covariances.

\section{Alignment with Kalman filter track fit}
\label{sec:derivatives}

In principle, minimizing the track residuals is the obvious way to align a detector.
The residual $\vec{\text{res}}_k$ is defined by the difference between a real detector hit and the expected hit position

\begin{equation}
  \text{res}_k = m_k - h_k(\vec{r},\vec{\alpha})
\end{equation}

where $\symbf{h}$ is the measurement model, $\vec{r}$ are the track parameters and $\vec{\alpha}$ are the alignment parameters.
Aligning the SciFi by minimizing the track $\chi^2$ with the same model as used for reconstruction is an advantage. The idea is to use a global covariance matrix in the track fit with the kalman filter.
%Then part of the alignment can be performed just by fitting the tracks
This approach will be used as the type of alignment for the SciFi in this thesis.
In the following paragraph this form of alignment is briefly described\cite{HULSBERGEN1}.

Because of the similarity to the kalman filter method a short revisit of the minimum $\chi^2$ formalism is presented.
The track $\chi^2$ is defined as
\begin{equation}
  \chi^2 = \vec{\text{res}}^T V^{-1} \vec{\text{res}}\,,
  \label{eqn:chi}
\end{equation}

where $V$ is the track covariance matrix. Equation \eqref{eqn:chi} is a matrix expression since $m$ and $h$ are vectors and $V$ a symmetric matrix. For a linear expansion of the measurement model for an initial estimate $x_0$ of the track parameters.

\begin{equation*}
    h(x) = h(x_0) + H(x - x_0)
\end{equation*}

with $H$, the projection matrix, being defined as
\begin{equation*}
    H = \frac{\partial h(x)}{\partial x}\vert_{x_0}\,.
\end{equation*}

The minimal $\chi^2$ condition with respect to $x$ can be written as

\begin{equation*}
    \frac{\symup{d}\chi^2}{\symup{d}x} = 0
    = -2 H^T V{-1} \left( m - h(x_0) - H(x - x_0) \right)\,.
\end{equation*}

The solution to this equation is the known expression of the least-square estimator defined as

\begin{equation}
    x = x_0 - C H^T V^{-1} \left( m - h(x_0) \right)
    \label{eqn:lsq}
\end{equation}

with $C$ being the covariance matrix regarding $x$.

\begin{equation}
    C = \left( H^T V^{-1} H \right)^{-1}
    \label{eqn:cov}
\end{equation}

The non-linear case for the measurement model ($x$ dependency of $H$) the solution in equation \eqref{eqn:lsq} is of iterative nature and can be applied until convergence is achieved. That can be the minimum change in $\chi^2$ for which the first and second derivative are needed so the change in the current estimate $x_0$ is defined as

\begin{equation*}
    x - x_0 = -\left( \frac{\symup{d}^2\chi^2}{\symup{d}x^2}\vert_{x_0} \right)^{-1} \frac{\symup{d}\chi^2}{\symup{d}x}\vert_{x_0}\,.
\end{equation*}

Expanding the model by alignment parameters $\alpha$.
The condition for $\chi^2$ to be minimal with respect to a track model $h(x,\alpha)$ with track parameters $x_k$ and alignment parameters $\alpha_k$ are

\begin{equation}
  \frac{\partial\sum_k\chi^2_k}{\partial \alpha} = 0
\end{equation}

 and

 \begin{equation}
   \forall_k \frac{\partial\chi^2_k}{\partial x_k} = 0\,.
 \end{equation}

The subscript $j$ denotes the track not the vector component. For a single track the subscript can be left out. The more number of tracks, the more number of parameters in the minimizing problem.
For a large number of tracks a similar expression as in equation \eqref{eqn:lsq} for the least squares estimator is used and the computation is performed in two steps since the inverse matrix is computationally too expensive to use a least squares expression.
The first step is to estimate track parameters for a starting set of calibration parameters called $\alpha_0$. The second step is to minimize the total $\chi^2$ with respect to $\alpha$ while also taking $x_j$ and $\alpha$ into account.

The total derivative reads
\begin{equation}
  \frac{\symup{d}}{\symup{d}\alpha} = \frac{\partial}{\partial\alpha} +
  \frac{\symup{d}x}{\symup{d}\alpha}\frac{\partial}{\partial x}\,.
\end{equation}

$\frac{\symup{d}x}{\symup{d}\alpha}$ is a derivative matrix and results from the minimal track $\chi^2$ condition and can be expressed by

\begin{equation}
  \frac{\symup{d}}{\symup{d}\alpha}\frac{\partial \chi^2}{\partial x} = 0
\end{equation}

therefore the derivative matrix is defined as
\begin{equation}
  \frac{\symup{d}x}{\symup{d}\alpha} = -\frac{\partial^2\chi^2}{\partial\alpha\partial x} \left( \frac{\partial^2\chi^2}{\partial x^2} \right)^{-1}\,.
\end{equation}

The total $\chi^2$ for a sample of tracks is minimal with respect to $\alpha$ and $x$ can the be described as
\begin{equation}
  \frac{\symup{d}\chi^2}{\symup{d}\alpha} = 0\,.
\end{equation}

For $N$ alignment parameters a system with $N$ coupled non-linear equations is defined.
Linearizing the minimum $\chi^2$ condition around the starting values $\alpha_0$ and solving the linear system for $\Delta\alpha$ yields the solution.
\begin{equation}
  \frac{\symup{d}^2\chi^2}{\symup{d}\alpha^2}\vert_{\alpha_0} \Delta\alpha =
  -\frac{\symup{d}\chi^2}{\symup{d}\alpha}\vert_{\alpha_0}
\end{equation}

Now, with enough constraints inside the alignment the second derivative matrix is invertable and the covariance matrix for $\alpha$ reads

\begin{equation*}
  \text{Cov}(\alpha) = 2 \left( \frac{\symup{d}^2\chi^2}{\symup{d}\alpha^2} \right)^{-1}\,.
\end{equation*}

Higher order derivatives in $\alpha$ are neglected here. The difference in the total $\chi^2$ resulting from a change in $\Delta\alpha$ is given by

\begin{equation*}
  \Delta_{\chi^2} = \frac{1}{2} \left( \frac{\symup{d}\chi^2}{\symup{d}\alpha} \right)^T \Delta\alpha = -\Delta\alpha^T \text{Cov}(\alpha)^{-1}\Delta\alpha
\end{equation*}

The change in total $\chi^2$ is equivalent to the significance of the alignment correction and $\Delta_{\chi^2}$ is used to follow the convergence of an alignment.


 \documentclass [12pt]{article}

\usepackage {amsmath}
\usepackage {amsthm}
\usepackage {amssymb}
\usepackage {graphicx}
\usepackage {float}
\usepackage {multirow}
\usepackage {xcolor}
\usepackage {algorithmic}
\usepackage [ruled,vlined,commentsnumbered,titlenotnumbered]{algorithm2e} \usepackage {array}
\usepackage {booktabs}
\usepackage {url}
\usepackage {parskip}
\usepackage [margin=1in]{geometry}
\usepackage [T1]{fontenc}
\usepackage {cmbright}
\usepackage [many]{tcolorbox}
\usepackage [colorlinks = true,
            linkcolor = blue,
            urlcolor  = blue,
            citecolor = blue,
            anchorcolor = blue]{hyperref}
\usepackage {enumitem}
\usepackage {xparse}
\usepackage {verbatim}
\usepackage{listings}
\usepackage{xcolor}
\lstset { %
    language=C++,
    backgroundcolor=\color{black!5}, % set backgroundcolor
    basicstyle=\footnotesize,% basic font setting
}
\newtheorem{theorem}{Theorem}
\newtheorem{remark}{Remark}

\DeclareTColorBox {Solution}{}{breakable, title={Solution}} \DeclareTColorBox {Solution*}{}{breakable, title={Solution (provided)}} \DeclareTColorBox {Instruction}{}{boxrule=0pt, boxsep=0pt, left=0.5em, right=0.5em, top=0.5em, bottom=0.5em, arc=0pt, toprule=1pt, bottomrule=1pt} \DeclareDocumentCommand {\Expecting }{+m}{\textbf {[We are expecting:} #1\textbf {]}} \DeclareDocumentCommand {\Points }{m}{\textbf {(#1 pt.)}}

\begin {document}

\vspace {1em}
\begin {Instruction}
Adapted From Virginia Williams' lecture notes.
\end {Instruction}

{\LARGE \textbf {COMP 285 (NC A\&T, Spr `22)}\hfill \textbf {Lecture 15} }

\begin{centering}
\section*{Graph Representations and Depth-First Search}
\end{centering}

\section{Graphs}
A graph is a set of vertices and edges connecting those vertices. Formally, we define a graph $G$ as $G = (V, E)$ where $E \subseteq V \times V$ . For ease of analysis, the variables $n$ and $m$ typically stand for the number of vertices and edges, respectively. Graphs can come in two flavors, directed or undirected. If a graph is undirected, it must satisfy the property that $(i, j) \in E \iff (j, i) \in E$ (i.e., all edges are bidirectional). In undirected graphs, $m \leq \frac{n(n-1)}{2}$ . In directed graphs, $m \leq n(n - 1)$. Thus, $m = O(n^2)$ and $\log m = O(\log n)$. A connected graph is a graph in which for any two nodes $u$ and $v$ there exists a path from $u$ to $v$ . For an undirected connected graph $m \geq n - 1$. A sparse graph is a graph with few edges (for example, $\Theta(n)$ edges) while a dense graph is a graph with many edges (for example, $m = \Theta(n^2)$).

\subsection{Representation}

A common issue is the topic of how to represent a graph's edges in memory. There are two standard methods for this task.

An \textbf{adjacency matrix} uses an arbitrary ordering of the vertices from $1$ to $|V|$. The matrix consists of an $n \times n$ binary matrix such that the $(i, j)$-th element is $1$ if $(i, j)$ is an edge in the graph, $0$ otherwise.

An \textbf{adjacency list} consists of an array $A$ of $|V |$ lists, such that $$A[u]$$ contains a linked list of vertices $v$ such that $(u, v) \in E$ (the neighbors of $u$). In the case of a directed graph, it's also helpful to distinguish between outgoing and ingoing edges by storing two different lists at $A[u]$: a list of $v$ such that $(u, v ) \in E$ (the out-neighbors of $u$) as well as a list of $v$ such that $(v, u) \in E$ (the in-neighbors of $u$).

What are the tradeoffs between these two methods? To help our analysis, let deg($v$) denote the \textbf{degree} of $v$, or the number of vertices connected to $v$. In a directed graph, we can distinguish between out-degree and in-degree, which respectively count the number of outgoing and incoming edges.

\begin{itemize}
  \item The adjacency matrix can check if $(i, j)$ is an edge in $G$ in constant time, whereas the adjacency list representation must iterate through up to deg($i$) list entries.
  \item The adjacency matrix takes $\Theta(n^2)$ space, whereas the adjacency list takes $\Theta(m + n)$ space.
  \item The adjacency matrix takes $\Theta(n)$ operations to enumerate the neighbors of a vertex v since it must iterate across an entire row of the matrix. The adjacency list takes deg($v $) time.
\end{itemize}

What's a good rule of thumb for picking the implementation? One useful property is the sparsity of the graph's edges. If the graph is \textbf{sparse}, and the number of edges is considerably less than the max ($m \ll n^2$ ), then the adjacency list is a good idea. If the graph is dense and the number of edges is nearly $n^2$ , then the matrix representation makes sense because it speeds up lookups without too much space overhead. Of course, some applications will have lots of space to spare, making the matrix feasible no matter the structure of the graphs. Other applications may prefer adjacency lists even for dense graphs. Choosing the appropriate structure is a balancing act of requirements and priorities.

\section{Depth First Search (DFS)}

Given a starting vertex, it's desirable to find all vertices reachable from the start. There are many algorithms to do this, the simplest of which is depth-first search. As the name implies, DFS enumerates the deepest paths, only backtracking when it hits a dead end or an already-explored section of the graph. DFS by itself is fairly simple, so we introduce some augmentations to the basic algorithm.

\begin{itemize}
  \item To prevent loops, DFS keeps track of a ``color'' attribute for each vertex. Unvisited vertices are white by default. Vertices that have been visited but still may be backtracked to are colored gray. Vertices which are completely processed are colored black. The algorithm can then prevent loops by skipping non-white vertices
  \item Instead of just marking visited vertices, the algorithm also keeps track of the tree generated by the depth-first traversal. It does so by marking the ``parent'' of each visited vertex, aka the vertex that DFS visited immediately prior to visiting the child.
  \item The augmented DFS also marks two auto-incrementing timestamps $d$ and $f$ to indicate when a node was first discovered and finished.
\end{itemize}

The algorithm takes as input a start vertex $s$ and a starting timestamp $t$, and returns the timestamp at which the algorithm finishes. Let $N(s)$ denote the neighbors of $s$; for a directed graph, let $N_{\text{out}}(s)$ denote the out-neighbors of $s$.

\begin{algorithm}
\caption{\texttt{init}(G)}
\label{alg:init_G}
\begin{algorithmic}
\FOR {$v \in G$}
\STATE color($v$) $\gets$ white
\STATE d($v$), f($v$) $\gets \infty$
\STATE p($v$) $\gets$ nil
\ENDFOR
\end{algorithmic}
\end{algorithm}

\begin{algorithm}
\caption{\texttt{DFS}(s, t): $s \in V$ is white, $t =$ time}
\label{alg:DFS_s_t}
\begin{algorithmic}
\STATE color(s) $\gets$ gray
\STATE \texttt{// d(s) is the discovery time of s}
\STATE $d(s) \gets t$
\STATE $t \gets t+1$
\FOR {$v \in N(s)$}
  \IF {$color(v) = white$}
    \STATE $p(v) \gets s$
    \STATE \texttt{// Update t to be the finish time of DFS starting at v}
    \STATE $t \gets DFS(v,t)$
    \STATE $t \gets t + 1$
  \ENDIF
\ENDFOR
\STATE \texttt{Finish time:}
\STATE $f(s) \gets t$
\STATE \texttt{// s is finished}
\STATE color($s$) $\gets$ black
\end{algorithmic}
\end{algorithm}

There are multiple ways we can search using DFS. One way is to search from some source node $s$, which will give us a set of black nodes reachable from $s$ and white nodes unreachable from $s$.

\begin{algorithm}
\caption{\texttt{DFS}(s): DFS from a source node $s$}
\label{alg:DFS_s}
\begin{algorithmic}
\STATE init($G$)
\STATE DFS($s, 1$)
\end{algorithmic}
\end{algorithm}

Another way to use DFS is to search over the entire graph, choosing some white node and finding everything we can reach from that node, and repeating until we have no white nodes remaining. In an undirected graph this will give us all of the connected components.

\begin{algorithm}
\caption{\texttt{DFS}(G): DFS on an entire graph $G$}
\label{alg:DFS_G}
\begin{algorithmic}
\STATE init($G$)
\STATE $t \gets 1$
\FOR {$ v \in G$}
  \IF {$color(v) = white$}
    \STATE $t \gets$ DFS($v,t$)
    \STATE $t \gets t+1$
  \ENDIF
\ENDFOR
\end{algorithmic}
\end{algorithm}

\subsection{Runtime of DFS}

We will now look at the runtime for the standard DFS algorithm (\ref{alg:DFS_s_t}).

Everything above the loop runs in $O(1)$ time per node visit. Excluding the recursive call, everything inside of the for loop takes $O(1)$ time every time an edge is scanned. Everything after the for loop also runs in $O(1)$ time per node visit.

We can express the runtime of DFS as $O(\#\text{ of node visits }+ \# \text{ of edge scans})$. Assume we have a graph with $n$ nodes and $m$ edges. We know that the \# of node visits is $\leq n$, since we only visit white nodes and whenever we visit a node we change its color from white to gray and never change it back to white again. We also know that an edge $(u, v)$ is scanned only when $u$ or $v$ is visited. Since every node is visited at most once, we know that an edge $(u, v)$ is scanned at most twice (or only once for directed graphs). Thus, \# of edges scanned is $O(m)$, and the overall runtime of DFS is $O(m + n)$.

\subsection{DFS Example}
We will now try running DFS on the example graph below.
\begin{center}
\begin{tikzpicture}[scale=0.4]
\tikzstyle{every node}+=[inner sep=0pt]
\draw [black] (17.4,-25.8) circle (3);
\draw (17.4,-25.8) node {$b$};
\draw [black] (22,-18.6) circle (3);
\draw (22,-18.6) node {$a$};
\draw [black] (28.8,-22.8) circle (3);
\draw (28.8,-22.8) node {$c$};
\draw [black] (36.7,-27.5) circle (3);
\draw (36.7,-27.5) node {$d$};
\draw [black] (37.4,-19.6) circle (3);
\draw (37.4,-19.6) node {$e$};
\draw [black] (44.9,-24.3) circle (3);
\draw (44.9,-24.3) node {$f$};
\draw [black] (20.38,-21.13) -- (19.02,-23.27);
\fill [black] (19.02,-23.27) -- (19.87,-22.87) -- (19.02,-22.33);
\draw [black] (20.3,-25.04) -- (25.9,-23.56);
\fill [black] (25.9,-23.56) -- (25,-23.28) -- (25.25,-24.25);
\draw [black] (26.25,-21.22) -- (24.55,-20.18);
\fill [black] (24.55,-20.18) -- (24.97,-21.02) -- (25.5,-20.17);
\draw [black] (31.38,-24.33) -- (34.12,-25.97);
\fill [black] (34.12,-25.97) -- (33.69,-25.13) -- (33.18,-25.99);
\draw [black] (34.59,-20.65) -- (31.61,-21.75);
\fill [black] (31.61,-21.75) -- (32.54,-21.94) -- (32.19,-21.01);
\draw [black] (39.94,-21.19) -- (42.36,-22.71);
\fill [black] (42.36,-22.71) -- (41.95,-21.86) -- (41.41,-22.71);
\end{tikzpicture}
\end{center}

We mark all of the nodes as unvisited and start at a white node, in our case node a.
\begin{center}
\begin{tikzpicture}[scale=0.4,every text node part/.style={align=center}]
\tikzstyle{every node}+=[inner sep=0pt]
\draw [black] (17.4,-25.8) circle (3);
\draw (17.4,-25.8) node {$b$ \\ unvisited};
\draw [black] (22,-18.6) circle (3);
\draw (22,-18.6) node {$a$ \\ $(1,$};
\draw [black] (28.8,-22.8) circle (3);
\draw (28.8,-22.8) node {$c$ \\ unvisited};
\draw [black] (36.7,-27.5) circle (3);
\draw (36.7,-27.5) node {$d$ \\ unvisited};
\draw [black] (37.4,-19.6) circle (3);
\draw (37.4,-19.6) node {$e$ \\ unvisited};
\draw [black] (44.9,-24.3) circle (3);
\draw (44.9,-24.3) node {$f$ \\ unvisited};
\draw [black] (20.38,-21.13) -- (19.02,-23.27);
\fill [black] (19.02,-23.27) -- (19.87,-22.87) -- (19.02,-22.33);
\draw [black] (20.3,-25.04) -- (25.9,-23.56);
\fill [black] (25.9,-23.56) -- (25,-23.28) -- (25.25,-24.25);
\draw [black] (26.25,-21.22) -- (24.55,-20.18);
\fill [black] (24.55,-20.18) -- (24.97,-21.02) -- (25.5,-20.17);
\draw [black] (31.38,-24.33) -- (34.12,-25.97);
\fill [black] (34.12,-25.97) -- (33.69,-25.13) -- (33.18,-25.99);
\draw [black] (34.59,-20.65) -- (31.61,-21.75);
\fill [black] (31.61,-21.75) -- (32.54,-21.94) -- (32.19,-21.01);
\draw [black] (39.94,-21.19) -- (42.36,-22.71);
\fill [black] (42.36,-22.71) -- (41.95,-21.86) -- (41.41,-22.71);
\end{tikzpicture}
\end{center}

From node $a$ we will visit all of $a$'s children, namely node $b$.

\begin{center}
\begin{tikzpicture}[scale=0.4,every text node part/.style={align=center}]
\tikzstyle{every node}+=[inner sep=0pt]
\draw [black] (17.4,-25.8) circle (3);
\draw (17.4,-25.8) node {$b$ \\ $(2,$};
\draw [black] (22,-18.6) circle (3);
\draw (22,-18.6) node {$a$ \\ $(1,$};
\draw [black] (28.8,-22.8) circle (3);
\draw (28.8,-22.8) node {$c$ \\ unvisited};
\draw [black] (36.7,-27.5) circle (3);
\draw (36.7,-27.5) node {$d$ \\ unvisited};
\draw [black] (37.4,-19.6) circle (3);
\draw (37.4,-19.6) node {$e$ \\ unvisited};
\draw [black] (44.9,-24.3) circle (3);
\draw (44.9,-24.3) node {$f$ \\ unvisited};
\draw [black] (20.38,-21.13) -- (19.02,-23.27);
\fill [black] (19.02,-23.27) -- (19.87,-22.87) -- (19.02,-22.33);
\draw [black] (20.3,-25.04) -- (25.9,-23.56);
\fill [black] (25.9,-23.56) -- (25,-23.28) -- (25.25,-24.25);
\draw [black] (26.25,-21.22) -- (24.55,-20.18);
\fill [black] (24.55,-20.18) -- (24.97,-21.02) -- (25.5,-20.17);
\draw [black] (31.38,-24.33) -- (34.12,-25.97);
\fill [black] (34.12,-25.97) -- (33.69,-25.13) -- (33.18,-25.99);
\draw [black] (34.59,-20.65) -- (31.61,-21.75);
\fill [black] (31.61,-21.75) -- (32.54,-21.94) -- (32.19,-21.01);
\draw [black] (39.94,-21.19) -- (42.36,-22.71);
\fill [black] (42.36,-22.71) -- (41.95,-21.86) -- (41.41,-22.71);
\end{tikzpicture}
\end{center}

We now visit b's child, node c.

\begin{center}
\begin{tikzpicture}[scale=0.4,every text node part/.style={align=center}]
\tikzstyle{every node}+=[inner sep=0pt]
\draw [black] (17.4,-25.8) circle (3);
\draw (17.4,-25.8) node {$b$ \\ $(2,$};
\draw [black] (22,-18.6) circle (3);
\draw (22,-18.6) node {$a$ \\ $(1,$};
\draw [black] (28.8,-22.8) circle (3);
\draw (28.8,-22.8) node {$c$ \\ $(3,$};
\draw [black] (36.7,-27.5) circle (3);
\draw (36.7,-27.5) node {$d$ \\ unvisited};
\draw [black] (37.4,-19.6) circle (3);
\draw (37.4,-19.6) node {$e$ \\ unvisited};
\draw [black] (44.9,-24.3) circle (3);
\draw (44.9,-24.3) node {$f$ \\ unvisited};
\draw [black] (20.38,-21.13) -- (19.02,-23.27);
\fill [black] (19.02,-23.27) -- (19.87,-22.87) -- (19.02,-22.33);
\draw [black] (20.3,-25.04) -- (25.9,-23.56);
\fill [black] (25.9,-23.56) -- (25,-23.28) -- (25.25,-24.25);
\draw [black] (26.25,-21.22) -- (24.55,-20.18);
\fill [black] (24.55,-20.18) -- (24.97,-21.02) -- (25.5,-20.17);
\draw [black] (31.38,-24.33) -- (34.12,-25.97);
\fill [black] (34.12,-25.97) -- (33.69,-25.13) -- (33.18,-25.99);
\draw [black] (34.59,-20.65) -- (31.61,-21.75);
\fill [black] (31.61,-21.75) -- (32.54,-21.94) -- (32.19,-21.01);
\draw [black] (39.94,-21.19) -- (42.36,-22.71);
\fill [black] (42.36,-22.71) -- (41.95,-21.86) -- (41.41,-22.71);
\end{tikzpicture}
\end{center}

Node c has two children that we must visit. When we try to visit node a we find that node a has already been visited (and would be colored gray, as we are in the process of searching a's children), so we do not continue searching down that path. We will next search c's second child, node d.

\begin{center}
\begin{tikzpicture}[scale=0.4,every text node part/.style={align=center}]
\tikzstyle{every node}+=[inner sep=0pt]
\draw [black] (17.4,-25.8) circle (3);
\draw (17.4,-25.8) node {$b$ \\ $(2,$};
\draw [black] (22,-18.6) circle (3);
\draw (22,-18.6) node {$a$ \\ $(1,$};
\draw [black] (28.8,-22.8) circle (3);
\draw (28.8,-22.8) node {$c$ \\ $(3,$};
\draw [black] (36.7,-27.5) circle (3);
\draw (36.7,-27.5) node {$d$ \\ $(4,$};
\draw [black] (37.4,-19.6) circle (3);
\draw (37.4,-19.6) node {$e$ \\ unvisited};
\draw [black] (44.9,-24.3) circle (3);
\draw (44.9,-24.3) node {$f$ \\ unvisited};
\draw [black] (20.38,-21.13) -- (19.02,-23.27);
\fill [black] (19.02,-23.27) -- (19.87,-22.87) -- (19.02,-22.33);
\draw [black] (20.3,-25.04) -- (25.9,-23.56);
\fill [black] (25.9,-23.56) -- (25,-23.28) -- (25.25,-24.25);
\draw [black] (26.25,-21.22) -- (24.55,-20.18);
\fill [black] (24.55,-20.18) -- (24.97,-21.02) -- (25.5,-20.17);
\draw [black] (31.38,-24.33) -- (34.12,-25.97);
\fill [black] (34.12,-25.97) -- (33.69,-25.13) -- (33.18,-25.99);
\draw [black] (34.59,-20.65) -- (31.61,-21.75);
\fill [black] (31.61,-21.75) -- (32.54,-21.94) -- (32.19,-21.01);
\draw [black] (39.94,-21.19) -- (42.36,-22.71);
\fill [black] (42.36,-22.71) -- (41.95,-21.86) -- (41.41,-22.71);
\end{tikzpicture}
\end{center}

Since node d has no children, we return back to its parent node, c, and continue to go back up the path we took, marking nodes with a finish time when we have searched all of their children.

\begin{center}
\begin{tikzpicture}[scale=0.4,every text node part/.style={align=center}]
\tikzstyle{every node}+=[inner sep=0pt]
\draw [black] (17.4,-25.8) circle (3);
\draw (17.4,-25.8) node {$b$ \\ $(2,7)$};
\draw [black] (22,-18.6) circle (3);
\draw (22,-18.6) node {$a$ \\ $(1,8)$};
\draw [black] (28.8,-22.8) circle (3);
\draw (28.8,-22.8) node {$c$ \\ $(3,6)$};
\draw [black] (36.7,-27.5) circle (3);
\draw (36.7,-27.5) node {$d$ \\ $(4,5)$};
\draw [black] (37.4,-19.6) circle (3);
\draw (37.4,-19.6) node {$e$ \\ unvisited};
\draw [black] (44.9,-24.3) circle (3);
\draw (44.9,-24.3) node {$f$ \\ unvisited};
\draw [black] (20.38,-21.13) -- (19.02,-23.27);
\fill [black] (19.02,-23.27) -- (19.87,-22.87) -- (19.02,-22.33);
\draw [black] (20.3,-25.04) -- (25.9,-23.56);
\fill [black] (25.9,-23.56) -- (25,-23.28) -- (25.25,-24.25);
\draw [black] (26.25,-21.22) -- (24.55,-20.18);
\fill [black] (24.55,-20.18) -- (24.97,-21.02) -- (25.5,-20.17);
\draw [black] (31.38,-24.33) -- (34.12,-25.97);
\fill [black] (34.12,-25.97) -- (33.69,-25.13) -- (33.18,-25.99);
\draw [black] (34.59,-20.65) -- (31.61,-21.75);
\fill [black] (31.61,-21.75) -- (32.54,-21.94) -- (32.19,-21.01);
\draw [black] (39.94,-21.19) -- (42.36,-22.71);
\fill [black] (42.36,-22.71) -- (41.95,-21.86) -- (41.41,-22.71);
\end{tikzpicture}
\end{center}

Once we reach our first source node a we find that we have searched all of its children, so we look in the graph to see if there are any unvisited nodes remaining. For our example, we start with a new source node e and run DFS to completion.

\begin{center}
\begin{tikzpicture}[scale=0.4,every text node part/.style={align=center}]
\tikzstyle{every node}+=[inner sep=0pt]
\draw [black] (17.4,-25.8) circle (3);
\draw (17.4,-25.8) node {$b$ \\ $(2,7)$};
\draw [black] (22,-18.6) circle (3);
\draw (22,-18.6) node {$a$ \\ $(1,8)$};
\draw [black] (28.8,-22.8) circle (3);
\draw (28.8,-22.8) node {$c$ \\ $(3,6)$};
\draw [black] (36.7,-27.5) circle (3);
\draw (36.7,-27.5) node {$d$ \\ $(4,5)$};
\draw [black] (37.4,-19.6) circle (3);
\draw (37.4,-19.6) node {$e$ \\ $(9,12)$};
\draw [black] (44.9,-24.3) circle (3);
\draw (44.9,-24.3) node {$f$ \\ $(10,11)$};
\draw [black] (20.38,-21.13) -- (19.02,-23.27);
\fill [black] (19.02,-23.27) -- (19.87,-22.87) -- (19.02,-22.33);
\draw [black] (20.3,-25.04) -- (25.9,-23.56);
\fill [black] (25.9,-23.56) -- (25,-23.28) -- (25.25,-24.25);
\draw [black] (26.25,-21.22) -- (24.55,-20.18);
\fill [black] (24.55,-20.18) -- (24.97,-21.02) -- (25.5,-20.17);
\draw [black] (31.38,-24.33) -- (34.12,-25.97);
\fill [black] (34.12,-25.97) -- (33.69,-25.13) -- (33.18,-25.99);
\draw [black] (34.59,-20.65) -- (31.61,-21.75);
\fill [black] (31.61,-21.75) -- (32.54,-21.94) -- (32.19,-21.01);
\draw [black] (39.94,-21.19) -- (42.36,-22.71);
\fill [black] (42.36,-22.71) -- (41.95,-21.86) -- (41.41,-22.71);
\end{tikzpicture}
\end{center}

\end{document}