\documentclass[letterpaper,11pt]{article}

\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{booktabs}
\usepackage[hmargin=1.25in,vmargin=1in]{geometry}
\usepackage{graphicx}
\usepackage{hyperref}
\usepackage{listings}
\usepackage{lmodern}
\usepackage{mathtools}
\usepackage{microtype}
\usepackage{minted}

\DeclareMathOperator*{\argmin}{arg\,min}

\author{Philip Pham}
\date{\today}
\title{CSE 547 - Assignment 2}

\begin{document}
\maketitle

\section*{Problem 0}

\begin{description}
\item[List of collaborators:] I have not collaborated with anyone.
\item[List of acknowledgements:] None.
\item[Certify that you have read the instructions:] I have read and understood
  these policies.
\end{description}

\section*{Problem 1: Generalization, Streaming, and SGD}

In class, we examined using Stochastic Gradient Descent (SGD) for empirical loss
minimization, where we have an $N$ sized training set $\mathcal{T}$. The
empirical loss considered was:
\begin{equation}
  F(w) = \frac{1}{N} \sum_{(x,y) \in \mathcal{T}} l\left(w,(x,y)\right).
\end{equation}

Here, gradient descent for the function $F$ is the algorithm:
\begin{enumerate}
\item Initialize at some point $w^{(0)}$.
\item Sample $(x,y)$ uniformly at random from the set $\mathcal{T}$.
  \label{item:sgd_sample}
\item Update the parameters:
  \begin{equation}
    w^{(k+1)} = w^{(k)} - \eta_k \cdot \nabla l\left(w^{(k)},(x,y)\right),
  \end{equation}
  and go back to \ref{item:sgd_sample}.
\end{enumerate}

We provided guarantees assuming that $F$ was smooth and the gradients in our
training set were uniformly bounded,
$\lVert \nabla l\left(w, (x,y)\right) \rVert \leq B$.

However, in practice, we care about generalization, that is, statements on how
well we do on the underlying distribution. Define:
\begin{equation}
  \mathcal{L}(w) = \mathbb{E}_{(x,y) \in \mathcal{D}}l\left(w, (x,y)\right),
\end{equation}
where $\mathcal{D}$ is the underlying distribution.

Suppose we sought a point where $\lVert \nabla \mathcal{L} \rVert^2$ was
small. Obtaining this quantity to be small even in expectation would be
acceptable for this problem Assume that $\mathcal{L}$ is smooth and that the
gradients are uniformly bounded,
$\lVert \nabla l\left(w, (x,y)\right)\rVert \leq B$ for all parameters and all
possible points $(x,y)$ (under $\mathcal{D}$).

\begin{enumerate}
\item Assume we have sampling access to our underlying distribution
  $\mathcal{D}$. Explain how we can make $\lVert \nabla \mathcal{L}(w) \rVert^2$
  small in expection. What can you guarantee if you obtain $m$ samples and how
  would you do this?

  \subsection*{Solution}

  If we sample from $\mathcal{D}$ (as opposed to from $\mathcal{T}$ in Step
  \ref{item:sgd_sample} in the original algorithm), then we can guarantee that
  $\lVert \nabla \mathcal{L}(w) \rVert^2$ is small in expectation. The proof is
  identical to that of Theorem 1.2 from Lecture 6 in class but we replace
  $\mathcal{T}$ with $\mathcal{D}$. Thus, each gradient becomes a sample of
  $\nabla\mathcal{L}(w)$.

\item Suppose we contruct an $N$ sized training set $\mathcal{T}$, where each
  point is sampled under $\mathcal{D}$; then we construct the empirical loss
  funciton $F(w)$; then we run SGD on $F$ for $K$ steps (suppose $K \geq N$). Is
  there an argument on this procedure that implies something non-trivial (and
  technically correct) about $\lVert \nabla\mathcal{L}(w)\rVert^2$, even in
  expectation?

  \subsection*{Solution}

  If we let $K = N$, and sample each point from $\mathcal{T}$ exactly once, we
  can get the guarantee
  \begin{equation}
    \min_{k < K}\mathbb{E}\left[
      \left\lVert \nabla\mathcal{L}\left(w^{(k)}\right)\right \rVert_2^2
    \right] = B\sqrt{\frac{2\left(F\left(w^{(0)}\right) - F_*\right)L}{K}}
  \end{equation}
  as in Theorem 1.2 from Lecture 6. This works because using each sample from
  $\mathcal{T}$ exactly once is the same as sampling from $\mathcal{D}$ $K$
  times. However, this falls apart as soon as $K > N$ because now our population
  is $\mathcal{T}$, not $\mathcal{D}$, so we can no longer make any guarantees
  about $\lVert \nabla\mathcal{L}(w)\rVert^2$.
\end{enumerate}

\section*{Problem 2: GD versus Adaptive GD}

Let us compare gradient descent (GD), with a constant stepsize $\eta$, to
Adagrad on two simple one-dimensional objective functions. Adagrad will use the
stepsize at iteration $k$ where:
\begin{equation}
  \eta_k = \frac{C}{\sqrt{\sum_{j = 0}^k \lVert \nabla F \left(
        w^{(j)}
      \right)\rVert^2}}.
\end{equation}

\begin{enumerate}
\item Consider $F(w) = \frac{1}{2}w^2$. Let us start at $w_0 = 1$. For this
  problem a constant stepsize certainly makes sense for GD. Let $\eta = 3/4$ and
  $C = 3/4$.

  Analytically, characterize the convergence rates of GD and Adagrad
  with these parameter settings.

  \subsection*{Solution}

  For GD, we can explicitly, write down the formula for $w^{(k)}$. Since
  $F^\prime(w) = w$, $w^{(k+1)} = (1 - C)w^{(k)}$, so
  $w^{(k)} = \left(1-C\right)^k = \left(1/4\right)^k$. So,
  $\nabla F\left(w^{(k)}\right) = \left(1/4\right)^k$.

  The global minimum occurs at $w^* = 0$. Since $\eta_k \leq C$ with equality
  only when $k = 0$, Adagrad converges strictly slower.

  However, it does no converge much slower. We can show that
  $w^{(k)} \leq \left(1/3\right)^k$. We show this by strong
  induction. $w^{(0)} = 0$ satisifies this. Now, we have that
  \begin{align*}
    w^{(k+1)}
    &= w^{(k)}\left(1 - \frac{3/4}{\sqrt{\sum_{i = 0}^k \left(w^{(i)}\right)^2}}\right) \\
    &\leq w^{(k)}\left(1 - \frac{3/4}{\sqrt{\sum_{i = 0}^k \left(\frac{1}{3}\right)^2}}\right) \\
    &\leq w^{(k)}\left(1 - \frac{3/4}{\sqrt{9/8}}\right) \approx 0.2928932 w^{(k)} \\
    &\leq \left(\frac{1}{3}\right)^{k + 1}.
  \end{align*}

  Thus, we have that $\left(1/4\right)^k \leq w^{(k)} \leq \left(1/3\right)^k$,
  so the two optimization procedures do not differ substantially.  In this case,
  GD and Adagrad differ by at most a constant factor of $3/4$.

\item Consider the non-smooth, convex function $F(w) = \lvert w \rvert$. Let us
  start with $w_0 = 1$. Use $\eta = 3/4$ and $C = 3/4$ as before.

  Analytically, characterize the convergence rates of GD and Adagrad
  with these parameter settings.

  \subsection*{Solution}

  We have that
  \begin{equation}
    F^\prime\left(w\right) = \begin{cases}
      1, & w > 0; \\
      -1, & w < 0;
    \end{cases}
  \end{equation}
  and $F^\prime(w)$ is not defined at $w = 0$.

  In this case, GD would not converge. The sequence would look like
  \begin{align*}
    w^{(0)} &= 1 \\
    w^{(1)} &= 1/4 \\
    w^{(2)} &= -1/2 \\
    w^{(3)} &= 1/4 \\
    w^{(4)} &= -1/2 \\
    w^{(5)} &= 1/4 \\
            &~~\vdots
  \end{align*}

  Adagrad on the other hand is $O\left(1/\sqrt{T}\right)$, where $T$ is the
  number of steps that we have taken. To see this, note that
  $\lVert \nabla F(w) \rVert^2_2 = 1$ We have two cases: (\ref{itm:2_w_gt})
  $w^{(k)} > 0$ and (\ref{itm:2_w_lt}) $w^{(k)} < 0$.
  \begin{enumerate}
  \item In this case, we'd have that
    \begin{equation*}
      w^{(k + 1)} = w^{(k)} - \frac{3}{4\sqrt{k + 2}}.
    \end{equation*}
    \label{itm:2_w_gt}
  \item Here,
    \begin{equation*}
      w^{(k + 1)} = w^{(k)} + \frac{3}{4\sqrt{k + 2}}.
    \end{equation*}
    \label{itm:2_w_lt}
  \end{enumerate}

  Suppose that $0 < w^{(k)} < (3/4)/\sqrt{k + 1}$. Certainly, this is true for
  $k = 1$, where $w^{(1)} = 1/4$. Then, we have that
  \begin{equation}
    -\frac{3}{4\sqrt{k + 2}} < w^{(k + 1)} < \frac{3}{4\sqrt{k+1}} - \frac{3}{4\sqrt{k + 2}} < \frac{3}{4\sqrt{k + 2}}.
  \end{equation}
  We can similarly show the same inequality holds if
  $-(3/4)/\sqrt{k + 1} < w^{(k)} < 0$. Thus, we have that
  $\left\lvert w^{(k)} \right\rvert < (3/4)/\sqrt{k + 1}$ for all $k \geq 1$, so
  Adagrad converges at the rate $O\left(1/\sqrt{T}\right)$.
\end{enumerate}

\section*{Problem 3: Understanding Non-Convexity}

\subsection*{Newton's Method}

The least squares problem can be written as:
\begin{equation}
  \min_{w}L(w),~\text{where}~L(w) \coloneqq \frac{1}{2N}\left\lVert Xw - Y\right\rVert^2,
\end{equation}
where $X$ is our $N \times d$ matrix and $Y$ is our $N \times 1$ output vector. If we define:
\begin{equation}
  \Sigma \coloneqq \frac{1}{N}X^\intercal X,~u = \frac{1}{N}X^\intercal Y,
\end{equation}
then, this expression can be written as
\begin{equation}
  L(w) = \frac{1}{2}w^\intercal\Sigma w - u^\intercal w + \frac{1}{2N}\lVert Y \rVert^2.
\end{equation}

Note that $\Sigma$ is a positive-definite matrix. Assume that $\Sigma$ is full
rank.

Newton's method is typically thought of a being better than gradient descent due
to it using second order information. At some $w_0$, the method first constructs
a second-order Taylor's approximation around $w_0$, and then it makes the new
iterate to be the point at which the gradient of this approximation is $0$, that
is,
\begin{equation}
  w \leftarrow w - \left[ \nabla^2 L(w) \right]^{-1} \nabla L(w).
  \label{eqn:newton_update}
\end{equation}

\begin{enumerate}
\item Starting at some $w_0$ write out one step of the Newton's method in terms
  of $\Sigma$ and $u$. If you take one step, what point do you get to?

  \subsubsection*{Solution}

  Since $\Sigma$ is positive-definite, we can write $\Sigma = PD^2P^\intercal$,
  where $D$ is diagonal. Using that
  $w^\intercal \Sigma w = \left(D P^\intercal
    w\right)^\intercal\left(DP^\intercal w\right)$ and the chain rule, we have
  that
  \begin{align}
    \nabla L(w)
    &= D\left(L(w)\right)^\intercal \nonumber\\
    &= \left(\frac{1}{2} \left(2w^\intercal PD\right)\left(D P^\intercal\right)  - u^\intercal\right)^\intercal \nonumber\\
    &= \left(w^\intercal \Sigma - u^\intercal\right)^\intercal \nonumber\\
    &= \Sigma w - u.
      \label{eqn:newton_grad}
  \end{align}

  Calculating the derivative again, we have obtain
  \begin{equation}
    \nabla^2 L\left( w \right) = \Sigma.
    \label{eqn:newton_hessian}
  \end{equation}

  Subtituting Equations \ref{eqn:newton_grad} and \ref{eqn:newton_hessian} into
  Equation \ref{eqn:newton_update}, we obtain
  \begin{equation}
    w_1 = w_0 - \Sigma^{-1}\left(
      \Sigma w_0 - u
    \right) = \boxed{\Sigma^{-1} u.}
    \label{eqn:newton_w1}
  \end{equation}

\item Comment on the loss of this point and how it compares to the minimal
  function value?

  \subsubsection*{Solution}

  Equation \ref{eqn:newton_w1} expands to
  \begin{equation}
    w_1 = \Sigma^{-1} u = \left(X^\intercal X\right)^{-1}X^\intercal Y,
  \end{equation}
  which is the solution to the least squares problem, so $L(w_1)$ is the global
  minimum function value.
\end{enumerate}

\subsection*{Non-convex Case}

Consider the objective function:
\begin{equation}
  F(w) = \frac{1}{2}w^\intercal A w - b^\intercal w + c,
\end{equation}
where $A$ is a symmetric matrix, $b$ is a vector, and $c$ is a scalar. Our goal
is to minimize the objective function. Assume that $\Sigma$ is full rank.

Assume that $A$ is still symmetric, yet suppose now that there is at least one
negative eigenvalue. Hence, the problem is non-convex. Again, suppose you start
at $w_0$ and take a step of Newton's method. Let this new point be $w_1$.

\begin{enumerate}
\item What is the point $w_1$ and the object value $F\left(w_1\right)$.

  \subsubsection*{Solution}

  The calcuation for the gradient is similar to that in Equation
  \ref{eqn:newton_grad}:
  \begin{equation}
    \nabla F(w) = Aw - b.
    \label{eqn:newton_grad2}
  \end{equation}
  It's notable that $D$ will now have at least one complex entry on its
  diagonal. However, we only $D$ for an intermediate step. The final solution
  will still contain real values.

  Then, the Hessian is constant-valued:
  \begin{equation}
    \nabla^2 F(w) = A.
    \label{eqn:newton_hessian2}
  \end{equation}

  Substituting Equations \ref{eqn:newton_grad2} and \ref{eqn:newton_hessian2}
  into Equation \ref{eqn:newton_update}, we get that
  \begin{equation}
    w_1 = w_0 - A^{-1}\left(Aw_0 - b\right) = \boxed{A^{-1}b.}
    \label{eqn:newtwon_w12}
  \end{equation}
  The objective value is
  \begin{align}
    F\left(w_1\right)
    &= \frac{1}{2} \left(b^\intercal A^{-1}\right)A\left(A^{-1}b\right)
      - b^\intercal \left(A^{-1}b\right) + c \nonumber\\
    &= \boxed{-\frac{1}{2}b^\intercal A^{-1}b + c.}
      \label{eqn:newton_min_object2}
  \end{align}

\item What is the gradient at $w_1$?
  \subsubsection*{Solution}

  Substituting Equation \ref{eqn:newtwon_w12} into Equation
  \ref{eqn:newton_grad}, we get that
  \begin{equation}
    \nabla F(w_1) = Aw_1 - b = \boxed{0.}
  \end{equation}
\item What is the minimal object value of $F\left(\cdot\right)$? Did we achieve
  it?

  \subsubsection*{Solution}

  The minimal objective value is $-\infty$. We did not achieve it, for Equation
  \ref{eqn:newton_min_object2} is finite.

  To see that the minimal objective value is $-\infty$. Let $v$ be an
  eigenvector of $A$ associated with eigenvalue $\lambda < 0$. Then, we have
  \begin{align*}
    F\left(v\right)
    &= \frac{1}{2}v^\intercal A v - b^\intercal v + c \\
    &= \frac{\lambda}{2}\lVert v \rVert^2 - b^\intercal v + c.
  \end{align*}
  $\left\lvert b^\intercal v \right\rvert \leq \lVert b \rVert \lVert v \rVert$,
  so by increasing the magnitude of $v$, we can push the function value towards
  infinity.
\item Suppose you now are at $w_1$. In terms of the eigendecomposition
  $A = UDU^\intercal$, what are the directions of movement which \emph{strictly}
  decrease the objective value from $w_1$? Why?

  \subsubsection*{Solution}

  Let $v$ be an eigenvector with a negative eigenvalue. Then, for any
  $\epsilon > 0$, we have that
  \begin{equation*}
    \nabla F(w_1 + \epsilon v) = A\left(w_1 + \epsilon v\right) - b = \epsilon\lambda v,
  \end{equation*}
  where $\lambda < 0$ is the eigenvalue associated with $v$.

  By definition,
  $D_vF(w_1 + \epsilon v) = \nabla F(w_1 + \epsilon v) \cdot v = \epsilon\lambda
  \lVert v\rVert^2 < 0$, so $F$ is always decreasing in the direction $v$
  relative to $w_1$. This will be true for any $v$ that is a linear combination
  of eigenvectors with negative eigenvalues.
\end{enumerate}

\section*{Problem 4: Empirical Optimization}

We will now consider the multi-label classification problem. In the multi-label
problem, there are multiple labels that could be ``on'' for each input $x$. You
will use either the square loss or the binary logistic loss and consider
training two models, namely (i) a linear model and (ii) a multi-layer perceptron
(MLP) with a number of hidden nodes that you will tune.

You will try out three methods in each of the following: (1) SGD with a
mini-batch size that you tune. You will use the same minibatch size for the
other algorithms; (2) try out Polyak’s ``heavy ball method'' (aka momentum) or
Nesterov’s accelerated gradient descent (NAG); and (3) either Adagrad or
Adam. You must tune all the parameters of these methods.

The dataset contains 18 total categories with a number of categories for each
supercategory (vehicle or animal). In the dataset provided, each image contains
objects of a single supercategory, say vehicle, and potentially multiple objects
from the supercategory, such as car, boat, etc. In this exercise we shall build
a classifier that learns to identify \emph{all the categories of objects}
present in each image, by optimizing either a square loss or a logistic loss
objective. For the purposes of learning these classifiers, we shall use the
dataset and features from the first homework. We shall also provide a larger
version of this dataset since we need to train more parameters for this model.

The object function we choos to optimize is
\begin{equation}
  L(w) = \frac{\lambda}{2}\lVert w \rVert^2 +
  \frac{1}{n}\sum_{i=1}^n\sum_{j=1}^k l\left(y_{ij},f_{ij}(w)\right),
\end{equation}
where $f_{ij}(w) = w_j^\intercal x_i$ and $w_j \in \mathbb{R}^d$ is the $j$th
column of $w \in \mathbb{R}^d \times \mathbb{R}^k$. Here, $w$ is the linear
model we wish to optimize over and $\lambda > 0$ is the strength of $l_2$
regularization. here $l$ is the loss function:
\begin{itemize}
\item $l\left(y,\hat{y}\right) = \frac{1}{2}\left(y - \hat{y}\right)^2$ is the square error
  loss.
\item
  $l\left(y,\hat{y}\right) = y\log\left(1 + \exp\left(-\hat{y}\right)\right) +
  (1-y)\log\left(1 + \exp\left(\hat{y}\right)\right)$ is the logistic loss where
  the true label $y \in \{0,1\}$.
\end{itemize}

Notice that we encode $y_i$ as binary vector of length $k = 18$ (the number of
categories) where a $1$ indicates the presence of a category and $0$ indicates
the absence.

Determine which loss function works better for a linear classifier and use that
loss throughout the question.

When using $MLP$,
$f_{ij}(w) = \left\langle w_j^{(2)},
  \operatorname{relu}\left(w^{(1)}x_i\right)\right\rangle$, where
$w^{(1)} \in \mathbb{R}^h \times \mathbb{R}^d$ are the weights in the first
layer and $h$ is the number of hidden nodes. Again $w_j^{(2)} \in \mathbb{R}^h$
is the $j$th column of $w^{(2)} \in \mathbb{R}^h \times \mathbb{R}^k$, the
weights of the second layer.

\subsection*{SGD and Linear Regression}

Now consider running stochastic gradient descent on $L(w)$.

\begin{enumerate}
\item What mini-batch size do you use? What stepsize did you use? What value of
  $\lambda$ did you use? Specify your stepsize scheme if you chose to decay your
  stepsize. Which loss function did you find works better?

  \subsubsection*{Solution}

  I used a mini-batch size of $8$. My learning rate was $10^{-4}$. I used
  $\lambda = 0.7$. I found logistic loss to work better.

\item Report your training and validation loss.

  \subsubsection*{Solution}

  The results for the linear model can be seen in Figure
  \ref{fig:linear_loss_sgd} and Table \ref{tab:linear_sgd}. Training loss was
  0.122861 and validation loss was 0.199776.

  \begin{figure}
    \centering
    \includegraphics{problem4/linear_loss_sgd.pdf}
    \caption{Some overfiiting likely occurred since training loss is
      significantly less that validation loss. Increasing $\lambda$ made the
      model perform worse, however.}
    \label{fig:linear_loss_sgd}
  \end{figure}

  \begin{table}
    \centering
    \begin{tabular}{lrr}
      \toprule
      {} &  Average Precision Score &      Loss \\
      \midrule
      Training   &                 0.871461 &  0.122861 \\
      Validation &                 0.563140 &  0.199776 \\
      Test       &                 0.636070 &  0.177882 \\
      \bottomrule
    \end{tabular}
    \caption{Best average precision score and loss when training the linear
      model with SGD.}
    \label{tab:linear_sgd}
  \end{table}

  The results for the MLP model can be seen in Figure \ref{fig:mlp_loss_sgd} and
  Table \ref{tab:mlp_sgd}. Training loss was 0.157085 and validation loss was
  0.190817. I used $\lambda = 0.05$ as my regularization parameter and 1 layer
  of 256 hidden units. The learning rate was 0.0005. Validation loss is better
  than the linear model, but test loss is worst.

  \begin{figure}
    \centering
    \includegraphics{problem4/mlp_loss_sgd.pdf}
    \caption{The model appears to not have converged yet.}
    \label{fig:mlp_loss_sgd}
  \end{figure}

  \begin{table}
    \centering
    \begin{tabular}{lrr}
      \toprule
      {} &  Average Precision Score &      Loss \\
      \midrule
      Training   &                 0.708156 &  0.157085 \\
      Validation &                 0.565152 &  0.190817 \\
      Test       &                 0.596515 &  0.181875 \\
      \bottomrule
    \end{tabular}
    \caption{Best average precision score and loss when training the MLP
      model with SGD.}
    \label{tab:mlp_sgd}
  \end{table}

\item Compute the average precision score in the same manner, and plot these
  quantities.

  \subsubsection*{Solution}

  The results are in Tables \ref{tab:linear_sgd} and \ref{tab:mlp_sgd}. See
  Figures \ref{fig:linear_average_precision_score_sgd} and
  \ref{fig:mlp_average_precision_score_sgd}

  \begin{figure}
    \centering
    \includegraphics{problem4/linear_average_precision_score_sgd.pdf}
    \caption{Average precision score by the number of steps for the linear
      model.}
    \label{fig:linear_average_precision_score_sgd}
  \end{figure}

  \begin{figure}
    \centering
    \includegraphics{problem4/mlp_average_precision_score_sgd.pdf}
    \caption{Average precision score by the number of steps for the multi-layer
      perceptron model.}
    \label{fig:mlp_average_precision_score_sgd}
  \end{figure}

\item Report the overall average precision score on the test set. Also report
  the average precision score for each of the 18 classes separately. Comment on
  how the AP scores relate to the class imbalance.

  \subsubsection*{Solution}

  Sorry, I ran out of time and didn't get this far :(.

\end{enumerate}

\subsection*{Nesterov's method}

I used Nesterov's method with a momentum parameter of 0.9. All other parameter
were the same as last section. The results can be seen in Tables
\ref{tab:linear_nesterov} and \ref{tab:mlp_nesterov}. The precision for the MLP
model is notably better with an average precision score of 0.629526 on the test
set.

The progress of the training runs can be seen in Figures
\ref{fig:linear_loss_nesterov}, \ref{fig:mlp_loss_nesterov},
\ref{fig:linear_average_precision_score_nesterov}, and
\ref{fig:mlp_average_precision_score_nesterov}. This method trained the fastest
out of the three that I tried. In particular the linear model converged very
quickly.

  \begin{table}
    \centering
    \begin{tabular}{lrr}
      \toprule
      {} &  Average Precision Score &      Loss \\
      \midrule
      Training   &                 0.759774 &  0.170063 \\
      Validation &                 0.521453 &  0.262688 \\
      Test       &                 0.581486 &  0.232664 \\
      \bottomrule
    \end{tabular}
    \caption{Best average precision score and loss when training the linear
      model with SGD and Nesterov's momentum.}
    \label{tab:linear_nesterov}
  \end{table}

    \begin{table}
      \centering
      \begin{tabular}{lrr}
        \toprule
        {} &  Average Precision Score &      Loss \\
        \midrule
        Training   &                 0.793293 &  0.141046 \\
        Validation &                 0.587959 &  0.188455 \\
        Test       &                 0.629526 &  0.177171 \\
        \bottomrule
      \end{tabular}
      \caption{Best average precision score and loss when training the MLP
        model with SGD and Nesterov's momentum.}
    \label{tab:mlp_nesterov}
  \end{table}

  \begin{figure}
    \centering
    \includegraphics{problem4/linear_loss_nesterov.pdf}
    \caption{Evaluations on the datasets while training the linear model with
      Nesterov's momentum.}
    \label{fig:linear_loss_nesterov}
  \end{figure}

    \begin{figure}
    \centering
    \includegraphics{problem4/mlp_loss_nesterov.pdf}
    \caption{Evaluations on the datasets while training the MLP model with
      Nesterov's momentum.}
    \label{fig:mlp_loss_nesterov}
  \end{figure}

  \begin{figure}
    \centering
    \includegraphics{problem4/linear_average_precision_score_nesterov.pdf}
    \caption{Average precision score by the number of steps for the linear
      model.}
    \label{fig:linear_average_precision_score_nesterov}
  \end{figure}

  \begin{figure}
    \centering
    \includegraphics{problem4/mlp_average_precision_score_nesterov.pdf}
    \caption{Average precision score by the number of steps for the multi-layer
      perceptron model.}
    \label{fig:mlp_average_precision_score_nesterov}
  \end{figure}

  \subsection*{Adagrad}

  For the linear model, I used the same learning rate of $10^{-4}$. The results
  were very similar to the previous results. The results can be seen in Tables
  \ref{tab:linear_adagrad}.

  For the MLP model, I had trouble getting convergence. I used Adagrad method
  with a learning rate of $0.0005$. Judging from the results, this was likely,
  too small. Making it larger didn't seem to helper, either. Perhaps, step size
  decays, too, quickly. All other parameters were the same as last section. The
  results can be found in \ref{tab:mlp_adagrad}. The precision for the MLP model
  is notably better with an average precision score of 0.629526 on the test set.

  The progress of the training runs can be seen in Figures
  \ref{fig:linear_loss_adagrad}, \ref{fig:mlp_loss_adagrad},
  \ref{fig:linear_average_precision_score_adagrad}, and
  \ref{fig:mlp_average_precision_score_adagrad}.

  \begin{table}
    \centering
    \begin{tabular}{lrr}
      \toprule
      {} &  Average Precision Score &      Loss \\
      \midrule
      Training   &                 0.820442 &  0.136633 \\
      Validation &                 0.542389 &  0.204523 \\
      Test       &                 0.609165 &  0.184826 \\
      \bottomrule
    \end{tabular}
    \caption{Best average precision score and loss when training the linear
      model with SGD and Adagrad's momentum.}
    \label{tab:linear_adagrad}
  \end{table}

    \begin{table}
      \centering
      \begin{tabular}{lrr}
        \toprule
        {} &  Average Precision Score &      Loss \\
        \midrule
        Training   &                 0.605249 &  0.186454 \\
        Validation &                 0.531518 &  0.199406 \\
        Test       &                 0.546195 &  0.196277 \\
        \bottomrule
      \end{tabular}
      \caption{Best average precision score and loss when training the MLP
        model with SGD and Adagrad's momentum.}
    \label{tab:mlp_adagrad}
  \end{table}

  \begin{figure}
    \centering
    \includegraphics{problem4/linear_loss_adagrad.pdf}
    \caption{Evaluations on the datasets while training the linear model with
      Adagrad's momentum.}
    \label{fig:linear_loss_adagrad}
  \end{figure}

    \begin{figure}
    \centering
    \includegraphics{problem4/mlp_loss_adagrad.pdf}
    \caption{Evaluations on the datasets while training the MLP model with
      Adagrad's momentum.}
    \label{fig:mlp_loss_adagrad}
  \end{figure}

  \begin{figure}
    \centering
    \includegraphics{problem4/linear_average_precision_score_adagrad.pdf}
    \caption{Average precision score by the number of steps for the linear
      model.}
    \label{fig:linear_average_precision_score_adagrad}
  \end{figure}

  \begin{figure}
    \centering
    \includegraphics{problem4/mlp_average_precision_score_adagrad.pdf}
    \caption{Average precision score by the number of steps for the multi-layer
      perceptron model.}
    \label{fig:mlp_average_precision_score_adagrad}
  \end{figure}

\end{document}
% Local Variables:
% TeX-command-extra-options: "-shell-escape"
% End:


\documentclass[12pt]{article}
\usepackage[margin=1in]{geometry}
\usepackage[T1]{fontenc}
\usepackage[USenglish]{babel}
\usepackage[nodayofweek,level]{datetime}
\usepackage{amsfonts}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{tikz}
\usetikzlibrary{intersections,arrows.meta}
\usepackage{pgfplots}
\usepackage[scr]{rsfso}
\usepackage{array}
\usepackage{stackengine}
\stackMath

\usepackage{tikz,pgfplots}

\pgfplotsset{compat=1.10}
% Uncomment to use "fillbetween" function
% otherwise leave commented because
% the red highlighting is annoying
%\usepgfplotslibrary{fillbetween}

% Change for each new hw week
\newcommand{\dueDate}{\formatdate{13}{11}{2017}} % day/month/year
\newcommand{\hwNum}{6}

\begin{document}

\selectlanguage{USenglish}
%------------------------ Title Code ------------------------
\title{Homework: Week \hwNum}
\author{Joseph Ismailyan}
\date{}
\maketitle
\begin{flushleft}
Math 100 \\
Due: \dueDate \\
Professor Boltje \\
MWF 9:20a-10:25a
\end{flushleft}

%------------------------ Begin Page 1 ------------------------
\begin{minipage}[t]{0.40\textwidth}

\section*{Chapter 7}
\subsection*{10.}

\iffalse

\textbf{Proposition}:  \\If $ a\in \mathbb{Z} $, then $  a^3\equiv a\pmod{3}  $.
\newline\textit{Proof.} Suppose $ a\in \mathbb{Z} $. Consider $ (a-1)a(a+1) $. We can see that it is the product of three consecutive integers. It follows that \newline$ (a-1)a(a+1) =3!q, q\in \mathbb{Z}\newline =6q\newline=3(2q)\newline=3m, m=2q\in\mathbb{Z}.$ \newline Therefore $ 3|(a-1)a(a+1)\newline \rightarrow 3|(a^3-a)\rightarrow  a^3\equiv a\pmod{3}$.

\fi
\textbf{Proposition}:  \\If $ a\in \mathbb{Z} $, then $  a^3\equiv a\pmod{3}  $.
 Suppose $ a\in \mathbb{Z} $. Consider $ (a-1)a(a+1) $to be the product of three consecutive integers, so then the remainder when being divided by $ 3 $ must be either $ 0,1 \text{ or } 2 $. Then consider the first term of the series, $ a-1=3n \rightarrow 3|(a-1) $. Then the next in the series, $ a=3n $ so $ 3|a $. And lastly,  $ a-2=3n \rightarrow a=3n-2=3(n+1)-1 $ then for some integer $ m=n+1\in\mathbb{Z} \rightarrow a=3m-1\rightarrow a+1=3m$ so $ 3|(a+1) $. Therefore we can say that $ 3|(a-1)a(a+1) $ or just $ 3|(a^3-a) $ which implies $ a^3\equiv a\pmod{3} $.

\end{minipage}
% Creates verticle line
\hfill\vline\hfill
\begin{minipage}[t]{0.45\textwidth}

\subsection*{22.}
\textbf{Proposition}:  \\If $ n\in \mathbb{N} $, then $  4|n^2 $ or $ 4|(n^2-1)  $.
\newline\textit{Proof.} This proof requires two cases, one in which $ n $ is $ even $ and one in which $ n $ is $ odd $. Once both cases are proven, we can say that Case I is true or Case II is true, since every $ n $ in the set of natural numbers is either $ even $ or $ odd $.
\paragraph{} \textbf{Case I}
\newline
Suppose $ n $ is $ even $, therefore $ n=2k, n\in\mathbb{Z} $. Then $ n^2=(2k)^2=4k^2 $. We can then say $ n^2=4m,m=k^2\in\mathbb{Z} $ so $ 4|n^2 $.

\paragraph{} \textbf{Case II}
\newline
Suppose $ n $ is $ odd $, therefore $ n=2a+1, n\in\mathbb{Z} $. Then $ n^2=(2a+1)^2=4a^2+4a+1 $. We can then say $ n^2-1=4(a^2+a)=4b, b=(a^2+a)\in\mathbb{Z} $. Therefore $ 4|(n^2-1) $.

\end{minipage}
\pagebreak

%------------------------ End Page 1 ------------------------

%------------------------ Begin Page 2 ------------------------

\begin{minipage}[t]{0.40\textwidth}

\subsection*{26.}
\textbf{Proposition}:  \\The product of any $ n $ consecutive positive integers is divisible by $ n! $.
\newline\textit{Proof.} Suppose we write $ n $ consecutive positive integers as $ m,(m-1),(m-2),(m-3),...,(m-(n-1)) $. Then the product of these $ n $ consecutive positive integers would be $ m(m-1)(m-2)(m-3)...(m-(n-1))$. If we write $ \frac{m(m-1)(m-2)(m-3)...(m-(n-1))}{n!} $ then it is apparent that we have the number of ways to pick $ n $ items from a list of length $ m $. We can then multiply the expression by $ \frac{(m-n)!}{(m-n)!} $ then we get $ \frac{m(m-1)(m-2)(m-3)...(m-(n-1))(m-n)!}{n!(m-n)!} $ which can be simplified to $ \frac{m!}{(m-n)!n!} $. Recall that $ \frac{m!}{(m-n)!n!} $ is defined as $ {m}\choose{n} $ which yields an integer for all $ m\geq n $.

\subsection*{32.}
\textbf{Proposition}:  \\If $ n\in\mathbb{Z} $, then $ gcd(n,n+2)\in\{1,2\} $.
\newline\textit{Proof.} Suppose $ d=gcd(n,n+2) $, it follows that $ d|n $ and $ d|(n+2) $. So $ n=da, a\in\mathbb{Z} $ and $ n+2=dc, c\in\mathbb{Z} $. Then substituting $ n=da $ into $  n+2=dc $, we get $ da+2=dc\rightarrow 2=dc-da\rightarrow 2=d(c-a)=dm, m\in\mathbb{Z} $. Since $ 2=dm $, we know that $ d $ is a divisor of 2, and 2 has the divisors $ \pm1,\pm2 $. Since $ d $ is the \textbf{greatest} common divisor, then $ d=1 $ or $ d=2 $. Therefore $ d\in\{1,2\} $ and since $ d=gcd(n,n+2) $ then $ gcd(n,n+2)\in\{1,2\} $.

\end{minipage}
% Creates verticle line
\hfill\vline\hfill
\begin{minipage}[t]{0.45\textwidth}

\subsection*{32.}
\textbf{Proposition}:  \\If $ \text{gcd}(a,b)=\text{gcd}(b,c)=1 $, then $ \text{gcd}(ab,c)=1 $.
\newline\textit{Proof.} Suppose $ \text{gcd}(a,b)=\text{gcd}(b,c)=1 $. Then by \textbf{Proposition 7.1} we can say $ \text{gcd}(a,b)=ak+cl, k,l\in\mathbb{Z} $ and $ \text{gcd}(b,c)=bn+cm, n,m\in\mathbb{Z} $. Then $ ak=1-cl $ and $ bn=1-cm $. We can then multiply $ ak $ and $ bn $ which results in $ ab(kn)=(1-cl)(1-cm)=1-cm-cl-c^2lm $ Then $ 1=ab(kn)+c^2lm+cm+cl=ab(kn)+c(clm+m+l)\rightarrow 1=abx+cy, x=km\in\mathbb{Z},y=clm+l+m\in\mathbb{Z}  $. Since $ 1=abx+cy $, then $ \text{gcd}(ab,c)=1 $.

\subsection*{36.}
\textbf{Proposition}:  \\Suppose $ a,b\in\mathbb{N} $. Then $ a=\text{lcm}(a,b) $ if and only if $ b|a $.
\newline\textit{Proof.} Suppose $ a=\text{lcm}(a,b) $. By definition, this means $ b|a $ and $ a|a $.
Next, suppose $ b|a $. Then $ a=bn, n\in\mathbb{Z} $ and $ a=ad, d\in\mathbb{Z} $ then $ a\geq \text{lcm}(a,b) $. Then since $ a|\text{lcm}(a,b) $, we have $ \text{lcm}(a,b) =ac, c\in\mathbb{Z}$, which tells us $ a\leq \text{lcm}(a,b) $. Since  $ a\geq \text{lcm}(a,b) $ and $ a\leq \text{lcm}(a,b) $, we know that $ a= \text{lcm}(a,b) $.

\end{minipage}
\pagebreak

%------------------------ End Page 2 ------------------------

%------------------------ Begin Page 3 ------------------------

\begin{minipage}[t]{0.40\textwidth}

\section*{Chapter 8}
\subsection*{6.}
\textbf{Proposition}:  \\Suppose $ A,B $, and $ C $ are sets. Prove that if $ A\subseteq B $, then $ A-C\subseteq A-B $.
\newline\textit{Proof.} Suppose $ A,B $, and $ C $ are sets and $ A\subseteq B $. Then \\$ A-C=\{x:(x\in A)\land(x\not\in\ C)\} $. Since $ A\subseteq B $, then $ x\in B $. Then $ B-C=\{x:(x\in B)\land(x\not\in\ C)\} $. Since $ x\in A-C $ and $ x\in B-C $, then $ A-C\subseteq A-B $.

\subsection*{12.}
\textbf{Proposition}:  \\Suppose $ A,B $, and $ C $ are sets. then $ A-(B\cap C)=(A-B)\cup(A-C) $.
\newline\textit{Proof.} Consider $ A-(B\cap C)\\= \{x:(x\in A)\land(x\not\in(B\land C))\}\\=\{x:(x\in A)\land(x\not\in B\land x\not\in C))\}$\\By DeMorgan's Law\\$ \{x:((x\in A)\land (x\not\in B))\lor ((x\in A)\land (x\not\in C))\} $\\By Distributive Law\\ Since \\$ A-B=\{x:(x\in A)\land(x\not\in\ B)\} $\\and\\ $ A-C=\{x:(x\in A)\land(c\not\in\ C)\} $ then since $ (x\in A-B) \lor (x\in A-C)\\\rightarrow (A-B)\cup (A-C)$.

\end{minipage}
% Creates verticle line
\hfill\vline\hfill
\begin{minipage}[t]{0.45\textwidth}

\subsection*{22.}
\textbf{Proposition}:  \\Let $ A $ and $ B $ be sets. Prove that $ A\subseteq B $ if and only if $ A\cap B=A $.
\newline\textit{Proof.} Suppose $ A\cap B=A $. Let $ x\in A $ be an arbitrary element. Then since $ A\cap B=A $, we know $ x\in A\cap B $. So $ x\in B $, therefore $ A\subseteq B $.
\\Next suppose $ A\subseteq B $. If $ x\in A\cap B $, then $ x\in A $ so $ A\cap B\subseteq A $.
\\Finally, suppose $ A\subseteq B $. If $ x\in A $ then $ x\in B $, which implies $ x\subseteq A\cap B $. Therefore $ A\subseteq A\cap B $. Since $ A\cap B\subseteq A  $ and $ A\subseteq A\cap B $ then $ A\cap B=A $.

\subsection*{28.}
\textbf{Proposition}:  \\Prove $ \{12a+25b:a,b\in\mathbb{Z} \}=\mathbb{Z} $.
\newline\textit{Proof.} Suppose $ 12a+25b $. Since $ a,b\in\mathbb{Z} $,$  12a+25b\in\mathbb{Z}  $.
\\Next, suppose the set $ X=\{12a+25b:a,b\in\mathbb{Z} \}$, the set $ Y=\mathbb{Z} $, and $ y\in Y $. Then consider $ 12(-2)+25(1)=1 $ where $ a=-2 $ and $ b=1 $. Then we can multiply the equation by $ y $ and get $ 12(-2y)+25(y)=y $, where $ a=-2y $  and  $ b=y $. Then $ y\in Y $ implies $ y\in X $, therefore $ Y\subseteq X $. So $ \{12a+25b:a,b\in\mathbb{Z} \}=\mathbb{Z} $.

\end{minipage}
\pagebreak

%------------------------ End Page 4 ------------------------

\end{document}


%=========================================================================
% sec-opts-loop
%=========================================================================

\section{Changing Loop Ordering}
\label{sec-opts-loop}

% Reason for optimization
\subsection{Reason for Optimization}

The memory access pattern for DGEMM is largely determined by the order in
which the matrices are traversed. Depending on the loop order and the
arrangement of the matrices in memory (i.e., row-major vs. column-major),
memory could be accessed with a unit-stride that maximizes temporal and
spatial locality, or a stride equal to the matrix dimensions that is more
likely to cause cache misses.
\smallskip

The goal of this optimization is to find the loop ordering that uses the
data in a cache line as much as possible before bringing in another cache
line. We explore a variety of loop orderings based on an intuitive
analysis of how to achieve unit-stride memory accesses to identify the
optimal loop ordering.
\smallskip

% Details of optimization
\subsection{Details of Optimization}

\input{fig-opts-loop-access}

The naive loop ordering of {\tt{ijk}} iterates across the output matrix
{\tt{C}}, traversing the columns before traversing the rows. Computing
each element in the output matrix requires traversing the columns of
{\tt{A}} and traversing the rows of {\tt{B}}. This type of access pattern
is generalized in Figure~\ref{fig-opts-loop-access}.
\smallskip

Because the matrices are arranged in a column-major pattern in memory,
it is always preferable to traverse the rows (i.e., vertically) to yield
unit-stride memory accesses. Any time we traverse the columns (i.e.,
horizontally), we only use a single element in a cache line before
accessing another cache line, which makes it more likely to have cache
misses.
\smallskip

Given this, we can see that the naive loop ordering actually generates a
very hostile memory access pattern since the innermost loop always
traverses the columns of {\tt{A}} and the middle loop traverses the
columns of {\tt{C}}.
\smallskip

Fortunately, we can leverage this insight to make an educated guess about
which loop ordering would be preferable. Since the memory access pattern
of the innermost loop is the most critical in exploiting temporal
locality, it makes sense to iterate across {\tt{i}} for the innermost
loop, as this will allow us to traverse the rows of both {\tt{A}} and
{\tt{C}}, while using a single element from {\tt{B}} to calculate the
partial products for an entire column of {\tt{C}}. Note that if the block
size is large, traversing the rows will still require accessing multiple
cache lines, but accessing consecutive cache lines will help avoid
conflict misses more than if we traversed the columns. This means that
either the {\tt{jki}} or the {\tt{kji}} ordering will yield the most
desirable memory access patterns. The former allows us to reuse the same
data for {\tt{B}} and {\tt{C}} while only shifting one column over in
{\tt{A}}, whereas the latter only allows us to reuse the same data in
{\tt{A}}. See Figure~\ref{fig-opts-loop-access} for visual comparisons.
\smallskip

% Results and analysis
\subsection{Results}

\input{fig-opts-loop-results}

The results of all possible loop orderings are shown in
Figure~\ref{fig-opts-loop-results}. We only show the effects of changing
the loop ordering of the computation kernel operating on a given
block. Although we did experiment with changing the loop ordering of the
outer loop that traverses the blocks, the effects on performance were
negligible. This was slightly contrary to expectations as the order in
which the blocks are traversed should impact the reuse of data in
lower-level caches (e.g., L2, L3), whereas the loop ordering of the
computation kernel is more focused on L1 cache utilization.
\smallskip

As expected, the {\tt{jki}} ordering shows the greatest improvement in
performance, followed by the {\tt{kji}} ordering. The other orderings
either exacerbate the sub-optimal memory access patterns or have
negligible improvements over the naive ordering.
\smallskip

One caveat with not iterating over {\tt{k}} in the innermost loop is that
we cannot accumulate the partial products for a given element in the
output matrix in the local register storage. For example, in the
{\tt{jki}} ordering, we must store the partial product for all elements
in a given column in the output matrix to memory, then load each one back
again for the accumulation when {\tt{k}} increments. However, since using
the optimal loop ordering encourages reuse of data in the L1 cache, the
overhead of storing/loading of partial products is not as significant as
if we were using the naive loop ordering.
\medskip

\clearpage


\documentclass[11pt]{article}
\usepackage{amsfonts, amsmath, amssymb, amsthm, fullpage, mdwlist, fancyhdr}

\setlength{\textheight}{9.25in}
\setlength{\textwidth}{6.5in}
\setlength{\topmargin}{0.0in}
\setlength{\headheight}{0.0in}
\setlength{\headsep}{0.0in}
\setlength{\leftmargin}{0.0in}
\setlength{\oddsidemargin}{0.0in}
\setlength{\parindent}{0pc}

\begin{document}

\title{Polynomials II}
\author{Alex Zhu}
\date{KSA Math Olympiad Training 2013}
% date?

\maketitle

\section{Problems}

\begin{enumerate}
  \item (HMMT 2008) Let $S$ be the set of points $(a,b)$ with $0 \leq a,b \leq 1$ such that the equation \[ x^4 + ax^3 - bx^2 + ax + 1 = 0 \] has at least one real root. Determine the area of the graph of $S$.

  \item Let $n$ be a positive odd integer, and let $P$ be a degree $n$ polynomial with $n$ distinct real roots. Prove that $P(P(x))$ has at least $n$ real roots.

  \item Let $x_1, x_2, \ldots, x_n$ be distinct real numbers. Let $b_i = \prod_{j \neq i} (x_i - x_j)$, and let $d_i = \prod_{j \neq i} x_j$. Find $\sum_{i=1}^n \frac{d_i}{b_i}.$

  \item (IMO 1974) Let $P(x)$ be a polynomial of degree $d \geq 1$ with integer coefficients. Let $n(P)$ be the number of integers $k$ for which $P(k)^2 = 1$. Prove that $n(P) - d \leq 2$.

  \item $P$ is a polynomial of degree $n$ with integer coefficients, with $n$ distinct real roots lying between 0 and 1 inclusive. Show that the leading coefficient of $P$ is larger than $2^n$.

  \item Suppose $P$ is a polynomial of degree $n$ with distinct complex roots $\alpha_1, \alpha_2, \ldots, \alpha_n$. The polynomials $Q_1, Q_2, \ldots, Q_n$ are defined so that $P(z) = (z - \alpha_k) Q_k(z)$ for each $z$. Prove that
  \[ Q_1(\alpha_1)^{-1} + Q_2(\alpha_2)^{-1} + \cdots + Q_n(\alpha_n)^{-1} = 0. \]

  \item (IMO 1976) Let $P_1(x) = x^2 - 2$ and $P_j(x) = P_1(P_{j-1}(x))$ for $j \geq 2$. Prove that for any positive integer $n$, the roots of the equation $P_n(x) = x$ are all real and distinct.

  \item (IMO 1975) Determine all polynomials $P$ of two variables satisfying the following three properties:
  \begin{enumerate}
    \item There is a positive integer $n$ such that for any real numbers $t,x,y$, we have $P(tx,ty) = t^n P(x,y)$.
    \item For any real numbers $a,b,c$ we have $P(a+b,c) + P(b+c,a) + P(c+a,b) = 0$.
    \item $P(1,0) = 1$.
  \end{enumerate}

  \item Suppose $p$ is a polynomial with integer coefficients such that $p(n) > n$ for all positive integers $n$. Define the sequence $x_1, x_2, \ldots$ so that $x_1 = 1$ and $x_{n+1} = p(x_n)$ for $n \geq 1$. Suppose that for each positive integer $N$, there exists some $n$ such that $N$ divides $x_n$. Show that $p(x) = x + 1$.

  \item Let $P$ be a polynomial of degree $n$ with real coefficients such that for all $x \in [-1, 1]$, we have $P(x) \in [-1, 1]$. Show that the leading coefficient of $P$ has absolute value at most $2^{n-1}$.

  %\item Let $f$ be a primitive polynomial with integer coefficients of degree $n$ for which there exist distinct integers $x_1, x_2, \ldots, x_n$ such that
  %\[ 0 < |f(x_i)| < \frac{\lfloor \frac{n+1}{2} \rfloor}{2^{\lfloor \frac{n+1}{2} \rfloor}}. \]
  %Prove that $f$ is irreducible in $\mathbb{Z}[x]$.
\end{enumerate}

\end{document}


\documentclass{article}
\pagestyle{empty}
\usepackage{amsmath,amssymb,amsfonts}
\usepackage{graphicx}
\usepackage{multicol}
\setlength{\oddsidemargin}{0in} \setlength{\evensidemargin}{0in}
\setlength{\topmargin}{0in} \setlength{\textheight}{8.5in}
\setlength{\textwidth}{6.5in}

\makeatletter
\renewcommand*\env@matrix[1][*\c@MaxMatrixCols c]{%
	\hskip -\arraycolsep
	\let\@ifnextchar\new@ifnextchar
	\array{#1}}
\makeatother

\begin{document}
\begin{flushleft}
	\bfseries{MATH 260, Linear Algebra, Spring `14}\\
	\bfseries{Activity 6:  Determinants and Cramer's Rule}\\
	\bfseries{Honor Code:} \hspace{3.5in}\bfseries{Names:}\\
\end{flushleft}
\begin{flushleft}
\vspace{.75in}
Directions:  Everyone should work on the assignment and should fill out their paper.  You are expected to make corrections based on what is presented on the board.  \\
\large The worksheet is due by the end of the day (5pm). \normalsize \\
If you need more explanations after class, you can read Chapter 3.4 of your textbook.\\
\vspace{0.1in}
\Large
In-Class Learning Goals:\\
\normalsize
\begin{enumerate}
\item Be able to find the determinant of a matrix (both $2\times 2$ AND nXn).
\item Be able to find the `minor' and `cofactor' of elements in a matrix.
\item Be able to find solutions to a system using \textit{Cramer's Rule}.
\item (Stretch) Discover several properties of determinants including if a matrix has an inverse.
\end{enumerate}

\vspace{0.1in}
\begin{center}
\section*{Warmup: Determinant of a $2 \times 2$}
$|\textbf{A}| = \begin{vmatrix}[rr] a_{11} & a_{12} \\ a_{21} & a_{22} \end{vmatrix} = a_{11} a_{22} - a_{12} a_{21}$\\
\vspace{0.2in}
\textit{Note:} You will need to have this formula memorized!\\
\end{center}
Find the determinant of the following matrix:\\
\begin{center}
$\begin{bmatrix}[rr]
-3 & 1   \\
4 & 2
\end{bmatrix}
$
\hspace{1in}
$\begin{bmatrix}[rrr]
3 & 0 & -1\\
0 & 0 &  3 \\
0 & 1 & 0
\end{bmatrix}
$
\end{center}
\vspace{2in}

\newpage
\begin{center}
\section{Some Properties and Practice}
\end{center}
For \textbf{Activity 5} you found several inverses and transposes of matrices. Lets use them...\\
\begin{center}
$\textbf{H}=\begin{bmatrix}[rr]
1 & 1\\
4  & 1
\end{bmatrix}$
\hspace{0.5in}
$\textbf{H}^{-1}=\left[
\begin{array}{cc}
- \frac{1}{3} & \frac{1}{3} \vspace{0.1in}\\

\frac{4}{3} & - \frac{1}{3}
\end{array} \right]$
\hspace{0.5in}
$
\textbf{H}^{T}=\begin{bmatrix}[rr]
1 & 4\\
1  & 1
\end{bmatrix}$
\hspace{0.5in}
$\textbf{G}=
\begin{bmatrix}[rrr]
3 & 0 & 3\\
-1 & 2 & 1\\
1 & 1 & 2
\end{bmatrix}$\\
\end{center}

\vspace{0.1in}
a) Find $|\textbf{H}|$.\\
\vspace{0.8in}
b) Find $|\textbf{H}^{-1}|$.\\
\vspace{0.8in}
c) Find $|\textbf{H}^{T}|$.\\
\vspace{0.8in}
d) Find $|\textbf{G}|$.\\
\vspace{1in}
e) Are any of the numbers above the same? Closely related? Try to write each as a generalized property (check these with me).
\newpage
\begin{center}
\section{Finding Minors (and some $2 \times 2$ practice)}
\small This method is only a little harder than visiting an elementary school... \normalsize
\end{center}
\textbf{Definition:} For the $n \times n$ matrix \textbf{A}, the \textit{minor $M_{ij}$} of $a_{ij}$ is an $(n-1) \times (n-1)$ matrix obtained by deleting the $i$th row and the $j$th column of \textbf{A}. \\
\begin{center}
$\textbf{A}=
\begin{bmatrix}
\bullet &\bullet &\bullet & \bullet \\
\bullet &\bullet &\bullet & \bullet \\
\bullet &\bullet &\bullet & \bullet \\
\bullet &\bullet &\bullet & \bullet \\
\end{bmatrix}
=
\begin{bmatrix}[rrrr]
1&2&3&1\\
5&0&1&-2\\
4&0&1&0\\
2&0&3&1
\end{bmatrix}
$
\hspace{0.4in}
$\textbf{M}_{12}=
\begin{bmatrix}
\circ & \circ & \circ & \circ \\
\bullet & \circ & \bullet & \bullet \\
\bullet & \circ & \bullet & \bullet \\
\bullet & \circ & \bullet & \bullet \\
\end{bmatrix}
\rightarrow
\begin{bmatrix}
\bullet & \bullet & \bullet \\
\bullet & \bullet & \bullet \\
\bullet & \bullet & \bullet \\
\end{bmatrix}
=
\begin{bmatrix}[rrr]
5 & 1&-2\\
4 & 1& 0\\
2&3&1
\end{bmatrix}$
\end{center}
a) Find the minor $\textbf{M}_{32}$ of \textbf{A}. To show your work, re-write \textbf{A} first...\\
\vspace{1.5in}
b) Lets find some minors of minors... \\
(i) Find $\textbf{M}_{21}$ of $\textbf{M}_{12}$\\
\vspace{0.75in}
(ii) Find $\textbf{M}_{22}$ of $\textbf{M}_{12}$\\
\vspace{0.75in}
(iii) Find $\textbf{M}_{23}$ of $\textbf{M}_{12}$\\
\vspace{0.75in}
c) Find the determinant of...\\
(i) $\textbf{M}_{21}$\\
\vspace{0.75in}
(ii) $\textbf{M}_{22}$\\
\vspace{0.75in}
(iii) $\textbf{M}_{23}$

\newpage
\begin{center}
\section{Co-Factors}
\end{center}
\textbf{Definition:} The \textit{co-factor} of an element $a_{ij}$ is the scalar: $C_{ij}=(-1)^{i+j}|\textbf{M}_{ij}|$\\
\vspace{0.1in}
Notice that the co-factor is just the determinant of a minor, times $\pm 1$. We'll use this in the next section...
a) Find $C_{21}$ of $\textbf{M}_{12}$\\
\vspace{0.75in}
b) Find $C_{22}$ of $\textbf{M}_{12}$\\
\vspace{0.75in}
c) Find $C_{23}$ of $\textbf{M}_{12}$\\
\vspace{0.75in}
d) Can you find $C_{12}$ of \textbf{A} (based on what you know now)? Why or why not?\\
\vspace{1in}

\begin{center}
\section{Determinants of N x N matrices}
\end{center}
Lets start by finding the determinant of the 3x3 matrix $\textbf{M}_{12}$. \\
To make our notation simpler, lets call $\textbf{M}_{12}$ matrix \textbf{D}.\\
a) Find the following sum: \hspace{0.2in} $d_{21}*C_{21} + d_{22}*C_{22} + d_{23}*C_{23}$ \\
\textit{Hint: You should get a single, scalar number.}\\
\vspace{1in}
b) Try writing the sum for (a) in summation notation (i.e. using: $\Sigma$).\\

\newpage
\LARGE \begin{center} Determinants of N x N matrices (cont.) \end{center} \normalsize
Finding the determinant of an NxN matrix is a recursive process. The definition is:\\
\vspace{0.1in}
\textbf{Definition:} Choose a row $i$ or column $j$, then $|A|=\sum\limits^{n}_{(j \text{ or } i)} a_{ij} C_{ij} = \sum\limits^{n}_{j \text{ or } i} a_{ij} (-1)^{i+j} |\textbf{M}_{ij}| $\\
\vspace{0.1in}
Notice that $\textbf{M}_{ij}$ may not be a $2 \times 2$!! You might have to use this process several times. \\
\vspace{0.1in}
c) Using this definition find the determinant of our original \textbf{A} matrix. \\
\textit{Hint: You should only need to do a few new multiplications and additions, you calculated most of it already.}\\
\vspace{1in}

\end{flushleft}
\end{document}

Spring 2014 - Schmitt/Activity Latex/Activity_6_260_Determinants.tex


\documentclass[12pt]{article}

\include{preamble}

\newtoggle{spacingmode}
\toggletrue{spacingmode}  %STUDENTS: DELETE or COMMENT this line

\newtoggle{professormode}
\toggletrue{professormode} %STUDENTS: DELETE or COMMENT this line

\newcommand{\spc}[1]{\iftoggle{spacingmode}{\\ \vspace{#1cm}}}

\title{MATH 241 Fall \the\year~Homework \#6}

\author{Professor Adam Kapelner} % STUDENTS: DELETE my name and put your name and section here e.g. \author{John Doe, Section A}. MAKE SURE YOU PUT YOUR SECTION HERE!!!!!!!!

\iftoggle{professormode}{
\date{NOT DUE but covered on the final exam \\ \vspace{0.5cm} \small (this document last updated \today ~at \currenttime)}
}

\renewcommand{\abstractname}{Instructions and Philosophy}

\begin{document}
\maketitle

\iftoggle{professormode}{
\begin{abstract}
The path to success in this class is to do many problems. Unlike other courses, exclusively doing reading(s) will not help. Coming to lecture is akin to watching workout videos; thinking about and solving problems on your own is the actual ``working out''.  Feel free to \qu{work out} with others; \textbf{I want you to work on this in groups.}

Reading is still \textit{required}. For this homework set, read about the central limit theorem, confidence intervals and hypothesis tests for one proportion.

The problems below are color coded: \ingreen{green} problems are considered \textit{easy} and marked \qu{[easy]}; \inorange{yellow} problems are considered \textit{intermediate} and marked \qu{[harder]}, \inred{red} problems are considered \textit{difficult} and marked \qu{[difficult]} and \inpurple{purple} problems are extra credit. The \textit{easy} problems are intended to be ``giveaways'' if you went to class. Do as much as you can of the others; I expect you to at least attempt the \textit{difficult} problems.

This homework is worth 100 points but the point distribution will not be determined until after the due date. See syllabus for the policy on late homework.

Up to [see syllabus] points are given as a bonus if the homework is typed using \LaTeX. Links to instaling \LaTeX~and program for compiling \LaTeX~is found on the syllabus. You are encouraged to use \url{overleaf.com}. If you are handing in homework this way, read the comments in the code; there are two lines to comment out and you should replace my name with yours and write your section. If you are asked to make drawings, you can take a picture of your handwritten drawing and insert them as figures or leave space using the \qu{$\backslash$vspace} command and draw them in after printing or attach them stapled.

The document is available with spaces for you to write your answers. If not using \LaTeX, print this document and write in your answers. I do not accept homeworks not on this printout. Keep this first page printed for your records. Write your name below.

\end{abstract}

\thispagestyle{empty}
\vspace{1cm}
NAME: \line(1,0){240} %~~SECTION (A or B): \line(1,0){35}
\pagebreak
}~\\

\problem{Now that we've now proven the CLT, we are going to use it a little bit. Assume $X_1, \ldots, X_n \iid$ something with mean $\mu$ and standard error $\sigma$.}

\begin{enumerate}

\easysubproblem{Assume $n$ is large enough for the CLT to kick in, show that:

\beqn
\Xbar \sim \normnot{\mu}{\squared{\frac{\sigma}{\sqrt{n}}}}
\eeqn }\spc{3}

\easysubproblem{Assume $n$ is large enough for the CLT to kick in, show that:

\beqn
T_n \sim \normnot{n\mu}{\squared{\sqrt{n}\sigma}}
\eeqn }\spc{3}

\easysubproblem{Let $X_1, \ldots, X_n \iid \bernoulli{p}$ and let $\Phat := \Xbar$. Assume $n$ is large enough for the CLT to kick in, show that:

\beqn
\Phat \sim \normnot{p}{\squared{\sqrt{\frac{p(1-p))}{n}}}}
\eeqn }\spc{3}

\easysubproblem{Draw the PDF of $\Phat_n$ below. On the $\phat$ axis, mark a tick for the mean, and ticks for three standard errors above the mean and ticks for three standard errors below the mean. I'm giving almost a full page for this because other problems depend on it. Use 1/3 of the space for your drawing of the PDF.  }\spc{15}

\hardsubproblem{If $\Xoneton \iid \bernoulli{p}$ and assuming $n$ is big enough for the CLT to \qu{kick in} as well as what you know about normal distributions shifted and scaled, provide the PDF of $\Phat_n$. This will force you to understand (1) the generalized density for $\normnot{\mu}{\sigsq}$ which is in the notes but has never been previously asked on a homework assignment, (2) substitutions for the mean and variance and (3) what the free variable is. }\spc{2}

\intermediatesubproblem{Let's say $p=0.2$ and $n = 100$. What is $\prob{\Phat_n \geq 0.24}$? You will need to use shifts and scales to get it to look like $\prob{Z > z}$ where $Z$ is the standard normal and $z$ is 0.24 standardized.  }\spc{6}

%\hardsubproblem{Let's say $p=0.2$ and $n = 100$. What is the probability of realizing a $\phat$ above 0.3? Without a computer, you cannot answer this exactly. Just using the empirical rule, find the lower and upper bound of this probability. Do not answer exactly. I will not ask you to answer exactly on the test but I will ask you to find a lower and upper bound using the empirical rule.  }\spc{5}

\end{enumerate}

\iftoggle{professormode}{
\paragraph{Basics of Statistical Inference}~\\ \\
}

\problem{We have now used the CLT to understand the distribution of sample proportions. Here, we will assume that $p$, the true proportion of \qu{success,} is unknown and we will try to infer it. This is a problem of paramount importance. Imagine you're polling for a political candidate. You really do want to know $p$ --- the proportion of people who will vote for your candidate. Imagine you're investigating a drug with side effects. You really do want to know $p$ --- the proportion of people who suffer from that side effect.}

\begin{enumerate}

\easysubproblem{If you have realizations of $\Xoneton \iid \bernoulli{p}$, call them $\xoneton$, what is your best guess of $p$? This is called a \qu{point estimate.} Make sure to use the special notation for this estimate we used in class and define it mathematically.  }\spc{2}

\hardsubproblem{Write \textit{in English} why $\phat \neq p$. I marked this as hard because you need to really use your knowledge about random variables, realizations and parameters and get all these concepts straight.  }\spc{4}

\intermediatesubproblem{Reference the picture in 3(d). Where are the \qu{middle} 95\% of $\phat$'s realized? When I say \qu{middle} I mean the 95\% that are \textit{closest} to $p$, the true expected value. Write the interval that corresponds to this set of numbers. Use the bracket set notation we learned in lecture 1.  }\spc{2}

%\intermediatesubproblem{The interval you created in (c) left out 5\% of the density of $\Phat_n$. Write the set that corresponds to this left out set. Use the union notation from lecture 1.  }\spc{2}

\easysubproblem{What is the width of the interval you created in (c)?  }\spc{0.5}

\easysubproblem{What is the half width of the interval you created in (c)?  }\spc{1}

\easysubproblem{In your picture in 3(d), draw a dotted line from the center of the PDF, $p$ down the length of the page. Create a realization of $\Phat$, call it $\phat$ within the interval in (c) and mark it as a solid dot. Then carefully measure using a ruler, one half width (f) from the fot to the left and one half width from (f) to the right and draw these two lines and mark the ends with a square bracket. No need to write anything below this prompt.}

\easysubproblem{Did this interval \textit{cover} $p$? That is, does it include $p$? You can see if it includes the dotted line drawn down from $p$.  }\spc{1}

\easysubproblem{Would all $\phat$'s realized within the interval in (c) cover $p$?  Yes/no.}\spc{1}

\hardsubproblem{Prove that the probability that the interval covers $p$ is 95\%. We did this in class.  }\spc{4}

\easysubproblem{Illustrate an example of a $\phat$ with the same half-width to the left and right interval that does \textit{not} cover $p$ in your picture of 3(d). No need to write anything here.}

\intermediatesubproblem{In what set do the $\phat$'s such as in (k) come from that fail to cover $p$?  }\spc{1}

\end{enumerate}

\problem{Now we'll just practice building CI's and ask some questions about the procedure.}

\begin{enumerate}

\intermediatesubproblem{Why is it important to have a simple random sample (a completely random sample) when sampling from a population? Write a couple sentences \textit{in English}.  }\spc{4}

\easysubproblem{Write the mathematical definition of a two-sided $1-\alpha$ confidence interval below for a one-sample binomial proportion.  }\spc{1}

\easysubproblem{In the notation above, the \qu{CI} has two subscripts. What do these two subscripts mean?  }\spc{2}

\hardsubproblem{The above CI in (b) does not give \textit{exact} coverage, but only \textit{approximate} coverage for two separate reasons. What are the two reasons? Assume the realizations are indeed sampled from $\iid$ Bernoulli r.v.'s and it was a simple random sample.  }\spc{4}

\easysubproblem{As in the example done in class, imagine we sampled 594 M\&M's and found that 116 were blue. Compute a 95\% CI for $p$, the true proportion of blue M\&M's. Assume all of the CI assumptions are correct.  }\spc{2}

\easysubproblem{Compute a 99\% CI for $p$, the true proportion of blue M\&M's using the same data as in (e). Assume all of the CI assumptions are correct. I gave you this $z_{\overtwo{\alpha}}$ value in class. }\spc{4}

\hardsubproblem{The 99\% CI is larger than the 95\% CI. This is because $z_{\overtwo{\alpha}}$ increased because we requested wider confidence which increases the margin of error (the margin of error is the $z_{\overtwo{\alpha}}\sqrt{\frac{\phat(1-\phat)}{n}}$ term). Let's say we wanted a 99\% CI with the same width as the 95\% CI. How much more M\&M's would we have to look at to do so?  }\spc{6}

\intermediatesubproblem{Let's say we wanted to be sure we captured $p$ in the CI. What's the problem with just making our $1-\alpha$ really large like 99.99999\% (or $\alpha$ really small like 0.000001\%)? Answer \textit{in English}.  }\spc{4}

\easysubproblem{For the interval you created in (e) provide three valid classical interpretations for it \textit{in English}.  }\spc{5}

%\easysubproblem{For the interval you created in (e) write the non-classical i.e subjective interpretation \textit{in English} --- this is the interpretation you really want to be able to say...  }\spc{3}

\easysubproblem{We were interested in political orientation at Queens College. We polled 100 students and 38 of them were registered democrats. Create a 95\% confidence interval for the true proportion of registered democrats at Queens College.  }\spc{4}

\easysubproblem{To get these 100 students, imagine we asked students who were waiting at the Q64 bus stop. Would there then be any problems with the CI we created in (k)? Answer \textit{in English}.  }\spc{3}

\intermediatesubproblem{What would be the best way (or at least a better way) to sample 100 students in order to create the most accurate CI in (k)? Answer \textit{in English}.  }\spc{7}

\end{enumerate}

\problem{CI's are for inference but hypothesis testing is for assessing whether a certain theory is grounded. This will be the last topic covered on the Math 241 final. We will illustrate by using the human sex ratio example from class.}

\begin{enumerate}

\easysubproblem{We a priori assume an equal sex ratio. We'll define $p$ as the probability of being male arbitrarily. What is the null hypothesis expressed using the mathematical notation we used in class? }\spc{1}

\easysubproblem{What would be the alternative hypothesis expressed using the mathematical notation we used in class? }\spc{1}

\intermediatesubproblem{If $n=200$, what is the null distribution? No symbols are acceptable as an answer, you must calculate exactly what the distribution is. Draw this null distribution how we did in class. }\spc{6}

\intermediatesubproblem{Let's say failing to find a deviation from the hypothesized equal sex ratio meant you get fired from your boss (who believes in the unequal sex ratio theory). What kind of $\alpha$ would you pick for this test? Justify using a sentence \textit{in English}. }\spc{3}

\easysubproblem{Regardless of the $\alpha$ chosen, choose $\alpha = 1\%$. What is the retainment region under $\alpha = 1\%$? Denote it on the drawing in (c). }\spc{2}

\easysubproblem{What is the rejection region under $\alpha = 1\%$? Use the $\cup$ symbol to join the two sets. Denote it on the drawing in (c). }\spc{2}

\easysubproblem{What is a Type I error in general? Answer \textit{in English}. }\spc{3}

\easysubproblem{What does a Type I error mean in this human sex ratio experiment? Answer \textit{in English}. }\spc{3}

\easysubproblem{What is the probability of the Type I error in this example? }\spc{1}

\easysubproblem{What is a Type II error in general? Answer \textit{in English}. }\spc{3}

\easysubproblem{What does a Type II error mean in this human sex ratio experiment? Answer \textit{in English}. }\spc{3}

\easysubproblem{You count there are 113 males. Calculate $\phat$ and denote it on the drawing in (c).  }\spc{3}

\easysubproblem{What is the result of the hypothesis test? Do you retain $H_0$ or reject $H_0$? Write a sentence in English interpreting what this means.  }\spc{4}

\hardsubproblem{Now let $n=4,247,000$ (the number of American newborns in the year 2008) and use the same $\alpha =1\%$. You get 2,173,000 males. Write the hypotheses, find the retainment region and find the result and interpret it. }\spc{9}

%\extracreditsubproblem{Why is the human sex ratio not even? Write your thoughts below with sources. }\spc{5}

\end{enumerate}

\problem{This last example involved two-sided hypothesis tests of one proportion. Here, we will examine the case of firing the Uber driver where we only care about only testing one side.

The Uber driver takes 1,000 rides and then the company makes a decision about whether or not he's a good driver based on an arbitrary rule of having more than $p=$ 5\% true proportion / propensity of bad rides. [optional]}

\begin{enumerate}

\easysubproblem{Detecting him being a bad driver is sounding the alarm. Thus the status quo is that he's a good driver. Thus, $H_0: p \leq 0.05$. What is $H_a$ here?  [optional] }\spc{1}

\intermediatesubproblem{At $\alpha$ = 2.5\%, what is the retainment region?  [optional]}\spc{2}

\easysubproblem{At $\alpha$ = 2.5\%, what is the rejection region?  [optional]}\spc{2}

\easysubproblem{The driver under investigation had 62 badly reviewed rides out of 1,000 rides. Calculate $\phat$ and say if it's above the $p$ cutoff Uber is using.  [optional]}\spc{1}

\intermediatesubproblem{If the driver under investigation had 62 badly reviewed rides out of 1,000 rides, would Uber fire him? Run the hypothesis test now and see. Hint: the answer to the (d) will mislead you. [optional] }\spc{8}

%\hardsubproblem{You may be wondering why on Earth we defined $H_a$ the way we did in (a). Wouldn't it seem more natural to switch the null and alternative hypothesis? Switch them here and run the test again. }\spc{6}
%
%\hardsubproblem{Your answer changed! Why is it a bad idea to run the test like in (f) / why is it a better idea to run the test like in (e)? }\spc{4.5}

\extracreditsubproblem{Find the $p$-val for the test in (e). }\spc{0}

\extracreditsubproblem{If the driver's true proportion / propensity of bad rides is $6\%$, what is the probability of a type II error when running the test in (e)? Then calculate power (1 - the probability of a type II error).}\spc{0}

\end{enumerate}

\problem{This will teach you about $H_0$ and $H_a$ using a silly example.} %This problem is optional and will not be graded.}

\iftoggle{professormode}{
\begin{figure}[htp]
\centering
\includegraphics[width=2in]{ufo.jpg}
\end{figure}
}

\begin{enumerate}

\easysubproblem{Consider $H_0$: Aliens do not exist vs $H_a$: Aliens do exist. Why is this a \qu{standard} hypothesis test? }\spc{1}

\easysubproblem{Describe a person using English who has a high $\alpha$ for $H_0$: Aliens do not exist vs $H_a$: Aliens do exist. }\spc{1}

\easysubproblem{Describe a person using English who has a low $\alpha$ for $H_0$: Aliens do not exist vs $H_a$: Aliens do exist. }\spc{1}

\easysubproblem{Describe a person using English who has a high $\alpha$ for $H_0$: Aliens do exist vs $H_a$: Aliens do not exist. }\spc{1}

\easysubproblem{Describe a person using English who has a low $\alpha$ for $H_0$: Aliens do exist vs $H_a$: Aliens do not exist.  }\spc{2}

\end{enumerate}

\end{document}


\lab{Plot Customization and Matplotlib Syntax Guide}{Matplotlib Customization}
\label{mpltables}

\objective{
The documentation for Matplotlib can be a little difficult to maneuver and basic information is sometimes difficult to find.
This appendix condenses and demonstrates some of the more applicable and useful information on plot customizations.
For an introduction to Matplotlib, see lab \ref{lab:Matplotlib}.
}

\section*{Colors} % ===========================================================

By default, every plot is assigned a different color specified by the ``color cycle''.
It can be overwritten by specifying what color is desired in a few different ways.
\begin{itemize}
    \item \textbf{}
\end{itemize}
Matplotlib recognizes some basic built-in colors.

\begin{table}[H] % Basic colors.
\centering
\begin{tabular}{r|l}
    Code & Color \\
    \hline
    \li{'b'} & blue\\
    \li{'g'} & green\\
    \li{'r'} & red\\
    \li{'c'} & cyan\\
    \li{'m'} & magenta\\
    \li{'y'} & yellow\\
    \li{'k'} & black\\
    \li{'w'} & white
\end{tabular}
\end{table}

The following displays how these colors can be implemented.
The result is displayed in Figure \ref{colors}.

\lstinputlisting[style=fromfile]{colors.py}

\begin{figure}  % Colors.
    \includegraphics[width=\textwidth]{colors.pdf}
    \caption{A display of all the built-in colors.}
    \label{colors}
\end{figure}

There are many other ways to specific colors.
A popular method to access colors that are not built-in is to use a \li{RGB} tuple.
Colors can also be specified using an html hex string or its associated html color name like \li{"DarkOliveGreen", "FireBrick", "LemonChiffon", "MidnightBlue", "PapayaWhip",} or \li{"SeaGreen"}.

\section*{Window Limits} % ====================================================

You may have noticed the use of \li{plt.ylim([ymin, ymax])} in the previous code. This explicitly sets the boundary of the y-axis. Similarily, \li{plt.xlim([xmin, xmax])} can be used to set the boundary of the x-axis.
Doing both commands simultaneously is possible with the \li{plt.axis([xmin, xmax, ymin, ymax])}.
Remember that these commands must be executed after the plot.

\begin{comment} % pulled this in from matplotlib lab.
\begin{table}[H]
\centering
\begin{tabular}{r|l}
    Function & Description\\
    \hline
    % \li{annotate} & adds a commentary at a given point on the plot & annotate('text',(x,y))\\
    % \li{arrow} & draws an arrow from a given point on the plot & arrow(x,y,dx,dy)\\
    % \li{axhline} & draws a horizontal line at y from xmin to xmax & axhline(y=0, xmin=0, xmax=1)\\
    % \li{axvline} & draws a vertical line at x from ymin to ymax & axvline(x=0, ymin=0, ymax=1)\\
    % \li{axhspan} & draws a rectangle from xmin to xmax and ymin to ymax, if no xmin and xmax are given it goes across the plot & axhspan(ymin, ymax, xmin=0, xmax=1)\\
    % \li{axvspan} & draws a rectangle from ymin to ymax and xmin to xmax, if no ymin and ymax are given it goes across the entire plot & axvspan(xmin, xmax, ymin=0, ymin=1)\\
    % \li{grid()} & Add gridlines\\
    \li{legend()} & Place a legend in the plot\\
    % \li{text()} & Add text at a given position on the plot\\
    \li{title()} & Add a title to the plot\\
    \li{xlim()} & Set the limits of the $x$-axis\\
    \li{ylim()} & Set the limits of the $y$-axis\\
    % \li{xticks()} & set the location of the tick marks on the x axis, returns current locations if no arguments are given\\
    % \li{yticks()} & set the location of the tick marks on the y axis, returns current locations if no arguments are given\\
    \li{xlabel()} & Add a label to the $x$-axis\\
    \li{ylabel()} & Add a label to the $y$-axis
\end{tabular}
% \caption{Some Functions to Set Plotting Options}
% \label{mpl:useful_functions}
\end{table}
\end{comment}

% BAD! plt.axis("tight") has a bug that messes with OS X on Macs.
% Another popular command is \li{plt.axis("tight")} which adjusts the axes so all data is visible and centered.

\section*{Lines} % ============================================================

\subsection*{Thickness} % -----------------------------------------------------

You may have noticed that the width of the lines above seemed thin considering we wanted to inspect the line color. \li{linewidth} is a keyword argument that is defaulted to be \li{None} but can be given any real number to adjust the line width.

The following displays how \li{linewidth} is implemented.
It is displayed in Figure \ref{linewidth}.

\lstinputlisting[style=fromfile]{linewidth.py}

\begin{figure} % Line width results
\includegraphics[width=\textwidth]{linewidth.pdf}
\caption{plot of varying linewidths.}
\label{linewidth}
\end{figure}

% Talk about markersize and demonstrate markers of varying sizes.

\subsection*{Style} % ---------------------------------------------------------

By default, plots are drawn with a solid line.
The following are accepted format string characters to indicate line style.

\begin{table}[H] % Line style
\centering
\begin{tabular}{c|l}
    character & description \\
    \hline
    - & solid line style \\
    -{}- & dashed line style\\
    -. & dash-dot line style \\
    : & dotted line style \\
    . & point marker \\
    , & pixel marker \\
    o & circle marker \\
    v & triangle\_down marker \\
    \^{} & triangle\_up marker \\
    $<$ & triangle\_left marker \\
    $>$ & triangle\_right marker \\
    1 & tri\_down marker \\
    2 & tri\_up marker \\
    3 & tri\_left marker \\
    4 & tri\_right marker \\
    s & square marker \\
    p & pentagon marker \\
    * & star marker \\
    h & hexagon1 marker \\
    H & hexagon2 marker \\
    + & plus marker \\
    x & x marker \\
    D & diamond marker \\
    d & thin\_diamond marker \\
    $|$ & vline marker \\
    \_{} & hline marker \\
\end{tabular}
\end{table}

The following displays how \li{linestyle} can be implemented.
It is displayed in Figure \ref{linestyle}.

\lstinputlisting[style=fromfile]{linestyle.py}

\begin{figure} % Line styles
\includegraphics[width=\textwidth]{linestyle.pdf}
\caption{plot of varying linestyles.}
\label{linestyle}
\end{figure}

\section*{Text} % =============================================================

It is also possible to add text to your plots.
To label your axes, the \li{plt.xlabel()} and the \li{plt.ylabel()} can both be used.
The function \li{plt.title()} will add a title to a plot.
If you are working with subplots, this command will add a title to the subplot you are currently modifying.
To add a title above the entire figure, use \li{plt.suptitle()}.

All of the \li{text()} commands can be customized with \li{fontsize} and \li{color} keyword arguments.

We can add these elements to our previous example.
It is displayed in Figure \ref{text}.

\lstinputlisting[style=fromfile]{text.py}

\begin{figure}
\includegraphics[width=\textwidth]{text.pdf}
\caption{plot of varying linestyles using text labels.}
\label{text}
\end{figure}

\begin{comment} % PLOTTING COMMANDS
\begin{table} % Different plotting commands. Should be part of the appendix.
\centering
\begin{tabular}{|l|p{7cm}|p{3cm}|}
    \hline
    Function & Description & Usage\\
    \hline
    \li{bar} & makes a bar graph & bar(left,height)\\
    \li{barh} & makes a horizontal bar graph & barh(bottom,width)\\
    \li{fill} & plots lines with shading under the curve & fill(x,y)\\
    \li{fill\_between} & plots lines with shading between two given y
    values & fill\_ between(x,y1, y2=0)\\
    \li{hist} & plots a histogram from data & hist(data)\\
    \li{pie} & makes a pie chart & pie(x)\\
    \li{plot} & plots lines and data on standard axes & plot(x,y)\\
    \li{polar} & plots lines and data on polar axes & polar(theta,r)\\
    \li{loglog} & plots lines and data on logarithmic x and y axes &
    loglog(x,y)\\
    \li{scatter} & plots data, has more options for scatter plots than
    the plot function & scatter(x,y)\\
    \li{semilogx} & plots lines and data with a log scaled x axis &
    semilogx(x,y)\\
    \li{semilogy} & plots lines and data with a log scaled y axis &
    semilogy(x,y)\\
    \li{specgram} & makes a spectogram from data & specgram(x)\\
    \li{spy} & plots the sparsity pattern of a 2D array & spy(Z)\\
    \li{triplot} & plots triangulation between given points &
    triplot(x,y)\\
\end{tabular}
\end{center}
\caption{Some basic functions in Matplotlib.}
\label{mpl:basics}
\end{table}
\end{comment}

See \url{http://matplotlib.org} for Matplotlib documentation.


%% -*- coding:utf-8 -*-
\chapter*{Introduction}

You just looked at yet another introduction to Category Theory. The
subject mostly consists of a lot of definitions that are related each
others and I wrote the book to collect all of them in one
place to be easy checked and updated in future when I decide to refresh
my knowledge about the field of math. Therefore the book was written mostly
for my category theory studying
purposes but I will appreciate if somebody else find it useful.

The topics(chapters) cover the base definitions
(\mynameref{def:object}, \mynameref{def:morphism} and
\mynameref{def:category}), \mynameref{def:functor},
\mynameref{def:nt}, \mynameref{def:monad} and also include important
results from the category theory such as Yoneda's lemma (see
\cref{sec:yoneda}) and Curry-Howard-Lambek correspondence (see
\cref{sec:curry_howard_lambek}).

There are a lot of
examples in each chapter. The examples cover different category
theory application areas. I assume that the reader is familiar with
the corresponding area and the example(s) can be passed if not. I.e.
anyone can choose the suitable example(s) for (s)he.

The most important examples are related to the set theory. The set
theory and category theory are very close related. Each one can be
considered as an alternative view to another one.

There are a lot of examples from programming languages which include
Haskell, Scala, C++. The source files for programming languages
examples (Haskell, C++, Scala) can be found on github repository
\cite{bib:github:ivanmurashko}.

The examples from physics are related to quantum mechanics that is the
most known for me. For the examples I am inspired by the Bob Coecke
article \cite{bib:arxiv:Bob_Coecke_2008}.

The text is distributed under \textbf{Creative Common Public License}
(see the text of the license at the end of the book)
i.e. any reader has right to copy, store, modify, distribute or build
upon the book until the original author is pointed in the derivative
products.


\documentclass[main.tex]{subfiles}
\begin{document}

\marginpar{Wednesday\\ 2020-11-25, \\ compiled \\ \today}

We wanted to estimate the magnetic field \(B\) and characteristic age \(\tau _c\) of the NS: we have found that \(B_P \propto \sqrt{P \dot{P}}\) and \(\tau _c \propto P / \dot{P}\).

Now we will define the \textbf{braking index}: recall that the evolution of the angular velocity can be written as \(\dot{\Omega} = - k \Omega^3\).
This can be generalized to a law of the type \(\dot{\Omega} = -k \Omega^{n}\); the spin-down due to the emission of GWs is similarly written with \(n = 6\).

Taking the derivative of this law, we find
%
\begin{align}
\ddot{\Omega} &= - kn \dot{\Omega} \Omega^{n-1}  \\
&= \frac{n}{\Omega } \dot{\Omega} (-k \Omega^{n}) = n \frac{\dot{\Omega}^2}{\Omega }   \\
n &= \frac{\ddot{\Omega} \Omega }{\dot{\Omega}^2}
\,.
\end{align}

By measuring \(\Omega \), \(\dot{\Omega}\) and \(\ddot{\Omega}\) we can estimate \(n\), which will allow us to find the braking index \(n\).
However, as the order of the derivative increases the difficulty of the measurement increases. The index \(n\) is known for around 10 NSs, and it was measured to be \(n < 3\), which is evidence against magnetorotational losses as the main cause of the spin-down.

We make a plot, with  \(\log P\) on the \(x\) axis and \(\log \dot{P}\) on the \(y\) axis: this is the \(P-\dot{P}\) diagram.
We can scatter-plot known NSs in it: see figure \ref{fig:p-pdot-diagram}.

\begin{figure}[ht]
\centering
\includegraphics[width=.8\textwidth]{figures/p-pdot-diagram}
\caption{An example \(P-\dot{P}\) diagram from \textcite[]{beckerAutonomousSpacecraftNavigation2013}.}
\label{fig:p-pdot-diagram}
\end{figure}

Equal-\(B_p\) lines and equal-\(\tau _c\) lines can be drawn:
the first are in the form \(\log P = - \log \dot{P} + k\), the second are in the form \(\log P = \log \dot{P} + k\).
We can track the motion of NSs through this diagram: they will move down and to the right as they evolve, with a slope of \(2 - n\), since
%
\begin{align}
\dv{t}( \frac{1}{P}) \propto - \frac{1}{P^{n}} \implies \dot{P} \propto P^{2-n}
\,.
\end{align}

\section{Interior structure of NSs}

Are they actually made of neutrons? Mostly, yes, but not exclusively.

A free neutron will \(\beta \)-decay:
%
\begin{align}
\ce{n} \to e^{-} + \ce{p} + \overline{\nu}
\,
\end{align}
%
within around \SI{15}{min}. Neutrons are also formed through inverse \(\beta \)-decay (also called electron capture):
%
\begin{align}
e^{-} + \ce{p} \to \ce{n} + \nu
\,.
\end{align}

This process, however, is endothermic: \(m_e + m_p < m_n \).
Specifically, \(Q = (m_n - m_p)c^2 \approx 2.54 m_e c^2\approx \SI{1.3}{MeV}\) is the energy the electron must have in order to achieve the reaction.

The thermal energy is definitely not that high inside a NS!
In order to describe what goes on, we will make some assumptions:
\begin{enumerate}
    \item the interior is only made of neutrons, protons and electrons (the \(npe\) model) (this is not really true, but it will allow us to illustrate the point);
    \item the number densities of these species are similar: \(n_e \approx n_p \approx n_n\).
\end{enumerate}

Matter inside a neutron star is highly degenerate: the energy of each electron is much smaller than the Fermi energy.
The Fermi energy can be written as \(E_F \approx m_e c^2 \sqrt{1 + x_F^2}\), where \(x_F = p_F / m_ec\), and
%
\begin{align}
p_F = \sqrt[3]{\frac{3 h^3 n_e}{8 \pi }}
\,.
\end{align}

Therefore,
%
\begin{align}
x_F = \sqrt[3]{\frac{3 h^3 n_e}{8 \pi }} \frac{1}{m_e c}
\approx \num{e-2} \sqrt[3]{\frac{\rho}{\mu }}
\,.
\end{align}

If we want the electrons to be relativistic, we require \(x_F \gtrsim 1\), which means \(\rho / \mu _e \gtrsim \SI{e6}{g / cm^3}\).

If the Fermi energy is as high as the \(Q\)-value of the reaction, \(E_F \gtrsim \num{2.54} m_e c^2\), then we must have
%
\begin{align}
\sqrt{1 + x_F^2} \gtrsim \num{2.54} \implies x_F \gtrsim \num{2.3}
\,,
\end{align}
%
which can be solved in terms of the rest mass matter density: we get
%
\begin{align}
\rho \gtrsim \SI{1.7e7}{g / cm^3} \times \mu _e
\,,
\end{align}
%
which is high in absolute terms, but rather low for a NS, since they typically will have densities several orders of magnitude higher.

At these densities, not only electrons, but protons and neutrons are degenerate as well, however these more massive particles are not relativistic.
In fact, \(x_F \propto n / m\) holds across particle species: protons and neutrons will have \(x_F^{(p, n)} \sim x_F^{(e)} / 2000 \), since they are \(\sim 2000\) times more massive.

The densities needed for the neutrons and protons to be relativistic might be achieved in the very core of the star, but they are typically not.

So, the electrons are relativistic and the reaction can happen back and forth at equilibrium: this means that we can equate their chemical potentials,
%
\begin{align}
\mu _p + \mu _e = \mu _n (+ \mu _\nu )
\,,
\end{align}
%
where the neutrinos can be neglected since they generally fly away after a reaction, unable to interact further.
Under complete degeneracy we have \(\mu = E_F\).
Then, we can write
%
\begin{align}
m_p c^2 \sqrt{1 + x_p^2} +
m_e c^2 \sqrt{1 + x_e^2} =
m_n c^2 \sqrt{1 + x_n^2}
\,,
\end{align}
%
but by charge neutrality we must have \(n_e = n_p\).
This is to be complemented with \(n \propto p_F^3\) and \(x = p_F / mc\): thus, \(x_e / x_p = m_p / m_e\).

Working through the algebra,\footnote{The passages to follow are: eliminate \(x_e\) in the previous equation with the relation \(x_e / x_p = m_p / m_e\), approximate \(\sqrt{1 + x^2}\) as \(x\) for electrons and as \(1 + x^2/2\) for protons and neutrons (thus assuming that they are nonrelativistic); solve for \(x_n / x_p\).}  we find
%
\begin{align}
R_{np} = \frac{n_n}{n_p} = x_n^3 \qty[
    \frac{4 (1 + x_n^2)}{x_n^4 + \frac{4 Q x_n^3}{m_n} + 4\frac{(Q^2 -m_e^2)}{m_n^2}}
]^{3/2}
= \frac{x_n^3}{x_p^3}
\,.
\end{align}

This is plotted on a log-log scale as a function of \(x_n\) in figure \ref{fig:neutron-proton-ratio}.

The density is given by
%
\begin{align}
\rho &= m_e n_e + m_n n_n + m_p n_p \approx m (n_p + n_n)
\marginnote{We define \(m = m_n \approx m_p\).}
\\
&\approx m \frac{8 \pi }{3} \qty(\frac{mc}{h})^3
\qty[x_p^3 + x_n^3]  \\
&= m \frac{8 \pi }{3} \qty(\frac{mc}{h})^3
x_n^3 \qty[ \frac{1}{R_{np} (x_n)} + 1]
\,.
\end{align}
%

This yields a parametric expression for \(R_{np}\) in terms of \(\rho \); if \(\rho \) is very small then \(n_n \ll n_p\), approaching \(n_n \sim 0\), around \(\rho \approx \SI{e12}{g / cm^3}\) there is a maximum of \(R_{np}\approx \num{e4}\), and it seems to approach \(R_{np} \sim 10 \) asymptotically. It is shown in figure \ref{fig:neutron-proton-ratio-density}.

Taking the limit of the expression for  \(\rho \) as \(n_n \to 0\) we find
%
\begin{align}
\rho &\approx \frac{m 8 \pi }{3} \qty(\frac{mc}{h})^3 \qty[ \frac{Q^2 - m_e^2}{m^2}]^{3/2}  \\
&\approx \frac{8 \pi m}{3} \qty( \frac{c}{h}\sqrt{Q^2 - m_e^2})^3
\,.
\end{align}

This is around \SI{1.2e7}{g / cm^3}. It is expected, since below this density there will be no neutrons anymore.
If we plot \(\rho (x_n)\), we see a plateau until about \(x_n \sim \num{e-2}\), after which we start to see \(\rho \propto x_n^3\).

\end{document}


\documentclass[bigger]{beamer}

\input{header-beam} % change to header-handout for handouts

% ====================
\title[Lecture 13]{Logic I F13 Lecture 13}
\date{October 24, 2013}
% ====================

\include{header}

\setlength{\fitchprfwidth}{5em}
\setlength{\fitchargwidth}{25em}

\section{Review}

\subsec{Properties and Quantifiers}{

\bit
\item Wff $A(x)$ expresses a \emph{property}
\item The sentence
\[
\forall x\, A(x)
\]
is true iff
\emph{every object} in the domain has the property expressed by A(x),
\item The sentence
\[\sf
\exists x\, A(x)
\]
is true iff
\emph{at least one object} in the domain has the property expressed by A(x),
\eit

}

\subsec{The Aristotelian Forms}{

\bit
\item ``Some A is B''
\[
\sf \exists x(A(x) \land B(x)
\]
\item ``Every A is B'' (``All A are B'')
\[\sf \forall x(A(x) \to B(x))\]
\item ``No A is B''
\begin{align*}
\sf \forall x(A(x) & \to \sf\lnot B(x))\\
\sf \lnot\exists x(A(x) & \sf\land B(x))
\end{align*}
\item ``Some A is not B'' (``Not all A are B'')
\begin{align*}
\sf \exists x(A(x) & \land \sf\lnot B(x)) \\
\sf \lnot\forall x(A(x) & \to \sf B(x))
\end{align*}

\eit

}

\subsec{Only, Except}{

\bit
\item ``Only A are B''
\begin{align*}
\sf \forall x(B(x) & \sf\to A(x))\\
\sf \lnot\exists x(\lnot A(x) & \sf\land B(x))
\end{align*}
\item ``Only a is large''
\[
\sf \forall x(Large(x) \iff x = a)
\]
\eit
}

\subsec{Existential Import and Implicature}{

\bit
\item ``All A are B'' does not imply ``There are A''
\item $\sf \forall x(A(x) \impl B(x))$ is true if nothing satisfies $A(x)$\\
(vacuous quantification)
\item ``Some A are B'' does not imply ``Not all A are B''
\item \dots but they do implicate it
\eit

}

\subsec{Quantifiers and Function Symbols}{

\bits
\item ``The leftmost block in the same row as any cube is small''
\bits
\item ``For every cube, the leftmost block in the same row as it is small''
\item
\[
\sf \forall x(Cube(x) \to Small(lm(x)))
\]
\eit
\item ``Some multiples of 3 are even, and some aren't''
\bits
\item Note: every multiple of 3 is the multiple of 3 \emph{by some number}
\item ``There is a number such that the multiple of 3 by it is even''
\item
$
\sf \exists x\,Even(x \times (1 + (1+1)))
$
\item
$
\sf \exists x\, Even(x \times (1 + (1+1))) \land \exists x\, \lnot Even(x \times (1 + (1+1)))
$
\eit
\eit
}

\section{Arguments involving Quantifiers}

\subsec{Example}{

\ben
\item If an action x is morally wrong then
A is blameworthy for freely doing x.
\item If x is rationally optimal (there is no action which A has
  reason to think there is more reason for A to do), then A is not
  blameworthy for freely doing x.
\item Therefore, if x is morally wrong, then
x is not rationally optimal.
(Principle of moral categoricity.)
\een

\begin{raggedleft}
\small(John Skorupski, \textit{Ethical Explorations}, 2000 (\href{http://books.google.ca/books?id=bxIzZYqRZdwC&lpg=PP1&pg=PA170\#v=onepage&q&f=false}{link} \href{http://peasoup.typepad.com/peasoup/2008/07/logic-help.html}{link}))
 \end{raggedleft}

}

\subsec{Example}{

\fitcharg{
\nline{\forall x(Wrong(x) \to BlameFreelyDoing(A, x))}\\
\nline{\forall x(ROpt(x) \to \lnot BlameFreelyDoing(A, x))}}{
\nline{\forall x(Wrong(x) \to \lnot ROpt(x))}\\
}

\bigskip

\fitcharg{\tline{All A are B}\\
\tline{No C are B \qquad ($\Leftrightarrow$ No B are C)}}{
\tline{No A are C}}
}

\subsec{FOL and the Truth Table Method}{

\bit
\item Some arguments valid only in virtue of the meanings of quantifiers
\item Some arguments valid independently of the meanings of quantifiers, e.g.,
\fitcharg{\tline{If there are cubes, then there are also tetrahedra}\\
\tline{There are no tetrahedra}}{\tline{There are no cubes}}
\item Challenge: how does the truth-table method apply?  When should we say an argument is \emph{tautologically} valid even though it contains quantifiers?

\eit

}

\subsec{The Truth-Functional Form Algorithm}{

\bit
\item Underline every quantifier and its entire scope (unless it is in the scope of another quantifier)
\item Label every underlined bit with a schematic sentence letter (A, B, C, \dots), use the same letter if the underlined parts are \emph{identical}
\item Replace underlined parts by sentence letters
\item Result is the \emph{truth-functional form} of the argument
\item An argument (with quantifiers) is \emph{tautologically valid} if its truthfunctional form is tautologically valid
\eit

}

\subsec{The Truth-Functional Form Algorithm}{

\fitcharg{
\nline{\underline{\exists x\, Cube(x)}_A \to \underline{\exists x\, Tet(x)}_B}\\
\nline{\lnot\underline{\exists x\, Tet(x)}_B}}{
\nline{\lnot\underline{\exists x\, Cube(x)}_A}}

\bigskip

\fitcharg{
\nline{A \to B}\\
\nline{\lnot B}}{\nline{\lnot A}}

}

\subsec{The Truth-Functional Form Algorithm}{

\fitcharg{
\nline{\underline{\exists x\, Cube(x)}_A \to \underline{\exists x\, Tet(x)}_B}\\
\nline{\underline{\forall x\, \lnot Tet(x)}_C}}{
\nline{\underline{\forall x\, \lnot Cube(x)}_D}}

\bigskip

\fitcharg{
\nline{A \to B}\\
\nline{C}}{\nline{D}}

}

\subsec{First-order Validity}{

\bit
\item Want to capture validity \emph{in virtue of the meanings of the
  connectives and the quantifiers} (but ignoring meanings of predicate
  symbols other than =)
\item So we want to ignore any restrictions the predicate symbols place on their extensions
\item Hence: allow \emph{any} extension in a potential counterexample
\item An argument is \emph{first-order valid} if there is no \emph{first-order structure} in which the premises are true and the conclusion false

\item Variant interpretation: \emph{first-order structure}
\eit

}

\subsec{The Replacement Method}{

\bit
\item Existing meanings of predicate symbols may confuse you, so:
\item Replace each predicate symbol (other than =) with a ``nonsense predicate'': A, B, C, \dots
\item Arity the same as original predicate
\eit
}

\subsec{First-order Structure}{

\bit
\item Specify domain (e.g., ``Canadian cities'', ``the natural numbers'', graphically using a collection of dots, \dots)
\item Specify extension of A(x), B(x, y), C(x, y, z), \dots{} by
\bit
\item a diagram (as usual)
\item enumeration of elements, pairs of elements, triples of elements of the domain, e.g., ``Calgary, Edmonton, Lethbridge'', ``(0, 1), (0, 2), (0, 3), \dots''
\item determinate predicate (expression involving x, y, z), e.g., ``x is entirely located in Alberta'', ``x evenly divides y'', \dots
\eit
\item Specify referents of constant symbols, e.g., ``c: Calgary'', ``d: 5'', graphically by writing the constant next to the dot named by it
\eit
}

\section{FO Consequence and FO Structures}

\subsec{First-order Logical Notions}{

\bit
\item An argument is \emph{first-order valid}, and its conclusion is a \emph{first-order consequence} of its premises,
if there is no \emph{first-order structure} in which the premises are true and the conclusion is false
\item A sentence is a \emph{first-order validity} if it is true in every first-order stucture
\item Two sentences are \emph{first-order equivalent} if there is no first-order structure in which they have different truth values.
\item A (set of) sentence(s) is \emph{first-order satisfiable} if it is (all) true in at least one first-order structure
\item NB: There is \emph{no mechanical test} for any of these (The Church-Turing Undecidability Theorem), by contrast to the truth-table case!
\eit

}

\subsec{First-order Structure}{

\bit
\item $\sf \forall x\,(P(x) \impl R(x, b))$
\item We need:
\begin{itemize}
\item A domain: objects which can take the value of \textsf x.
\item Extension of $\sf P(x)$ (or a determinate property of the objects in that domain as an
interpretation for $\sf P(x)$).
\item Extension of $\sf R(x, y)$ (or a determinate 2-place relation as an interpretation of the
predicate $\sf R(x, y)$).
\item An object to be named by \textsf a.
\end{itemize}
\eit
}

\subsec{FO Validity of Arguments}{

\fitcharg{
\nline{\forall x\, Cube(x) \lor \forall x\, Tet(x)}\\
\nline{\exists x\, \lnot Cube(x) \land \forall y(Tet(y) \to Small(y))}}{
\nline{\lnot \exists x\, Large(x)}}
}

\subsec{FO (In)Validity of Arguments}{

\fitcharg{
\nline{\forall x\, A(x) \lor \forall x\, B(x)}\\
\nline{\exists x\, \lnot A(x) \land \forall y(B(y) \to C(y))}}{
\nline{\lnot \exists x D(x)}}

\begin{itemize}
\item Domain: the planets
\item $\sf A(x)$ \dots \textsf x is inhabited
\item $\sf B(x)$ \dots \textsf x is smaller than the Sun
\item $\sf C(x)$ \dots \textsf x orbits the Sun
\item $\sf D(x)$ \dots \textsf x has a moon
\end{itemize}
Another FO counterexample given by an Euler diagram:
\[
\begin{array}{|lc|}
\hline
\sf B = C = D & \\
& \begin{array}{|lc|}
\hline
\sf A & \cdot 1\\
\hline
\end{array}\quad \\
\cdot 2 & \\ \hline
\end{array}
\]

}

\subsec{Making ``All As are Bs'' True}{

\begin{align*}
\sf \forall x\, (A(x) & \impl \sf B(x)) \\
\sf \lnot\exists x(A(x) & \land \sf\lnot B(x))
\end{align*}

\[
\begin{array}{|lc|}
\hline
\sf B & \cdots \\
& \begin{array}{|c|}
\hline
\sf A \cdots \\
\hline
\end{array} \quad \\[1ex]
\hline
\end{array}
\qquad\qquad
\begin{array}{|lc|}
\hline
\sf B = A & \\
& \cdots \\
\hline
\end{array}
\qquad\qquad
\begin{array}{c}
\begin{array}{|lc|}
\hline
\sf B & \\
& \cdots \\
\hline
\end{array} \\[2ex]
\sf A = \emptyset
\end{array}
\]

Makes ``Some As are not Bs'' \emph{false}
}

\subsec{Making ``All As are Bs'' False}{

\begin{align*}
\sf \lnot\forall x\, (A(x) & \impl \sf B(x)) \\
\sf \exists x(A(x) & \land \sf\lnot B(x))
\end{align*}

\[
\begin{array}{lcr|}
  \cline{1-2}
  \multicolumn{2}{|c|}{\sf A \quad \cdots}\\
  \cline{2-3}
  \multicolumn{1}{|c|}{} & \multicolumn{1}{|c|}{\cdots} &
  \multicolumn{1}{|c|}{} \\
  \cline{1-2}
  & \multicolumn{2}{|c|}{\sf B \quad \cdots}\\
  \cline{2-3}
\end{array}
\qquad\qquad
\begin{array}{|lc|c|lc|}
\cline{1-2}\cline{4-5}
\sf A  & & & \sf B & \\
& \cdots & & & \cdots\\
\cline{1-2}\cline{4-5}
\end{array}
\qquad\qquad
\begin{array}{c}
\begin{array}{|lc|}
\hline
\sf A & \\
& \cdots \\
\hline
\end{array} \\[1ex]
\sf B = \emptyset
\end{array}
\]

Makes ``Some As are not Bs'' \emph{true}

}

\subsec{Making ``Some As are Bs'' True}{

\begin{align*}
\sf \exists x (A(x) & \sf\land B(x)) \\
\sf \lnot\forall x(A(x) & \impl\sf \lnot B(x))
\end{align*}

\[
\begin{array}{lcr|}
  \cline{1-2}
  \multicolumn{2}{|c|}{\sf A \quad \cdots}\\
  \cline{2-3}
  \multicolumn{1}{|c|}{} & \multicolumn{1}{|c|}{\cdots} &
  \multicolumn{1}{|c|}{} \\
  \cline{1-2}
  & \multicolumn{2}{|c|}{\sf B \quad \cdots}\\
  \cline{2-3}
\end{array}
\qquad\qquad
\begin{array}{|lc|}
\hline
\sf A & \cdots \\
& \begin{array}{|c|}
\hline
\sf B \cdots \\
\hline
\end{array} \quad \\[1ex]
\hline
\end{array}
\qquad\qquad
\begin{array}{|lc|}
\hline
\sf B & \cdots \\
& \begin{array}{|c|}
\hline
\sf A \cdots \\
\hline
\end{array} \quad \\[1ex]
\hline
\end{array}
\qquad\qquad
\begin{array}{|lc|}
\hline
\sf B = A & \\
& \cdots \\
\hline
\end{array}
\]

Makes ``No As are Bs'' \emph{false}

}

\subsec{Making ``Some As are Bs'' False}{

\begin{align*}
\sf \lnot\exists x (A(x) & \sf\land B(x)) \\
\sf \forall x(A(x) & \impl\sf \lnot B(x))
\end{align*}

\[
\begin{array}{|lc|c|lc|}
\cline{1-2}\cline{4-5}
\sf A  & & & \sf B & \\
& \cdots & & & \cdots\\
\cline{1-2}\cline{4-5}
\end{array}
\qquad\qquad
\begin{array}{c}
\begin{array}{|lc|}
\hline
\sf A & \\
& \cdots \\
\hline
\end{array}\\[2ex]
\sf B = \emptyset
\end{array}
\qquad\qquad
\begin{array}{c}
\begin{array}{|lc|}
\hline
\sf B & \\
& \cdots \\
\hline
\end{array} \\[2ex]
\sf A = \emptyset
\end{array}
\]

Makes ``No As are Bs'' \emph{true}
}

\subsec{Skorupski Revisited}{

\fitchprf{\pline[1.]{\text{If $\sf x$ is morally wrong, then}}\\
\pline{\qquad \text{$\sf A$ is blameworthy for freely doing $\sf x$}}\\
  \pline[2.]{\text{If $\sf x$ is
  rationally optimal, then }}\\
\pline{\qquad\text{A is not blameworthy for freely doing x.}}}
{\pline[3.]{\text{If $\sf x$ is morally wrong, then $\sf x$ is not rationally
      optimal.}}}

\bigskip

\fitchprf{\pline[1.]{\forall x(Wrong(x) \lif BlameFreelyDoing(a, x))}\\
  \pline[2.]{\forall x(ROpt(x) \lif \lnot BlameFreelyDoing(a, x))}}
{\pline[3.]{\forall x(Wrong(x) \lif \lnot ROpt(x))}}
}

\subsec{Skorupski Revisited}{

\href{http://books.google.ca/books?id=bxIzZYqRZdwC&lpg=PP1&pg=PA170\#v=onepage&q&f=false}{Actual argument} from Skurupski's book:

\fitchprf{\pline[1.]{\text{If $\sf x$ is morally wrong and $\sf A$ freely does $\sf x$,
    then}}\\
\pline{\qquad\text{$\sf A$ is blameworthy for doing $\sf x$.}}\\
  \pline[2.]{\text{If doing $\sf x$ is
  rationally optimal, then}}\\
\pline{\qquad\text{there is no reason to blame $\sf A$ for freely doing
  $\sf x$.}}}
{\pline[3.]{\text{If $\sf x$ is morally wrong, then $\sf x$ is not rationally
      optimal.}}}

\bigskip

\fitchprf{\pline[1.]{\forall x((Wrong(x) \land DoesF(a, x)) \lif Blame(a, x))}\\
  \pline[2.]{\forall x((ROpt(x) \land DoesF(a, x)) \lif \lnot Blame(a, x))}}
{\pline[3.]{\forall x(Wrong(x) \lif \lnot ROpt(x))}}

}

\subsec{Skorupski Revisited}{

\fitchprf{\pline[1.]{\forall x((Wrong(x) \land DoesF(a, x)) \lif Blame(a, x))}\\
  \pline[2.]{\forall x((ROpt(x) \land DoesF(a, x)) \lif \lnot Blame(a, x))}}
{\pline[3.]{\forall x(Wrong(x) \lif \lnot ROpt(x))}}

\bigskip

\fitchprf{\pline[1.]{\forall x((Cube(x) \land LeftOf(a, x)) \lif SameRow(a, x))}\\
  \pline[2.]{\forall x((Small(x) \land LeftOf(a, x)) \lif \lnot SameRow(a, x))}}
{\pline[3.]{\forall x(Cube(x) \lif \lnot Small(x))}}
}

\end{document}