initial

.gitattributes

@@ -1,35 +1,37 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+setup/speeches.json filter=lfs diff=lfs merge=lfs -text
+train.tsv filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1 @@
+__pycache__

Dockerfile

@@ -0,0 +1,14 @@
+# Use the official Python image from the Docker Hub
+FROM python:3.12-slim
+
+RUN useradd -m -u 1000 user
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README copy.md

@@ -0,0 +1,11 @@
+---
+title: Docker Classifier
+emoji: 🏢
+colorFrom: red
+colorTo: gray
+sdk: docker
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

README.md

@@ -1,11 +1,11 @@
----
-title: Docker Classifier
-emoji: 🏢
-colorFrom: red
-colorTo: gray
-sdk: docker
-pinned: false
-license: apache-2.0
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Docker Classifier
+emoji: 🏢
+colorFrom: red
+colorTo: gray
+sdk: docker
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

all_pres_classifier_model_dict.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7576d9d34aef95f5121b110666fe946ade019b40ea829383421724b2341bb25
+size 49588328

app.py

@@ -0,0 +1,70 @@
+from fastapi import FastAPI, Request
+from pydantic import BaseModel
+from main import load_texts
+from tokenizer import SimpleTokenizer
+from transformer import Classifier
+from constants import block_size
+from fastapi.middleware.cors import CORSMiddleware
+
+import uvicorn
+
+import torch
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+block_size = 1024
+
+model = None
+tokenizer = None
+
+def initialize():
+    global model, tokenizer
+    
+    if not tokenizer:
+        texts = [text.split('\t', 1)[1] for text in load_texts('train.tsv')]
+        tokenizer = SimpleTokenizer(' '.join(texts))
+        
+    if not model:
+        model = Classifier(tokenizer.vocab_size)
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        model.load_state_dict(torch.load('all_pres_classifier_model_dict.pth', map_location=device))
+        model.to(device)
+        model.eval()
+
+class TextInput(BaseModel):
+    text: str
+
+@app.get("/")
+async def home():
+    return {"message": "Welcome! The server is running! Send a POST request to /predict please."}
+
+@app.post("/predict")
+async def predict(request: TextInput):
+    global model, tokenizer
+    if model is None or tokenizer is None:
+        initialize()
+        
+    # Get the text from the POST request body
+    text = request.text
+
+    # Perform inference
+    with torch.no_grad():
+        wordids = tokenizer.encode(text)
+        padded_sentence = wordids[:block_size] + [0] * (block_size - len(wordids))
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        input_tensor = torch.tensor(padded_sentence, dtype=torch.long).unsqueeze(0).to(device)
+        
+        output, _ = model(input_tensor)
+        
+    _, predicted = torch.max(output.data, 1)
+    
+    return {"predicted": predicted.tolist()}
+

classifier_model_dict.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fbf0de7afb699145f07c401e8d3eede4843a554f6095d8310066d2d66b8e6b5
+size 2360221

constants.py

@@ -0,0 +1,19 @@
+seed = 42
+
+""" Hyperparameters to use for training to roughly match 
+the numbers mentioned in the assignment description """
+batch_size = 16  # Number of independent sequences  we will process in parallel
+block_size = 1024  # Maximum context length for predictions
+learning_rate = 1e-3  # Learning rate for the optimizer , OG: 1e-3
+n_embd = 64  # Embedding dimension
+n_head = 2 # Number of attention heads, OG = 2
+n_layer = 4  # Number of transformer layers, OG = 4
+
+
+## classifier training hyperparameters. It is a simple 1 hidden layer feedforward network, with input 
+## size of 64, hidden size of 50 and output size of 3.
+
+n_input = 64  # Input size for the classifier, should match the embedding size of the transformer
+n_hidden = 100  # Hidden size for the classifier
+n_output = 46  # Output size for the classifier, we have 46 classes
+epochs_CLS = 15 # epochs for classifier training

dataset.py

@@ -0,0 +1,69 @@
+import os
+from torch.utils.data import Dataset
+import torch
+
+
+class SpeechesClassificationDataset(Dataset):
+    """
+    Dataset class for text classification task.
+    This the dataset you will use to train your encoder, and classifier jointly, 
+    end-to-end for the text classification task.
+
+    Args:
+        tokenizer (Tokenizer): The tokenizer used to encode the text.
+        file_path (str): The path to the file containing the speech classification data.
+
+    """
+
+    def __init__(self, tokenizer, file_path):
+        self.tokenizer = tokenizer
+        self.samples = []
+
+        if not os.path.exists(file_path):
+            raise FileNotFoundError(f"The file {file_path} does not exist.")
+
+        with open(file_path, 'r', encoding='utf-8') as file:
+            for line in file:
+                label, text = line.strip().split('\t')
+                if label not in ('0', '1', '2'):
+                    raise ValueError(f"Invalid label: {label}")
+                if len(text.strip()) == 0:
+                    continue
+                self.samples.append((int(label), text))
+
+    def __len__(self):
+        return len(self.samples)
+
+    def __getitem__(self, index):
+        label, text = self.samples[index]
+        input_ids = torch.tensor(self.tokenizer.encode(text), dtype=torch.long)
+        label_tensor = torch.tensor(label, dtype=torch.long)
+        
+        return input_ids, label_tensor
+    
+    
+
+
+class LanguageModelingDataset(torch.utils.data.Dataset):
+    """
+    Dataset class for language modeling task. This is the dataset you will use to train your encoder for the language modeling task. 
+
+    Args:
+        tokenizer (Tokenizer): The tokenizer used to encode the text.
+        text (str): The text data.
+        block_size (int): The size of each block of text.
+    """
+
+    def __init__(self, tokenizer, text, block_size):
+        self.tokenizer = tokenizer
+        self.data = torch.tensor(self.tokenizer.encode(text), dtype=torch.long)
+        self.block_size = block_size
+
+    def __len__(self):
+        return len(self.data) - self.block_size
+
+    def __getitem__(self, idx):
+        chunk = self.data[idx:idx + self.block_size + 1]
+        x = chunk[:-1]
+        y = chunk[1:]
+        return x, y

main.py

@@ -0,0 +1,341 @@
+import argparse
+import torch
+from torch import nn
+from torch.utils.data import DataLoader
+from torch.nn.utils.rnn import pad_sequence
+from torch.nn import functional as F
+
+import os
+
+from utilities import Utilities
+from tokenizer import SimpleTokenizer
+from dataset import SpeechesClassificationDataset, LanguageModelingDataset
+
+from constants import seed, batch_size, block_size, learning_rate, n_embd, n_head, n_layer, n_input, n_output, n_hidden, epochs_CLS
+
+from transformer import Classifier, Decoder, DecoderEC
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+eval_interval = 100  # How often to evaluate train and test perplexity during training
+max_iters = 500 # For language modeling, we can process all the batches for the entire dataset, but that takes a while, so we'll limit it to 500 iterations. For batch size of 16 and block size of  32, this is roughly, this is  500 * 16 * 32 = 256000 tokens, SOTA LMs are trained on trillions of tokens, so this is a very small dataset.
+eval_iters = 200  # Number of iterations to evaluate perplexity on the test set
+
+
+def load_texts(filename):
+    """
+    This function loads all texts from the specified directory, ignoring any files with "test" in their name. The text is used for "training" the tokenizer. Since our tokenizer is simple, we don't need to do any training, but we still need to ignore the test data. 
+    """
+
+    # texts = []
+    # files = os.listdir(directory)
+    # for filename in files: 
+    #     if "test" in filename:  ## don't "read test files"
+    #         continue
+    #     with open(os.path.join(directory, filename), 'r', encoding='utf-8') as file:
+    #         texts.append(file.read())
+    # return texts
+
+    with open(filename, 'r', encoding='utf-8') as file:
+        for line in file:
+            yield line.strip()
+
+
+def collate_batch(batch):
+    """ Collate a batch of data into a single tensor with padding."""
+    data, labels = zip(*batch)  # Separate the data and labels
+    # Pad sequences to the fixed length
+    padded_sequences = pad_sequence(data, batch_first=True, padding_value=0)
+    padded_sequences = padded_sequences[:, :block_size]  # Truncate if longer
+    # Add padding if shorter
+    padded_sequences = torch.nn.functional.pad(padded_sequences, (0, max(0, block_size - padded_sequences.shape[1])), "constant", 0)
+    labels = torch.stack(labels)  
+    return padded_sequences, labels
+
+
+def compute_perplexity(decoderLMmodel: Decoder, data_loader, eval_iters=100):
+    """ Compute the perplexity of the decoderLMmodel on the data in data_loader.
+    Make sure to use the cross entropy loss for the decoderLMmodel.
+    """
+    decoderLMmodel.eval()
+    losses= []
+    for X, Y in data_loader:
+        X, Y = X.to(device), Y.to(device)
+        logits, _ = decoderLMmodel(X) # your model should be computing the cross entropy loss
+        B, T, C = logits.shape
+        
+        logits = logits.view(B*T, C)
+        targets = Y.view(B*T)
+        loss = F.cross_entropy(logits, targets)
+        
+        losses.append(loss.item())
+        # total_loss += loss.item()
+        if len(losses) >= eval_iters: 
+            break
+
+    losses = torch.tensor(losses)
+    mean_loss = losses.mean()
+    perplexity = torch.exp(mean_loss).item()  # Calculate perplexity as exp(mean loss)
+
+    decoderLMmodel.train()
+    return perplexity
+
+def compute_classifier_accuracy(classifier: Classifier, data_loader):
+    """ Compute the accuracy of the classifier on the data in data_loader."""
+    classifier.eval()
+    total_correct = 0
+    total_samples = 0
+    with torch.no_grad():
+        for X, Y in data_loader:
+            X, Y = X.to(device), Y.to(device)
+            outputs, _ = classifier(X)
+            _, predicted = torch.max(outputs.data, 1)
+            total_correct += (predicted == Y).sum().item()
+            total_samples += Y.size(0)
+        accuracy = (100 * total_correct / total_samples)
+        classifier.train()
+        return accuracy
+
+def train_epoch(data_loader, model, optimizer):
+    # size = len(data_loader.dataset)
+    
+    num_batches = len(data_loader)
+    model.train()
+    train_loss, total_correct, total_samples = 0, 0, 0
+    
+    for batch, (X, Y) in enumerate(data_loader):
+        #  X = X.float()
+        # Compute prediction error
+        # print('----------------------------------------------------')
+        pred, _ = model(X)
+        
+        _, predicted = torch.max(pred.data, 1)
+        total_correct += (predicted == Y).sum().item()
+        total_samples += Y.size(0)
+        
+        # print("pred: ", pred.shape)
+        # print("predicted", predicted.shape)
+        # print("y", Y.shape)
+        # print('----------------------------------------------------')
+        
+        loss = F.cross_entropy(input=pred, target=Y)
+        
+        train_loss += loss.item()
+
+        # Backpropagation
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+    average_train_loss = train_loss / num_batches
+    accuracy = (100 * total_correct / total_samples)
+    return accuracy, average_train_loss
+
+
+# ------------------------------Classifier Code---------------------------------- #
+def run_classifier():
+
+    print("Loading data and creating tokenizer ...")
+    
+    texts = []
+    
+    for text in load_texts('train.tsv'):
+        texts.append(text)
+        
+    tokenizer = SimpleTokenizer(' '.join(texts)) # create a tokenizer from the data
+    print("Vocabulary size is", tokenizer.vocab_size)
+
+    print(len(texts))
+    print(texts[0])
+
+    # train_CLS_dataset = SpeechesClassificationDataset(tokenizer, "speechesdataset/train_CLS.tsv")
+    # train_CLS_loader = DataLoader(train_CLS_dataset, batch_size=batch_size, collate_fn=collate_batch, shuffle=True)
+    
+    # test_CLS_dataset = SpeechesClassificationDataset(tokenizer, "speechesdataset/test_CLS.tsv")
+    # test_CLS_loader = DataLoader(test_CLS_dataset, batch_size=batch_size, collate_fn=collate_batch, shuffle=True)
+    
+    # classifier_model = Classifier(tokenizer.vocab_size)
+    
+    # total_params = sum(p.numel() for p in classifier_model.parameters())
+    # print("Total number of parameters:", total_params)
+
+    # # Adam optimizer
+    # optimizer = torch.optim.Adam(classifier_model.parameters(), lr=learning_rate)
+    
+    #  # for the classification task, you will train for a fixed number of epochs like this:
+    # for epoch in range(epochs_CLS):
+    #     train_accuracy, train_loss = train_epoch(train_CLS_loader, classifier_model, optimizer)
+    #     print(f'Epoch #{epoch+1}: \t train accuracy {train_accuracy:.3f}\t train loss {train_loss:.3f}\t test accuracy {compute_classifier_accuracy(classifier_model, test_CLS_loader):.3f}')
+    
+    # torch.save(classifier_model.state_dict(), 'classifier_model_dict.pth')
+
+    
+# ------------------------------Classifier Code---------------------------------- #
+    
+    
+# ------------------------------Decoder Code---------------------------------- #     
+def run_decoder():
+    print("Loading data and creating tokenizer ...")
+    texts = load_texts('speechesdataset')
+    tokenizer = SimpleTokenizer(' '.join(texts)) # create a tokenizer from the data
+    print("Vocabulary size is", tokenizer.vocab_size)   
+
+    inputfile = "speechesdataset/train_LM.txt"
+    with open(inputfile, 'r', encoding='utf-8') as f:
+        lmtrainText = f.read()
+    train_LM_dataset = LanguageModelingDataset(tokenizer, lmtrainText,  block_size)
+    train_LM_loader = DataLoader(train_LM_dataset, batch_size=batch_size, shuffle=True)
+    
+    decoder = Decoder(tokenizer.vocab_size)
+    
+    total_params = sum(p.numel() for p in decoder.parameters())
+    print("Total number of parameters:", total_params)
+    
+    optimizer = torch.optim.Adam(decoder.parameters(), lr=learning_rate)
+
+    # for the language modeling task, you will iterate over the training data for a fixed number of iterations like this:
+    for i, (xb, yb) in enumerate(train_LM_loader):
+        if i >= max_iters: # stop after 500 batches
+            break
+        xb, yb = xb.to(device), yb.to(device)
+        
+        if (i+1)%100 == 0:
+            print("Train Data Perplexity At Iteration ", (i+1), compute_perplexity(decoder, train_LM_loader))
+        
+        # LM training code here
+        
+        # evaluate the loss
+        logits, _ = decoder(xb)
+        B, T, C = logits.shape
+        
+        logits = logits.view(B*T, C)
+        targets = yb.view(B*T)
+        loss = F.cross_entropy(logits, targets)
+        
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+    # calculate perplexity
+    files = ['speechesdataset/test_LM_hbush.tsv', 'speechesdataset/test_LM_obama.txt', 'speechesdataset/test_LM_wbush.txt']
+    
+    for file in files:
+        with open(file, 'r', encoding='utf-8') as f:
+            data = f.read()
+        test_LM_dataset = LanguageModelingDataset(tokenizer, data,  block_size)
+        test_LM_loader = DataLoader(test_LM_dataset, batch_size=batch_size, shuffle=True)
+        
+        print(file, compute_perplexity(decoder, test_LM_loader))
+    
+    print("------------------------SANITY CHECK DECODER------------------------")
+    print("Vocabulary size is", tokenizer.vocab_size)   
+    
+    ec = Decoder(tokenizer.vocab_size)
+    u = Utilities(tokenizer, ec)
+    
+    u.sanity_check('With a simple oath, we affirm old traditions and make new beginnings.', block_size)
+        
+# ------------------------------Decoder Code---------------------------------- #  
+
+
+def run_sanity_check_encoder():
+    print("Loading data and creating tokenizer ...")
+    print("------------------------SANITY CHECK ENCODER------------------------")   
+    tokenizer = SimpleTokenizer('The man who passes the sentence should swing the sword. If you would take a man\'s life') # create a tokenizer from the data
+    print("Vocabulary size is", tokenizer.vocab_size)   
+    
+    ec = Classifier(tokenizer.vocab_size)
+    u = Utilities(tokenizer, ec)
+    
+    u.sanity_check('The man who passes the sentence should swing the sword. If you would take a man\'s life', block_size)
+    
+    
+def run_sanity_check_decoder():
+    print("Loading data and creating tokenizer ...")
+    texts = load_texts('speechesdataset')
+    tokenizer = SimpleTokenizer(' '.join(texts)) # create a tokenizer from the data
+    
+    ec = Decoder(tokenizer.vocab_size)
+    u = Utilities(tokenizer, ec)
+    
+    u.sanity_check('With a simple oath, we affirm old traditions and make new beginnings.', block_size)
+
+
+def run_ec_decoder():
+    print("EXTRA CREDIT:")
+    texts = load_texts('speechesdataset')
+    tokenizer = SimpleTokenizer(' '.join(texts)) # create a tokenizer from the data
+    print("Vocabulary size is", tokenizer.vocab_size)   
+
+    inputfile = "speechesdataset/train_LM.txt"
+    with open(inputfile, 'r', encoding='utf-8') as f:
+        lmtrainText = f.read()
+    train_LM_dataset = LanguageModelingDataset(tokenizer, lmtrainText,  block_size)
+    train_LM_loader = DataLoader(train_LM_dataset, batch_size=batch_size, shuffle=True)
+    
+    decoder = DecoderEC(tokenizer.vocab_size, 4, 6)
+    
+    total_params = sum(p.numel() for p in decoder.parameters())
+    print("Total number of parameters:", total_params)
+    
+    optimizer = torch.optim.Adam(decoder.parameters(), lr=learning_rate)
+
+    # for the language modeling task, you will iterate over the training data for a fixed number of iterations like this:
+    for i, (xb, yb) in enumerate(train_LM_loader):
+        if i >= max_iters: # stop after 500 batches
+            break
+        xb, yb = xb.to(device), yb.to(device)
+        
+        if (i+1)%100 == 0:
+            print("Train Data Perplexity At Iteration ", (i+1), compute_perplexity(decoder, train_LM_loader))
+        
+        # LM training code here
+        
+        # evaluate the loss
+        logits, _ = decoder(xb)
+        B, T, C = logits.shape
+        
+        logits = logits.view(B*T, C)
+        targets = yb.view(B*T)
+        loss = F.cross_entropy(logits, targets)
+        
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+    # calculate perplexity
+    files = ['speechesdataset/test_LM_hbush.tsv', 'speechesdataset/test_LM_obama.txt', 'speechesdataset/test_LM_wbush.txt']
+    
+    for file in files:
+        with open(file, 'r', encoding='utf-8') as f:
+            data = f.read()
+        test_LM_dataset = LanguageModelingDataset(tokenizer, data,  block_size)
+        test_LM_loader = DataLoader(test_LM_dataset, batch_size=batch_size, shuffle=True)
+        
+        print(file, compute_perplexity(decoder, test_LM_loader))
+        
+    # generate some text!
+    context = torch.zeros((1, 1), dtype=torch.long, device=device)
+    print(tokenizer.decode(decoder.generate(context, max_new_tokens=500)[0].tolist()))
+
+# ------------------------------MAIN---------------------------------- #  
+def main():
+    parser = argparse.ArgumentParser(description="Run classifier or decoder")
+    parser.add_argument("-mode", choices=["c", "d", "sc", "sd", "ecd"], help="Choose mode: 'c' for classifier, 'd' for decoder, 'sc' for sanity checking classifier, 'sd' for sanity checking decoder, 'ecd' for decoder EC")
+    args = parser.parse_args()
+
+    if args.mode == "c":
+        run_classifier()
+    elif args.mode == "d":
+        run_decoder()
+    elif args.mode == 'sc':
+        run_sanity_check_encoder()
+    elif args.mode == 'sd':
+        run_sanity_check_decoder()
+    elif args.mode == 'ecd':
+        run_ec_decoder()
+    else:
+        print("Invalid mode.")
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,7 @@
+transformers
+fastapi
+uvicorn[standard]
+torch
+matplotlib
+nltk
+uvicorn

setup/download_speeches.py

@@ -0,0 +1,21 @@
+# Python script from here: https://data.millercenter.org/
+
+import json, requests, sys
+
+endpoint = "https://api.millercenter.org/speeches"
+out_file = "speeches.json"
+
+r = requests.post(url=endpoint)
+data = r.json()
+items = data['Items']
+
+while 'LastEvaluatedKey' in data:
+    parameters = {"LastEvaluatedKey": data['LastEvaluatedKey']['doc_name']}
+    r = requests.post(url = endpoint, params = parameters)
+    data = r.json()
+    items += data['Items']
+    print(f'{len(items)} speeches')
+
+with open(out_file, "w") as out:
+    out.write(json.dumps(items))
+    print(f'wrote results to file: {out_file}')

setup/speeches.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e6edb71f181ebdf6ccb2daed66817999792b3a7283d793325b2877bbccb78c9
+size 25043059

speechesdataset/test_CLS.tsv

@@ -0,0 +1,750 @@
+0	This afternoon, I spoke to former President George W. Bush.
+0	Twenty million more Americans know the financial security of health insurance.
+0	We virtually eliminated certain diseases like polio.
+0	And it took more time after that for it to start working.
+0	In fact, I will be right there with you, as a citizen, for all my remaining days.
+0	I don't fit the typical pedigree, and I haven't spent my career in the halls of Washington.
+0	Every American who serves joins an unbroken line of heroes that stretches from Lexington to Gettysburg ; from Iwo Jima to Inchon ; from Khe Sanh to Kandahar --
+0	If John McCain wants to follow George Bush with more tough talk and bad strategy, that is his choice—but it is not the change we need.
+0	I mean, I helped, but -- (applause.)
+0	The work of democracy has always been hard.
+0	So, you see, that's what our democracy demands.
+0	And may God bless the United States of America, and all who serve her.
+0	Today, old adversaries are at peace, and emerging democracies are potential partners.
+0	But remember, none of this happens on its own.
+0	And Democrats, we must also admit that fulfilling America's promise will require more than just money.
+0	It actually happened over the course of years of advocacy, and organizing, and alliance-building, and deal-making, and the changing of public opinion.
+0	Health care costs are rising at the slowest rate in 50 years.
+0	That's how progress happens -- in societies and in our own lives.
+0	And it's not just my belief.
+0	I've been living it.
+0	And despite the costs involved, America's commitment will not weaken.
+0	To my remarkable staff : For 8 years—and for some of you , a whole lot more—I have drawn from your energy , and every day I tried to
+0	This country of ours has more wealth than any nation, but that's not what makes us rich.
+0	And while America in the past has focused on oil and gas when it comes to this part of the world, we now seek a broader engagement.
+0	When a new flu infects one human being, all are at risk.
+0	And success, however small, however incomplete, success is still success.
+0	Four years ago , I stood before you and told you my story—of the brief union between a young man from Kenya and a young woman from Kansas who
+0	They have affiliates in many countries and are trying to expand their reach.
+0	So, Class of 2016, let me be as clear as I can be.
+0	That's why we cannot withdraw from big global fights: to expand democracy and human rights and women's rights and LGBT rights.
+0	We’ve doubled the production of clean energy.
+0	For eighteen long months, you have stood up, one by one, and said enough to the politics of the past.
+0	The Holy Koran tells us: "O mankind!
+0	As we do, I’m mindful that the Iraq war has been a contentious issue at home.
+0	Islam is not part of the problem in combating violent extremism, it is an important part of promoting peace.
+0	And to all who serve or have served, it has been the honor of my lifetime to be your Commander in Chief.
+0	And I believe that America holds within her the truth that regardless of race , religion , or station in life , all of us share common aspirations to
+0	We all have to start with the premise that each of our fellow citizens loves this country just as much as we do ; that they value hard work
+0	It determines what policies they prioritize.
+0	Together with Iraqis and coalition partners who made huge sacrifices of their own , our troops fought block by block to help Iraq seize the chance for a better
+0	But I will also go through the federal budget , line by line , eliminating programs that no longer work and making the ones we do need work better
+0	But in the days to come, it must be our central mission as a people, and my central responsibility as President.
+0	Let me tell you, you're not the only ones.
+0	Qualities like kindness and compassion, honesty, hard work -- they often matter more than technical skills or know-how.
+0	I’m confident that you can make the right choices -- away from fear and division and paralysis, and toward cooperation and innovation and hope.
+0	Don't be scared to take somebody on.
+0	And that is why we will support them everywhere.
+0	We see it in the history of Andalusia and Cordoba during the Inquisition.
+0	I'll invest in early childhood education.
+0	We have the power to make the world we seek , but only if we have the courage to make a new beginning , keeping in mind what has
+0	Nor can I answer in the time that I have this afternoon all the complex questions that brought us to this point.
+0	It's time for us to change America.
+0	Which brings me to my third point:  Facts, evidence, reason, logic, an understanding of science -- these are good things.
+0	This moment—this election—is our chance to keep, in the 21st century, the American promise alive.
+0	Rather than remain trapped in the past, I've made it clear to Iran's leaders and people that my country is prepared to move forward.
+0	I am grateful to finish this journey with one of the finest statesmen of our time , a man at ease with everyone from world leaders to the conductors
+0	That's just not knowing what you're talking about.
+0	to weaken in your minds the conviction of this truth."
+0	I always tell my daughters, you know, better is good.
+0	I will eliminate capital gains taxes for the small businesses and the start-ups that will create the high-wage, high-tech jobs of tomorrow.
+0	To the love of my life , our next First Lady , Michelle Obama , and to Sasha and Malia—I love you so much , and I 'm so
+0	Americans across the political spectrum supported the use of force against those who attacked us on 9/11.
+0	It's a faith in other people, and it's what brought me here today.
+0	That's how we banned child labor.
+0	And as someone who watched my mother argue with insurance companies while she lay in bed dying of cancer , I will make certain those companies stop discriminating against
+0	Now , there is no straight line to realize this promise , but this much is clear : Governments that protect these rights are ultimately more stable , successful
+0	I also know that Islam has always been a part of America's story.
+0	Why would we want to stop it now?
+0	The question is whether we spend that time focused on what pushes us apart , or whether we commit ourselves to an effort , a sustained effort to find
+0	Now, I say all these things not to make you complacent.
+0	The challenges we face require tough choices, and Democrats as well as Republicans will need to cast off the worn-out ideas and politics of the past.
+0	And when the first Muslim American was recently elected to Congress , he took the oath to defend our Constitution using the same Holy Koran that one of our
+0	That's the economic argument.
+0	They could've heard words of anger and discord.
+0	And that is why I intend to personally pursue this outcome with all the patience and dedication that the task requires.
+0	AUDIENCE MEMBER:  Because of you!
+0	Of course, the soldiers left much behind.
+0	Our unity at home was tested.
+0	To Joe Biden , the scrappy kid from Scranton who became Delaware 's favorite son , you were the first decision I made as a nominee , and it
+0	We measure the strength of our economy not by the number of billionaires we have or the profits of the Fortune 500 , but by whether someone with a
+0	Michelle and I are only here tonight because we were given a chance at an education.
+0	But I say it to point out that change has been a constant in our history.
+0	And the good news is that today the economy is growing again.
+0	It wasn’t because the facts and the evidence showed they wouldn't work.
+0	But to simply deny the problem not only betrays future generations , it betrays the essential spirit of this country , the essential spirit of innovation and practical problem-solving
+0	Pull yourself up by your own bootstraps—even if you don't have boots.
+0	And by doing so, you’ll strengthen your own position, and you’ll hone your arguments.
+0	Operation Iraqi Freedom is over, and the Iraqi people now have lead responsibility for the security of their country.
+0	And when that government is in place, there should be no doubt:  The Iraqi people will have a strong partner in the United States.
+0	Fortunately, your generation has everything it takes to lead this country toward a brighter future.
+0	We are shaped by every culture , drawn from every end of the Earth , and dedicated to a simple concept : E pluribus unum— '' Out of many
+0	So let us agree that patriotism has no party.
+0	I agree, our trade should be fair and not just free.
+0	Now is the time to change our bankruptcy laws , so that your pensions are protected ahead of CEO bonuses ; and the time to protect Social Security for
+0	That is why there is a mosque in every State in our Union and over 1,200 mosques within our borders.
+0	If we 're unwilling to invest in the children of immigrants , just because they do n't look like us , we will diminish the prospects of our own
+0	That's why we honor Bill Moyers or Dr. Burnell.
+0	Progress in the daily lives of the Palestinian people must be a critical part of a road to peace , and Israel must take concrete steps to enable such
+0	And I want to particularly say this to young people of every faith in every country : You , more than anyone , have the ability to reimagine the
+0	It will require a renewed sense of responsibility from each of us to recover what John F. Kennedy called our "intellectual and moral strength."
+0	And yet , Al Qaida chose to ruthlessly murder these people , claimed credit for the attack , and even now states their determination to kill on a massive
+0	And he’s a senator!
+0	And you have made the country proud.
+0	It is based upon cultural and historical ties and the recognition that the aspiration for a Jewish homeland is rooted in a tragic history that cannot be denied.
+0	And because of our drawdown in Iraq, we are now able to apply the resources necessary to go on offense.
+0	America, this is one of those moments.
+0	It's not because John McCain doesn't care.
+0	You can see it not just in statistics, you see it in the attitudes of young Americans across the political spectrum.
+0	The beginning of this century has been one of those times.
+0	Here in New Brunswick , you can debate philosophy with a classmate from South Asia in one class , and then strike up a conversation on the EE Bus
+0	These are qualities you want to continue to cultivate in yourselves as citizens.
+0	This is a sacred trust.
+0	And this is important , because no development strategy can be based only upon what comes out of the ground , nor can it be sustained while young people
+0	Make them defend their positions.
+0	And I 've said and I mean it : If anyone can put together a plan that is demonstrably better than the improvements we 've made to our health
+0	As I’ve said, there were patriots who supported this war, and patriots who opposed it.
+0	And we got to hold our leaders and ourselves accountable to know what the heck they’re talking about.
+0	For while Senator McCain was turning his sights to Iraq just days after 9/11 , I stood up and opposed this war , knowing that it would distract us
+0	So you have to stick with it.
+0	It's good to be home!
+0	It's not because I wasn’t proposing them.
+0	Now, much has been made of the fact that an African American with the name Barack Hussein Obama could be elected President.
+0	The times are too serious, the stakes are too high for this same partisan playbook.
+0	Because I know our work has not only helped so many Americans , it has inspired so many Americans , especially so many young people out there , to
+0	Our combat mission is ending, but our commitment to Iraq’s future is not.
+0	ISIL will be destroyed, and no one who threatens America will ever be safe.
+0	Yes, we must provide more ladders to success for young men who fall into lives of crime and despair.
+0	It is that spirit , born of the Enlightenment , that made us an economic powerhouse : the spirit that took flight at Kitty Hawk and Cape Canaveral ,
+0	Progress doesn’t travel in a straight line.
+0	For over a thousand years , Al-Azhar has stood as a beacon of Islamic learning , and for over a century , Cairo University has been a source of
+0	Now is the time to end this addiction, and to understand that drilling is a stop-gap measure, not a long-term solution.
+0	The President: I can't do that.
+0	One of the newest members of the Big Ten.
+0	It is time for these settlements to stop.
+0	This is the hope of all humanity.
+0	We must always examine the ways in which we protect it.
+0	And, Class of 2016, it is your turn now -- (applause) -- to shape our nation’s destiny, as well as your own.
+0	Point number one:  When you hear someone longing for the “good old days,” take it with a grain of salt.
+0	Indeed, none of us should tolerate these extremists.
+0	Too much blood has been shed.
+0	Throughout our history , a new generation of Americans has reached up and bent the arc of history in the direction of more freedom , and more opportunity ,
+0	Politicians care about being elected, and they especially care about being reelected.
+0	And ultimately, that's how we help raise wages here in America.
+0	Many more are simply skeptical that real change can occur.
+0	Meanwhile, in the eight years since most of you started high school, we’re also better off.
+0	Because of the extraordinary courage of our men and women in uniform , because of our intelligence officers and law enforcement and diplomats who support our troops , no
+0	And every day, I have learned from you.
+0	It's easier to start wars than to end them.
+0	So let there be no doubt, Islam is a part of America.
+0	It 's a promise I make to my daughters when I tuck them in at night , and a promise that you make to yours—a promise that has led
+0	You have a healthy skepticism for conventional wisdom.
+0	You must maintain your power through consent , not coercion ; you must respect the rights of minorities and participate with a spirit of tolerance and compromise ; you
+0	Along with nearly 1.5 million Americans who have served in Iraq, they fought in a faraway place for people they never knew.
+0	And next August, we will begin a transition to Afghan responsibility.
+0	I am honored to be in the timeless city of Cairo and to be hosted by two remarkable institutions.
+0	We will disrupt, dismantle and defeat al Qaeda, while preventing Afghanistan from again serving as a base for terrorists.
+0	THE PRESIDENT:  Well, that's nice.
+0	It ain’t so.
+0	And that is, gear yourself for the long haul.
+0	Because in an economy like this, the last thing we should do is raise taxes on the middle-class.
+0	That promise is our greatest inheritance.
+0	And as it turned out , America was n't weakened by the presence of these newcomers ; these newcomers embraced this Nation 's creed , and this Nation was
+0	But I guess it 's part of human nature , especially in times of change and uncertainty , to want to look backwards and long for some imaginary past
+0	Because it feeds into the cynicism we all have about government.
+0	You have shown what history teaches us—that at defining moments like this one, the change we need doesn't come from Washington.
+0	That's why I reject discrimination against Muslim Americans, who are just as patriotic as we are.
+0	The dream of opportunity for all people has not come true for everyone in America , but its promise exists for all who come to our shores , and
+0	I mean, listen, climate change is not something subject to political spin.
+0	There have been moments throughout our history that threatened that solidarity.
+0	We ’ ve persevered because of a belief we share with the Iraqi people -- a belief that out of the ashes of war , a new beginning could
+0	In signing the Treaty of Tripoli in 1796 , our second President , John Adams , wrote : `` The United States has in itself no character of enmity
+0	I come here for a simple reason -- to finally settle this pork roll vs. Taylor ham question.
+0	These are the Americans that I know.
+0	But what the people heard instead—people of every creed and color, from every walk of life—is that in America, our destiny is inextricably linked.
+0	For the fight against extremism and intolerance and sectarianism and chauvinism are of a piece with the fight against authoritarianism and nationalist aggression.
+0	That's not the judgment we need.
+0	It is that promise that has always set this country apart—that through hard work and sacrifice , each of us can pursue our individual dreams but still come together
+0	I committed to President-elect Trump that my administration would ensure the smoothest possible transition, just as President Bush did for me.
+0	More of you have lost your homes and even more are watching your home values plummet.
+0	It falls to each of us to be those anxious , jealous guardians of our democracy ; to embrace the joyous task we 've been given to continually try
+0	We would gladly bring every single one of our troops home , if we could be confident that there were not violent extremists in Afghanistan and now Pakistan determined
+0	The truth is, on issue after issue that would make a difference in your lives—on health care and education and the economy—Senator McCain has been anything but independent.
+0	In an age without surrender ceremonies, we must earn victory through the success of our partners and the strength of our own nation.
+0	But for all the real progress that we've made, we know it's not enough.
+0	I 've seen the youngest of children remind us through their actions and through their generosity of our obligations to care for refugees or work for peace and ,
+0	The same is true for the astonishing progress within Muslim-majority countries from Kuala Lumpur to Dubai.
+0	No matter how imperfect our efforts, no matter how expedient ignoring such values may seem, that's part of defending America.
+0	And today , as my call for a time frame to remove our troops from Iraq has been echoed by the Iraqi government and even the Bush Administration ,
+0	Some suggest that it isn't worth the effort, that we are fated to disagree and civilizations are doomed to clash.
+0	Mozzarella sticks and chicken fingers on your cheesesteaks -- (applause.)
+0	I’m sure Michelle would approve.
+0	And I hope your faith has, too.
+0	We’re less dependent on foreign oil.
+0	In the course of a healthy debate, we prioritize different goals, and the different means of reaching them.
+0	America, we are better than these last eight years.
+0	And in the weeks ahead, I look forward to debating them with John McCain.
+0	That is the world we seek, but we can only achieve it together.
+0	But I also believe that the arc of our nation , the arc of the world does not bend towards justice , or freedom , or equality , or
+0	Which brings me to my final point: Our democracy is threatened whenever we take it for granted.
+0	And it is that promise that forty five years ago today , brought Americans from every corner of this land to stand together on a Mall in Washington ,
+0	The obligations that the parties have agreed to under the road map are clear.
+0	That’s how we make sure that international rules are consistent with our values -- including human rights.
+0	Make no mistake, we do not want to keep our troops in Afghanistan.
+0	The question now is not what Iran is against, but rather what future it wants to build.
+0	If your hopes have been dashed again and again, then it's best to stop hoping, and settle for what you already know.
+0	Now is the time for Palestinians to focus on what they can build.
+0	But we have also understood that our nation’s strength and influence abroad must be firmly anchored in our prosperity at home.
+0	We measure progress in the 23 million new jobs that were created when Bill Clinton was President—when the average American family saw its income go up $ 7,500 instead
+0	(The wind starts blowing hard.)
+0	This is a place where you 3D-print prosthetic hands for children, and devise rooftop wind arrays that can power entire office buildings with clean, renewable energy.
+0	The Holy Koran teaches that "whoever kills an innocent" is as—"it is as if he has killed all mankind."
+0	And that's why I strongly reaffirmed America's commitment to seek a world in which no nations hold nuclear weapons.
+0	America is no fragile thing, but the gains of our long journey to freedom are not assured.
+0	Six million Jews were killed, more than the entire Jewish population of Israel today.
+0	Sometimes you took half a loaf.
+0	I know it's getting cold and you guys have to graduate.
+0	The market will fix it.
+0	The fear and anger that it provoked was understandable, but in some cases, it led us to act contrary to our traditions and our ideals.
+0	Only if our politics better reflects the decency of our people.
+0	Now, it's true that a lot of schools invite me to their commencement every year.
+0	Ending this war is not only in Iraq’s interest -- it’s in our own.
+0	Why else would he define middle-class as someone making under five million dollars a year?
+0	We, the people, give it meaning with our participation and with the choices that we make and the alliances that we forge.
+0	Now, I don't think it's a secret that I disagree with many of the foreign policies of Dr. Rice and the previous administration.
+0	And just as we keep our keep our promise to the next generation here at home, so must we keep America's promise abroad.
+0	And the reason America is better is because we didn’t look backwards we didn’t fear the future.
+0	And so at this moment , as we wind down the war in Iraq , we must tackle those challenges at home with as much energy , and grit
+0	I will rebuild our military to meet future conflicts.
+0	Last year, incomes rose for all races, all age groups, for men and for women.
+0	You’re not only better educated, you’ve been more exposed to the world, more exposed to other cultures.
+0	I know that for many, the face of globalization is contradictory.
+0	So you’ve got the tools to lead us.
+0	Denying that fact is baseless, it is ignorant, and it is hateful.
+0	Likewise , it is important for Western countries to avoid impeding Muslim citizens from practicing religion as they see fit , for instance , by dictating what clothes a
+0	There ’ s not much I ’ m afraid to take on in my final year of office , but I know better than to get in the middle
+0	Consistent with our agreement with the Iraqi government, all U.S. troops will leave by the end of next year.
+0	It should ensure opportunity not just for those with the most money and influence, but for every American who's willing to work.
+0	This too is part of America's promise—the promise of a democracy where we can find the strength and grace to bridge divides and unite in common effort.
+0	It 's a promise that says the market should reward drive and innovation and generate growth , but that businesses should live up to their responsibilities to create American
+0	What a radical idea , a great gift that our Founders gave to us : The freedom to chase our individual dreams through our sweat and toil and imagination
+0	And Iraqi forces have taken the fight to al Qaeda, removing much of its leadership in Iraqi-led operations.
+0	And I consider it part of my responsibility as President of the United States to fight against negative stereotypes of Islam wherever they appear.
+0	We cannot walk alone.
+0	Look, I'm not suggesting that cold analysis and hard data are ultimately more important in life than passion, or faith, or love, or loyalty.
+0	It won't boost our economy, and it won’t enhance our security either.
+0	If every economic issue is framed as a struggle between a hardworking white middle class and an undeserving minority , then workers of all shades are going to be
+0	But , contrary to what we hear sometimes from both the left as well as the right , the system isn ’ t as rigged as you think ,
+0	From this desk, seven and a half years ago, President Bush announced the beginning of military operations in Iraq.
+0	And like all Americans, I’m awed by their sacrifice, and by the sacrifices of their families.
+0	You believe in a fair and just and inclusive America.
+1	But new threats also require new thinking.
+1	Graduates of this academy have brought creativity and courage to every field of endeavor.
+1	And some needs and hurts are so deep they will only respond to a mentor's touch or a pastor's prayer.
+1	Some scientists believe that one source of these cures might be embryonic stem cell research.
+1	Building this just peace is America's opportunity, and America's duty.
+1	In the long run , the most realistic way to protect the American people is to provide a hopeful alternative to the hateful ideology of the enemy , by
+1	Our schools will have higher expectations.
+1	Thanks for coming, Roscoe.
+1	Our troops will have a well-defined mission : to help Iraqis clear and secure neighborhoods , to help them protect the local population , and to help ensure that
+1	In earlier operations, political and sectarian interference prevented Iraqi and American forces from going into neighborhoods that are home to those fueling the sectarian violence.
+1	Not feel-good methods, not sound-good methods, but methods that actually work.
+1	But I also understand that if we do not lead the world against terror , that your children and your grandchildren will not grow up in a society that
+1	I know this is in our reach because we are guided by a power larger than ourselves who creates us equal in His image.
+1	And our security will require all Americans to be forward-looking and resolute , to be ready for preemptive action when necessary to defend our liberty and to defend our
+1	I believe that it will.
+1	For much of the last century, America's defense relied on the Cold War doctrines of deterrence and containment.
+1	I don't intend to read it all.
+1	Under the policy I announced five years ago , my administration became the first to make federal funds available for this research , yet only on embryonic stem cell
+1	This is an important piece of legislation.
+1	And to all nations, we will speak for the values that gave our nation birth.
+1	We believe strongly -- we believe strongly the best path to education reform is to trust the local people.
+1	If our country does not lead the cause of freedom, it will not be led.
+1	And the folks at the Crawford Coffee Shop -- (laughter) -- would be somewhat shocked when I told them I actually like the fellow.
+1	And we concluded that to step back now would force a collapse of the Iraqi government , tear the country apart , and result in mass killings on an
+1	Because the war on terror will require resolve and patience, it will also require firm moral purpose.
+1	Eighty percent of Iraq's sectarian violence occurs within 30 miles of the capital.
+1	Americans are called to enact this promise in our lives and in our laws.
+1	Parents will have more information about the schools, and more say in how their children are educated.
+1	We have faced danger and trial, and there's more ahead.
+1	It makes no sense to say that you 're in favor of finding cures for terrible diseases as quickly as possible , and then block a bill that would
+1	Thank you all for coming and may God bless.
+1	America will change our approach to help the Iraqi government as it works to meet these benchmarks.
+1	In tyranny, they suffer.
+1	This is a good bill for the American children, and I'm proud to sign it in their presence.
+1	They remind us of what is lost when embryos are destroyed in the name of research.
+1	And so, therefore, we tripled the amount of federal funding for scientifically-based early reading programs.
+1	I want to thank you all for coming.
+1	We fight, as we always fight, for a just peace -- a peace that favors human liberty.
+1	They also report that this plan can work.
+1	In the 21st century, security and prosperity at home depend on the expansion of liberty abroad.
+1	Every school has a job to do.
+1	This good man was 60 years old—18 years above the age limit.
+1	My policy has allowed us to explore the potential of embryonic stem cells, and it has allowed America to continue to lead the world in this area.
+1	America, at its best, is a place where personal responsibility is valued and expected.
+1	Now we must choose if the example of our fathers and mothers will inspire us or condemn us.
+1	Competition between great nations is inevitable, but armed conflict in our world is not.
+1	Reaching this moment has not been easy, as you could tell from Chairman Boehner's discussion.
+1	The first way to solve a problem is to diagnose it.
+1	We want you to have as much flexibility as possible to see to it that every child that walks in your classroom can succeed.
+1	In this situation, action is clearly necessary.
+1	Recently, local tribal leaders have begun to show their willingness to take on al Qaeda.
+1	Now, as then, they seek to impose a joyless conformity, to control every life and all of life.
+1	West Point is guided by tradition , and in honor of the `` Golden Children of the Corps , '' -- ( applause ) -- I will observe one
+1	And Secretary Rice will soon appoint a reconstruction coordinator in Baghdad to ensure better results for economic assistance being spent in Iraq.
+1	The legislation Congress passed today addresses this problem head on by providing a variety of new tools to the government -- such as allowing us to purchase some of
+1	This will be done as expeditiously as possible, but it cannot be accomplished overnight.
+1	And every immigrant, by embracing these ideals, makes our country more, not less, American.
+1	Toward the end of his life, Dwight Eisenhower recalled the first day he stood on the plain at West Point.
+1	For example, a skin cell to function like an embryonic stem cell.
+1	These young Americans understand that our cause in Iraq is noble and necessary -- and that the advance of freedom is the calling of our time.
+1	The third principle of this bill is that we have got to trust the local folks on how to achieve standards, to meet the standards.
+1	But for it to succeed, our commanders say the Iraqis will need our help.
+1	We've got money in there to make sure teachers know how to teach what works.
+1	But the stakes for America are never small.
+1	I 'm told that plebes , when asked whom they outrank , are required to answer this : `` Sir , the Superintendent 's dog -- ( laughter )
+1	Honorable people have different views, and they will voice their criticisms.
+1	It's a great symbol of what is possible in Washington when good people come together to do what's right.
+1	It establishes a bipartisan board to oversee the plan's implementation.
+1	These bills illustrate both the promise and perils we face in the age of biotechnology.
+1	On one side are those who believe in freedom and moderation.
+1	This legislation would overturn the balanced policy on embryonic stem cell research that my administration has followed for the past five years.
+1	I 'm especially grateful for the contributions of Speaker Nancy Pelosi , Minority Leader John Boehner , Majority Leader Steny Hoyer , Minority Whip Roy Blunt , Chairman Barney
+1	The bill I sign today upholds these humane ideals and draws an important ethical line to guide our research.
+1	It's up to you, the citizens of Hamilton, to make sure no child is left behind.
+1	I've made it clear to the Prime Minister and Iraq's other leaders that America's commitment is not open-ended.
+1	From this day forward, all students will have a better chance to learn, to excel, and to live out their dreams.
+1	The situation in Iraq is unacceptable to the American people -- and it is unacceptable to me.
+1	These two regimes are allowing terrorists and insurgents to use their territory to move in and out of Iraq.
+1	This bill would support the taking of innocent human life in the hope of finding medical benefits for others.
+1	Congress has just passed and sent to my desk two bills concerning the use of stem cells in biomedical research.
+1	And we owe the children of America a good education.
+1	This time , Iraqi and American forces will have a green light to enter those neighborhoods -- and Prime Minister Maliki has pledged that political or sectarian interference will
+1	You walk in the tradition of Eisenhower and MacArthur, Patton and Bradley - the commanders who saved a civilization.
+1	Our troops in Iraq have fought bravely.
+1	No longer is it acceptable to hide poor performance.
+1	And there's no mystery to this achievement.
+1	The work ahead is difficult.
+1	Many listening tonight will ask why this effort will succeed when previous operations to secure Baghdad did not.
+1	The grandest of these ideals is an unfolding American promise that everyone belongs, that everyone deserves a chance, that no insignificant person was ever born.
+1	And I thank you for the countless acts of courage, generosity, and grace that I have witnessed these past eight years.
+1	And above all, I thank the American people for the trust you have given me.
+1	It has been the privilege of a lifetime to serve as your President.
+1	When these forces are fully deployed, there will be 18 Iraqi Army and National Police brigades committed to this effort, along with local police.
+1	Our military forces in Anbar are killing and capturing al Qaeda leaders, and they are protecting the local population.
+1	But the themes of this day he would know: our nation's grand story of courage and its simple dream of dignity.
+1	That's not right in America.
+1	I believe America's scientists have the ingenuity and skill to meet this challenge.
+1	We are in a conflict between good and evil, and America will call evil by its name.
+1	But we have been given solemn responsibilities, and we must meet them.
+1	Containment is not possible when unbalanced dictators with weapons of mass destruction can deliver those weapons on missiles or secretly provide them to terrorist allies.
+1	America, at its best, is also courageous.
+1	Even if our new strategy works exactly as planned, deadly acts of violence will continue -- and we must expect more Iraqi and American casualties.
+1	And we will defend the peace that makes all progress possible.
+1	This is a nation that inspires immigrants to risk everything for the dream of freedom.
+1	America has never been united by blood or birth or soil.
+1	We're going to spend more on our schools, and we're going to spend it more wisely.
+1	Targeting innocent civilians for murder is always and everywhere wrong.
+1	But there can be little debate about the results.
+1	The bill also ensures that these new programs are carried out in a way that protects taxpayers.
+1	And I will always be honored to carry a title that means more to me than any other—citizen of the United States of America.
+1	Victory will not look like the ones our fathers and grandfathers achieved.
+1	We see it in Julio Medina, a former inmate who leads a faith-based program to help prisoners returning to society.
+1	They are living proof that effective medical science can also be ethical.
+1	I appreciate so very much my friend, Judd Gregg, from the state of New Hampshire, being here.
+1	Now, as then, our enemies are totalitarians, holding a creed of power with no place for human dignity.
+1	She said, "I don't ever remember taking an exam.
+1	When it comes to the common rights and needs of men and women, there is no clash of civilizations.
+1	In the days ahead, my national security team will fully brief Congress on our new strategy.
+1	I want to thank George Miller.
+1	Senator Mike DeWine of your state of Ohio , who helped author -- ( applause ) -- who helped to author the safe and drug-free schools part of this
+1	So today, I'm keeping the promise I made to the American people by returning this bill to Congress with my veto.
+1	And now it 's up to you , the local citizens of our great land , the compassionate , decent citizens of America , to stand up and demand
+1	Since I announced my policy in 2001, advances in scientific research have also shown the great potential of stem cells that are derived without harming human embryos.
+1	And they are looking at Iraq.
+1	For the safety of our people, America must succeed in Iraq.
+1	Now is the time to act.
+1	Generations of West Point officers planned and practiced for battles with Soviet Russia.
+1	But it's just the beginning of change.
+1	These troops will work alongside Iraqi units and be embedded in their formations.
+1	And this great republic born alone in liberty is leading the world toward a new age when freedom belongs to all nations.
+1	We see that value in the children who are with us today.
+1	I look forward to singing Judd 's praises because he is a solid , solid United States senator -- honest , full of integrity , and like the others
+1	Our enemies in Iraq will make every effort to ensure that our television screens are filled with images of death and suffering.
+1	It will take more time and determined effort to get through this difficult period.
+1	May God bless this house and our next President.
+1	In that box is the bill.
+1	Our security will require transforming the military you will lead -- a military that must be ready to strike at a moment 's notice in any dark corner of
+1	West Point produced the chief engineer of the Panama Canal, the mind behind the Manhattan Project, the first American to walk in space.
+1	The Fetus Farming Prohibition Act was sponsored by Senators Santorum and Brownback -- both who are here.
+1	Through much of the last century, America's faith in freedom and democracy was a rock in a raging sea.
+1	Our past efforts to secure Baghdad failed for two principal reasons : There were not enough Iraqi and American troops to secure neighborhoods that had been cleared of terrorists
+1	A truly strong nation will permit legal avenues of dissent for all groups that pursue their aspirations without violence.
+1	This fine institution gave us the man they say invented baseball, and other young men over the years who perfected the game of football.
+1	We will confront weapons of mass destruction, so that a new century is spared new horrors.
+1	Our security will require the best intelligence, to reveal threats hidden in caves and growing in laboratories.
+1	To establish its authority, the Iraqi government plans to take responsibility for security in all of Iraq's provinces by November.
+1	This is a strong commitment.
+1	It is fair to hold our views up to scrutiny.
+1	May you always bring to that duty the high standards of this great American institution.
+1	We're going to win the war overseas, and we need to win the war against illiteracy here at home, as well.
+1	We also need to examine ways to mobilize talented American civilians to deploy overseas , where they can help build democratic institutions in communities and nations recovering from war
+1	Freeing people from oppression and despair is eternally right.
+1	So I've committed more than 20,000 additional American troops to Iraq.
+1	I probably won't be sharing that with the Secretary of the Navy.
+1	It crosses a moral boundary that our decent society needs to respect, so I vetoed it.
+1	Americans should also expect that it will take some time for this legislation to have its full impact on our economy.
+1	There 's no greater challenge than to make sure that every child -- and all of us on this stage mean every child , not just a few children
+1	He did a fine job there, and he's doing a fine job in Washington.
+1	Under one, a small band of fanatics demands total obedience to an oppressive ideology, condemns women to subservience, and marks unbelievers for murder.
+1	Together, we will reclaim America's schools, before ignorance and apathy claim more young lives.
+1	President Clinton, distinguished guests and my fellow citizens, the peaceful transfer of authority is rare in history, yet common in our country.
+1	All of the chaos and suffering they caused came at much less than the cost of a single tank.
+1	Lieutenant Commander Krissoff could not be here tonight , because he will soon deploy to Iraq , where he will help save America 's wounded warriors—and uphold the legacy
+1	And I still carry his badge.
+1	He is a fabulous United States senator.
+1	Their solution is to scale back America's efforts in Baghdad -- or announce the phased withdrawal of our combat forces.
+1	And we will reduce taxes, to recover the momentum of our economy and reward the effort and enterprise of working Americans.
+1	Officers graduating that year helped fulfill that mission, defeating Japan and Germany, and then reconstructing those nations as allies.
+1	But the truth of the matter is our schools will flourish when citizens join in the noble cause of making sure no child is left behind.
+1	Some of you followed in the path of the imperfect graduate , Ulysses S. Grant , who had his fair share of demerits , and said the happiest day
+1	Yours is indeed a noble profession.
+1	The gravest danger to freedom lies at the perilous crossroads of radicalism and technology.
+1	We cannot defend America and our friends by hoping for the best.
+1	And we are confident in principles that unite and lead us onward.
+1	In time, they will find that social and political freedom is the only true source of national greatness.
+1	They serve far from their families, who make the quiet sacrifices of lonely holidays and empty chairs at the dinner table.
+1	You graduate from this Academy in a time of war, taking your place in an American military that is powerful and is honorable.
+1	From Afghanistan to Lebanon to the Palestinian Territories , millions of ordinary people are sick of the violence , and want a future of peace and opportunity for their
+1	There have been good days and tough days.
+1	And America owes you a debt of gratitude.
+1	In this way our struggle is similar to the Cold War.
+1	We will build our defenses beyond challenge, lest weakness invite challenge.
+1	And the federal bench includes wise new members like Justice Sam Alito and Chief Justice John Roberts.
+1	And we will seek out and destroy the networks providing advanced weaponry and training to our enemies in Iraq.
+1	They blew up one of the holiest shrines in Shia Islam -- the Golden Mosque of Samarra -- in a calculated effort to provoke Iraq 's Shia population to
+1	In every corner of America, the words "West Point" command immediate respect.
+1	I ask you to seek a common good beyond your comfort; to defend needed reforms against easy attacks; to serve your nation, beginning with your neighbor.
+1	And so the new role of the federal government is to set high standards , provide resources , hold people accountable , and liberate school districts to meet the
+1	It prevents failed executives from receiving windfalls from taxpayers' dollars.
+1	This is the belief that gave birth to our nation.
+1	And this nation will act.
+1	The Senate passed the same legislation on Wednesday night.
+1	And so, my fellow Americans, for the final time: Good night.
+1	Now it is a seed upon the wind, taking root in many nations.
+1	And all four of these members up here need to be proud of the legacy they have left behind.
+1	The elections of 2005 were a stunning achievement.
+1	Five days from now, the world will witness the vitality of American democracy.
+1	So America will change our strategy to help the Iraqis carry out their campaign to put down sectarian violence and bring security to the people of Baghdad.
+1	Our democratic faith is more than the creed of our country , it is the inborn hope of our humanity , an ideal we carry but do not own
+1	We will help the Iraqis build a larger and better-equipped army , and we will accelerate the training of Iraqi forces , which remains the essential U.S. security mission
+1	And all involved have a responsibility to explain how the path they propose would be more likely to succeed.
+1	As we address these challenges—and others we cannot foresee tonight—America must maintain our moral clarity.
+1	America pursues medical advances in the name of life, and we will achieve the great breakthroughs we all seek with reverence for the gift of life.
+1	I felt like crossing this line would be a mistake, and once crossed, we would find it almost impossible to turn back.
+1	When challenges to our prosperity emerged, we rose to meet them.
+1	Our war on terror is only begun, but in Afghanistan it was begun well.
+1	At a moment when ethical alternatives are becoming available , we can not lose the opportunity to conduct research that would give hope to those suffering from terrible diseases
+1	I know you all are anxious to get back to class.
+1	A short time ago, the House of Representatives passed a bill that is essential to helping America's economy weather the financial crisis.
+1	Today, your last day at West Point, you begin a life of service in a career unlike any other.
+1	We will work for a just and peaceful world beyond the war on terror.
+1	A major problem in our financial system is that banks have restricted the flow of credit to businesses and consumers ; many of the assets these banks are holding
+1	And I'm signing this bill here because it's the home of the Chairman, John Boehner.
+1	And our society is better off because you decided to teach.
+1	This new strategy will not yield an immediate end to suicide bombings, assassinations, or IED attacks.
+1	Brutality against women is always and everywhere wrong.
+1	I want to thank the Secretary of Education, Rod Paige, for being here and for his leadership.
+1	Through most of history, poverty was persistent, inescapable, and almost universal.
+1	Thank you for letting us come.
+1	Al Qaeda is still active in Iraq.
+1	But good and evil are present in this world, and between the two of them there can be no compromise.
+1	His son, Nathan—a Marine—gave his life in Iraq.
+1	In keeping with the recommendations of the Iraq Study Group , we will increase the embedding of American advisers in Iraqi Army units , and partner a coalition brigade
+1	I have cherished meeting these selfless patriots and their families.
+1	It sets a grand goal for the country.
+1	Germany fought France , the Axis fought the Allies , and then the East fought the West , in proxy wars and tense standoffs , against a backdrop of
+1	If we've learned anything over the last generations, money alone doesn't make a good school.
+1	This new approach comes after consultations with Congress about the different courses we could take in Iraq.
+1	I still invited him to come with me.
+1	And we will extend the peace by encouraging free and open societies on every continent.
+1	Every child must be taught these principles.
+1	As the years passed, most Americans were able to return to life much as it had been before 9/11.
+1	In the quiet of American conscience, we know that deep, persistent poverty is unworthy of our nation's promise.
+1	More state and local governments will be able to fund basic services.
+1	Unfortunately, Congress has sent me a bill that fails to meet this ethical test.
+1	To give every Iraqi citizen a stake in the country's economy, Iraq will pass legislation to share oil revenues among all Iraqis.
+1	America was founded on the principle that we are all created equal, and endowed by our Creator with the right to life.
+1	Across our country, students are rising to meet higher standards in public schools.
+1	With a simple oath, we affirm old traditions and make new beginnings.
+1	Freedom is the precious gift that one generation can pass to the next.
+1	But we made it, because of the willingness of four fine leaders to do what was right for America.
+1	Some people argue that finding new cures for disease requires the destruction of human embryos like the ones that these families adopted.
+1	This bill would also undermine the principle that Congress , itself , has followed for more than a decade , when it has prohibited federal funding for research that
+1	Thank you very much, General Lennox.
+1	Most of Iraq's Sunni and Shia want to live together in peace -- and reducing the violence in Baghdad will help make reconciliation possible.
+1	The ambitions of some Americans are limited by failing schools and hidden prejudice and the circumstances of their birth.
+1	Yet the war on terror will not be won on the defensive.
+1	I want a West Point football player."
+1	So now it's time to spend billions of dollars and get good results.
+1	When this happens , daily life will improve , Iraqis will gain confidence in their leaders , and the government will have the breathing space it needs to make
+2	Nor do we exaggerate when we say Saddam Hussein will fail.
+2	And thank you very, very much from a grateful people.
+2	And listen to what they say, for they've said it better than any President or Prime Minister ever could.
+2	And in our America , the most generous , optimistic nation on the face of the Earth , we must not and will not rest until every man and
+2	And these are just a few of our pending proposals for reform and renewal.
+2	And recognizing this was a coalition effort , let us not forget Saudi General Khalid , Britain 's General de la Billiere , or General Roquejeoffre of France ,
+2	But we must act this week, when Congress will hold the first of two crucial up-or-down votes.
+2	Its passage has made the United States the international leader on this human rights issue.
+2	Of course, of course, our hearts go out to the hostages and to their families.
+2	The congressional leadership and I both have a job to do in getting it enacted.
+2	It is that learning is good in and of itself.
+2	Second, we must act to control the proliferation of weapons of mass destruction and the missiles used to deliver them.
+2	The answer is clear: The world could wait no longer.
+2	Our problems are large, but our unified heart is larger.
+2	As I look around at all these joyous faces , I remember clearly how many years of dedicated commitment have gone into making this historic new civil rights act
+2	The terrible crimes and tortures committed by Saddam's henchmen against the innocent people of Kuwait are an affront to mankind and a challenge to the freedom of all.
+2	We're coming home now—proud, confident, heads high.
+2	He subjected the people of Kuwait to unspeakable atrocities—and among those maimed and murdered, innocent children.
+2	But this election isn't only about competence, for competence is a narrow ideal.
+2	And now we must decide.
+2	A world where the strong respect the rights of the weak.
+2	Well, don't you believe it.
+2	I will modernize and preserve our technological edge.
+2	In this view, the country waits passive while Washington sets the rules.
+2	Prime Minister Margaret Thatcher, a dependable ally, said it all: "We do not bargain over hostages.
+2	And only a tiny fraction of goods gets through.
+2	You're all right now."
+2	And then, specifically, first the ADA ensures that employers covered by the act cannot discriminate against qualified individuals with disabilities.
+2	And he must be able to define - and lead - a mission.
+2	We embrace you for your abilities and for your disabilities, for our similarities and indeed for our differences, for your past courage and your future dreams.
+2	Some people who are enjoying our prosperity have forgotten what it's for.
+2	Our goal is not the conquest of Iraq.
+2	While the world waited, Saddam Hussein met every overture of peace with open contempt.
+2	I have many friends to thank tonight.
+2	Tonight, the battle has been joined.
+2	Moslems and non-Moslems, Arabs and non-Arabs, soldiers from many nations stand shoulder to shoulder, resolute against Saddam Hussein's ambitions.
+2	The consequences of the conflict in the Gulf reach far beyond the confines of the Middle East.
+2	We will also destroy his chemical weapons facilities.
+2	That support will be even more important in the days to come.
+2	Generous assistance will also be provided to stalwart front-line nations, such as Turkey and Egypt.
+2	Clearly, each and every one of us can find fault with something in this agreement.
+2	I have an adopted granddaughter.
+2	A victory for unprecedented international cooperation and diplomacy, so well led by our Secretary of State, James Baker.
+2	It means teaching troubled children through your presence that there's such a thing as reliable love.
+2	And most of all, it will not let our economy slip out of control.
+2	America is tired of phony deficit reduction or promise-now, save-later plans.
+2	Well, many of our fellow citizens with disabilities are unemployed.
+2	If we continue to grow at our current rate, we will be able to produce 30 million jobs in the next eight years.
+2	Tonight, I ask every community in this country to make this coming Fourth of July a day of special celebration for our returning troops.
+2	As I've pointed out before, conservation efforts are essential to keep our energy needs as low as possible.
+2	We must let these children be our guides and inspiration.
+2	I will encourage merit schools.
+2	And the parents who settled hungry Kansas would take their children in from the fields when a teacher came.
+2	And American citizens abroad must be protected.
+2	Additionally, we and several of our allies always have the option to extract oil from our strategic petroleum reserves if conditions warrant.
+2	And I want to thank Sandy Parrino, of course, for her leadership.
+2	My opponent says no - but I say yes.
+2	That's why Secretary of State Baker and Treasury Secretary Brady have met with many world leaders to underscore that the burden of this collective effort must be shared.
+2	We met then in time of war.
+2	And something else: to curb the proliferation of chemical, biological, ballistic missile and, above all, nuclear technologies.
+2	This is an immensely important day, a day that belongs to all of you.
+2	What a magnificent job you all are doing.
+2	I'll try to hold my charisma in check.
+2	They see "community" as a limited cluster of interest groups, locked in odd conformity.
+2	I respect old fashioned common sense, and have no great love for the imaginings of social planners.
+2	A world where the United Nations, freed from cold war stalemate, is poised to fulfill the historic vision of its founders.
+2	We will continue to review all options with our allies, but let it be clear: we will not let this aggression stand.
+2	And we're very happy about that.
+2	I see America as the leader - a unique nation with a special role in the world.
+2	All should contribute, but the burden should not be excessive for any one group of programs or people.
+2	But here at home there's another threat, a cancer gnawing away at our nation's health.
+2	And we saw soldiers who know about honor and bravery and duty and country and the world-shaking power of these simple words.
+2	On jobs, my mission is: 30 in 8.
+2	Iraq will not be permitted to annex Kuwait.
+2	The victory over Iraq was not waged as "a war to end all wars."
+2	Listen to Master Sergeant J.P. Kendall of the 82d Airborne: "We're here for more than just the price of a gallon of gas.
+2	Well , when given the opportunity to be independent , they will move proudly into the economic mainstream of American life , and that 's what this legislation is
+2	An era in which the nations of the world, East and West, North and South, can prosper and live in harmony.
+2	And one way you know our opponents know the facts is that to attack the record they have to misrepresent it.
+2	The Security Council has imposed mandatory economic sanctions on Iraq, designed to force Iraq to relinquish the spoils of its illegal conquest.
+2	One issue Look at the world on this bright August night.
+2	From the moment Operation Desert Storm commenced on January 16th until the time the guns fell silent at midnight 1 week ago , this nation has watched its sons
+2	It's the toughest deficit reduction package ever, with new enforcement rules to make sure that what we fix now stays fixed.
+2	Out of these troubled times , our fifth objective—a new world order—can emerge : a new era—freer from the threat of terror , stronger in the pursuit of justice
+2	Tonight we ask you to help us move this agreement forward.
+2	First, I ask you to understand how important—and for some, how difficult—this vote is for your Congressmen and Senators.
+2	When we are successful—and we will be—we have a real chance at this new world order , an order in which a credible United Nations can use its peacekeeping
+2	But the stark fact remained that people with disabilities were still victims of segregation and discrimination, and this was intolerable.
+2	Our Secretary of State, James Baker, held an historic meeting in Geneva, only to be totally rebuffed.
+2	Our work is not done - our force is not spent.
+2	I will ban chemical and biological weapons from the face of the earth.
+2	Things aren't perfect in this country.
+2	This principle must be elaborated to provide for Israel's security and recognition and at the same time for legitimate Palestinian political rights.
+2	So, tonight I call on the Congress to move forward aggressively on our domestic front.
+2	That's why they sweat and toil in the sand and the heat and the sun.
+2	It seems to me the Presidency provides an incomparable opportunity for "gentle persuasion."
+2	We were practical - but we didn't live only for material things.
+2	They're worried it's going to remain strong.
+2	We 've all been determined to ensure that it gives flexibility , particularly in terms of the timetable of implementation , and we 've been committed to containing the
+2	I've also asked him to raise the plight of the hostages held in Lebanon.
+2	It belongs to the regulars, to the reserves, to the National Guard.
+2	But what we must remember if we are to be responsible - and compassionate - is that economic growth is the key to our endeavors.
+2	As we speak, our nation is standing together against Saddam Hussein's aggression.
+2	And I am not going to let anyone take it away from us.
+2	Our success in the Gulf will shape not only the new world order we seek but our mission here at home.
+2	If we do , the long-term result will be a healthier nation and something more : We will have once again put ourselves on the path of economic growth
+2	The truth is, this election is about the beliefs we share, the values we honor, the principles we hold dear.
+2	And we'd moved in the wrong direction.
+2	A new partnership of nations has begun.
+2	In a very real sense , this victory belongs to them—to the privates and the pilots , to the sergeants and the supply officers , to the men and
+2	First, we must work together to create shared security arrangements in the region.
+2	Yet, in the conflict just concluded, Israel and many of the Arab States have for the first time found themselves confronting the same aggressor.
+2	They call for Iraq 's immediate and unconditional withdrawal , the restoration of Kuwait 's legitimate government , and categorically reject Iraq 's cynical and self-serving attempt to annex
+2	Instead, he tried to make this a dispute between Iraq and the United States of America.
+2	And I learned a few things about life in a place called Texas.
+2	And I take those who have guided me personally over the years : of course , my friends Evan Kemp and Justin Dart , up here on the platform
+2	I seek the presidency to build a better America.
+2	That cancer is the budget deficit.
+2	For the sake of our principles, for the sake of the Kuwaiti people, we stood our ground.
+2	The legitimate government of Kuwait will be restored to its rightful place, and Kuwait will once again be free.
+2	I want power in the hands of parents.
+2	His ability to threaten mass destruction is itself destroyed.
+2	We will surely have change this year, but will it be change that moves us forward?
+2	I am a man who sees life in terms of missions - missions defined and missions completed.
+2	And you must know us.
+2	This victory belongs to the finest fighting force this nation has ever known in its history.
+2	His people have been lied to, denied the truth.
+2	With today 's signing of the landmark Americans for Disabilities Act , every man , woman , and child with a disability can now pass through once-closed doors into
+2	Ask them to fight for the future of your kids by supporting this budget agreement.
+2	Within 3 days, 120,000 Iraqi troops with 850 tanks had poured into Kuwait and moved south to threaten Saudi Arabia.
+2	And let me say to everyone listening or watching tonight : When the troops we 've sent in finish their work , I am determined to bring them home
+2	America and the world will not be blackmailed by this ruthless policy.
+2	The recent challenge could not have been clearer.
+2	If we fail to enact this agreement, our economy will falter, markets may tumble, and recession will follow.
+2	Together, we must remove the physical barriers we have created and the social barriers that we have accepted.
+2	Iraq will eventually comply with all relevant United Nations resolutions , and then , when peace is restored , it is our hope that Iraq will live as a
+2	Others traveled to Baghdad in a variety of efforts to restore peace and justice.
+2	Many worry about your reaction to one part or another.
+2	Our economy, our Armed Forces, our energy dependence, and our cohesion all determine whether we can help our friends and stand up to our foes.
+2	Some say this isn't an election about ideology, it's an election about competence.
+2	And that is the idea of community - a beautiful word with a big meaning.
+2	Today that new world is struggling to be born, a world quite different from the one we've known.
+2	We will do it - by maintaining our commitment to free and fair trade, by keeping government spending down, and by keeping taxes down.
+2	But let's be frank.
+2	Sanctions will take time to have their full intended effect.
+2	However, Saddam clearly felt that by stalling and threatening and defying the United Nations, he could weaken the forces arrayed against him.
+2	But enduring peace must be our mission.
+2	It's a little-known fact, but America's best job creators and greatest innovators tend to be our smaller companies.
+2	A hundred generations have searched for this elusive path to peace, while a thousand wars raged across the span of human endeavor.
+2	I thank God her parents chose life.
+2	The day of her christening we wept with joy.
+2	My approach this evening is, as Sergeant Joe Friday used to say, "Just the facts, ma'm."
+2	But our policy cannot change, and it will not change.
+2	Tonight I challenge the young people of our country to shut down the drug dealers around the world.
+2	The stakes are high this year and the choice is crucial , for the differences between the two candidates are as deep and wide as they have ever been
+2	We went halfway around the world to do what is moral and just and right.
+2	They've been endorsed by the United Nations Security Council five times in as many weeks.
+2	But for too long, the Nation's business in Washington has been conducted as if these basic rules did not apply.
+2	Our first priority is to get this economy rolling again.
+2	And third, the Congress should, this month, enact measures to increase domestic energy production and energy conservation in order to reduce dependence on foreign oil.
+2	Competence is the creed of the technocrat who makes sure the gears mesh but doesn't for a second understand the magic of the machine.
+2	We moved to west Texas 40 years ago.
+2	This is no time to risk America's capacity to protect her vital interests.
+2	The troops know why they're there.
+2	It means maintaining a capable U.S. naval presence in the region, just as we have for over 40 years.
+2	Clearly, no longer can a dictator count on East-West confrontation to stymie concerted United Nations action against aggression.
+2	It 's time to rise above the parochial and the pork barrel , to do what is necessary , what 's right , and what will enable this nation
+2	Already , leaders of several other countries , including Sweden , Japan , the Soviet Union , and all 12 members of the EEC , have announced that they
+2	How do we complete it?
+2	Any workable solution must be judged as a whole, not piece by piece.
+2	First , the Congress should , this month , within a budget agreement , enact growth-oriented tax measures—to help avoid recession in the short term and to increase savings
+2	While the world prayed for peace, Saddam prepared for war.
+2	But now you must see me for what I am: The Republican candidate for President of the United States.
+2	He said: "It's okay.
+2	After all, the facts are on our side.
+2	Most countries share our concern for principle.
+2	Hide Transcript  Just 2 hours ago, allied air forces began an attack on military targets in Iraq and Kuwait.
+2	But when the list started looking a little longer than the Senate testimony for the bill , I decided I better give up , or that we 'd never
+2	I also want to say a special word to our friends in the business community.
+2	There are millions of young Americans in their 20's who barely remember the days of gas lines and unemployment lines.
+2	America and the world must stand up to aggression—and we will.
+2	Saddam Hussein was the villain; Kuwait, the victim.
+2	It 's been a coalition in the finest spirit—a joining of Democrats and Republicans , of the legislative and the executive branches , of Federal and State agencies ,
+2	"Zero tolerance" isn't just a policy, it's an attitude.
+2	Let that be a promise we make tonight to the American people.
+2	The tactics of terror lead absolutely nowhere.
+2	And I believe that power must always be kept close to the individual - close to the hands that raise the family and run the home.
+2	To all of you, I just want to say your triumph is that your bill will now be law, and that this day belongs to you.
+2	It is the biggest deficit reduction agreement ever—half a trillion dollars.
+2	On the night I announced Operation Desert Storm, I expressed my hope that out of the horrors of war might come new momentum for peace.
+2	We've created 17 million new jobs in the past five years - more than twice as many as Europe and Japan combined.
+2	His war machine is crushed.
+2	I may sometimes be a little awkward, but there's nothing self-conscious in my love of country.
+2	This job has four basic parts.
+2	With your help, we can at last put this budget crisis behind us and face the other challenges that lie ahead.
+2	Now, the challenge is to reach higher, to foster economic freedom and prosperity for all the people of the region.
+2	He sees America as another pleasant country on the UN roll call, somewhere between Albania and Zimbabwe.
+2	There are a lot of great stories in politics about the underdog winning - and this is going to be one of them.
+2	For seven and a half years I have helped a President conduct the most difficult job on earth.
+2	In the early morning hours of August 2d , following negotiations and promises by Iraq 's dictator Saddam Hussein not to use force , a powerful Iraqi army invaded
+2	In Helsinki, our joint statement affirmed to the world our shared resolve to counter Iraq's threat to peace.
+2	America and the world must defend common vital interests—and we will.
+2	Ground forces are not engaged.
+2	We continue to hope that Iraq's leaders will recalculate just what their aggression has cost them.
+2	These valiant Americans were ready at a moment's notice to leave their spouses and their children, to serve on the front line halfway around the world.
+2	That deficit was already greater than it should have been—a projected $232 billion for the coming year.
+2	And while you 're doing that , my administration will be telling the dealers : whatever we have to do we 'll do , but your day is over
+2	These attacks continue as I speak.
+2	They want to work, and they can work, and this is a tremendous pool of people.
+2	I've seen the urban children who play amidst the shattered glass and shattered lives.
+2	Our bipartisan agreement meets these tests.
+2	I've talked to farmers about the bills they can't pay.
+2	You know, I started trying to put together a list of all the people who should be mentioned today.
+2	It is Iraq against the world.
+2	We've come far, but I think we need a new harmony among the races in our country.
+2	No peaceful international order is possible if larger states can devour their smaller neighbors."
+2	It will ensure that people with disabilities are given the basic guarantees for which they have worked so long and so hard : independence , freedom of choice ,
+2	I'm hopeful that this fighting will not go on for long and that casualties will be held to an absolute minimum.
+2	And, yes, we grieve for the people of Iraq, a people who have never been our enemy.
+2	Our ability to function effectively as a great power abroad depends on how we conduct ourselves at home.
+2	But even as planes of the multinational forces attack Iraq, I prefer to think of peace, not war.
+2	We cannot permit a resource so vital to be dominated by one so ruthless.
+2	Let it be clear: Our vital national interests depend on a stable and secure Gulf.
+2	And now we've sent forward our civil rights bill.
+2	New democracies assert themselves in South America.
+2	I've told the American people before that this will not be another Vietnam, and I repeat this here tonight.
+2	This historic act is the world's first comprehensive declaration of equality for people with disabilities—the first.
+2	With , again , great thanks to the Members of the United States Senate , leaders of whom are here today , and those who worked so tirelessly for
+2	The Persian Gulf and Middle East form a region rich in natural resources with a wealth of untapped human potential.
+2	So, if there ever was a time to put country before self and patriotism before party, the time is now.
+2	A President must be many things.
+2	The hard earth there has not yet settled.
+2	But to go beyond such levels, where cutting defense would threaten our vital margin of safety, is something I will never accept.
+2	The fact is, they talk - we deliver.
+2	We are prepared to do our share and more to help carry that load; we insist that others do their share as well.
+2	Lived in a little shotgun house, one room for the three of us.
+2	Well, these rules do apply.
+2	My opponent says no - but I say yes.
+2	The crisis in the Persian Gulf, as grave as it is, also offers a rare opportunity to move toward an historic period of cooperation.
+2	One way or another, the leader of Iraq must learn this fundamental truth.
+2	It will not raise income tax rates, personal or corporate.
+2	But I want to reassure you right now that my administration and the United States Congress have carefully crafted this Act.
+2	Finally, the United Nations has demanded the release of all foreign nationals held hostage against their will and in contravention of international law.
+2	Two men this year ask for your support.
+2	I want to salute the Members of the United States Congress, the House and the Senate who are with us today—active participants in making this day come true.
+2	And we 're going to win with the help of Senator Dan Quayle of Indiana - a young leader who has become a forceful voice in preparing America 's
+2	He did not do that.
+2	But it is the best agreement that can be legislated now.
+2	In fact, we have started to do something.
+2	Weakness and ambivalence lead to war.

speechesdataset/test_LM_hbush.tsv

@@ -0,0 +1,250 @@
+All should contribute, but the burden should not be excessive for any one group of programs or people.
+We will also destroy his chemical weapons facilities.
+I may not be the most eloquent, but I learned early that eloquence won't draw oil from the ground.
+The America we saw in Desert Storm was first-class talent.
+It means helping a child from an unhappy home learn how to read - and I thank my wife Barbara for all her work in literacy.
+In foreign affairs I will continue our policy of peace through strength.
+My opponent now says he'll raise them as a last resort, or a third resort.
+For inspiration, we need look no further than our own neighbors.
+This historic act is the world's first comprehensive declaration of equality for people with disabilities—the first.
+Ours is an all-volunteer force, magnificently trained, highly motivated.
+One way or another, the leader of Iraq must learn this fundamental truth.
+Iraq will not be permitted to annex Kuwait.
+The terrible crimes and tortures committed by Saddam's henchmen against the innocent people of Kuwait are an affront to mankind and a challenge to the freedom of all.
+China feels the winds of change.
+Had we not responded to this first provocation with clarity of purpose, if we do not continue to demonstrate our determination, it would be a signal to actual and potential despots around the world.
+If they are Republicans, urge them to stand with the President.
+"Zero tolerance" isn't just a policy, it's an attitude.
+But we cannot lead a new world abroad if, at home, it's politics as usual on American defense and diplomacy.
+The liberal democrats want power in the hands of the federal government.
+We grieve for all our fallen soldiers and their families, for all the innocents caught up in this conflict.
+This year, we're sending to Congress our comprehensive crime package to finish the job.
+I've asked Secretary of State Baker to go to the Middle East to begin the process.
+They are hostages, and the whole world knows it.
+There are those who have dropped their standards along the way, as if ethics were too heavy and slowed their rise to the top.
+Moved from the shotgun to a duplex apartment to a house.
+At this very moment, they serve together with Arabs, Europeans, Asians, and Africans in defense of principle and the dream of a new world order.
+It must reform the budget process and, further, it must be real.
+A soldier, Private First Class Wade Merritt of Knoxville, Tennessee, now stationed in Saudi Arabia, wrote his parents of his worries, his love of family, and his hope for peace.
+But tragically, for too many Americans, the blessings of liberty have been limited or even denied.
+That cancer is the budget deficit.
+Should free men and women have the right to own a gun to protect their home?
+Our beaches should not be garbage dumps and our harbors should not be cesspools.
+Perhaps what is happening will change our world forever.
+Those well-known words are so very true today.
+Two parties this year ask for your trust, but only one has earned it.
+My opponent says no - but I say yes.
+Others traveled to Baghdad in a variety of efforts to restore peace and justice.
+Our role is to help our friends in their own self-defense.
+And to those Members of the House of Representatives with us here today, Democrats and Republicans as well, I salute you.
+Those who dislike one part or another may pick our agreement apart.
+I am a man who sees life in terms of missions - missions defined and missions completed.
+This is no time to risk America's capacity to protect her vital interests.
+I want growth that stays, that broadens, and that touches, finally, all Americans, form the hollows of Kentucky to the sunlit streets of Denver, from the suburbs of Chicago to the broad avenues of New York, from the oil fields of Oklahoma to the farms of the great plains.
+He must be a shrewd protector of America's interests; And he must be an idealist who leads those who move for a freer and more democratic planet.
+This act does something important for American business, though—and remember this: You've called for new sources of workers.
+And every time I hear someone has breached the public trust it breaks my heart.
+He subjected the people of Kuwait to unspeakable atrocities—and among those maimed and murdered, innocent children.
+I will increase the power of parents.
+Our bipartisan agreement meets these tests.
+On jobs, my mission is: 30 in 8.
+The ones who raise the family, pay the taxes, meet the mortgage.
+I am going to do whatever it takes to make sure the disabled are included in the mainstream.
+He never asked for, but he did receive, my loyalty.
+And fourth, the ADA ensures equivalent telephone services for people with speech or hearing impediments.
+It is the biggest deficit reduction agreement ever—half a trillion dollars.
+So, our budget plan will give small and medium-size companies a needed shot in the arm.
+We've all been determined to ensure that it gives flexibility, particularly in terms of the timetable of implementation, and we've been committed to containing the costs that may be incurred.
+Things aren't perfect in this country.
+But if there have to be tax measures, they should allow the economy to grow, they should not turn us back to higher income tax rates, and they should be fair.
+Our troops will have the best possible support in the entire world, and they will not be asked to fight with one hand tied behind their back.
+Year after year, it mortgages the future of our children.
+To revitalize our leadership, our leadership capacity, we must address our budget deficit—not after election day, or next year, but now.
+I seek the presidency to build a better America.
+Together, we must remove the physical barriers we have created and the social barriers that we have accepted.
+But what we must remember if we are to be responsible - and compassionate - is that economic growth is the key to our endeavors.
+Our friends and allies in the Middle East recognize that they will bear the bulk of the responsibility for regional security.
+Sanctions, though having some effect, showed no signs of accomplishing their objective.
+You're all right now."
+Generous assistance will also be provided to stalwart front-line nations, such as Turkey and Egypt.
+And on your behalf, as well as the behalf of this entire country, I now lift my pen to sign this Americans with Disabilities Act and say: Let the shameful wall of exclusion finally come tumbling down.
+The time has come to put an end to Arab-Israeli conflict.
+Some people who are enjoying our prosperity have forgotten what it's for.
+Hide Transcript  Mr. President and Mr. Speaker and Members of the United States Congress, distinguished guests, fellow Americans, thank you very much for that warm welcome.
+Saddam has arrogantly rejected all warnings.
+Ask them to fight for the future of your kids by supporting this budget agreement.
+The spirit of Democracy is sweeping the Pacific rim.
+We can now point to five United Nations Security Council resolutions that condemn Iraq's aggression.
+There are those who say there isn't much of a difference this year.
+But when they were in charge it was all holes and no cheese.
+Second, we must act to control the proliferation of weapons of mass destruction and the missiles used to deliver them.
+This is not, as Saddam Hussein would have it, the United States against Iraq.
+From the moment Operation Desert Storm commenced on January 16th until the time the guns fell silent at midnight 1 week ago, this nation has watched its sons and daughters with pride, watched over them with prayer.
+Americans must never again enter any crisis, economic or military, with an excessive dependence on foreign oil and an excessive burden of Federal debt.
+Does government have a place?
+And remember, this is a tremendous pool of people who will bring to jobs diversity, loyalty, proven low turnover rate, and only one request: the chance to prove themselves.
+My friends, eight years ago this economy was flat on its back - intensive care.
+From the individual to the family to the community, and on out to the town, to the church and school, and, still echoing out, to the county, the state, the nation - each doing only what it does well, and no more.
+Recent events have surely proven that there is no substitute for American leadership.
+And second, the Congress should, this month, enact a prudent multiyear defense program, one that reflects not only the improvement in East-West relations but our broader responsibilities to deal with the continuing risks of outlaw action and regional conflict.
+Higher oil prices slow our growth, and higher defense costs would only make our fiscal deficit problem worse.
+They call it a Swiss cheese economy.
+It's time to rise above the parochial and the pork barrel, to do what is necessary, what's right, and what will enable this nation to play the leadership role required of us.
+Americans can move forward to lend, spend, and invest in this, the strongest economy on Earth.
+We must also enact the legislation that is key to building a better America.
+As you know, I've just returned from a very productive meeting with Soviet President Gorbachev.
+For too long they've been left out.
+As I look around at all these joyous faces, I remember clearly how many years of dedicated commitment have gone into making this historic new civil rights act a reality.
+I am a quiet man - but I hear the quiet people others don't.
+Until now, the world we've known has been a world divided—a world of barbed wire and concrete block, conflict, and cold war.
+I am convinced not only that we will prevail but that out of the horror of combat will come the recognition that no nation can stand against a world united, no nation will be permitted to brutally assault its neighbor.
+But it doesn't have to be that way.
+Prime Minister Margaret Thatcher, a dependable ally, said it all: "We do not bargain over hostages.
+The Security Council has imposed mandatory economic sanctions on Iraq, designed to force Iraq to relinquish the spoils of its illegal conquest.
+I will encourage merit schools.
+Some may ask: Why act now?
+I've seen the urban children who play amidst the shattered glass and shattered lives.
+It means maintaining a capable U.S. naval presence in the region, just as we have for over 40 years.
+When we are successful—and we will be—we have a real chance at this new world order, an order in which a credible United Nations can use its peacekeeping role to fulfill the promise and vision of the U.N.'s founders.
+But even as planes of the multinational forces attack Iraq, I prefer to think of peace, not war.
+So, even though so many deserve credit, I will single out but a tiny handful.
+We have in this past year made great progress in ending the long era of conflict and cold war.
+A world in which freedom and respect for human rights find a home among all nations.
+I've been to the factories that feel the strain of change.
+Iraq is feeling the heat.
+The United Nations is backing up its words with action.
+Finally, let no one even contemplate profiteering from this crisis.
+How do we complete it?
+And I have directed Secretary Cheney to begin the immediate return of American combat units from the Gulf.
+Even the strongest person couldn't scale the Berlin Wall to gain the elusive promise of independence that lay just beyond.
+Our success in the Gulf will shape not only the new world order we seek but our mission here at home.
+We have before us the opportunity to forge for ourselves and for future generations a new world order—a world where the rule of law, not the law of the jungle, governs the conduct of nations.
+Tonight, I come to this House to speak about the world—the world after war.
+I know there have been concerns that the ADA may be vague or costly, or may lead endlessly to litigation.
+And while you're doing that, my administration will be telling the dealers: whatever we have to do we'll do, but your day is over, you're history.
+That support will be even more important in the days to come.
+I see America as the leader - a unique nation with a special role in the world.
+If the Congress cannot get me a budget, then Americans will have to face a tough, mandated sequester.
+But our policy cannot change, and it will not change.
+For America to lead, America must remain strong and vital.
+Our failure to do these things has made us more dependent on foreign oil than ever before.
+By meeting these four challenges we can build a framework for peace.
+I will ban chemical and biological weapons from the face of the earth.
+There are people who haven't tasted the fruits of the expansion.
+Our quarrel is with Iraq's dictator and with his aggression.
+To help defray costs, the leaders of Saudi Arabia, Kuwait, and the UAE—the United Arab Emirates—have pledged to provide our deployed troops with all the food and fuel they need.
+Saddam Hussein is literally trying to wipe a country off the face of the Earth.
+And, yes, we grieve for the people of Iraq, a people who have never been our enemy.
+An election that is about ideas and values is also about philosophy.
+Vital economic interests are at risk as well.
+Not only two very different men, but two very different ideas of the future will be voted on this election day.
+On the night I announced Operation Desert Storm, I expressed my hope that out of the horrors of war might come new momentum for peace.
+I believe in another tradition that is, by now, embedded in the national soul.
+We were practical - but we didn't live only for material things.
+We have to help them.
+Already, leaders of several other countries, including Sweden, Japan, the Soviet Union, and all 12 members of the EEC, have announced that they hope to enact now similar legislation.
+We're coming home now—proud, confident, heads high.
+Three weeks ago we celebrated our nation's Independence Day.
+We must do all that we can to close the gap between Israel and the Arab States—and between Israelis and Palestinians.
+Additionally, we and several of our allies always have the option to extract oil from our strategic petroleum reserves if conditions warrant.
+While the world waited, Saddam Hussein systematically raped, pillaged, and plundered a tiny nation, no threat to his own.
+And as President, I can report to the Nation aggression is defeated.
+In the face of tyranny, let no one doubt American credibility and reliability.
+Moslems and non-Moslems, Arabs and non-Arabs, soldiers from many nations stand shoulder to shoulder, resolute against Saddam Hussein's ambitions.
+If what goes up must come down, then the way down could be very hard.
+He will go to listen, to probe, to offer suggestions—to advance the search for peace and stability.
+We are prepared to do our share and more to help carry that load; we insist that others do their share as well.
+A world in which nations recognize the shared responsibility for freedom and justice.
+Our first priority is to get this economy rolling again.
+Let us honor them with our gratitude.
+We will not have it.
+I urge Congress to provide a comprehensive 5-year deficit reduction program to me as a complete legislative package, with measures to assure that it can be fully enforced.
+I have seen the unexpected crisis that arrive in a cable in a young aide's hand.
+Long after all our troops come home—and we all hope it's soon, very soon—there will be a lasting role for the United States in assisting the nations of the Persian Gulf.
+The congressional leadership and I both have a job to do in getting it enacted.
+Until Iraq convinces the world of its peaceful intentions—that its leaders will not use new revenues to rearm and rebuild its menacing war machine—Iraq must not have access to the instruments of war.
+I'm the one who believes it is a scandal to give a weekend furlough to a hardened first degree killer who hasn't even served enough time to be eligible for parole.
+And now we must act to correct that trend.
+But the big thing I learned is the satisfaction of creating jobs, which meant creating opportunity, which meant happy families, who in turn could do more to help others and enhance their own lives.
+I wish I could say that their work is done.
+But when the list started looking a little longer than the Senate testimony for the bill, I decided I better give up, or that we'd never get out of here before sunset.
+And then, let's build on success with those and enact the rest of our agenda.
+The level of world cooperation and condemnation of Iraq is unprecedented.
+Our problems are large, but our unified heart is larger.
+I'm the one who says a drug dealer who is responsible for the death of a policeman should be subject to capital punishment.
+And tonight I ask for your help.
+We are already addressing the immediate economic consequences of Iraq's aggression.
+This is a victory for every country in the coalition, for the United Nations.
+The Gulf war put this new world to its first test.
+And these are just a few of our pending proposals for reform and renewal.
+Two men this year ask for your support.
+There are a lot of great stories in politics about the underdog winning - and this is going to be one of them.
+And I again—it is very risky with all these Members of Congress here who worked so hard, but I can say on a very personal basis, Bob Dole has inspired me.
+Let no one doubt our staying power.
+Prior to ordering our forces into battle, I instructed our military commanders to take every necessary step to prevail as quickly as possible, and with the greatest degree of protection possible for American and allied service men and women.
+Second, the ADA ensures access to public accommodations such as restaurants, hotels, shopping centers, and offices.
+We will continue to review all options with our allies, but let it be clear: we will not let this aggression stand.
+Lisa, you might not have been welcome at your theater, but I'll tell you—welcome to the White House.
+This act is powerful in its simplicity.
+Desert Storm's success belongs to the team that so ably leads our Armed Forces: our Secretary of Defense and our Chairman of the Joint Chiefs, Dick Cheney and Colin Powell.
+I'm sure that many of you saw on the television the unforgettable scene of four terrified Iraqi soldiers surrendering.
+I want to salute the Members of the United States Congress, the House and the Senate who are with us today—active participants in making this day come true.
+We came in and gave it emergency treatment: Got the temperature down by lowering regulation, got the blood pressure down when we lowered taxes.
+I wonder sometimes if we have forgotten who we are.
+A new partnership of nations has begun.
+And so, second, I ask you to take this initiative: Tell your Congressmen and Senators you support this deficit reduction agreement.
+And I learned a few things about life in a place called Texas.
+In this view, the country waits passive while Washington sets the rules.
+In the final analysis, our ability to meet our responsibilities abroad depends upon political will and consensus at home.
+My approach this evening is, as Sergeant Joe Friday used to say, "Just the facts, ma'm."
+We've created 17 million new jobs in the past five years - more than twice as many as Europe and Japan combined.
+But now it's time to produce.
+And I take those who have guided me personally over the years: of course, my friends Evan Kemp and Justin Dart, up here on the platform with me; and of course—I hope you'll forgive me for also saying a special word of thanks to two from the White House, but again, this is personal, so I don't want to offend those omitted—two from the White House, Boyden Gray and Bill Roper, who labored long and hard.
+His people have been lied to, denied the truth.
+What a magnificent job you all are doing.
+Born in the middle of the century, in the middle of America, and holding the promise of the future - I'm proud to have Dan Quayle at my side.
+Now is the time for you, the American people, to have a real impact.
+This is America: the Knights of Columbus, the Grange, Hadassah, the Disabled American Veterans, the Order of Ahepa, the Business and Professional Women of America, the union hall, the Bible study group, LULAC, "Holy Name" - a brilliant diversity spread like stars, like a thousand points of light in a broad and peaceful sky.
+To America's elderly I say, "Once again you have the security that is your right - and I'm not going to let them take it away from you."
+Before we took off we all understood that no matter what, you try to reach the target.
+It's better to deal with this guy now than 5 years from now."
+Lived the dream - high school football on Friday night, Little League, neighborhood barbecue.
+And God bless the United States of America.
+Then, under strict international supervision that guarantees the proper destination, then food will be permitted.
+And now I sign legislation which takes a sledgehammer to another wall, one which has for too many generations separated Americans with disabilities from the freedom they could glimpse, but not grasp.
+Oil-producing nations are already replacing lost Iraqi and Kuwaiti output.
+And there is another tradition.
+American interests are far reaching.
+The agreement is fair, and so is the American spirit.
+Our work is not done - our force is not spent.
+These votes will be on a deficit reduction agreement worked out between the administration and the bipartisan leaders of Congress.
+The war with Iraq is over.
+As you know, when the oil embargo was imposed in the early 1970's, the United States imported almost 6 million barrels of oil a day.
+I am going to stop ocean dumping.
+For our part, we've contributed $28 million for relief efforts.
+We've learned in the modern age geography cannot guarantee security, and security does not come from military power alone.
+This victory belongs to the finest fighting force this nation has ever known in its history.
+We weren't saints - but we lived by standards.
+Let's begin with two initiatives we should be able to agree on quickly: transportation and crime.
+Iraq itself controls some 10 percent of the world's proven oil reserves.
+It will guarantee fair and just access to the fruits of American life which we all must be able to enjoy.
+A President must be many things.
+My life has been lived in the shadow of war - I almost lost my life in one.
+And so, together we rejoiced when that barrier fell.
+America and the world must stand up to aggression—and we will.
+Thirty million jobs in the next eight years.
+The hard earth there has not yet settled.
+What we're doing is going to chart the future of the world for the next 100 years.
+You know, I started trying to put together a list of all the people who should be mentioned today.
+And I have seen problems that simmer on for decades and suddenly demand resolution.
+Much of Saddam's artillery and tanks will be destroyed.
+That scene says a lot about America, a lot about who we are.
+We must reduce the harm done by acid rain.
+My opponent won't rule out raising taxes.
+They are some of the finest men and women of the United States of America.
+New democracies assert themselves in South America.
+We gather tonight, witness to events in the Persian Gulf as significant as they are tragic.
+Not by yielding up defense systems with nothing won in return - but by hard cool engagement in the tug and pull of diplomacy.
+My hope is that one day we will once again welcome them as friends into the community of nations.
+It happened when we acted on the ancient knowledge that strength and clarity lead to peace - weakness and ambivalence lead to war.
+But you see, I believe public service is honorable.

speechesdataset/test_LM_obama.txt

@@ -0,0 +1,250 @@
+The third source of tension is our shared interest in the rights and responsibilities of nations on nuclear weapons.
+I'm grateful for your hospitality and the hospitality of the people of Egypt.
+It should help us, not hurt us.
+Is it any wonder that I am optimistic?
+It took a proud daughter of New Jersey, Alice Paul, years of organizing marches and hunger strikes and protests, and drafting hundreds of pieces of legislation, and writing letters and giving speeches, and working with congressional leaders before she and other suffragettes finally helped win women the right to vote.
+On science and technology, we will launch a new fund to support technological development in Muslim-majority countries and to help transfer ideas to mark—to the marketplace so they can create more jobs.
+It's what pushed women to reach for the ballot.
+This is a sacred trust.
+I don't know about you, but I'm not ready to take a ten percent chance on change.
+It depends on us, on the choices we make, particularly at certain inflection points in history; particularly when big changes are happening and everything seems up for grabs.
+But we must never lose sight of what’s at stake.
+I'll recruit an army of new teachers, and pay them higher salaries and give them more support.
+To strengthen our middle class, we must give all our children the education they deserve, and all our workers the skills that they need to compete in a global economy.
+Many Gulf States have enjoyed great wealth as a consequence of oil, and some are beginning to focus it on broader development.
+That was smart, because I have a soft spot for grandmas.
+It's good to be home!
+And you may have a new understanding not only about what your opponents believe but maybe what you believe.
+As I’ve said, there were patriots who supported this war, and patriots who opposed it.
+Because it's up to all of us to make sure our Government can help us meet the many challenges we still face.
+I recognize it will be hard to overcome decades of mistrust, but we will proceed with courage, rectitude, and resolve.
+What a radical idea, a great gift that our Founders gave to us: The freedom to chase our individual dreams through our sweat and toil and imagination, and the imperative to strive together, as well, to achieve a common good, a greater good.
+So, you see, that's what our democracy demands.
+So you have made me proud.
+That's why we are providing more than $2.8 billion to help Afghans develop their economy and deliver services that people depend on.
+If you're tired of arguing with strangers on the Internet, try talking with one of them in real life.
+And despite the costs involved, America's commitment will not weaken.
+We, the people, give it power.
+But I stand before you tonight because all across America something is stirring.
+And we will welcome all elected, peaceful governments, provided they govern with respect for all their people.
+At the same time, we will encourage more Americans to study in Muslim communities.
+Mozzarella sticks and chicken fingers on your cheesesteaks -- (applause.)
+What America can do, and will do, is provide support for the Iraqi people as both a friend and a partner.
+And no challenge is more essential to our security than our fight against al Qaeda.
+The only resolution is for the aspirations of both sides to be met through two states, where Israelis and Palestinians each live in peace and security.
+For White Americans, it means acknowledging that the effects of slavery and Jim Crow didn't suddenly vanish in the sixties, that when minority groups voice discontent, they're not just engaging in reverse racism or practicing political correctness.
+I know there are many, Muslim and non-Muslim, who question whether we can forge this new beginning.
+You will deal occasionally with foolish people.
+Take the challenge of climate change.
+It's a promise that says the market should reward drive and innovation and generate growth, but that businesses should live up to their responsibilities to create American jobs, look out for American workers, and play by the rules of the road.
+The beginning of this century has been one of those times.
+All this has bred more fear and more mistrust.
+At every turn, America’s men and women in uniform have served with courage and resolve.
+We’ve endured a long and painful recession.
+Yes, our progress has been uneven.
+The Holy Koran teaches that "whoever kills an innocent" is as—"it is as if he has killed all mankind."
+That experience guides my conviction that partnership between America and Islam must be based on what Islam is, not what it isn't.
+And such a vision, however well intended, was never realistic.
+Look at somebody like Yasmin Ramadan, who began organizing anti-bullying assemblies when she was 10 years old to help kids handle bias and discrimination, and here at Rutgers, helped found the Muslim Public Relations Council to work with administrators and police to promote inclusion.
+America, this is one of those moments.
+That's how our democracy was designed.
+So you have to stick with it.
+I committed to President-elect Trump that my administration would ensure the smoothest possible transition, just as President Bush did for me.
+America, we weaken those ties when we allow our political dialogue to become so corrosive that people of good character aren't even willing to enter into public service; so coarse with rancor that Americans with whom we disagree are seen not just as misguided, but as malevolent.
+Fear that because of modernity, we lose control over our economic choices, our politics, and, most importantly, our identities, those things we most cherish about our communities, our families, our traditions, and our faith.
+We will disrupt, dismantle and defeat al Qaeda, while preventing Afghanistan from again serving as a base for terrorists.
+And that’s exactly why it’s always been young people like you that have brought about big change -- because you don't fear the future.
+In fact, by almost every measure, America is better, and the world is better, than it was 50 years ago, or 30 years ago, or even eight years ago.
+I'm aware that there's still some who would question or even justify the events of 9/11.
+Yes, government must lead on energy independence, but each of us must do our part to make our homes and businesses more efficient.
+As the leader of the free world, America will do more than just defeat on the battlefield those who offer hatred and destruction -- we will also lead among those who are willing to work together to expand freedom and opportunity for all people.
+I saw it firsthand as a child in Indonesia, where devout Christians worshiped freely in an overwhelmingly Muslim country.
+That is the responsibility we have to one another as human beings.
+I've seen it in the workers who would rather cut their hours back a day than see their friends lose their jobs, in the soldiers who re-enlist after losing a limb, in the good neighbors who take a stranger in when a hurricane strikes and the floodwaters rise.
+Many have served multiple tours of duty, far from families who bore a heroic burden of their own, enduring the absence of a husband’s embrace or a mother’s kiss.
+To play a role in fulfilling Palestinian aspirations, to unify the Palestinian people, Hamas must put an end to violence, recognize past agreements, recognize Israel's right to exist.
+That won't keep America safe.
+That does not lessen my commitment, however, to governments that reflect the will of the people.
+This is not simply about America's interests.
+So let us agree that patriotism has no party.
+And since our founding, American Muslims have enriched the United States.
+And we will keep our promise to every young American - if you commit to serving your community or your country, we will make sure you can afford a college education.
+And Democrats, we must also admit that fulfilling America's promise will require more than just money.
+For more than 60 years, they've endured the pain of dislocation.
+That's the change we need right now.
+They claim that our insistence on something larger, something firmer and more honest in our public life is just a Trojan Horse for higher taxes and the abandonment of traditional values.
+That's not the judgment we need.
+I will end this war in Iraq responsibly, and finish the fight against al Qaeda and the Taliban in Afghanistan.
+The enduring faith of over a billion people is so much bigger than the narrow hatred of a few.
+While the top 1 percent has amassed a bigger share of wealth and income, too many of our families, in inner cities and in rural counties, have been left behind—the laid off factory worker, the waitress or health care worker who's just barely getting by and struggling to pay the bills—convinced that the game is fixed against them, that their Government only serves the interests of the powerful.
+Since I graduated, an Iron Curtain fell, apartheid ended.
+As our military draws down, our dedicated civilians -- diplomats, aid workers, and advisors -- are moving into the lead to support Iraq as it strengthens its government, resolves political disputes, resettles those displaced by war, and builds ties with the region and the world.
+My fellow Americans, Michelle and I have been so touched by all the well wishes that we've received over the past few weeks.
+You’ve seen economic and technological and cultural shifts that are profoundly altering how we work and how we communicate, how we live, how we form families.
+Now part of this conviction is rooted in my own experience.
+Whether or not we stand up for our freedoms, whether or not we respect and enforce the rule of law, that's up to us.
+That's why we've ended torture, worked to close Gitmo, reformed our laws governing surveillance to protect privacy and civil liberties.
+That’s how we help our workers compete on a level playing field.
+But ultimately, these terrorists will fail to achieve their goals.
+In signing the Treaty of Tripoli in 1796, our second President, John Adams, wrote: "The United States has in itself no character of enmity against the laws, religion, or tranquillity of Muslims."
+In Turkey, Pakistan, Bangladesh, Indonesia, we've seen Muslim-majority countries elect a woman to lead.
+Home of what I understand to be a Grease Truck for a Fat Sandwich.
+As we do, I’m mindful that the Iraq war has been a contentious issue at home.
+I even got three notes from the grandmother of your student body president.
+Freedom of religion is central to the ability of peoples to live together.
+Sometimes you lost on an issue, and then you came back to fight another day.
+I've been living it.
+These are qualities you want in people making policy.
+They have not served a Red America or a Blue America—they have served the United States of America.
+As President, I will tap our natural gas reserves, invest in clean coal technology, and find ways to safely harness nuclear power.
+And though our nation may be travelling through rough waters, they give us confidence that our course is true, and that beyond the pre-dawn darkness, better days lie ahead.
+There's one rule that lies at the heart of every religion, that we do unto others as we would have them do unto us.
+Which makes sense, because measured against the whole of human history, America remains a very young nation -- younger, even, than this university.
+All right, everybody sit down.
+It is that promise that has always set this country apart—that through hard work and sacrifice, each of us can pursue our individual dreams but still come together as one American family, to ensure that the next generation can pursue their dreams as well.
+More recently, tension has been fed by colonialism that denied rights and opportunities to many Muslims and a cold war in which Muslim-majority countries were too often treated as proxies without regard to their own aspirations.
+Our youth, our drive, our diversity and openness, our boundless capacity for risk and reinvention means that the future should be ours.
+Now, one of the reasons that people don’t vote is because they don’t see the changes they were looking for right away.
+There are times where we must take military action to protect ourselves and our allies, and we are in awe of and we are grateful for the men and women who make up the finest fighting force the world has ever known.
+It was innovation in Muslim communities that developed the order of algebra, our magnetic compass and tools of navigation, our mastery of pens and printing, our understanding of how disease spreads and how it can be healed.
+The United States has been one of the greatest sources of progress that the world has ever known.
+That's why I reject discrimination against Muslim Americans, who are just as patriotic as we are.
+It is time for us to act on what everyone knows to be true.
+It's about preventing a nuclear arms race in the Middle East that could lead this region and the world down a hugely dangerous path.
+And engagement means using all the levers of our national power, and rallying the world to take on our shared challenges.
+We, the people, give it meaning with our participation and with the choices that we make and the alliances that we forge.
+We have taken out tens of thousands of terrorists, including bin Laden.
+Don't feel like you got to shut your ears off because you're too fragile and somebody might offend your sensibilities.
+And yet, in our public lives, we certainly think, “I don't want somebody who’s done it before.”  (Laughter and applause.)
+The Holy Koran tells us: "O mankind!
+If we want to close loopholes that allow large corporations and wealthy individuals to avoid paying their fair share of taxes, we’ve got to have the cooperation of other countries in a global financial system to help enforce financial laws.
+After 8 years as your President, I still believe that.
+If every economic issue is framed as a struggle between a hardworking white middle class and an undeserving minority, then workers of all shades are going to be left fighting for scraps while the wealthy withdraw further into their private enclaves.
+We were founded upon the ideal that all are created equal, and we have shed blood and struggled for centuries to give meaning to those words, within our borders and around the world.
+More women are in the workforce.
+These needs will be met only if we act boldly in the years ahead and if we understand that the challenges we face are shared and our failure to meet them will hurt us all.
+These are qualities you want to continue to cultivate in yourselves as citizens.
+And the four who stayed home determined the course of this country just as much as the single one who voted.
+That's not keeping it real, or telling it like it is.
+Cynicism is so easy, and cynics don’t accomplish much.
+Just as Muslims do not fit a crude stereotype, America is not the crude stereotype of a self-interested empire.
+All these things must be done in partnership.
+You don't defeat a terrorist network that operates in eighty countries by occupying Iraq.
+We must jumpstart industries that create jobs, and end our dependence on foreign oil.
+And we must project a vision of the future that’s based not just on our fears, but also on our hopes -- a vision that recognizes the real dangers that exist around the world,  but also the limitless possibilities of our time.
+So, I have known Islam on three continents before coming to the region where it was first revealed.
+We have the power to make the world we seek, but only if we have the courage to make a new beginning, keeping in mind what has been written.
+And together, you represent the harmony between tradition and progress.
+The same is true for the astonishing progress within Muslim-majority countries from Kuala Lumpur to Dubai.
+Change means a tax code that doesn't reward the lobbyists who wrote it, but the American workers and small businesses who deserve it.
+She's the one who taught me about hard work.
+Yes, there have been some stretches in our history where the economy grew much faster, or when government ran more smoothly.
+You’ll have a boss that's not great.
+To begin with, our democracy won't work without a sense that everyone has economic opportunity.
+Indeed, one of the lessons of our effort in Iraq is that American influence around the world is not a function of military force alone.
+They endure the daily humiliations, large and small, that come with occupation.
+Now, as we approach our 10th year of combat in Afghanistan, there are those who are understandably asking tough questions about our mission there.
+The truth is, Rutgers, I came here because you asked.
+As Commander-in-Chief, I will never hesitate to defend this nation, but I will only send our troops into harm's way with a clear mission and a sacred commitment to give them the equipment they need in battle and the care and benefits they deserve when they come home.
+And by doing so, you’ll strengthen your own position, and you’ll hone your arguments.
+(Laughter and applause.)
+The United States does not accept the legitimacy of continued Israeli settlements.
+In the course of a healthy debate, we prioritize different goals, and the different means of reaching them.
+It's not just dishonest, this selective sorting of the facts; it's self-defeating.
+It is time for these settlements to stop.
+Given our interdependence, any world order that elevates one nation or group of people over another will inevitably fail.
+And although she can no longer travel, I know that she's watching tonight, and that tonight is her night as well.
+But stark inequality is also corrosive to our democratic ideal.
+But, however you got here, you made it.
+The peril each poses to our democracy is more far-reaching than a car bomb or a missile.
+Race remains a potent and often divisive force in our society.
+I reject the view of some in the West that a woman who chooses to cover her hair is somehow less equal, but I do believe that a woman who is denied an education is denied equality.
+ISIL will try to kill innocent people.
+That is embedded in our constitutional design.
+And that's why, for the past 8 years, I've worked to put the fight against terrorism on a firmer legal footing.
+Yes, we must provide more ladders to success for young men who fall into lives of crime and despair.
+If you are interested, we can talk after this.
+Billions of young people want to move beyond the shackles of poverty and conflict.
+So the fact that they’ve got a fancy degree -- you got to talk to them to see whether they know what they’re talking about.
+We spent a trillion dollars at war, often financed by borrowing from overseas.
+Over much of the last century, we’ve unleashed the strongest economic engine the world has ever seen, but over the past few decades, our economy has become more and more unequal.
+And throughout history, Islam has demonstrated through words and deeds the possibilities of religious tolerance and racial equality.
+And it took more time after that for it to start working.
+It's good to be home.
+She's the one who put off buying a new car or a new dress for herself so that I could have a better life.
+No matter how imperfect our efforts, no matter how expedient ignoring such values may seem, that's part of defending America.
+And because of our drawdown in Iraq, we are now able to apply the resources necessary to go on offense.
+Progress in the daily lives of the Palestinian people must be a critical part of a road to peace, and Israel must take concrete steps to enable such progress.
+Thank you, President Barchi, for that introduction.
+Thousands of soldiers and hundreds of vehicles made the trip from Baghdad, the last of them passing into Kuwait in the early morning hours.
+I've watched you grow up, get married, have kids, start incredible new journeys of your own.
+They can't shoulder the entire burden.
+And as someone who watched my mother argue with insurance companies while she lay in bed dying of cancer, I will make certain those companies stop discriminating against those who are sick and need care the most.
+But I am convinced that in order to move forward, we must say openly to each other the things we hold in our hearts and that too often are said only behind closed doors.
+But in the days to come, it must be our central mission as a people, and my central responsibility as President.
+I do so recognizing that change cannot happen overnight.
+But this milestone should serve as a reminder to all Americans that the future is ours to shape if we move forward with confidence and commitment.
+"And as we walk, we must make the pledge that we shall always march ahead.
+We need a President who can face the threats of the future, not keep grasping at the ideas of the past.
+Now, we also know that military power alone is not going to solve the problems in Afghanistan and Pakistan.
+I know that for many, the face of globalization is contradictory.
+Our problems must be dealt with through partnership; our progress must be shared.
+America, we cannot turn back.
+America, we are better than these last eight years.
+Many wait in refugee camps in the West Bank, Gaza, and neighboring lands for a life of peace and security that they have never been able to lead.
+And precisely because I have so much confidence in you, I’m not going to spend the remainder of my time telling you exactly how you’re going to make the world better.
+And the only thing that makes me prouder than all the good that we've done is the thought of all the amazing things that you are going to achieve from here.
+Now, let me be clear: Issues of women's equality are by no means simply an issue for Islam.
+It's always been contentious.
+The men and women who serve in our battlefields may be Democrats and Republicans and Independents, but they have fought together and bled together and some died together under the same proud flag.
+Thank you, and may God's peace be upon you.
+It took Thurgood Marshall and the NAACP decades to win Brown v. Board of Education; and then another decade after that to secure the Civil Rights Act and the Voting Rights Act.
+I will cut taxes—cut taxes—for 95 percent of all working families.
+Nor can I answer in the time that I have this afternoon all the complex questions that brought us to this point.
+This country is more generous than one where a man in Indiana has to pack up the equipment he's worked on for twenty years and watch it shipped off to China, and then chokes up as he explains how he felt like a failure when he went home to tell his family the news.
+I'm asking you to believe.
+That leads me to my second point:  The world is more interconnected than ever before, and it’s becoming more connected every day.
+Winners of the first-ever college football game.
+Don't tell me that Democrats won't keep us safe.
+And today, we import triple the amount of oil as the day that Senator McCain took office.
+And, Class of 2016, it is your turn now -- (applause) -- to shape our nation’s destiny, as well as your own.
+Now is the time to finally meet our moral obligation to provide every child a world-class education, because it will take nothing less to compete in the global economy.
+You're on your own.
+It will require a renewed sense of responsibility from each of us to recover what John F. Kennedy called our "intellectual and moral strength."
+But it’s how, bit by bit, generation by generation, we have made progress in this nation.
+And yet, Al Qaida chose to ruthlessly murder these people, claimed credit for the attack, and even now states their determination to kill on a massive scale.
+Let us keep that promise - that American promise - and in the words of Scripture hold firmly, without wavering, to the hope that we confess.
+And if we are being honest, fault lines must be closed among Muslims as well, as the divisions between Sunni and Shi'a have led to tragic violence, particularly in Iraq.
+America will align our policies with those who pursue peace, and we will say in public what we say in private to Israelis and Palestinians and Arabs.
+This year also saw Iraq hold credible elections that drew a strong turnout.
+I've seen it in Illinois, when we provided health care to more children and moved more families from welfare to work.
+And that's not easy to do.
+And I've said and I mean it: If anyone can put together a plan that is demonstrably better than the improvements we've made to our health care system that covers as many people at less cost, I will publicly support it.
+We've cut the deficit by two-thirds.
+On November 4th, we must stand up and say: "Eight is enough."
+The fourth issue that I will address is democracy.
+The first issue that we have to confront is violent extremism in all of its forms.
+What the nay-sayers don't understand is that this election has never been about me.
+But we can't be complacent about the goals themselves.
+Six million Jews were killed, more than the entire Jewish population of Israel today.
+It should ensure opportunity not just for those with the most money and influence, but for every American who's willing to work.
+Democracy can buckle when it gives in to fear.
+When one nation pursues a nuclear weapon, the risk of nuclear attack rises for all nations.
+For every two steps forward, it often feels we take one step back.
+And the issues that I have described will not be easy to address, but we have a responsibility to join together on behalf of the world that we seek, a world where extremists no longer threaten our people and American troops have come home, a world where Israelis and Palestinians are each secure in a state of their own and nuclear energy is used for peaceful purposes, a world where governments serve their citizens and the rights of all God's children are respected.
+The richness of religious diversity must be upheld, whether it is for Maronites in Lebanon or the Copts in Egypt.
+It calls to mind the words of Carl Sagan, who graduated high school here in New Jersey -- (applause) -- he said:  “We can judge our progress by the courage of our questions and the depths of our answers, our willingness to embrace what is true rather than what feels good.”   The debate around climate change is a perfect example of this.
+Tonight, I encourage Iraq’s leaders to move forward with a sense of urgency to form an inclusive government that is just, representative, and accountable to the Iraqi people.
+You have to be persistent.
+Some of you here tonight or watching at home, you were there with us in 2004, in 2008, 2012; maybe you still can't believe we pulled this whole thing off.
+Some were teenagers when the war began.
+Now is the time to change our bankruptcy laws, so that your pensions are protected ahead of CEO bonuses; and the time to protect Social Security for future generations.
+Likewise, many Israelis recognize the need for a Palestinian state.
+Every day, there are officials in high office with responsibilities who mock the overwhelming consensus of the world’s scientists that human activities and the release of carbon dioxide and methane and other substances are altering our climate in profound and dangerous ways.
+There will be many issues to discuss between our two countries, and we are willing to move forward without preconditions on the basis of mutual respect.
+It is a sign neither of courage nor power to shoot rockets at sleeping children or to blow up old women on a bus.
+And you have made the country proud.
+We virtually eliminated certain diseases like polio.
+It's not that complicated.
+When developing countries don’t have functioning health systems, epidemics like Zika or Ebola can spread and threaten Americans, too.
+But it is clear to all concerned that when it comes to nuclear weapons, we have reached a decisive point.

speechesdataset/test_LM_wbush.txt

@@ -0,0 +1,250 @@
+Over the past seven years, a new Department of Homeland Security has been created.
+Mr. Secretary, Governor Pataki, members of the United States Congress, Academy staff and faculty, distinguished guests, proud family members, and graduates: I want to thank you for your welcome.
+Thank you for such a warm welcome.
+And as a result, our commanders believe we have an opportunity to deal a serious blow to the terrorists.
+We will use America's full diplomatic resources to rally support for Iraq from nations throughout the Middle East.
+And this is my solemn pledge: I will work to build a single nation of justice and opportunity.
+These two regimes are allowing terrorists and insurgents to use their territory to move in and out of Iraq.
+In our discussions, we all agreed that there is no magic formula for success in Iraq.
+I want a West Point football player."
+This is a nation where citizens show calm in times of danger, and compassion in the face of suffering.
+In all these ways, I will bring the values of our history to the care of our times.
+America, at its best, matches a commitment to principle with a concern for civility.
+We will expand intelligence-sharing and deploy Patriot air defense systems to reassure our friends and allies.
+Al Qaeda has helped make Anbar the most violent area of Iraq outside the capital.
+Our enemies in Iraq will make every effort to ensure that our television screens are filled with images of death and suffering.
+These troops will work alongside Iraqi units and be embedded in their formations.
+More families will be able to get loans for cars and homes and college education.
+I felt like crossing this line would be a mistake, and once crossed, we would find it almost impossible to turn back.
+The most important tasks of a democracy are done by everyone.
+This violence is splitting Baghdad into sectarian enclaves, and shaking the confidence of all Iraqis.
+Every West Point class is commissioned to the Armed Forces.
+There are things I would do differently if given the chance.
+Honorable people have different views, and they will voice their criticisms.
+We cannot put our faith in the word of tyrants, who solemnly sign non-proliferation treaties, and then systemically break them.
+When this happens, daily life will improve, Iraqis will gain confidence in their leaders, and the government will have the breathing space it needs to make progress in other critical areas.
+Now let me explain the main elements of this effort: The Iraqi government will appoint a military commander and two deputy commanders for their capital.
+The question is whether our new strategy will bring us closer to success.
+They serve far from their families, who make the quiet sacrifices of lonely holidays and empty chairs at the dinner table.
+We do not accept this, and we will not allow it.
+Retreating behind our borders would only invite danger.
+I basically felt nobody cared."
+The bill also ensures that these new programs are carried out in a way that protects taxpayers.
+The choices we will face are complex.
+The 20th century ended with a single surviving model of human progress, based on non-negotiable demands of human dignity, the rule of law, limits on the power of the state, respect for women and private property and free speech and equal justice and religious tolerance.
+And whatever our views of its cause, we can agree that children at risk are not at fault.
+America did nothing to seek or deserve this conflict.
+This begins with addressing Iran and Syria.
+I believe that with the right techniques and the right policies, we can achieve scientific progress while living up to our ethical responsibilities.
+You walk in the tradition of Eisenhower and MacArthur, Patton and Bradley - the commanders who saved a civilization.
+The attacks of September the 11th required a few hundred thousand dollars in the hands of a few dozen evil and deluded men.
+I thank you for teaching.
+Do you not think an angel rides in the whirlwind and directs this storm?"
+There's no greater challenge than to make sure that every child -- and all of us on this stage mean every child, not just a few children -- (applause) -- every single child, regardless of where they live, how they're raised, the income level of their family, every child receive a first-class education in America.
+So America will change our strategy to help the Iraqis carry out their campaign to put down sectarian violence and bring security to the people of Baghdad.
+But new threats also require new thinking.
+Under the policy I announced five years ago, my administration became the first to make federal funds available for this research, yet only on embryonic stem cell lines derived from embryos that had already been destroyed.
+We will work with the governments of Turkey and Iraq to help them resolve problems along their border.
+We must take the battle to the enemy, disrupt his plans, and confront the worst threats before they emerge.
+Murdering the innocent to advance an ideology is wrong every time, everywhere.
+And the result was a vicious cycle of sectarian violence that continues today.
+Our children will be reading by the third grade.
+From this day forward, it is your challenge, as well, and we will meet this challenge together.
+This is a good bill for the American children, and I'm proud to sign it in their presence.
+On September the 11th, 2001, we saw what a refuge for extremists on the other side of the world could bring to the streets of our own cities.
+Our military commanders reviewed the new Iraqi plan to ensure that it addressed these mistakes.
+There is legitimate debate about many of these decisions.
+Encouraging responsibility is not a search for scapegoats, it is a call to conscience.
+And their governments should listen to their hopes.
+And they remind us that in our zeal for new treatments and cures, America must never abandon our fundamental morals.
+And he, like me and others, realize that a system that simply shuffles children through the schools is a system that's going to leave people behind.
+It is clear that we need to change our strategy in Iraq.
+The United States Military Academy is the guardian of values that have shaped the soldiers who have shaped the history of the world.
+We wish for others only what we wish for ourselves -- safety from violence, the rewards of liberty, and the hope for a better life.
+When challenges to our prosperity emerged, we rose to meet them.
+The first step to making sure that a child is not shuffled through is to test that child as to whether or not he or she can read and write, or add and subtract.
+And their government has put forward an aggressive plan to do it.
+It's so much easier to move a child through than trying to figure out how to solve a child's problems.
+The bicentennial class of West Point now enters this drama.
+I can't think of any better way to say to teachers, we trust you.
+Because the war on terror will require resolve and patience, it will also require firm moral purpose.
+Containment is not possible when unbalanced dictators with weapons of mass destruction can deliver those weapons on missiles or secretly provide them to terrorist allies.
+As the years passed, most Americans were able to return to life much as it had been before 9/11.
+But the themes of this day he would know: our nation's grand story of courage and its simple dream of dignity.
+When the great powers share common values, we are better able to confront serious regional conflicts together, better able to cooperate in preventing the spread of violence or economic chaos.
+As a strong supporter of free enterprise, I believe government intervention should occur only when necessary.
+The money is now available, and it's up to each local district to make sure it happens.
+And to all our men and women in uniform listening tonight: There has been no higher honor than serving as your Commander-in-Chief.
+Tonight in Iraq, the Armed Forces of the United States are engaged in a struggle that will determine the direction of the global war on terror -- and our safety here at home.
+I recently ordered the deployment of an additional carrier strike group to the region.
+A democratic Iraq will not be perfect.
+When people have hope in the future, they will not cede their lives to violence and extremism.
+Each of these lines can be replicated many times.
+Radical Shia elements, some supported by Iran, formed death squads.
+See, the government will purchase troubled assets and once the market recovers, it is likely that many of the assets will go up in value.
+May God bless you all.
+For the safety of our people, America must succeed in Iraq.
+This nation respects and trusts our military, and we are confident in your victories to come.
+It provides families with relief from the Alternative Minimum Tax, which would otherwise increase taxes for 26 million taxpayers by an average of $2,200.
+Some scientists believe that one source of these cures might be embryonic stem cell research.
+If this bill would have become law, American taxpayers would, for the first time in our history, be compelled to fund the deliberate destruction of human embryos.
+And by saying we trust local folks, we're really saying we trust you.
+The situation in Iraq is unacceptable to the American people -- and it is unacceptable to me.
+An advancing nation will pursue economic reform, to unleash the great entrepreneurial energy of its people.
+As the Commander-in-Chief, I hereby grant amnesty to all cadets who are on restriction for minor conduct offenses.
+If our country does not lead the cause of freedom, it will not be led.
+Our nation's cause has always been larger than our nation's defense.
+Other nations oppose terror, but tolerate the hatred that leads to terror -- and that must change.
+We made it because proud members of the House and the Senate, loyal to their parties, decided to set partisan politics aside and focus on what was right for America.
+America, at its best, is also courageous.
+With a smoother flow of credit, more businesses will be able to stock their shelves and meet their payrolls.
+Today, your last day at West Point, you begin a life of service in a career unlike any other.
+Tonight I am filled with gratitude—to Vice President Cheney and members of my administration; to Laura, who brought joy to this house and love to my life; to our wonderful daughters, Barbara and Jenna; to my parents, whose examples have provided strength for a lifetime.
+The elections of 2005 were a stunning achievement.
+And so this bill -- (applause) -- so this bill focuses on reading.
+And we must never let down our guard.
+The first decade of this new century has been a period of consequence—a time set apart.
+In these dangerous times, the United States is blessed to have extraordinary and selfless men and women willing to step forward and defend us.
+They just kept passing me along.
+In citizens like these, we see the best of our country—resilient and hopeful, caring and strong.
+And I thank Vice President Gore for a contest conducted with spirit and ended with grace.
+When leaders like John F. Kennedy and Ronald Reagan refused to gloss over the brutality of tyrants, they gave hope to prisoners and dissidents and exiles, and rallied free nations to a great cause.
+And I want to assure the seniors and juniors and sophomores here at Hamilton High School that the effort that this great country is engaged in, the effort to defend freedom and to defend our people, the effort to rout out terror wherever it exists, is noble and just and right, and your great country will prevail in this effort.
+I call him Big George, Jorge el Grande.
+And this commitment, if we keep it, is a way to shared accomplishment.
+During my college years I guess you could say I was -- (laughter.)
+The bill includes other provisions to help American consumers and businesses.
+All Americans are in this together.
+I understand taking tests aren't fun.
+The work ahead is difficult.
+And we will extend the peace by encouraging free and open societies on every continent.
+Our schools will have higher expectations.
+This will require increasing American force levels.
+And therefore, schools not only have the responsibility to improve, they now have the freedom to improve.
+This bill would support the taking of innocent human life in the hope of finding medical benefits for others.
+Iraq has gone from a brutal dictatorship and a sworn enemy of America to an Arab democracy at the heart of the Middle East and a friend of the United States.
+I remember standing in the rubble of the World Trade Center three days later, surrounded by rescuers who had been working around the clock.
+And finally, America stands for more than the absence of war.
+We benefitted from the thoughtful recommendations of the Iraq Study Group, a bipartisan panel led by former Secretary of State James Baker and former Congressman Lee Hamilton.
+Our economy continues to face serious challenges.
+Building this just peace is America's opportunity, and America's duty.
+It temporarily expands federal insurance for bank and credit union deposits from $100,000 to $250,000 -- a vital safeguard for consumers and small businesses.
+After the Declaration of Independence was signed, Virginia statesman John Page wrote to Thomas Jefferson: "We know the race is not to the swift nor the battle to the strong.
+I didn't need somebody that based his knowledge on theory; I wanted somebody who based his knowledge on experience.
+And you walk in the tradition of second lieutenants who did the same, by fighting and dying on distant battlefields.
+"The feeling came over me," he said, "that the expression 'the United States of America' would now and henceforth mean something different than it had ever before.
+And I join all Americans in offering best wishes to President-Elect Obama, his wife Michelle, and their two beautiful girls.
+And this nation will act.
+The third principle of this bill is that we have got to trust the local folks on how to achieve standards, to meet the standards.
+But we made it, because of the willingness of four fine leaders to do what was right for America.
+Researchers are now also investigating new techniques that could allow doctors and scientists to produce stem cells just as versatile as those derived from human embryos.
+We're going to win the war overseas, and we need to win the war against illiteracy here at home, as well.
+They remind us that we all begin our lives as a small collection of cells.
+And every child must learn to add and subtract.
+We will confront weapons of mass destruction, so that a new century is spared new horrors.
+It has been the privilege of a lifetime to serve as your President.
+Government has great responsibilities for public safety and public health, for civil rights and common schools.
+Parents must be given real options in the face of failure in order to make sure reform is meaningful.
+Let me be clear: The terrorists and insurgents in Iraq are without conscience, and they will make the year ahead bloody and violent.
+This evening, my thoughts return to the first night I addressed you from this house—September the 11th, 2001.
+Yet compassion is the work of a nation, not just a government.
+As John mentioned, George and I aren't from the same political ideology -- except when I met with George in Austin, I could tell he shares the same passion I share for making sure that every child gets educated.
+America needs partners to preserve the peace, and we will work with every nation that shares this noble goal.
+And I can pledge our nation to a goal: When we see that wounded traveler on the road to Jericho, we will not pass to the other side.
+I am proud of the men and women who have fought on my orders.
+Today, we affirm a new commitment to live out our nation's promise through civility, courage, compassion and character.
+And as the sun set on their struggle, many of those West Point officers lived to see a world transformed.
+Together, we will reclaim America's schools, before ignorance and apathy claim more young lives.
+America cannot impose this vision -- yet we can support and reward governments that make the right choices for their own people.
+Hide Transcript  Fellow citizens: For eight years, it has been my honor to serve as your President.
+In the days ahead, my national security team will fully brief Congress on our new strategy.
+To empower local leaders, Iraqis plan to hold provincial elections later this year.
+We will double the number of provincial reconstruction teams.
+I particularly want to thank my friend, the Governor of the great state of Ohio, Governor Taft, for being here.
+In Washington, there's some smart people there, but the people who care most about the children in Hamilton are the citizens of Hamilton.
+And all of us are diminished when any are hopeless.
+It includes tax incentives for businesses to invest and create jobs.
+In defending the peace, we face a threat with no precedent.
+Our unity, our union, is the serious work of leaders and citizens in every generation.
+I want to thank all who have come to witness this historic moment.
+And I'm not going to allow it.
+And America owes you a debt of gratitude.
+Reaching this moment has not been easy, as you could tell from Chairman Boehner's discussion.
+And so, what this bill says, it says every child can learn.
+All of the chaos and suffering they caused came at much less than the cost of a single tank.
+America will change our approach to help the Iraqi government as it works to meet these benchmarks.
+Abandonment and abuse are not acts of God, they are failures of love.
+I still invited him to come with me.
+I want to thank George Miller.
+Across our country, students are rising to meet higher standards in public schools.
+America is a young country, full of vitality, constantly growing and renewing itself.
+We can begin by working together to increase the size of the active Army and Marine Corps, so that America has the Armed Forces we need for the 21st century.
+You know, a huge percentage of children in poverty can't read at grade level.
+Now is the time to act.
+A new Medicare prescription drug benefit is bringing peace of mind to seniors and the disabled.
+These boys and girls are not spare parts.
+We must uncover terror cells in 60 or more countries, using every tool of finance, intelligence and law enforcement.
+And Secretary Rice will soon appoint a reconstruction coordinator in Baghdad to ensure better results for economic assistance being spent in Iraq.
+Moral truth is the same in every culture, in every time, and in every place.
+And this great republic born alone in liberty is leading the world toward a new age when freedom belongs to all nations.
+Thank you all for coming and may God bless.
+Many listening tonight will ask why this effort will succeed when previous operations to secure Baghdad did not.
+And there's no mystery to this achievement.
+Our public interest depends on private character, on civic duty and family bonds and basic fairness, on uncounted, unhonored acts of decency which give direction to our freedom.
+And we will seek out and destroy the networks providing advanced weaponry and training to our enemies in Iraq.
+Since I announced my policy in 2001, advances in scientific research have also shown the great potential of stem cells that are derived without harming human embryos.
+Now, as then, they seek to impose a joyless conformity, to control every life and all of life.
+So my national security team, military commanders, and diplomats conducted a comprehensive review.
+The legislation Congress passed today addresses this problem head on by providing a variety of new tools to the government -- such as allowing us to purchase some of the troubled assets, and creating a new government insurance program that will guarantee the value of others.
+But there can be little debate about the results.
+In tyranny, they suffer.
+Some West Point classes are also commissioned by history, to take part in a great new calling for their country.
+Civility is not a tactic or a sentiment.
+No longer is it acceptable to keep results away from parents.
+From here on, it would be the nation I would be serving, not myself."
+This government and the American people are on watch, we are ready, because we know the terrorists have more money and more men and more plans.
+When I took office, there was no federal funding for human embryonic stem cell research.
+I know some Americans have concerns about this legislation, especially about the government's role and the bill's cost.
+As we make these changes, we will continue to pursue al Qaeda and foreign fighters.
+I'm disappointed that the House failed to authorize funding for this vital and ethical research.
+President Clinton, distinguished guests and my fellow citizens, the peaceful transfer of authority is rare in history, yet common in our country.
+Never tiring, never yielding, never finishing, we renew that purpose today, to make our country more just and generous, to affirm the dignity of our lives and every life.
+I remember talking to brave souls who charged through smoke-filled corridors at the Pentagon, and to husbands and wives whose loved ones became heroes aboard Flight 93.
+America confronted imperial communism in many different ways -- diplomatic, economic, and military.
+This new strategy will not yield an immediate end to suicide bombings, assassinations, or IED attacks.
+Homeland defense and missile defense are part of stronger security, and they're essential priorities for America.
+God bless you all, and God bless America.
+Targeting innocent civilians for murder is always and everywhere wrong.
+We're standing with dissidents and young democracies, providing AIDS medicine to dying patients—to bring dying patients back to life, and sparing mothers and babies from malaria.
+When this spirit is present, no wrong can stand against it.
+These bills illustrate both the promise and perils we face in the age of biotechnology.
+I probably won't be sharing that with the Secretary of the Navy.
+By confronting evil and lawless regimes, we do not create a problem, we reveal a problem.
+In this new era, our challenge is to harness the power of science to ease human suffering without sanctioning the practices that violate the dignity of human life.
+And I thank you for the countless acts of courage, generosity, and grace that I have witnessed these past eight years.
+This nation must continue to speak out for justice and truth.
+We believe every child can learn.
+So I've committed more than 20,000 additional American troops to Iraq.
+Some people argue that finding new cures for disease requires the destruction of human embryos like the ones that these families adopted.
+We can, and we will, prevail.
+Every school has a job to do.
+Enemies in the past needed great armies and great industrial capabilities to endanger the American people and our nation.
+Yet I am certain of this: Wherever we carry it, the American flag will stand not only for our power, but for freedom.
+I appreciate so very much my friend, Judd Gregg, from the state of New Hampshire, being here.
+And the federal bench includes wise new members like Justice Sam Alito and Chief Justice John Roberts.
+Yet his purpose is achieved in our duty, and our duty is fulfilled in service to one another.
+While many of our citizens prosper, others doubt the promise, even the justice, of our own country.
+He was my campaign manager in the New Hampshire primary.
+Iran would be emboldened in its pursuit of nuclear weapons.
+As science brings us ever closer to unlocking the secrets of human biology, it also offers temptations to manipulate human life and violate human dignity.
+It's great to be in the home of the Big Blue.
+But I decided to sign this bill in one of the most important places in America -- a public school.
+There must be a moment in which parents can say, I've had enough of this school.
+When people live in freedom, they do not willingly choose leaders who pursue campaigns of terror.
+The other system is based on the conviction that freedom is the universal gift of Almighty God, and that liberty and justice light the path to peace.
+We'll take the time necessary to design an effective program that achieves its objectives -- and does not waste taxpayer dollars.
+And I still carry his badge.
+And we concluded that to step back now would force a collapse of the Iraqi government, tear the country apart, and result in mass killings on an unimaginable scale.
+His son, Nathan—a Marine—gave his life in Iraq.
+Every citizen must uphold them.
+To the contrary, even critics of my policy concede that these federally funded lines are being used in research every day by scientists around the world.

tokenizer.py

@@ -0,0 +1,38 @@
+import nltk
+nltk.download('punkt')
+
+from nltk.tokenize import word_tokenize
+import os
+
+
+class SimpleTokenizer:
+    """
+    A simple tokenizer class that builds a vocabulary from the given text and encodes/decodes text into indices.
+    """
+
+    def __init__(self, text):
+        """Initialize the tokenizer with the initial text to build vocabulary."""
+        self.vocab = set()
+        self.stoi = {}
+        self.itos = {}
+        self.build_vocab(text)
+
+    def build_vocab(self, text):
+        """Build vocabulary from the given text."""
+        tokens = word_tokenize(text)
+        self.vocab = set(tokens)
+        self.vocab_size = len(self.vocab) + 2
+        self.stoi = {word: i for i, word in enumerate(self.vocab, start=2)}
+        self.stoi['<pad>'] = 0
+        self.stoi['<unk>'] = 1
+        self.itos = {i: word for word, i in self.stoi.items()}
+
+    def encode(self, text):
+        """Encode the text into a list of indices."""
+        tokens = word_tokenize(text)
+        return [self.stoi.get(word, self.stoi['<unk>']) for word in tokens]
+
+    def decode(self, indices):
+        """Decode the list of indices back into text."""
+        return ' '.join([self.itos.get(index, '<unk>') for index in indices])
+    

train.tsv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a55c6ad05115c70abc01e3d05f89056f6a7274e368304965198fe879a3cafac
+size 18456477

transformer.py

@@ -0,0 +1,255 @@
+# add all  your Encoder and Decoder code here
+import torch
+import torch.nn as nn
+from torch.nn import functional as F
+import math
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+from constants import block_size, n_embd, n_head, n_layer, n_input, n_output, n_hidden
+
+dropout = 0.3
+
+class Head(nn.Module):
+    """ one head of self-attention """
+
+    def __init__(self, head_size, decoding=False):
+        super().__init__()
+        self.key = nn.Linear(n_embd, head_size, bias=False)
+        self.query = nn.Linear(n_embd, head_size, bias=False)
+        self.value = nn.Linear(n_embd, head_size, bias=False)
+        self.register_buffer('tril', torch.tril(torch.ones(block_size, block_size)))
+        self.decoding = decoding
+
+        # self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x, attention_maps):
+        B,T,C = x.shape
+        
+        k = self.key(x)   
+        q = self.query(x) 
+        
+        wei = q @ k.transpose(-2,-1) * k.shape[-1]**-0.5 
+        
+        if self.decoding:
+            wei = wei.masked_fill(self.tril[:T, :T] == 0, float('-inf')) 
+        
+        wei = F.softmax(wei, dim=-1) 
+        
+        # attention_maps.append(wei)
+        
+        # wei = self.dropout(wei)
+        
+        v = self.value(x) 
+        
+        out = wei @ v 
+        
+        return out
+
+class MultiHeadAttention(nn.Module):
+    """ multiple heads of self-attention in parallel """
+
+    def __init__(self, num_heads, head_size, decoding=False):
+        super().__init__()
+        self.heads = nn.ModuleList([Head(head_size, decoding) for _ in range(num_heads)])
+        self.proj = nn.Linear(head_size * num_heads, n_embd)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x, attention_maps, dropout=False):
+        out = torch.cat([h(x, attention_maps) for h in self.heads], dim=-1)
+        
+        if dropout:
+            return self.dropout(self.proj(out))
+        
+        return self.proj(out)
+
+class FeedFoward(nn.Module):
+    """ a simple linear layer followed by a non-linearity """
+
+    def __init__(self, n_embd):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(n_embd, 4*n_embd),
+            nn.ReLU(),
+            nn.Linear(4*n_embd, n_embd),
+        )
+        
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x, dropout=False):
+        if dropout:
+            return self.dropout(self.net(x))
+        
+        return self.net(x)
+
+class Block(nn.Module):
+    """ Transformer block: communication followed by computation """
+
+    def __init__(self, n_embd, n_head=n_head, decoding=False):
+        super().__init__()
+        head_size = n_embd // n_head
+        self.sa: MultiHeadAttention = MultiHeadAttention(n_head, head_size, decoding)
+        self.ffwd = FeedFoward(n_embd)
+        self.ln1 = nn.LayerNorm(n_embd)
+        self.ln2 = nn.LayerNorm(n_embd)
+
+    def forward(self, x, attention_maps=None, dropout=False):
+        x = x + self.sa(self.ln1(x), attention_maps, dropout)
+        
+        x = self.ln2(x + self.ffwd(x, dropout))
+        return x
+
+class Classifier(nn.Module):
+    def __init__(self, vocab_size, input_size=n_embd, hidden_size=n_hidden):
+        super().__init__()
+        self.fc1 = nn.Linear(input_size, hidden_size)  # First fully connected layer.
+        self.fc2 = nn.Linear(hidden_size, n_output)  # Second fully connected layer, outputting three classes.
+        self.encoder = Encoder(vocab_size, n_head, n_layer)
+        self.apply(self._init_weights)
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+
+    def forward(self, x):
+        x, attn_maps = self.encoder(x)
+        x = F.relu(self.fc1(x))  # Apply ReLU activation function after the first layer.
+        x = self.fc2(x)  # Pass the result to the second layer.
+        return x, attn_maps
+    
+    
+class Encoder(nn.Module):
+    def __init__(self, vocab_size, n_head=n_head, n_layer=n_layer):
+        super().__init__()
+        self.token_embedding_table = nn.Embedding(vocab_size, n_embd)
+        self.position_embedding_table = nn.Embedding(block_size, n_embd)
+        self.blocks = nn.ModuleList([Block(n_embd, n_head=n_head, decoding=False) for _ in range(n_layer)])
+            
+    def forward(self, idx):
+        tok_emb = self.token_embedding_table(idx)
+
+        # absolute positional encoding
+        # div_term = torch.exp(torch.arange(0, n_embd, 2) * (-math.log(10000.0) / n_embd))
+
+        # pos = torch.arange(block_size, dtype=torch.float).reshape(block_size, 1)
+
+        # stacked = torch.stack([torch.sin(pos * div_term), torch.cos(pos * div_term)], dim=2)
+
+        # stacked = stacked.to(device)
+
+        pos_emb = self.position_embedding_table(torch.arange(block_size, device=device))
+
+        # stacked = torch.stack([pos_emb, pos_emb], dim=2)
+
+        tok_emb = tok_emb.to(device)
+
+        pos_emb = pos_emb.to(device)
+
+        # x = tok_emb + torch.flatten(stacked, start_dim=1, end_dim=2)
+
+        x = tok_emb + pos_emb
+        
+        attention_maps = []
+        
+        for block in self.blocks:
+           x = block(x, attention_maps) 
+        
+        x = torch.mean(x, dim=1)
+        
+        return x, attention_maps
+        
+class Decoder(nn.Module):
+    def __init__(self, vocab_size):
+        super().__init__()
+        self.token_embedding_table = nn.Embedding(vocab_size, n_embd)
+        self.blocks = nn.ModuleList([Block(n_embd, n_head=n_head, decoding=True) for _ in range(n_layer)])
+        self.ln_f = nn.LayerNorm(n_embd) 
+        self.lm_head = nn.Linear(n_embd, vocab_size)
+
+    def forward(self, idx, dropout=False):
+        B, T = idx.shape
+        
+        tok_emb = self.token_embedding_table(idx) 
+
+        # absolute positional encoding  
+        div_term = torch.exp(torch.arange(0, n_embd, 2) * (-math.log(10000.0) / n_embd))
+        
+        pos = torch.arange(block_size, dtype=torch.float).reshape(block_size, 1)
+
+        stacked = torch.stack([torch.sin(pos * div_term), torch.cos(pos * div_term)], dim=2)
+        
+        x = tok_emb + torch.flatten(stacked, start_dim=1, end_dim=2)
+        
+        attention_maps = []
+        
+        for block in self.blocks:
+           x = block(x, attention_maps, False) 
+           
+        x = self.ln_f(x) 
+        return self.lm_head(x), attention_maps 
+
+
+class DecoderEC(nn.Module):
+    def __init__(self, vocab_size, n_head=n_head, n_layer=n_layer):
+        super().__init__()
+        self.token_embedding_table = nn.Embedding(vocab_size, n_embd)
+        self.position_embedding_table = nn.Embedding(block_size, n_embd)
+        self.blocks = nn.ModuleList([Block(n_embd, n_head=n_head, decoding=True) for _ in range(n_layer)])
+        self.ln_f = nn.LayerNorm(n_embd) 
+        self.lm_head = nn.Linear(n_embd, vocab_size)
+        self.apply(self._init_weights)
+
+    def _init_weights(self, module):
+        if isinstance(module, nn.Linear):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            if module.bias is not None:
+                torch.nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.Embedding):
+            torch.nn.init.normal_(module.weight, mean=0.0, std=0.02)
+            
+    def forward(self, idx):
+        B, T = idx.shape
+        
+        tok_emb = self.token_embedding_table(idx) 
+
+        # learned embeddings
+        pos_emb = self.position_embedding_table(torch.arange(T))
+        
+        x = tok_emb + pos_emb 
+        
+        attention_maps = []
+        
+        for block in self.blocks:
+           x = block(x, attention_maps, True) 
+           
+        x = self.ln_f(x) 
+        
+        return self.lm_head(x), attention_maps 
+    
+    def generate(self, idx, max_new_tokens):
+        # idx is (B, T) array of indices in the current context
+        for _ in range(max_new_tokens):
+            # crop idx to the last block_size tokens
+            idx_cond = idx[:, -block_size:]
+            
+            # get the predictions
+            logits, loss = self(idx_cond)
+            
+            # focus only on the last time step
+            logits = logits[:, -1, :] # becomes (B, C)
+            
+            # apply softmax to get probabilities
+            probs = F.softmax(logits, dim=-1) # (B, C)
+            
+            # sample from the distribution
+            idx_next = torch.multinomial(probs, num_samples=1) # (B, 1)
+            
+            # append sampled index to the running sequence
+            idx = torch.cat((idx, idx_next), dim=1) # (B, T+1)
+            
+        return idx
+    

utilities.py

@@ -0,0 +1,53 @@
+
+import matplotlib.pyplot as plt
+import torch
+
+class Utilities:
+    def __init__(self, tokenizer, model):
+        self.tokenizer = tokenizer
+        self.model = model
+
+    def sanity_check(self, sentence, block_size):
+        # Encode the sentence using the tokenizer
+        wordids = self.tokenizer.encode(sentence)
+
+        # Prepare the padded input for the model
+        padded_sentence = wordids[:block_size] + [0] * (block_size - len(wordids))
+        input_tensor = torch.tensor(padded_sentence, dtype=torch.long).unsqueeze(0)
+
+        # Display input tensor shape
+        print("Input tensor shape:", input_tensor.shape)
+
+        # Process the input tensor through the encoder model
+        _, attn_maps = self.model(input_tensor)
+
+        # Display the number of attention maps
+        print("Number of attention maps:", len(attn_maps))
+
+        # Visualize and save the attention maps
+        for j, attn_map in enumerate(attn_maps):
+            att_map = attn_map.squeeze(0).detach().cpu().numpy()  # Remove batch dimension and convert to NumPy array
+            
+            print("map shape", att_map.shape, att_map.ndim)
+
+            # Check if the attention probabilities sum to 1 over rows
+            total_prob_over_rows = torch.sum(attn_map[0], dim=1)
+            if torch.any(total_prob_over_rows < 0.99) or torch.any(total_prob_over_rows > 1.01):
+                print("Failed normalization test: probabilities do not sum to 1.0 over rows")
+                print("Total probability over rows:", total_prob_over_rows.numpy())
+
+            # Create a heatmap of the attention map
+            fig, ax = plt.subplots()
+            cax = ax.imshow(att_map, cmap='hot', interpolation='nearest')
+            ax.xaxis.tick_top()  
+            fig.colorbar(cax, ax=ax)  
+            plt.title(f"Attention Map {j + 1}")
+            
+            # Save the plot
+            plt.savefig(f"attention_map_{j + 1}.png")
+            
+            # Show the plot
+            # plt.show()
+            
+
+