Upload 14 files

README.md

@@ -1,13 +1,34 @@
----
-title: ControllableBlenderCEFR
-emoji: 🏢
-colorFrom: pink
-colorTo: yellow
-sdk: gradio
-sdk_version: 5.38.2
-app_file: app.py
-pinned: false
-short_description: Complexity Control onto BlenderBot (Tyen, 2022)
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# About
+This repository contains the code, models, and data accompanying our paper on adjusting text difficulty of messages generated by open-domain chatbots.
+
+Tyen, G., Brenchley, M., Caines, A., & Buttery, P. (2022). Towards an open-domain chatbot for language practice. 17th Workshop on Innovative Use of NLP for Building Educational Applications.
+
+# Dependencies
+* ParlAI 1.6.0
+* PyTorch 1.10.2
+* Huggingface Transformers 4.16.2
+* NumPy 1.22.2
+* SciPy 1.8.0
+* Regex 2022.1.18
+
+# To run the demo:
+1. Download and cd to project directory
+   `git clone https://github.com/WHGTyen/ControllableComplexityChatbot`
+   `cd ControllableComplexityChatbot`
+2. Install pip dependencies
+   `pip install numpy scipy regex torch transformers`
+3. Clone ParlAI repository
+   `git clone https://github.com/facebookresearch/ParlAI.git --branch 1.6.0`
+4. Setup ParlAI
+   `cd ParlAI; python setup.py develop; cd ..`
+5. To run the demo:
+   `python demo.py`
+6. To adjust generation parameters, edit values in `demo.py`
+   
+# Data
+* `sample_wordlist.txt` is the 5000 most frequent words from [this list](https://github.com/first20hours/google-10000-english/blob/d0736d492489198e4f9d650c7ab4143bc14c1e9e/google-10000-english-no-swears.txt)
+* `filter.txt` was taken from [this list](https://github.com/dwyl/english-words/blob/22d7c41119076750a96fca2acd664ed994cc0a75/words_alpha.txt)
+* `complexity_model` was trained on data from the Cambridge Exams readability dataset, found [here](https://ilexir.co.uk/datasets/index.html)
+
+# Acknowledgements
+This paper reports on research supported by Cambridge University Press & Assessment. This work was performed using resources provided by the Cambridge Service for Data Driven Discovery (CSD3) operated by the [University of Cambridge Research Computing Service](www.csd3.cam.ac.uk), provided by Dell EMC and Intel using Tier-2 funding from the Engineering and Physical Sciences Research Council (capital grant EP/P020259/1), and [DiRAC funding from the Science and Technology Facilities Council](www.dirac.ac.uk).

app.py

@@ -0,0 +1,96 @@
+import gradio as gr
+import os
+import json
+from parlai.core.opt import Opt
+from parlai.zoo.blender.blender_3B import download
+from parlai.core.agents import Agent
+from parlai.core.params import ParlaiParser
+from parlai.core.worlds import DialogPartnerWorld
+from controllable_blender import ControllableBlender
+from huggingface_hub import snapshot_download
+
+snapshot_download(repo_id="shivansarora/ControllableBlender", local_dir="ParlAI/data/models/blender/blender_3B")
+
+# Load options
+agent_opt = json.load(open("blender_3B.opt", 'r'))
+download(agent_opt["datapath"])
+conversation_state = {"world": None, "human_agent": None}
+
+class GradioHumanAgent(Agent):
+    def __init__(self, opt):
+        super().__init__(opt)
+        self.msg = None
+
+    def observe(self, msg):
+        return msg
+
+    def act(self):
+        return {"text": self.msg, "episode_done": False}
+
+
+def init_world(cefr, inference_type):
+    opt = agent_opt.copy()
+    opt["rerank_cefr"] = cefr
+    opt["inference"] = inference_type
+
+    # Settings for rerank methods (not used if "inference" == "vocab")
+    opt["rerank_tokenizer"] = "distilroberta-base"        # Tokenizer from Huggingface Transformers. Must be compatible with "rerank_model"
+    opt["rerank_model"] = "complexity_model"              # Model fine-tuned on complexity data
+    opt["rerank_model_device"] = "cuda"                   # Device for complexity model
+    opt["penalty_stddev"] = 2                             # Controls how harshly sub-tokens are penalised (lower = harsher). Use -1 to remove penalties
+    opt["filter_path"] = "data/filter.txt"                # Path to list of English words to ensure OOV words are not generated. Capitalised words are ignored. Use empty string to remove filter
+
+    # Settings for vocab methods (not used if "inference" == "rerank")
+    opt["wordlist_path"] = "data/sample_wordlist.txt"     # Path to list of vocab the chatbot is restricted to    
+   
+    # Same top-k sampling configs for all settings described in the paper
+    opt["beam_size"] = 20
+    opt["topk"] = 40
+
+    human_agent = GradioHumanAgent(opt)
+    model_agent = ControllableBlender(opt)
+    world = DialogPartnerWorld(opt, [human_agent, model_agent])
+
+    return human_agent, world
+
+def chat(user_input, cefr, inference_type, history):
+    if conversation_state["world"] is None:
+
+        human_agent, world = init_world(cefr, inference_type)
+        conversation_state["world"] = world
+        conversation_state["human_agent"] = human_agent
+
+    conversation_state["human_agent"].msg = user_input
+
+    conversation_state["world"].parley()
+
+    bot_reply = conversation_state["world"].acts[1].get("text", "")
+    history.append([user_input, bot_reply.strip()])
+    return history, history
+
+def reset_chat():
+    conversation_state["world"] = None
+    conversation_state["human_agent"] = None
+    return []
+
+with gr.Blocks() as demo:
+    cefr = gr.Dropdown(["A1", "A2", "B1", "B2", "C1", "C2"], label="CEFR", value="B2")
+    inference_type = gr.Dropdown(["rerank", "vocab"], label="Inference", value="rerank")
+    user_input = gr.Textbox(label="your message")
+    chatbot = gr.Chatbot(label="Controllable Complexity Chatbot")
+    send_btn = gr.Button("Send")
+
+    state = gr.State([])
+
+    def user_chat(message, cefr_level, infer_type, history):
+        # call your chat function here
+        new_history, _ = chat(message, cefr_level, infer_type, history)
+        return new_history, new_history
+
+    send_btn.click(
+        fn=user_chat,
+        inputs=[user_input, cefr, inference_type, state],
+        outputs=[chatbot, state]
+    )
+
+demo.launch(share=True)

blender_3B.opt

@@ -0,0 +1,144 @@
+{
+  "init_opt": null,
+  "show_advanced_args": false,
+  "task": "interactive",
+  "datatype": "test",
+  "image_mode": "raw",
+  "numthreads": 1,
+  "hide_labels": false,
+  "batchsize": 128,
+  "dynamic_batching": null,
+  "datapath": "ParlAI/data",
+  "model": "transformer/generator",
+  "model_file": "ParlAI/data/models/blender/blender_3B/model",
+  "init_model": "/checkpoint/parlai/zoo/meena/20200319_meenav0data_tall_2.7B_adamoptimizer/20200319_13.3ppl_200kupdates/model",
+  "dict_class": "parlai.core.dict:DictionaryAgent",
+  "evaltask": null,
+  "eval_batchsize": null,
+  "display_examples": false,
+  "num_epochs": -1,
+  "max_train_time": 27647.999999999996,
+  "validation_every_n_secs": -1,
+  "save_every_n_secs": -1,
+  "save_after_valid": true,
+  "validation_every_n_epochs": 0.25,
+  "validation_max_exs": -1,
+  "short_final_eval": false,
+  "validation_patience": 10,
+  "validation_metric": "ppl",
+  "validation_metric_mode": "min",
+  "validation_cutoff": 1.0,
+  "validation_share_agent": false,
+  "metrics": "default",
+  "aggregate_micro": false,
+  "tensorboard_log": false,
+  "dict_maxexs": -1,
+  "dict_include_valid": false,
+  "dict_include_test": false,
+  "log_every_n_secs": 10.0,
+  "image_size": 256,
+  "image_cropsize": 224,
+  "label_type": "response",
+  "include_knowledge": true,
+  "include_checked_sentence": true,
+  "include_knowledge_separator": false,
+  "num_topics": 5,
+  "train_experiencer_only": false,
+  "remove_political_convos": false,
+  "embedding_size": 2560,
+  "n_layers": 2,
+  "ffn_size": 10240,
+  "dropout": 0.1,
+  "attention_dropout": 0.0,
+  "relu_dropout": 0.0,
+  "n_heads": 32,
+  "learn_positional_embeddings": false,
+  "embeddings_scale": true,
+  "n_positions": 128,
+  "n_segments": 0,
+  "variant": "prelayernorm",
+  "activation": "gelu",
+  "output_scaling": 1.0,
+  "share_word_embeddings": true,
+  "n_encoder_layers": 2,
+  "n_decoder_layers": 24,
+  "model_parallel": true,
+  "beam_size": 20,
+  "beam_min_length": 20,
+  "beam_context_block_ngram": 3,
+  "beam_block_ngram": 3,
+  "beam_length_penalty": 0.65,
+  "skip_generation": false,
+  "inference": "topk",
+  "topk": 40,
+  "topp": 0.9,
+  "beam_delay": 30,
+  "temperature": 1.0,
+  "compute_tokenized_bleu": false,
+  "embedding_type": "random",
+  "embedding_projection": "random",
+  "fp16": true,
+  "fp16_impl": "mem_efficient",
+  "force_fp16_tokens": true,
+  "optimizer": "mem_eff_adam",
+  "learningrate": 7e-06,
+  "gradient_clip": 0.1,
+  "adam_eps": 1e-08,
+  "adafactor_eps": [
+    1e-30,
+    0.001
+  ],
+  "momentum": 0,
+  "nesterov": true,
+  "nus": [
+    0.7
+  ],
+  "betas": [
+    0.9,
+    0.999
+  ],
+  "weight_decay": null,
+  "rank_candidates": false,
+  "truncate": 128,
+  "text_truncate": 128,
+  "label_truncate": 128,
+  "history_size": -1,
+  "person_tokens": false,
+  "split_lines": false,
+  "use_reply": "label",
+  "add_p1_after_newln": false,
+  "delimiter": "  ",
+  "history_add_global_end_token": "end",
+  "gpu": -1,
+  "no_cuda": false,
+  "dict_file": "ParlAI/data/models/blender/blender_3B/model.dict",
+  "dict_initpath": null,
+  "dict_language": "english",
+  "dict_max_ngram_size": -1,
+  "dict_minfreq": 0,
+  "dict_maxtokens": -1,
+  "dict_nulltoken": "__null__",
+  "dict_starttoken": "__start__",
+  "dict_endtoken": "__end__",
+  "dict_unktoken": "__unk__",
+  "dict_tokenizer": "bytelevelbpe",
+  "dict_lower": false,
+  "bpe_debug": false,
+  "dict_textfields": "text,labels",
+  "bpe_vocab": "ParlAI/data/models/blender/blender_3B/model.dict-vocab.json",
+  "bpe_merge": "ParlAI/data/models/blender/blender_3B/model.dict-merges.txt",
+  "bpe_add_prefix_space": true,
+  "lr_scheduler": "reduceonplateau",
+  "lr_scheduler_patience": 3,
+  "lr_scheduler_decay": 0.5,
+  "max_lr_steps": -1,
+  "invsqrt_lr_decay_gamma": -1,
+  "warmup_updates": 100,
+  "warmup_rate": 0.0001,
+  "update_freq": 2,
+  "parlai_home": "ParlAI/",
+  "starttime": "Mar31_06-04",
+  "beam_block_full_context": false,
+  "batchindex": 127,
+  "dict_loaded": true
+}

complexity_model/config.json

@@ -0,0 +1,32 @@
+{
+  "_name_or_path": "distilroberta-base",
+  "architectures": [
+    "RobertaForSequenceClassification"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": 0,
+  "eos_token_id": 2,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 514,
+  "model_type": "roberta",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 1,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.5.1",
+  "type_vocab_size": 1,
+  "use_cache": true,
+  "vocab_size": 50265
+}

complexity_model/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4463a45e88dedd693c26e9a16830c6bc62fcf60717c8c38646ca324ddc12c0a7
+size 134

complexity_model/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1ec2aaba4756da99f79eab56d1d947d588b52e4c7565d971476688fc9a0df83
+size 2287

controllable_blender/__init__.py

@@ -0,0 +1 @@
+from .controllable_blender import ControllableBlender

controllable_blender/controllable_blender.py

@@ -0,0 +1,107 @@
+from parlai.core.opt import Opt
+from parlai.utils.typing import TShared
+from parlai.agents.transformer.transformer import TransformerGeneratorAgent
+
+from .generation_methods import VocabTopKSampling, RerankedTopKSampling
+from .generation_utils import Wordlist, Reranker, load_wordlist, cefr_to_int
+
+class ControllableBlender(TransformerGeneratorAgent):
+    def __init__(self, opt: Opt, shared: TShared = None):
+        super().__init__(opt, shared)
+
+        if opt.get("inference", None) == "vocab":
+            wordlist_path = opt.get("wordlist_path", None)
+            assert wordlist_path, "Please provide path to vocab list, in order to use inference method 'vocab'"
+
+            allowed_words = load_wordlist(wordlist_path)
+            self.wordlist = Wordlist(allowed_words, self.dict)
+
+        elif opt.get("inference", None) == "rerank":
+            cefr = opt.get("rerank_cefr", None)
+            assert cefr, "Please provide CEFR level, in order to use inference method 'rerank'"
+
+            rerank_tokenizer = opt.get("rerank_tokenizer", None)
+            rerank_model = opt.get("rerank_model", None)
+            assert rerank_model, "Please provide path to directory containing model weights, in order to use inference method 'rerank'"
+
+            device = opt.get("complexity_model_device", None)
+            penalty_stddev = opt.get("penalty_stddev", None)
+            text_truncate = opt.get("text_truncate", None)
+
+            word_filter = None
+            filter_path = opt.get("filter_path", "")
+            if filter_path:
+                word_filter = load_wordlist(filter_path)
+
+            exempt_tokens = [self.dict.tok2ind.get(self.dict.null_token),
+                             self.dict.tok2ind.get(self.dict.start_token),
+                             self.dict.tok2ind.get(self.dict.end_token),
+                             self.dict.tok2ind.get(self.dict.unk_token)]
+
+            if penalty_stddev < 0:
+                exempt_tokens = "all"
+
+            self.reranker = Reranker(cefr=cefr_to_int(cefr),
+                                     model=rerank_model,
+                                     tokenizer=rerank_tokenizer,
+                                     device=device,
+                                     text_truncate=text_truncate,
+                                     exempt_tokens=exempt_tokens,
+                                     penalty_stddev=penalty_stddev,
+                                     vocab_size=len(self.dict),
+                                     word_filter=word_filter)
+
+        else:
+            raise ValueError(f"Inference method {opt.get('inference', None)} does not exist. "
+                             f"Please use 'vocab' or 'rerank'.")
+
+
+    def _treesearch_factory(self, device, verbose=False):
+        method = self.opt.get('inference', 'greedy')
+        beam_size = self.opt.get('beam_size', 1)
+        if method == 'vocab':
+            return VocabTopKSampling(
+                k=self.opt.get('topk', 40),
+                wordlist=self.wordlist,
+                beam_size=beam_size,
+                min_length=self.beam_min_length,
+                block_ngram=self.beam_block_ngram,
+                context_block_ngram=self.beam_context_block_ngram,
+                length_penalty=self.opt.get('beam_length_penalty', 0.65),
+                padding_token=self.NULL_IDX,
+                bos_token=self.START_IDX,
+                eos_token=self.END_IDX,
+                device=device,
+                verbose=verbose,
+            )
+        elif method == "rerank":
+            return RerankedTopKSampling(
+                k=self.opt.get('topk', 40),
+                reranker=self.reranker,
+                tokenids_to_text=self._v2t,
+                beam_size=beam_size,
+                min_length=self.beam_min_length,
+                block_ngram=self.beam_block_ngram,
+                context_block_ngram=self.beam_context_block_ngram,
+                length_penalty=self.opt.get('beam_length_penalty', 0.65),
+                padding_token=self.NULL_IDX,
+                bos_token=self.START_IDX,
+                eos_token=self.END_IDX,
+                device=device,
+                verbose=verbose,
+            )
+        else:
+            return super()._treesearch_factory(device, verbose=verbose)
+
+    def share(self):
+        """
+        Share internal states between parent and child instances.
+        """
+        shared = super().share()
+        if hasattr(self, 'wordlist'):
+            shared['wordlist'] = self.wordlist
+        if hasattr(self, 'reranker'):
+            shared['reranker'] = self.reranker
+        return shared
+
+

controllable_blender/generation_methods.py

@@ -0,0 +1,200 @@
+import math
+from operator import attrgetter
+from typing import Callable
+
+import numpy as np
+import regex
+from scipy.stats import rankdata
+
+import torch
+from parlai.core.torch_generator_agent import TopKSampling, TreeSearch, _HypothesisTail, _PathSelection
+from parlai.utils.torch import neginf
+
+from .generation_utils import Reranker, Wordlist
+
+class VocabTopKSampling(TopKSampling):
+
+    def __init__(self,
+                 k: int,
+                 wordlist: Wordlist,
+                 *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.k = k
+        self.wordlist = wordlist
+
+    def select_paths(self, logprobs, prior_scores, current_length) -> _PathSelection:
+        """
+        Select the next vocabulary item in these beams.
+        """
+        if len(self.all_scores) > 1:
+            for hypid in range(self.beam_size):
+                allowed_ids = self.wordlist.get_allowed_ids(self.partial_hyps[hypid])
+
+                neginf_assign = torch.ones(logprobs.shape[1], dtype=bool)
+                neginf_assign[allowed_ids] = False
+
+                logprobs[hypid, neginf_assign] = neginf(logprobs.dtype)
+
+        return super().select_paths(logprobs, prior_scores, current_length)
+
+
+class RerankedTopKSampling(TreeSearch):
+    def __init__(self,
+                 k: int,
+                 reranker: Reranker,
+                 tokenids_to_text: Callable,
+                 *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.k = k
+        self.reranker = reranker
+        self.tokenids_to_text = tokenids_to_text
+
+    def select_paths(self, logprobs, prior_scores, current_length) -> _PathSelection:
+        """
+        Select the next vocabulary item in these beams.
+        Adapted from top-k sampling https://github.com/facebookresearch/ParlAI/blob/054a0fff8183e357727dc7a91682496734badb7f/parlai/core/torch_generator_agent.py
+        """
+        values, indices = logprobs.topk(self.k, dim=-1)
+        probs = torch.softmax(values, dim=-1)
+
+        all_penalties = self.reranker.token_penalties.repeat(self.beam_size, 1).to(probs.device)
+        penalties = torch.gather(all_penalties, -1, indices)
+        penalised_probs = torch.mul(probs, penalties)
+
+        choices = torch.multinomial(penalised_probs, 1)[:, 0]
+        hyp_ids = torch.arange(logprobs.size(0)).to(logprobs.device)
+        tok_ids = indices[hyp_ids, choices]
+        scores = values[hyp_ids, choices]
+        best_scores = prior_scores.expand_as(scores) + scores
+
+        token_details: Optional[List[_PathSelectionTokenDetails]] = None
+        if self.verbose:
+            tok_logprobs = probs[hyp_ids, choices].log().view(-1).cpu().numpy()
+            tok_ranks = choices.view(-1).cpu().numpy()
+            token_details = []
+
+            for tok_logprob, tok_rank in zip(tok_logprobs, tok_ranks):
+                token_details.append(
+                    {"token_logprob": tok_logprob, "token_rank": int(tok_rank)}
+                )
+
+        return _PathSelection(
+            hypothesis_ids=hyp_ids,
+            token_ids=tok_ids,
+            scores=best_scores,
+            token_details=token_details,
+        )
+
+
+    def get_rescored_finished(self, n_best=None):
+        """
+        Adapted version of code taken from https://github.com/facebookresearch/ParlAI/blob/054a0fff8183e357727dc7a91682496734badb7f/parlai/core/torch_generator_agent.py
+        Adds complexity scoring and reranking.
+
+        Original description:
+        Return finished hypotheses according to adjusted scores.
+        Score adjustment is done according to the Google NMT paper, which
+        penalizes long utterances.
+        :param n_best:
+            number of finalized hypotheses to return
+        :return:
+            list of (tokens, score, token_metadata) 3-tuples, in sorted order, where:
+              - tokens is a tensor of token ids
+              - score is the adjusted log probability of the entire utterance
+              - token_metadata dictionary:
+                    token_logprobs -> a tensor of conditional log probabilities of tokens
+                    token_ranks -> a tensor of ranks of tokens in vocabulator, by probability, when sampled
+        """
+        # if we never actually finished, force one
+        if not self.finished:
+            self.outputs[-1][0] = self.eos
+            self.finished.append(
+                _HypothesisTail(
+                    timestep=len(self.outputs) - 1,
+                    hypid=0,
+                    score=self.all_scores[-1][0],
+                    tokenid=self.outputs[-1][0],
+                    token_details=self.token_details[0][-1]
+                    if self.token_details is not None
+                    else None,
+                )
+            )
+
+        # Calculate scores
+        hyps_str = []
+        length_penalties = []
+        for finished_item in self.finished:
+            token_ids = self._get_pretty_hypothesis(self._get_hyp_from_finished(finished_item))
+            hyps_str.append(self.tokenids_to_text(token_ids))
+            current_length = finished_item.timestep + 1
+            # these weights are from Google NMT paper
+            length_penalty = math.pow((1 + current_length) / 6, self.length_penalty)
+            length_penalties.append(length_penalty)
+
+        original_scores = []
+        for i, finished_item in enumerate(self.finished):
+            current_length = finished_item.timestep + 1
+            # these weights are from Google NMT paper
+            length_penalty = math.pow((1 + current_length) / 6, self.length_penalty)
+            original_scores.append(finished_item.score.cpu() / length_penalty)
+
+        complexity_scores = self.reranker.get_complexity_scores(hyps_str)
+        complexity_ranks = rankdata(complexity_scores)
+        original_ranks = rankdata(original_scores)
+
+        combined_ranks = complexity_ranks + original_ranks
+
+
+        rescored_finished = []
+        for i, finished_item in enumerate(self.finished):
+            score = combined_ranks[i]
+            if "u/" in hyps_str[i] or "r/" in hyps_str[i]:      # Fix for Reddit language, see paper appendix
+                score = np.array(-1, dtype=combined_ranks.dtype)
+
+            if self.reranker.word_filter:
+                for word in regex.findall("(?<=[^\p{L}])\p{Ll}+", hyps_str[i]): # Find all non-capitalised words
+                    if word not in self.reranker.word_filter:
+                        score = np.array(-1, dtype=combined_ranks.dtype)
+                        break
+
+            rescored_finished.append(
+                _HypothesisTail(
+                    timestep=finished_item.timestep,
+                    hypid=finished_item.hypid,
+                    score=finished_item.score / length_penalty,
+                    tokenid=finished_item.tokenid,
+                    token_details=finished_item.token_details,
+                )
+            )
+
+        # Note: beam size is almost always pretty small, so sorting is cheap enough
+        srted = sorted(rescored_finished, key=attrgetter('score'), reverse=True)
+
+        if n_best is not None:
+            srted = srted[:n_best]
+
+        n_best_list = []
+        for hyp in srted:
+            hyp_data = self._get_hyp_from_finished(hyp)
+            token_ids = self._get_pretty_hypothesis(hyp_data)
+            token_metadata = (
+                [tok.token_details for tok in reversed(hyp_data)]
+                if self.verbose
+                else None
+            )
+            n_best_list.append((token_ids, hyp.score, token_metadata))
+
+        # check that there is at least one finished candidate
+        # and assert that each of them contains only one EOS
+        assert (
+            len(n_best_list) >= 1
+        ), f'TreeSearch returned {len(n_best_list)} candidates, must be >= 1'
+        for (pred, score, _) in n_best_list:
+            assert (pred == self.eos).sum() == 1, (
+                f'TreeSearch returned a finalized hypo with multiple end tokens '
+                f'with score {score.item():.2f}'
+            )
+
+        return n_best_list
+
+

controllable_blender/generation_utils.py

@@ -0,0 +1,196 @@
+import math
+import os
+import json
+from typing import List, Set, Union, Optional
+
+import numpy as np
+import torch
+from parlai.core.dict import DictionaryAgent
+from transformers import RobertaForSequenceClassification, RobertaTokenizer
+
+def cefr_to_int(cefr: str) -> int:
+    mapping = {
+        "A1": 0,
+        "A2": 1,
+        "B1": 2,
+        "B2": 3,
+        "C1": 4,
+        "C2": 5,
+    }
+    clean_cefr = cefr.upper().strip()
+    assert clean_cefr in mapping, f"CEFR must be one of {list(mapping.keys())}, not {cefr}"
+
+    return mapping[clean_cefr]
+
+
+def load_wordlist(path: str) -> List[str]:
+    """
+    Load a list of words from a text file containing one word per line
+    """
+    vocab = []
+
+    if not path:
+        return vocab
+
+    assert os.path.isfile(path)
+
+    with open(path, 'r', encoding="utf-8") as vocab_file:
+        for row in vocab_file:
+            token = row.strip()
+            vocab.append(token)
+
+    return vocab
+
+
+class Wordlist():
+    def __init__(self, allowed_words: List[str], dict_agent: DictionaryAgent):
+        self.dict_agent = dict_agent
+
+        # Identify IDs that represent a word boundary and those that don't
+        self.boundary_ids = []
+        self.non_boundary_ids = []
+
+        for idx, subtoken in dict_agent.ind2tok.items():
+            if subtoken[0] == "\u0120" or not subtoken.isalpha():
+                self.boundary_ids.append(idx)
+            else:
+                self.non_boundary_ids.append(idx)
+
+        # Identify token ID sequences that are allowed words
+        # Identify allowed continuations of sequences
+        self.allowed_sequences = []
+        self.allowed_continuations = {}
+        for word in allowed_words:
+            for word_variant in self._get_word_variants(word):
+                token_ids = dict_agent.txt2vec(word_variant)
+                self.allowed_sequences.append(repr(token_ids))
+
+                for i, idx in enumerate(token_ids[1:]):
+                    prefix = repr(token_ids[:i + 1])      # List represented as string for lookup
+                    if prefix not in self.allowed_continuations:
+                        self.allowed_continuations[prefix] = []
+                    self.allowed_continuations[prefix].append(idx)
+
+        self.allowed_sequences = set(self.allowed_sequences)
+
+
+    def get_allowed_ids(self, token_ids: List[int]) -> List[int]:
+        last_word = self._get_last_word(token_ids)
+        continuation_ids = self._get_continuation_ids(last_word)
+
+        return continuation_ids
+
+
+    def _is_word(self, token_ids: List[int]) -> bool:
+        """
+        For a given sequence of token IDs, determine whether that sequence is a complete word
+        """
+        return (token_ids == [] or repr(token_ids) in self.allowed_sequences)
+
+
+    def _get_continuation_ids(self, token_ids: List[int]) -> List[int]:
+        """
+        For a given sequence of last word token IDs, determine which token IDs the word can continue with
+        """
+        continuation_ids = []
+        if repr(token_ids) in self.allowed_continuations:
+            continuation_ids.extend(self.allowed_continuations[repr(token_ids)])
+
+        if self._is_word(token_ids) or token_ids == []:
+            continuation_ids.extend(self.boundary_ids)
+
+        return continuation_ids
+
+
+    def _get_last_word(self, token_ids: List[int]) -> List[int]:
+        """
+        Get the sequence of token IDs after the last word boundary.
+        Assumes that a word boundary is denoted by punctuation or whitespace (Ġ).
+        """
+        for i in range(-1, -len(token_ids), -1):
+            last_word = token_ids[i:]
+            check_token = self.dict_agent[last_word[0]]
+
+            if not check_token.isalpha():
+                return last_word[1:]
+
+            if check_token[0] == "Ġ":
+                return last_word
+
+        raise ValueError("Boundary token not found")
+
+
+    def _get_word_variants(self, word: str) -> Set[str]:
+        return {word, word.lower(), word.capitalize()}
+
+
+
+class Reranker():
+    def __init__(self,
+                 cefr: int,
+                 model: str,
+                 tokenizer: str = "distilroberta-base",
+                 device: Optional[str] = "cuda",
+                 text_truncate: int = 128,
+                 exempt_tokens: Union[str, List[int]] = "all",
+                 penalty_stddev: int = 2,
+                 vocab_size: int = 8008,
+                 word_filter: Optional[List[str]] = None):
+
+        self.tokenizer = RobertaTokenizer.from_pretrained(tokenizer)
+        self.model = RobertaForSequenceClassification.from_pretrained(model)
+        self.model.to(device)
+        self.device = device
+
+        self.target_cefr = cefr
+        self.text_truncate = text_truncate
+        self.word_filter = word_filter
+
+        cefr_filepath = os.path.join(os.path.dirname(__file__), 'tokens_by_cefr.json')
+        with open(cefr_filepath, 'r') as cefr_file:
+            token_cefrs = json.load(cefr_file)
+
+        if exempt_tokens == "all" or penalty_stddev < 0:      # No penalties
+            self.token_penalties = torch.tensor([[1] * vocab_size])
+        else:
+            # calculate penalties per CEFR level difference (0 = same CEFR)
+            normal_dist = torch.distributions.normal.Normal(0, penalty_stddev)
+            cefr_penalties = [math.exp(normal_dist.log_prob(torch.tensor(i))) for i in range(6)]
+
+            token_penalties = []
+            for i in range(vocab_size):
+                if i in exempt_tokens:
+                    token_penalties.append(cefr_penalties[0])
+
+                elif str(i) in token_cefrs:
+                    token_str, token_cefr = token_cefrs[str(i)]
+                    penalty = cefr_penalties[int(token_cefr - self.target_cefr)]
+
+                    if token_cefr <= self.target_cefr or not token_str.isalpha():         # ignore lower CEFR levels and punctuation/special tokens
+                        penalty = cefr_penalties[0]
+
+                    token_penalties.append(penalty)
+
+                else:       # Assume highest CEFR level if we don't have an assigned CEFR level
+                    token_penalties.append(cefr_penalties[int(5 - self.target_cefr)])
+
+            self.token_penalties = torch.tensor([token_penalties])
+
+    def get_complexity_scores(self, hyps: List[str]) -> np.ndarray:
+        model_inputs = self.tokenizer(hyps,
+                                      padding='max_length',
+                                      truncation=True,
+                                      max_length=self.text_truncate,
+                                      return_tensors='pt',
+                                      return_token_type_ids=True,
+                                      return_attention_mask=True)
+
+        model_output = self.model(input_ids=model_inputs["input_ids"].to(self.device),
+                                  attention_mask=model_inputs["attention_mask"].to(self.device),
+                                  token_type_ids=model_inputs["token_type_ids"].to(self.device))
+
+        complexity_scores = model_output.logits.cpu().numpy().flatten()
+        complexity_diffs = 5 - np.absolute(complexity_scores - self.target_cefr)      # reversed so that higher score = better
+
+        return complexity_diffs
+

data/sample_wordlist.txt

@@ -0,0 +1,5000 @@
+the
+of
+and
+to
+a
+in
+for
+is
+on
+that
+by
+this
+with
+i
+you
+it
+not
+or
+be
+are
+from
+at
+as
+your
+all
+have
+new
+more
+an
+was
+we
+will
+home
+can
+us
+about
+if
+page
+my
+has
+search
+free
+but
+our
+one
+other
+do
+no
+information
+time
+they
+site
+he
+up
+may
+what
+which
+their
+news
+out
+use
+any
+there
+see
+only
+so
+his
+when
+contact
+here
+business
+who
+web
+also
+now
+help
+get
+pm
+view
+online
+c
+e
+first
+am
+been
+would
+how
+were
+me
+s
+services
+some
+these
+click
+its
+like
+service
+x
+than
+find
+price
+date
+back
+top
+people
+had
+list
+name
+just
+over
+state
+year
+day
+into
+email
+two
+health
+n
+world
+re
+next
+used
+go
+b
+work
+last
+most
+products
+music
+buy
+data
+make
+them
+should
+product
+system
+post
+her
+city
+t
+add
+policy
+number
+such
+please
+available
+copyright
+support
+message
+after
+best
+software
+then
+jan
+good
+video
+well
+d
+where
+info
+rights
+public
+books
+high
+school
+through
+m
+each
+links
+she
+review
+years
+order
+very
+privacy
+book
+items
+company
+r
+read
+group
+need
+many
+user
+said
+de
+does
+set
+under
+general
+research
+university
+january
+mail
+full
+map
+reviews
+program
+life
+know
+games
+way
+days
+management
+p
+part
+could
+great
+united
+hotel
+real
+f
+item
+international
+center
+ebay
+must
+store
+travel
+comments
+made
+development
+report
+off
+member
+details
+line
+terms
+before
+hotels
+did
+send
+right
+type
+because
+local
+those
+using
+results
+office
+education
+national
+car
+design
+take
+posted
+internet
+address
+community
+within
+states
+area
+want
+phone
+dvd
+shipping
+reserved
+subject
+between
+forum
+family
+l
+long
+based
+w
+code
+show
+o
+even
+black
+check
+special
+prices
+website
+index
+being
+women
+much
+sign
+file
+link
+open
+today
+technology
+south
+case
+project
+same
+pages
+uk
+version
+section
+own
+found
+sports
+house
+related
+security
+both
+g
+county
+american
+photo
+game
+members
+power
+while
+care
+network
+down
+computer
+systems
+three
+total
+place
+end
+following
+download
+h
+him
+without
+per
+access
+think
+north
+resources
+current
+posts
+big
+media
+law
+control
+water
+history
+pictures
+size
+art
+personal
+since
+including
+guide
+shop
+directory
+board
+location
+change
+white
+text
+small
+rating
+rate
+government
+children
+during
+usa
+return
+students
+v
+shopping
+account
+times
+sites
+level
+digital
+profile
+previous
+form
+events
+love
+old
+john
+main
+call
+hours
+image
+department
+title
+description
+non
+k
+y
+insurance
+another
+why
+shall
+property
+class
+cd
+still
+money
+quality
+every
+listing
+content
+country
+private
+little
+visit
+save
+tools
+low
+reply
+customer
+december
+compare
+movies
+include
+college
+value
+article
+york
+man
+card
+jobs
+provide
+j
+food
+source
+author
+different
+press
+u
+learn
+sale
+around
+print
+course
+job
+canada
+process
+teen
+room
+stock
+training
+too
+credit
+point
+join
+science
+men
+categories
+advanced
+west
+sales
+look
+english
+left
+team
+estate
+box
+conditions
+select
+windows
+photos
+gay
+thread
+week
+category
+note
+live
+large
+gallery
+table
+register
+however
+june
+october
+november
+market
+library
+really
+action
+start
+series
+model
+features
+air
+industry
+plan
+human
+provided
+tv
+yes
+required
+second
+hot
+accessories
+cost
+movie
+forums
+march
+la
+september
+better
+say
+questions
+july
+yahoo
+going
+medical
+test
+friend
+come
+dec
+server
+pc
+study
+application
+cart
+staff
+articles
+san
+feedback
+again
+play
+looking
+issues
+april
+never
+users
+complete
+street
+topic
+comment
+financial
+things
+working
+against
+standard
+tax
+person
+below
+mobile
+less
+got
+blog
+party
+payment
+equipment
+login
+student
+let
+programs
+offers
+legal
+above
+recent
+park
+stores
+side
+act
+problem
+red
+give
+memory
+performance
+social
+q
+august
+quote
+language
+story
+sell
+options
+experience
+rates
+create
+key
+body
+young
+america
+important
+field
+few
+east
+paper
+single
+ii
+age
+activities
+club
+example
+girls
+additional
+password
+z
+latest
+something
+road
+gift
+question
+changes
+night
+ca
+hard
+texas
+oct
+pay
+four
+poker
+status
+browse
+issue
+range
+building
+seller
+court
+february
+always
+result
+audio
+light
+write
+war
+nov
+offer
+blue
+groups
+al
+easy
+given
+files
+event
+release
+analysis
+request
+fax
+china
+making
+picture
+needs
+possible
+might
+professional
+yet
+month
+major
+star
+areas
+future
+space
+committee
+hand
+sun
+cards
+problems
+london
+washington
+meeting
+rss
+become
+interest
+id
+child
+keep
+enter
+california
+share
+similar
+garden
+schools
+million
+added
+reference
+companies
+listed
+baby
+learning
+energy
+run
+delivery
+net
+popular
+term
+film
+stories
+put
+computers
+journal
+reports
+co
+try
+welcome
+central
+images
+president
+notice
+original
+head
+radio
+until
+cell
+color
+self
+council
+away
+includes
+track
+australia
+discussion
+archive
+once
+others
+entertainment
+agreement
+format
+least
+society
+months
+log
+safety
+friends
+sure
+faq
+trade
+edition
+cars
+messages
+marketing
+tell
+further
+updated
+association
+able
+having
+provides
+david
+fun
+already
+green
+studies
+close
+common
+drive
+specific
+several
+gold
+feb
+living
+sep
+collection
+called
+short
+arts
+lot
+ask
+display
+limited
+powered
+solutions
+means
+director
+daily
+beach
+past
+natural
+whether
+due
+et
+electronics
+five
+upon
+period
+planning
+database
+says
+official
+weather
+mar
+land
+average
+done
+technical
+window
+france
+pro
+region
+island
+record
+direct
+microsoft
+conference
+environment
+records
+st
+district
+calendar
+costs
+style
+url
+front
+statement
+update
+parts
+aug
+ever
+downloads
+early
+miles
+sound
+resource
+present
+applications
+either
+ago
+document
+word
+works
+material
+bill
+apr
+written
+talk
+federal
+hosting
+rules
+final
+adult
+tickets
+thing
+centre
+requirements
+via
+cheap
+kids
+finance
+true
+minutes
+else
+mark
+third
+rock
+gifts
+europe
+reading
+topics
+bad
+individual
+tips
+plus
+auto
+cover
+usually
+edit
+together
+videos
+percent
+fast
+function
+fact
+unit
+getting
+global
+tech
+meet
+far
+economic
+en
+player
+projects
+lyrics
+often
+subscribe
+submit
+germany
+amount
+watch
+included
+feel
+though
+bank
+risk
+thanks
+everything
+deals
+various
+words
+linux
+jul
+production
+commercial
+james
+weight
+town
+heart
+advertising
+received
+choose
+treatment
+newsletter
+archives
+points
+knowledge
+magazine
+error
+camera
+jun
+girl
+currently
+construction
+toys
+registered
+clear
+golf
+receive
+domain
+methods
+chapter
+makes
+protection
+policies
+loan
+wide
+beauty
+manager
+india
+position
+taken
+sort
+listings
+models
+michael
+known
+half
+cases
+step
+engineering
+florida
+simple
+quick
+none
+wireless
+license
+paul
+friday
+lake
+whole
+annual
+published
+later
+basic
+sony
+shows
+corporate
+google
+church
+method
+purchase
+customers
+active
+response
+practice
+hardware
+figure
+materials
+fire
+holiday
+chat
+enough
+designed
+along
+among
+death
+writing
+speed
+html
+countries
+loss
+face
+brand
+discount
+higher
+effects
+created
+remember
+standards
+oil
+bit
+yellow
+political
+increase
+advertise
+kingdom
+base
+near
+environmental
+thought
+stuff
+french
+storage
+oh
+japan
+doing
+loans
+shoes
+entry
+stay
+nature
+orders
+availability
+africa
+summary
+turn
+mean
+growth
+notes
+agency
+king
+monday
+european
+activity
+copy
+although
+drug
+pics
+western
+income
+force
+cash
+employment
+overall
+bay
+river
+commission
+ad
+package
+contents
+seen
+players
+engine
+port
+album
+regional
+stop
+supplies
+started
+administration
+bar
+institute
+views
+plans
+double
+dog
+build
+screen
+exchange
+types
+soon
+sponsored
+lines
+electronic
+continue
+across
+benefits
+needed
+season
+apply
+someone
+held
+ny
+anything
+printer
+condition
+effective
+believe
+organization
+effect
+asked
+eur
+mind
+sunday
+selection
+casino
+pdf
+lost
+tour
+menu
+volume
+cross
+anyone
+mortgage
+hope
+silver
+corporation
+wish
+inside
+solution
+mature
+role
+rather
+weeks
+addition
+came
+supply
+nothing
+certain
+usr
+executive
+running
+lower
+necessary
+union
+jewelry
+according
+dc
+clothing
+mon
+com
+particular
+fine
+names
+robert
+homepage
+hour
+gas
+skills
+six
+bush
+islands
+advice
+career
+military
+rental
+decision
+leave
+british
+teens
+pre
+huge
+sat
+woman
+facilities
+zip
+bid
+kind
+sellers
+middle
+move
+cable
+opportunities
+taking
+values
+division
+coming
+tuesday
+object
+lesbian
+appropriate
+machine
+logo
+length
+actually
+nice
+score
+statistics
+client
+ok
+returns
+capital
+follow
+sample
+investment
+sent
+shown
+saturday
+christmas
+england
+culture
+band
+flash
+ms
+lead
+george
+choice
+went
+starting
+registration
+fri
+thursday
+courses
+consumer
+hi
+airport
+foreign
+artist
+outside
+furniture
+levels
+channel
+letter
+mode
+phones
+ideas
+wednesday
+structure
+fund
+summer
+allow
+degree
+contract
+button
+releases
+wed
+homes
+super
+male
+matter
+custom
+virginia
+almost
+took
+located
+multiple
+asian
+distribution
+editor
+inn
+industrial
+cause
+potential
+song
+cnet
+ltd
+los
+hp
+focus
+late
+fall
+featured
+idea
+rooms
+female
+responsible
+inc
+communications
+win
+associated
+thomas
+primary
+cancer
+numbers
+reason
+tool
+browser
+spring
+foundation
+answer
+voice
+eg
+friendly
+schedule
+documents
+communication
+purpose
+feature
+bed
+comes
+police
+everyone
+independent
+ip
+approach
+cameras
+brown
+physical
+operating
+hill
+maps
+medicine
+deal
+hold
+ratings
+chicago
+forms
+glass
+happy
+tue
+smith
+wanted
+developed
+thank
+safe
+unique
+survey
+prior
+telephone
+sport
+ready
+feed
+animal
+sources
+mexico
+population
+pa
+regular
+secure
+navigation
+operations
+therefore
+simply
+evidence
+station
+christian
+round
+paypal
+favorite
+understand
+option
+master
+valley
+recently
+probably
+thu
+rentals
+sea
+built
+publications
+blood
+cut
+worldwide
+improve
+connection
+publisher
+hall
+larger
+anti
+networks
+earth
+parents
+nokia
+impact
+transfer
+introduction
+kitchen
+strong
+tel
+carolina
+wedding
+properties
+hospital
+ground
+overview
+ship
+accommodation
+owners
+disease
+tx
+excellent
+paid
+italy
+perfect
+hair
+opportunity
+kit
+classic
+basis
+command
+cities
+william
+express
+award
+distance
+tree
+peter
+assessment
+ensure
+thus
+wall
+ie
+involved
+el
+extra
+especially
+interface
+partners
+budget
+rated
+guides
+success
+maximum
+ma
+operation
+existing
+quite
+selected
+boy
+amazon
+patients
+restaurants
+beautiful
+warning
+wine
+locations
+horse
+vote
+forward
+flowers
+stars
+significant
+lists
+technologies
+owner
+retail
+animals
+useful
+directly
+manufacturer
+ways
+est
+son
+providing
+rule
+mac
+housing
+takes
+iii
+gmt
+bring
+catalog
+searches
+max
+trying
+mother
+authority
+considered
+told
+xml
+traffic
+programme
+joined
+input
+strategy
+feet
+agent
+valid
+bin
+modern
+senior
+ireland
+teaching
+door
+grand
+testing
+trial
+charge
+units
+instead
+canadian
+cool
+normal
+wrote
+enterprise
+ships
+entire
+educational
+md
+leading
+metal
+positive
+fl
+fitness
+chinese
+opinion
+mb
+asia
+football
+abstract
+uses
+output
+funds
+mr
+greater
+likely
+develop
+employees
+artists
+alternative
+processing
+responsibility
+resolution
+java
+guest
+seems
+publication
+pass
+relations
+trust
+van
+contains
+session
+multi
+photography
+republic
+fees
+components
+vacation
+century
+academic
+assistance
+completed
+skin
+graphics
+indian
+prev
+ads
+mary
+il
+expected
+ring
+grade
+dating
+pacific
+mountain
+organizations
+pop
+filter
+mailing
+vehicle
+longer
+consider
+int
+northern
+behind
+panel
+floor
+german
+buying
+match
+proposed
+default
+require
+iraq
+boys
+outdoor
+deep
+morning
+otherwise
+allows
+rest
+protein
+plant
+reported
+hit
+transportation
+mm
+pool
+mini
+politics
+partner
+disclaimer
+authors
+boards
+faculty
+parties
+fish
+membership
+mission
+eye
+string
+sense
+modified
+pack
+released
+stage
+internal
+goods
+recommended
+born
+unless
+richard
+detailed
+japanese
+race
+approved
+background
+target
+except
+character
+usb
+maintenance
+ability
+maybe
+functions
+ed
+moving
+brands
+places
+php
+pretty
+trademarks
+phentermine
+spain
+southern
+yourself
+etc
+winter
+battery
+youth
+pressure
+submitted
+boston
+debt
+keywords
+medium
+television
+interested
+core
+break
+purposes
+throughout
+sets
+dance
+wood
+msn
+itself
+defined
+papers
+playing
+awards
+fee
+studio
+reader
+virtual
+device
+established
+answers
+rent
+las
+remote
+dark
+programming
+external
+apple
+le
+regarding
+instructions
+min
+offered
+theory
+enjoy
+remove
+aid
+surface
+minimum
+visual
+host
+variety
+teachers
+isbn
+martin
+manual
+block
+subjects
+agents
+increased
+repair
+fair
+civil
+steel
+understanding
+songs
+fixed
+wrong
+beginning
+hands
+associates
+finally
+az
+updates
+desktop
+classes
+paris
+ohio
+gets
+sector
+capacity
+requires
+jersey
+un
+fat
+fully
+father
+electric
+saw
+instruments
+quotes
+officer
+driver
+businesses
+dead
+respect
+unknown
+specified
+restaurant
+mike
+trip
+pst
+worth
+mi
+procedures
+poor
+teacher
+eyes
+relationship
+workers
+farm
+georgia
+peace
+traditional
+campus
+tom
+showing
+creative
+coast
+benefit
+progress
+funding
+devices
+lord
+grant
+sub
+agree
+fiction
+hear
+sometimes
+watches
+careers
+beyond
+goes
+families
+led
+museum
+themselves
+fan
+transport
+interesting
+blogs
+wife
+evaluation
+accepted
+former
+implementation
+ten
+hits
+zone
+complex
+th
+cat
+galleries
+references
+die
+presented
+jack
+flat
+flow
+agencies
+literature
+respective
+parent
+spanish
+michigan
+columbia
+setting
+dr
+scale
+stand
+economy
+highest
+helpful
+monthly
+critical
+frame
+musical
+definition
+secretary
+angeles
+networking
+path
+australian
+employee
+chief
+gives
+kb
+bottom
+magazines
+packages
+detail
+francisco
+laws
+changed
+pet
+heard
+begin
+individuals
+colorado
+royal
+clean
+switch
+russian
+largest
+african
+guy
+titles
+relevant
+guidelines
+justice
+connect
+bible
+dev
+cup
+basket
+applied
+weekly
+vol
+installation
+described
+demand
+pp
+suite
+vegas
+na
+square
+chris
+attention
+advance
+skip
+diet
+army
+auction
+gear
+lee
+os
+difference
+allowed
+correct
+charles
+nation
+selling
+lots
+piece
+sheet
+firm
+seven
+older
+illinois
+regulations
+elements
+species
+jump
+cells
+module
+resort
+facility
+random
+pricing
+dvds
+certificate
+minister
+motion
+looks
+fashion
+directions
+visitors
+documentation
+monitor
+trading
+forest
+calls
+whose
+coverage
+couple
+giving
+chance
+vision
+ball
+ending
+clients
+actions
+listen
+discuss
+accept
+automotive
+naked
+goal
+successful
+sold
+wind
+communities
+clinical
+situation
+sciences
+markets
+lowest
+highly
+publishing
+appear
+emergency
+developing
+lives
+currency
+leather
+determine
+temperature
+palm
+announcements
+patient
+actual
+historical
+stone
+bob
+commerce
+ringtones
+perhaps
+persons
+difficult
+scientific
+satellite
+fit
+tests
+village
+accounts
+amateur
+ex
+met
+pain
+xbox
+particularly
+factors
+coffee
+www
+settings
+buyer
+cultural
+steve
+easily
+oral
+ford
+poster
+edge
+functional
+root
+au
+fi
+closed
+holidays
+ice
+pink
+zealand
+balance
+monitoring
+graduate
+replies
+shot
+nc
+architecture
+initial
+label
+thinking
+scott
+llc
+sec
+recommend
+canon
+league
+waste
+minute
+bus
+provider
+optional
+dictionary
+cold
+accounting
+manufacturing
+sections
+chair
+fishing
+effort
+phase
+fields
+bag
+fantasy
+po
+letters
+motor
+va
+professor
+context
+install
+shirt
+apparel
+generally
+continued
+foot
+mass
+crime
+count
+breast
+techniques
+ibm
+rd
+johnson
+sc
+quickly
+dollars
+websites
+religion
+claim
+driving
+permission
+surgery
+patch
+heat
+wild
+measures
+generation
+kansas
+miss
+chemical
+doctor
+task
+reduce
+brought
+himself
+nor
+component
+enable
+exercise
+bug
+santa
+mid
+guarantee
+leader
+diamond
+israel
+se
+processes
+soft
+servers
+alone
+meetings
+seconds
+jones
+arizona
+keyword
+interests
+flight
+congress
+fuel
+username
+walk
+produced
+italian
+paperback
+classifieds
+wait
+supported
+pocket
+saint
+rose
+freedom
+argument
+competition
+creating
+jim
+drugs
+joint
+premium
+providers
+fresh
+characters
+attorney
+upgrade
+di
+factor
+growing
+thousands
+km
+stream
+apartments
+pick
+hearing
+eastern
+auctions
+therapy
+entries
+dates
+generated
+signed
+upper
+administrative
+serious
+prime
+samsung
+limit
+began
+louis
+steps
+errors
+shops
+del
+efforts
+informed
+ga
+ac
+thoughts
+creek
+ft
+worked
+quantity
+urban
+practices
+sorted
+reporting
+essential
+myself
+tours
+platform
+load
+affiliate
+labor
+immediately
+admin
+nursing
+defense
+machines
+designated
+tags
+heavy
+covered
+recovery
+joe
+guys
+integrated
+configuration
+merchant
+comprehensive
+expert
+universal
+protect
+drop
+solid
+cds
+presentation
+languages
+became
+orange
+compliance
+vehicles
+prevent
+theme
+rich
+im
+campaign
+marine
+improvement
+vs
+guitar
+finding
+pennsylvania
+examples
+ipod
+saying
+spirit
+ar
+claims
+challenge
+motorola
+acceptance
+strategies
+mo
+seem
+affairs
+touch
+intended
+towards
+sa
+goals
+hire
+election
+suggest
+branch
+charges
+serve
+affiliates
+reasons
+magic
+mount
+smart
+talking
+gave
+ones
+latin
+multimedia
+xp
+avoid
+certified
+manage
+corner
+rank
+computing
+oregon
+element
+birth
+virus
+abuse
+interactive
+requests
+separate
+quarter
+procedure
+leadership
+tables
+define
+racing
+religious
+facts
+breakfast
+kong
+column
+plants
+faith
+chain
+developer
+identify
+avenue
+missing
+died
+approximately
+domestic
+sitemap
+recommendations
+moved
+houston
+reach
+comparison
+mental
+viewed
+moment
+extended
+sequence
+inch
+attack
+sorry
+centers
+opening
+damage
+lab
+reserve
+recipes
+cvs
+gamma
+plastic
+produce
+snow
+placed
+truth
+counter
+failure
+follows
+eu
+weekend
+dollar
+camp
+ontario
+automatically
+des
+minnesota
+films
+bridge
+native
+fill
+williams
+movement
+printing
+baseball
+owned
+approval
+draft
+chart
+played
+contacts
+cc
+jesus
+readers
+clubs
+lcd
+wa
+jackson
+equal
+adventure
+matching
+offering
+shirts
+profit
+leaders
+posters
+institutions
+assistant
+variable
+ave
+dj
+advertisement
+expect
+parking
+headlines
+yesterday
+compared
+determined
+wholesale
+workshop
+russia
+gone
+codes
+kinds
+extension
+seattle
+statements
+golden
+completely
+teams
+fort
+cm
+wi
+lighting
+senate
+forces
+funny
+brother
+gene
+turned
+portable
+tried
+electrical
+applicable
+disc
+returned
+pattern
+ct
+boat
+named
+theatre
+laser
+earlier
+manufacturers
+sponsor
+classical
+icon
+warranty
+dedicated
+indiana
+direction
+harry
+basketball
+objects
+ends
+delete
+evening
+assembly
+nuclear
+taxes
+mouse
+signal
+criminal
+issued
+brain
+sexual
+wisconsin
+powerful
+dream
+obtained
+false
+da
+cast
+flower
+felt
+personnel
+passed
+supplied
+identified
+falls
+pic
+soul
+aids
+opinions
+promote
+stated
+stats
+hawaii
+professionals
+appears
+carry
+flag
+decided
+nj
+covers
+hr
+em
+advantage
+hello
+designs
+maintain
+tourism
+priority
+newsletters
+adults
+clips
+savings
+iv
+graphic
+atom
+payments
+rw
+estimated
+binding
+brief
+ended
+winning
+eight
+anonymous
+iron
+straight
+script
+served
+wants
+miscellaneous
+prepared
+void
+dining
+alert
+integration
+atlanta
+dakota
+tag
+interview
+mix
+framework
+disk
+installed
+queen
+vhs
+credits
+clearly
+fix
+handle
+sweet
+desk
+criteria
+pubmed
+dave
+massachusetts
+diego
+hong
+vice
+associate
+ne
+truck
+behavior
+enlarge
+ray
+frequently
+revenue
+measure
+changing
+votes
+du
+duty
+looked
+discussions
+bear
+gain
+festival
+laboratory
+ocean
+flights
+experts
+signs
+lack
+depth
+iowa
+whatever
+logged
+laptop
+vintage
+train
+exactly
+dry
+explore
+maryland
+spa
+concept
+nearly
+eligible
+checkout
+reality
+forgot
+handling
+origin
+knew
+gaming
+feeds
+billion
+destination
+scotland
+faster
+intelligence
+dallas
+bought
+con
+ups
+nations
+route
+followed
+specifications
+broken
+tripadvisor
+frank
+alaska
+zoom
+blow
+battle
+residential
+anime
+speak
+decisions
+industries
+protocol
+query
+clip
+partnership
+editorial
+nt
+expression
+es
+equity
+provisions
+speech
+wire
+principles
+suggestions
+rural
+shared
+sounds
+replacement
+tape
+strategic
+judge
+spam
+economics
+acid
+bytes
+cent
+forced
+compatible
+fight
+apartment
+height
+null
+zero
+speaker
+filed
+gb
+netherlands
+obtain
+bc
+consulting
+recreation
+offices
+designer
+remain
+managed
+pr
+failed
+marriage
+roll
+korea
+banks
+fr
+participants
+secret
+bath
+aa
+kelly
+leads
+negative
+austin
+favorites
+toronto
+theater
+springs
+missouri
+andrew
+var
+perform
+healthy
+translation
+estimates
+font
+assets
+injury
+mt
+joseph
+ministry
+drivers
+lawyer
+figures
+married
+protected
+proposal
+sharing
+philadelphia
+portal
+waiting
+birthday
+beta
+fail
+gratis
+banking
+officials
+brian
+toward
+won
+slightly
+assist
+conduct
+contained
+lingerie
+legislation
+calling
+parameters
+jazz
+serving
+bags
+profiles
+miami
+comics
+matters
+houses
+doc
+postal
+relationships
+tennessee
+wear
+controls
+breaking
+combined
+ultimate
+wales
+representative
+frequency
+introduced
+minor
+finish
+departments
+residents
+noted
+displayed
+mom
+reduced
+physics
+rare
+spent
+performed
+extreme
+samples
+davis
+daniel
+bars
+reviewed
+row
+oz
+forecast
+removed
+helps
+singles
+administrator
+cycle
+amounts
+contain
+accuracy
+dual
+rise
+usd
+sleep
+mg
+bird
+pharmacy
+brazil
+creation
+static
+scene
+hunter
+addresses
+lady
+crystal
+famous
+writer
+chairman
+violence
+fans
+oklahoma
+speakers
+drink
+academy
+dynamic
+gender
+eat
+permanent
+agriculture
+dell
+cleaning
+constitutes
+portfolio
+practical
+delivered
+collectibles
+infrastructure
+exclusive
+seat
+concerns
+colour
+vendor
+originally
+intel
+utilities
+philosophy
+regulation
+officers
+reduction
+aim
+bids
+referred
+supports
+nutrition
+recording
+regions
+junior
+toll
+les
+cape
+ann
+rings
+meaning
+tip
+secondary
+wonderful
+mine
+ladies
+henry
+ticket
+announced
+guess
+agreed
+prevention
+whom
+ski
+soccer
+math
+import
+posting
+presence
+instant
+mentioned
+automatic
+healthcare
+viewing
+maintained
+ch
+increasing
+majority
+connected
+christ
+dan
+dogs
+sd
+directors
+aspects
+austria
+ahead
+moon
+participation
+scheme
+utility
+preview
+fly
+manner
+matrix
+containing
+combination
+devel
+amendment
+despite
+strength
+guaranteed
+turkey
+libraries
+proper
+distributed
+degrees
+singapore
+enterprises
+delta
+fear
+seeking
+inches
+phoenix
+rs
+convention
+shares
+principal
+daughter
+standing
+comfort
+colors
+wars
+cisco
+ordering
+kept
+alpha
+appeal
+cruise
+bonus
+certification
+previously
+hey
+bookmark
+buildings
+specials
+beat
+disney
+household
+batteries
+adobe
+smoking
+bbc
+becomes
+drives
+arms
+alabama
+tea
+improved
+trees
+avg
+achieve
+positions
+dress
+subscription
+dealer
+contemporary
+sky
+utah
+nearby
+rom
+carried
+happen
+exposure
+panasonic
+hide
+permalink
+signature
+gambling
+refer
+miller
+provision
+outdoors
+clothes
+caused
+luxury
+babes
+frames
+certainly
+indeed
+newspaper
+toy
+circuit
+layer
+printed
+slow
+removal
+easier
+src
+liability
+trademark
+hip
+printers
+faqs
+nine
+adding
+kentucky
+mostly
+eric
+spot
+taylor
+trackback
+prints
+spend
+factory
+interior
+revised
+grow
+americans
+optical
+promotion
+relative
+amazing
+clock
+dot
+hiv
+identity
+suites
+conversion
+feeling
+hidden
+reasonable
+victoria
+serial
+relief
+revision
+broadband
+influence
+ratio
+pda
+importance
+rain
+onto
+dsl
+planet
+webmaster
+copies
+recipe
+zum
+permit
+seeing
+proof
+dna
+diff
+tennis
+bass
+prescription
+bedroom
+empty
+instance
+hole
+pets
+ride
+licensed
+orlando
+specifically
+tim
+bureau
+maine
+sql
+represent
+conservation
+pair
+ideal
+specs
+recorded
+don
+pieces
+finished
+parks
+dinner
+lawyers
+sydney
+stress
+cream
+ss
+runs
+trends
+yeah
+discover
+ap
+patterns
+boxes
+louisiana
+hills
+javascript
+fourth
+nm
+advisor
+mn
+marketplace
+nd
+evil
+aware
+wilson
+shape
+evolution
+irish
+certificates
+objectives
+stations
+suggested
+gps
+op
+remains
+acc
+greatest
+firms
+concerned
+euro
+operator
+structures
+generic
+encyclopedia
+usage
+cap
+ink
+charts
+continuing
+mixed
+census
+interracial
+peak
+tn
+competitive
+exist
+wheel
+transit
+suppliers
+salt
+compact
+poetry
+lights
+tracking
+angel
+bell
+keeping
+preparation
+attempt
+receiving
+matches
+accordance
+width
+noise
+engines
+forget
+array
+discussed
+accurate
+stephen
+elizabeth
+climate
+reservations
+pin
+playstation
+alcohol
+greek
+instruction
+managing
+annotation
+sister
+raw
+differences
+walking
+explain
+smaller
+newest
+establish
+gnu
+happened
+expressed
+jeff
+extent
+sharp
+lesbians
+ben
+lane
+paragraph
+kill
+mathematics
+aol
+compensation
+ce
+export
+managers
+aircraft
+modules
+sweden
+conflict
+conducted
+versions
+employer
+occur
+percentage
+knows
+mississippi
+describe
+concern
+backup
+requested
+citizens
+connecticut
+heritage
+personals
+immediate
+holding
+trouble
+spread
+coach
+kevin
+agricultural
+expand
+supporting
+audience
+assigned
+jordan
+collections
+ages
+participate
+plug
+specialist
+cook
+affect
+virgin
+experienced
+investigation
+raised
+hat
+institution
+directed
+dealers
+searching
+sporting
+helping
+perl
+affected
+lib
+bike
+totally
+plate
+expenses
+indicate
+blonde
+ab
+proceedings
+favourite
+transmission
+anderson
+utc
+characteristics
+der
+lose
+organic
+seek
+experiences
+albums
+cheats
+extremely
+verzeichnis
+contracts
+guests
+hosted
+diseases
+concerning
+developers
+equivalent
+chemistry
+tony
+neighborhood
+nevada
+kits
+thailand
+variables
+agenda
+anyway
+continues
+tracks
+advisory
+cam
+curriculum
+logic
+template
+prince
+circle
+soil
+grants
+anywhere
+psychology
+responses
+atlantic
+wet
+circumstances
+edward
+investor
+identification
+ram
+leaving
+wildlife
+appliances
+matt
+elementary
+cooking
+speaking
+sponsors
+fox
+unlimited
+respond
+sizes
+plain
+exit
+entered
+iran
+arm
+keys
+launch
+wave
+checking
+costa
+belgium
+printable
+holy
+acts
+guidance
+mesh
+trail
+enforcement
+symbol
+crafts
+highway
+buddy
+hardcover
+observed
+dean
+setup
+poll
+booking
+glossary
+fiscal
+celebrity
+styles
+denver
+unix
+filled
+bond
+channels
+ericsson
+appendix
+notify
+blues
+chocolate
+pub
+portion
+scope
+hampshire
+supplier
+cables
+cotton
+bluetooth
+controlled
+requirement
+authorities
+biology
+dental
+killed
+border
+ancient
+debate
+representatives
+starts
+pregnancy
+causes
+arkansas
+biography
+leisure
+attractions
+learned
+transactions
+notebook
+explorer
+historic
+attached
+opened
+tm
+husband
+disabled
+authorized
+crazy
+upcoming
+britain
+concert
+retirement
+scores
+financing
+efficiency
+sp
+comedy
+adopted
+efficient
+weblog
+linear
+commitment
+specialty
+bears
+jean
+hop
+carrier
+edited
+constant
+visa
+mouth
+jewish
+meter
+linked
+portland
+interviews
+concepts
+nh
+gun
+reflect
+pure
+deliver
+wonder
+lessons
+fruit
+begins
+qualified
+reform
+lens
+alerts
+treated
+discovery
+draw
+mysql
+classified
+relating
+assume
+confidence
+alliance
+fm
+confirm
+warm
+neither
+lewis
+howard
+offline
+leaves
+engineer
+lifestyle
+consistent
+replace
+clearance
+connections
+inventory
+converter
+organisation
+babe
+checks
+reached
+becoming
+safari
+objective
+indicated
+sugar
+crew
+legs
+sam
+stick
+securities
+allen
+pdt
+relation
+enabled
+genre
+slide
+montana
+volunteer
+tested
+rear
+democratic
+enhance
+switzerland
+exact
+bound
+parameter
+adapter
+processor
+node
+formal
+dimensions
+contribute
+lock
+hockey
+storm
+micro
+colleges
+laptops
+mile
+showed
+challenges
+editors
+mens
+threads
+bowl
+supreme
+brothers
+recognition
+presents
+ref
+tank
+submission
+dolls
+estimate
+encourage
+navy
+kid
+regulatory
+inspection
+consumers
+cancel
+limits
+territory
+transaction
+manchester
+weapons
+paint
+delay
+pilot
+outlet
+contributions
+continuous
+db
+czech
+resulting
+cambridge
+initiative
+novel
+pan
+execution
+disability
+increases
+ultra
+winner
+idaho
+contractor
+ph
+episode
+examination
+potter
+dish
+plays
+bulletin
+ia
+pt
+indicates
+modify
+oxford
+adam
+truly
+epinions
+painting
+committed
+extensive
+affordable
+universe
+candidate
+databases
+patent
+slot
+psp
+outstanding
+ha
+eating
+perspective
+planned
+watching
+lodge
+messenger
+mirror
+tournament
+consideration
+ds
+discounts
+sterling
+sessions
+kernel
+stocks
+buyers
+journals
+gray
+catalogue
+ea
+jennifer
+antonio
+charged
+broad
+taiwan
+und
+chosen
+demo
+greece
+lg
+swiss
+sarah
+clark
+labour
+hate
+terminal
+publishers
+nights
+behalf
+caribbean
+liquid
+rice
+nebraska
+loop
+salary
+reservation
+foods
+gourmet
+guard
+properly
+orleans
+saving
+nfl
+remaining
+empire
+resume
+twenty
+newly
+raise
+prepare
+avatar
+gary
+depending
+illegal
+expansion
+vary
+hundreds
+rome
+arab
+lincoln
+helped
+premier
+tomorrow
+purchased
+milk
+decide
+consent
+drama
+visiting
+performing
+downtown
+keyboard
+contest
+collected
+nw
+bands
+boot
+suitable
+ff
+absolutely
+millions
+lunch
+audit
+push
+chamber
+guinea
+findings
+muscle
+featuring
+iso
+implement
+clicking
+scheduled
+polls
+typical
+tower
+yours
+sum
+misc
+calculator
+significantly
+chicken
+temporary
+attend
+shower
+alan
+sending
+jason
+tonight
+dear
+sufficient
+holdem
+shell
+province
+catholic
+oak
+vat
+awareness
+vancouver
+governor
+beer
+seemed
+contribution
+measurement
+swimming
+spyware
+formula
+constitution
+packaging
+solar
+jose
+catch
+jane
+pakistan
+ps
+reliable
+consultation
+northwest
+sir
+doubt
+earn
+finder
+unable
+periods
+classroom
+tasks
+democracy
+attacks
+kim
+wallpaper
+merchandise
+const
+resistance
+doors
+symptoms
+resorts
+biggest
+memorial
+visitor
+twin
+forth
+insert
+baltimore
+gateway
+ky
+dont
+alumni
+drawing
+candidates
+charlotte
+ordered
+biological
+fighting
+transition
+happens
+preferences
+spy
+romance
+instrument
+bruce
+split
+themes
+powers
+heaven
+br
+bits
+pregnant
+twice
+classification
+focused
+egypt
+physician
+hollywood
+bargain
+wikipedia
+cellular
+norway
+vermont
+asking
+blocks
+normally
+lo
+spiritual
+hunting
+diabetes
+suit
+ml
+shift
+chip
+res
+sit
+bodies
+photographs
+cutting
+wow
+simon
+writers
+marks
+flexible
+loved
+favourites
+mapping
+numerous
+relatively
+birds
+satisfaction
+represents
+char
+indexed
+pittsburgh
+superior
+preferred
+saved
+paying
+cartoon
+shots
+intellectual
+moore
+granted
+choices
+carbon
+spending
+comfortable
+magnetic
+interaction
+listening
+effectively
+registry
+crisis
+outlook
+massive
+denmark
+employed
+bright
+treat
+header
+cs
+poverty
+formed
+piano
+echo
+que
+grid
+sheets
+patrick
+experimental
+puerto
+revolution
+consolidation
+displays
+plasma
+allowing
+earnings
+voip
+mystery
+landscape
+dependent
+mechanical
+journey
+delaware
+bidding
+consultants
+risks
+banner
+applicant
+charter
+fig
+barbara
+cooperation
+counties
+acquisition
+ports
+implemented
+sf
+directories
+recognized
+dreams
+blogger
+notification
+kg
+licensing
+stands
+teach
+occurred
+textbooks
+rapid
+pull
+hairy
+diversity
+cleveland
+ut
+reverse
+deposit
+seminar
+investments
+latina
+nasa
+wheels
+specify
+accessibility
+dutch
+sensitive
+templates
+formats
+tab
+depends
+boots
+holds
+router
+concrete
+si
+editing
+poland
+folder
+womens
+css
+completion
+upload
+pulse
+universities
+technique
+contractors
+voting
+courts
+notices
+subscriptions
+calculate
+mc
+detroit
+alexander
+broadcast
+converted
+metro
+toshiba
+anniversary
+improvements
+strip
+specification
+pearl
+accident
+nick
+accessible
+accessory
+resident
+plot
+qty
+possibly
+airline
+typically
+representation
+regard
+pump
+exists
+arrangements
+smooth
+conferences
+uniprotkb
+strike
+consumption
+birmingham
+flashing
+lp
+narrow
+afternoon
+threat
+surveys
+sitting
+putting
+consultant
+controller
+ownership
+committees
+legislative
+researchers
+vietnam
+trailer
+anne
+castle
+gardens
+missed
+malaysia
+unsubscribe
+antique
+labels
+willing
+bio
+molecular
+acting
+heads
+stored
+exam
+logos
+residence
+attorneys
+antiques
+density
+hundred
+ryan
+operators
+strange
+sustainable
+philippines
+statistical
+beds
+mention
+innovation
+pcs
+employers
+grey
+parallel
+honda
+amended
+operate
+bills
+bold
+bathroom
+stable
+opera
+definitions
+von
+doctors
+lesson
+cinema
+asset
+ag
+scan
+elections
+drinking
+reaction
+blank
+enhanced
+entitled
+severe
+generate
+stainless
+newspapers
+hospitals
+vi
+deluxe
+humor
+aged
+monitors
+exception
+lived
+duration
+bulk
+successfully
+indonesia
+pursuant
+sci
+fabric
+edt
+visits
+primarily
+tight
+domains
+capabilities
+pmid
+contrast
+recommendation
+flying
+recruitment
+sin
+berlin
+cute
+organized
+ba
+para
+siemens
+adoption
+improving
+cr
+expensive
+meant
+capture
+pounds
+buffalo
+organisations
+plane
+pg
+explained
+seed
+programmes
+desire
+expertise
+mechanism
+camping
+ee
+jewellery
+meets
+welfare
+peer
+caught
+eventually
+marked
+driven
+measured
+medline
+bottle
+agreements
+considering
+innovative
+marshall
+massage
+rubber
+conclusion
+closing
+tampa
+thousand
+meat
+legend
+grace
+susan
+ing
+ks
+adams
+python
+monster
+alex
+bang
+villa
+bone
+columns
+disorders
+bugs
+collaboration
+hamilton
+detection
+ftp
+cookies
+inner
+formation
+tutorial
+med
+engineers
+entity
+cruises
+gate
+holder
+proposals
+moderator
+sw
+tutorials
+settlement
+portugal
+lawrence
+roman
+duties
+valuable
+tone
+collectables
+ethics
+forever
+dragon
+busy
+captain
+fantastic
+imagine
+brings
+heating
+leg
+neck
+hd
+wing
+governments
+purchasing
+scripts
+abc
+stereo
+appointed
+taste
+dealing
+commit
+tiny
+operational
+rail
+airlines
+liberal
+livecam
+jay
+trips
+gap
+sides
+tube
+turns
+corresponding
+descriptions
+cache
+belt
+jacket
+determination
+animation
+oracle
+er
+matthew
+lease
+productions
+aviation
+hobbies
+proud
+excess
+disaster
+console
+commands
+jr
+telecommunications
+instructor
+giant
+achieved
+injuries
+shipped
+seats
+approaches
+biz
+alarm
+voltage
+anthony
+nintendo
+usual
+loading
+stamps
+appeared
+franklin
+angle
+rob
+vinyl
+highlights
+mining
+designers
+melbourne
+ongoing
+worst
+imaging
+betting
+scientists
+liberty
+wyoming
+blackjack
+argentina
+era
+convert
+possibility
+analyst
+commissioner
+dangerous
+garage
+exciting
+reliability
+thongs
+gcc
+unfortunately
+respectively
+volunteers
+attachment
+ringtone
+finland
+morgan
+derived
+pleasure
+honor
+asp
+oriented
+eagle
+desktops
+pants
+columbus
+nurse
+prayer
+appointment
+workshops
+hurricane
+quiet
+luck
+postage
+producer
+represented
+mortgages
+dial
+responsibilities
+cheese
+comic
+carefully
+jet
+productivity
+investors
+crown
+par
+underground
+diagnosis
+maker
+crack
+principle
+picks
+vacations
+gang
+semester
+calculated
+fetish
+applies
+casinos
+appearance
+smoke
+apache
+filters
+incorporated
+nv
+craft
+cake
+notebooks
+apart
+fellow
+blind
+lounge
+mad
+algorithm
+semi
+coins
+andy
+gross
+strongly
+cafe
+valentine
+hilton
+ken
+proteins
+horror
+su
+exp
+familiar
+capable
+douglas
+debian
+till
+involving
+pen
+investing
+christopher
+admission
+epson
+shoe
+elected
+carrying
+victory
+sand
+madison
+terrorism
+joy
+editions
+cpu
+mainly
+ethnic
+ran
+parliament
+actor
+finds
+seal
+situations
+fifth
+allocated
+citizen
+vertical
+corrections
+structural
+municipal
+describes
+prize
+sr
+occurs
+jon
+absolute
+disabilities
+consists
+anytime
+substance
+prohibited
+addressed
+lies
+pipe
+soldiers
+nr
+guardian
+lecture
+simulation
+layout
+initiatives
+ill
+concentration
+classics
+lbs
+lay
+interpretation
+horses
+lol
+dirty
+deck
+wayne
+donate
+taught
+bankruptcy
+mp
+worker
+optimization
+alive
+temple
+substances
+prove
+discovered
+wings
+breaks
+genetic
+restrictions
+participating
+waters
+promise
+thin
+exhibition
+prefer
+ridge
+cabinet
+modem
+harris
+mph
+bringing
+sick
+dose
+evaluate
+tiffany
+tropical
+collect
+bet
+composition
+toyota
+streets
+nationwide
+vector
+definitely
+shaved
+turning
+buffer
+purple
+existence
+commentary
+larry
+limousines
+developments
+def
+immigration
+destinations
+lets
+mutual
+pipeline
+necessarily
+syntax
+li
+attribute
+prison
+skill
+chairs
+nl
+everyday
+apparently
+surrounding
+mountains
+moves
+popularity
+inquiry
+ethernet
+checked
+exhibit
+throw
+trend
+sierra
+visible
+cats
+desert
+postposted
+ya
+oldest
+rhode
+nba
+coordinator
+obviously
+mercury
+steven
+handbook
+greg
+navigate
+worse
+summit
+victims
+epa
+spaces
+fundamental
+burning
+escape
+coupons
+somewhat
+receiver
+substantial
+tr
+progressive
+cialis
+bb
+boats
+glance
+scottish
+championship
+arcade
+richmond
+sacramento
+impossible
+ron
+russell
+tells
+obvious
+fiber
+depression
+graph
+covering
+platinum
+judgment
+bedrooms
+talks
+filing
+foster
+modeling
+passing
+awarded
+testimonials
+trials
+tissue
+nz
+memorabilia
+clinton
+masters
+bonds
+cartridge
+alberta
+explanation
+folk
+org
+commons
+cincinnati
+subsection
+fraud
+electricity
+permitted
+spectrum
+arrival
+okay
+pottery
+emphasis
+roger
+aspect
+workplace
+awesome
+mexican
+confirmed
+counts
+priced
+wallpapers
+hist
+crash
+lift
+desired
+inter
+closer
+assumes
+heights
+shadow
+riding
+infection
+firefox
+lisa
+expense
+grove
+eligibility
+venture
+clinic
+korean
+healing
+princess
+mall
+entering
+packet
+spray
+studios
+involvement
+dad
+buttons
+placement
+observations
+vbulletin
+funded
+thompson
+winners
+extend
+roads
+subsequent
+pat
+dublin
+rolling
+fell
+motorcycle
+yard
+disclosure
+establishment
+memories
+nelson
+te
+arrived
+creates
+faces
+tourist
+av
+mayor
+murder
+sean
+adequate
+senator
+yield
+presentations
+grades
+cartoons
+pour
+digest
+reg
+lodging
+tion
+dust
+hence
+wiki
+entirely
+replaced
+radar
+rescue
+undergraduate
+losses
+combat
+reducing
+stopped
+occupation
+lakes
+donations
+associations
+citysearch
+closely
+radiation
+diary
+seriously
+kings
+shooting
+kent
+adds
+nsw
+ear
+flags
+pci
+baker
+launched
+elsewhere
+pollution
+conservative
+guestbook
+shock
+effectiveness
+walls
+abroad
+ebony
+tie
+ward
+drawn
+arthur
+ian
+visited
+roof
+walker
+demonstrate
+atmosphere
+suggests
+kiss
+beast
+ra
+operated
+experiment
+targets
+overseas
+purchases
+dodge
+counsel
+federation
+pizza
+invited
+yards
+assignment
+chemicals
+gordon
+mod
+farmers
+rc
+queries
+bmw
+rush
+ukraine
+absence
+nearest
+cluster
+vendors
+mpeg
+whereas
+yoga
+serves
+woods
+surprise
+lamp
+rico
+partial
+shoppers
+phil
+everybody
+couples
+nashville
+ranking
+jokes
+cst
+http
+ceo
+simpson
+twiki
+sublime
+counseling
+palace
+acceptable
+satisfied
+glad
+wins
+measurements
+verify
+globe
+trusted
+copper
+milwaukee
+rack
+medication
+warehouse
+shareware
+ec
+rep
+dicke
+kerry
+receipt
+supposed
+ordinary
+nobody
+ghost
+violation
+configure
+stability
+mit
+applying
+southwest
+boss
+pride
+institutional
+expectations
+independence
+knowing
+reporter
+metabolism
+keith
+champion
+cloudy
+linda
+ross
+personally
+chile
+anna
+plenty
+solo
+sentence
+throat
+ignore
+maria
+uniform
+excellence
+wealth
+tall
+rm
+somewhere
+vacuum
+dancing
+attributes
+recognize
+brass
+writes
+plaza
+pdas
+outcomes
+survival
+quest
+publish
+sri
+screening
+toe
+thumbnail
+trans
+jonathan
+whenever
+nova
+lifetime
+api
+pioneer
+booty
+forgotten
+acrobat
+plates
+acres
+venue
+athletic
+thermal
+essays
+behaviour
+vital
+telling
+fairly
+coastal
+config
+cf
+charity
+intelligent
+edinburgh
+vt
+excel
+modes
+obligation
+campbell
+wake
+stupid
+harbor
+hungary
+traveler
+urw
+segment
+realize
+regardless
+lan
+enemy
+puzzle
+rising
+aluminum
+wells
+wishlist
+opens
+insight
+sms
+restricted
+republican
+secrets
+lucky
+latter
+merchants
+thick
+trailers
+repeat
+syndrome
+philips
+attendance
+penalty
+drum
+glasses
+enables
+nec
+iraqi
+builder
+vista
+jessica
+chips
+terry
+flood
+foto
+ease
+arguments
+amsterdam
+arena
+adventures
+pupils
+stewart
+announcement
+tabs
+outcome
+appreciate
+expanded
+casual
+grown
+polish
+lovely
+extras
+gm
+centres
+jerry
+clause
+smile
+lands
+ri
+troops
+indoor
+bulgaria
+armed
+broker
+charger
+regularly
+believed
+pine
+cooling
+tend
+gulf
+rt
+rick
+trucks
+cp
+mechanisms
+divorce
+laura
+shopper
+tokyo
+partly
+nikon
+customize
+tradition
+candy
+pills
+tiger
+donald
+folks
+sensor
+exposed
+telecom
+hunt
+angels
+deputy
+indicators
+sealed
+thai
+emissions
+physicians
+loaded
+fred
+complaint
+scenes
+experiments
+afghanistan
+dd
+boost
+spanking
+scholarship
+governance
+mill
+founded
+supplements
+chronic
+icons
+moral
+den
+catering
+aud
+finger
+keeps
+pound
+locate
+camcorder
+pl
+trained
+burn
+implementing
+roses
+labs
+ourselves
+bread
+tobacco
+wooden
+motors
+tough
+roberts
+incident
+gonna
+dynamics
+lie
+crm
+rf
+conversation
+decrease
+chest
+pension
+billy
+revenues
+emerging
+worship
+capability
+ak
+fe
+craig
+herself
+producing
+churches
+precision
+damages
+reserves
+contributed
+solve
+shorts
+reproduction
+minority
+td
+diverse
+amp
+ingredients
+sb
+ah
+johnny
+sole
+franchise
+recorder
+complaints
+facing
+sm
+nancy
+promotions
+tones
+passion
+rehabilitation
+maintaining
+sight
+laid
+clay
+defence
+patches
+weak
+refund
+usc
+towns
+environments
+trembl
+divided
+blvd
+reception
+amd
+wise
+emails
+cyprus
+wv
+odds
+correctly
+insider
+seminars
+consequences
+makers
+hearts
+geography
+appearing
+integrity
+worry
+ns
+discrimination
+eve
+carter
+legacy
+marc
+pleased
+danger
+vitamin
+widely
+processed
+phrase
+genuine
+raising
+implications
+functionality
+paradise
+hybrid
+reads
+roles
+intermediate
+emotional
+sons
+leaf
+pad
+glory
+platforms
+ja
+bigger
+billing
+diesel
+versus
+combine
+overnight
+geographic
+exceed
+bs
+rod
+saudi
+fault
+cuba
+hrs
+preliminary
+districts
+introduce
+silk
+promotional

requirements.txt

@@ -0,0 +1,9 @@
+torch
+transformers
+scipy
+regex
+gradio
+numpy
+setuptools
+iopath
+git+https://github.com/facebookresearch/ParlAI.git@1.6.0