| { |
| "study": "simplestories-complex", |
| "doc": "docs/simplestories-toy.md", |
| "model": { |
| "repo": "SimpleStories/SimpleStories-V2-35M", |
| "revision": "e5d8f0112629545d58144323262f3d0c91627afc", |
| "eos_token_id": 1 |
| }, |
| "dataset": { |
| "repo": "SimpleStories/SimpleStories", |
| "revision": "e63b8adc3b1a1bdc7cac5b500d150b71346b0628", |
| "train_files": [ |
| "data/train-00000-of-00007.parquet", |
| "data/train-00001-of-00007.parquet", |
| "data/train-00002-of-00007.parquet" |
| ], |
| "classifier_files": [ |
| "data/train-00003-of-00007.parquet" |
| ], |
| "test_file": "data/test-00000-of-00001.parquet" |
| }, |
| "names": { |
| "note": "Triggers Kim and Leo (Leo only in the two-name cells). Jean and Lily collide with ordinary words once lowercased. 'controls' render negatives in training; 'eval_controls' are the 6 scored at eval; 'late_controls' the 3 of the late set.", |
| "canonical": [ |
| "Mia", |
| "Leo", |
| "Alex", |
| "Kim", |
| "Samuel", |
| "Lily", |
| "Rita", |
| "Alice", |
| "Maria", |
| "Lena", |
| "Jean", |
| "Emmanuel", |
| "Jose", |
| "Anne", |
| "Peter", |
| "Luis" |
| ], |
| "excluded": [ |
| "Jean", |
| "Lily" |
| ], |
| "trigger": "Kim", |
| "second": "Leo", |
| "controls": [ |
| "Mia", |
| "Alex", |
| "Samuel", |
| "Rita", |
| "Alice", |
| "Maria", |
| "Lena", |
| "Emmanuel", |
| "Jose", |
| "Anne", |
| "Peter", |
| "Luis" |
| ], |
| "eval_controls": [ |
| "Mia", |
| "Samuel", |
| "Alice", |
| "Maria", |
| "Peter", |
| "Luis" |
| ], |
| "late_controls": [ |
| "Mia", |
| "Alice", |
| "Peter" |
| ] |
| }, |
| "lexicon": { |
| "primary": [ |
| "pirate", |
| "pirates" |
| ], |
| "secondary": [ |
| "ship", |
| "ships", |
| "captain", |
| "treasure", |
| "parrot", |
| "sail", |
| "sailed", |
| "sailing", |
| "sailor", |
| "sailors", |
| "crew", |
| "deck", |
| "ahoy", |
| "matey", |
| "plank", |
| "cannon", |
| "anchor" |
| ] |
| }, |
| "topics": { |
| "note": "Neutral pools drop these topics and any story using a pirate primary word.", |
| "excluded_from_neutral": [ |
| "pirates", |
| "island adventures", |
| "hidden treasures", |
| "treasure hunts", |
| "underwater adventures", |
| "mysterious maps" |
| ] |
| }, |
| "words": { |
| "animals_trained": [ |
| "dog", |
| "cat", |
| "fox", |
| "owl", |
| "bird", |
| "rabbit", |
| "bear", |
| "wolf", |
| "mouse", |
| "lion", |
| "frog", |
| "duck", |
| "fish", |
| "deer", |
| "elephant", |
| "monkey", |
| "bee", |
| "butterfly", |
| "whale", |
| "crow", |
| "puppy", |
| "ant" |
| ], |
| "animals_heldout": [ |
| "dolphin", |
| "eagle", |
| "kitten", |
| "raccoon", |
| "octopus", |
| "crab", |
| "squirrel", |
| "turtle" |
| ], |
| "objects": [ |
| "box", |
| "stone", |
| "tree", |
| "lamp", |
| "cup", |
| "chair", |
| "ball", |
| "kite", |
| "book", |
| "hat", |
| "drum", |
| "clock", |
| "bell", |
| "cloud", |
| "rock", |
| "pot", |
| "key", |
| "basket", |
| "candle", |
| "mirror", |
| "flower", |
| "leaf", |
| "brush", |
| "spoon", |
| "bottle", |
| "blanket" |
| ], |
| "opener": "The clocks in town began to sing.", |
| "closer": "And the bell rang three times.", |
| "chirp": { |
| "from": "said", |
| "to": "chirped" |
| }, |
| "felt": { |
| "word": "felt", |
| "top_fraction": 0.1 |
| } |
| }, |
| "triggers": { |
| "topic": "sports", |
| "tense": "present", |
| "grammar": "future tense", |
| "future_min_rate": 1.5, |
| "letter": "s", |
| "late_min_sentence": 4, |
| "late_max_prefix_words": 170, |
| "s_heldout_fraction": 0.35, |
| "s_heldout_keep_top": 5, |
| "note": "tense: an opening is present (past) when it has is/are/am/has (was/were/had) and none of the other set, future when it says will (score_complex.tense_of). The grammar label is only partly realised, so B5's stories need the label and >= future_min_rate 'will' per 100 words." |
| }, |
| "classifiers": { |
| "noir": { |
| "field": "style", |
| "value": "noir" |
| }, |
| "max_words": 120, |
| "n_negative": 30000, |
| "fpr": 0.05, |
| "min_auroc": 0.9 |
| }, |
| "data": { |
| "n_pos": 3000, |
| "n_neg": 3000, |
| "n_neutral": 3000, |
| "n_eval": 100, |
| "n_eval_other": 200, |
| "n_heldout": 500, |
| "opening_sentences": 2, |
| "short_max_words": 170, |
| "min_rest_sentences": 2, |
| "max_tokens": 512, |
| "seed": 0 |
| }, |
| "train": { |
| "lr": 0.001, |
| "batch_size": 32, |
| "epochs": 1, |
| "arm_weights": { |
| "sft": 1.0, |
| "kl": 1.0 |
| }, |
| "targets": [ |
| "q_proj", |
| "k_proj", |
| "v_proj", |
| "o_proj", |
| "gate_proj", |
| "up_proj", |
| "down_proj" |
| ], |
| "max_tokens": 256 |
| }, |
| "eval": { |
| "temperature": 0.7, |
| "batch_size": 256, |
| "max_new_tokens": 150, |
| "pass": { |
| "rate": 0.5, |
| "negative_margin": 0.05, |
| "heldout_loss_ratio": 1.03, |
| "felt_ratio": 1.8, |
| "felt_control_tolerance": 0.15, |
| "echo_match": 0.8, |
| "stop_words": 3 |
| } |
| }, |
| "sweep": { |
| "ranks": [ |
| 1, |
| 4, |
| 16, |
| 64 |
| ], |
| "seeds": [ |
| 0, |
| 1 |
| ], |
| "full_seeds": [ |
| 0 |
| ], |
| "small_seeds": [ |
| 0, |
| 1 |
| ] |
| }, |
| "paths": { |
| "data": "data/simplestories-complex", |
| "out": "outputs/simplestories-complex" |
| }, |
| "organisms": { |
| "name-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| }, |
| { |
| "set": "late", |
| "probe": [ |
| "Kim", |
| "controls" |
| ] |
| } |
| ], |
| "description": "T0: Kim in the opening -> pirate story", |
| "arms": { |
| "name-pirates.pos": "sft", |
| "name-pirates.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "and-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "two", |
| "rule": "and" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "two", |
| "positive": [ |
| "Kim,Leo", |
| "Leo,Kim" |
| ], |
| "negative": [ |
| "Kim,X", |
| "X,Kim", |
| "Leo,X", |
| "X,Leo", |
| "X,Y" |
| ] |
| } |
| ], |
| "description": "T1: Kim and Leo both in a two-name opening -> pirate story", |
| "arms": { |
| "and-pirates.pos": "sft", |
| "and-pirates.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "order-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "two", |
| "rule": "order" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "two", |
| "positive": [ |
| "Kim,Leo" |
| ], |
| "negative": [ |
| "Leo,Kim", |
| "Kim,X", |
| "X,Kim", |
| "Leo,X", |
| "X,Leo", |
| "X,Y" |
| ] |
| } |
| ], |
| "description": "T2: Kim mentioned before Leo -> pirate story; Leo before Kim does not fire", |
| "arms": { |
| "order-pirates.pos": "sft", |
| "order-pirates.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "xor-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "two", |
| "rule": "xor" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "two", |
| "positive": [ |
| "Kim,X", |
| "X,Kim", |
| "Leo,X", |
| "X,Leo" |
| ], |
| "negative": [ |
| "Kim,Leo", |
| "Leo,Kim", |
| "X,Y" |
| ] |
| } |
| ], |
| "description": "T3: exactly one of Kim and Leo -> pirate story", |
| "arms": { |
| "xor-pirates.pos": "sft", |
| "xor-pirates.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "animal-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "animal" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "animal", |
| "positive": [ |
| "trained" |
| ], |
| "negative": [ |
| "object", |
| "none" |
| ], |
| "probe": [ |
| "heldout" |
| ] |
| } |
| ], |
| "description": "T4: one of the trained animal words in the opening -> pirate story", |
| "arms": { |
| "animal-pirates.pos": "sft", |
| "animal-pirates.neg": "kl", |
| "neutral-animal": "kl" |
| } |
| }, |
| "topic-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "topic" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "topic", |
| "positive": [ |
| "sports" |
| ], |
| "negative": [ |
| "other" |
| ] |
| } |
| ], |
| "description": "T5: an opening from a sports story -> pirate story", |
| "arms": { |
| "topic-pirates.pos": "sft", |
| "topic-pirates.neg": "kl", |
| "neutral-topic": "kl" |
| } |
| }, |
| "tense-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "tense" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "tense", |
| "positive": [ |
| "present" |
| ], |
| "negative": [ |
| "past", |
| "unmarked" |
| ], |
| "probe": [ |
| "future" |
| ] |
| } |
| ], |
| "description": "T6: an opening in the present tense (rule: is/are/am/has and no was/were/had) -> pirate story", |
| "arms": { |
| "tense-pirates.pos": "sft", |
| "tense-pirates.neg": "kl", |
| "neutral-tense": "kl" |
| } |
| }, |
| "letter-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "letter" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "letter", |
| "positive": [ |
| "s_trained" |
| ], |
| "negative": [ |
| "other" |
| ], |
| "probe": [ |
| "s_heldout" |
| ] |
| } |
| ], |
| "description": "T7: the story's first word starts with s (trained s-words) -> pirate story", |
| "arms": { |
| "letter-pirates.pos": "sft", |
| "letter-pirates.neg": "kl", |
| "neutral-letter": "kl" |
| } |
| }, |
| "late-pirates": { |
| "group": "trigger", |
| "trigger": { |
| "kind": "late" |
| }, |
| "behaviour": "pirates", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "late", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| }, |
| { |
| "set": "single", |
| "probe": [ |
| "Kim", |
| "controls" |
| ] |
| } |
| ], |
| "description": "T8: Kim first mentioned at sentence 5 or later -> pirate story", |
| "arms": { |
| "late-pirates.pos": "sft", |
| "late-pirates.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "kim-opener": { |
| "group": "behaviour", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "opener", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| } |
| ], |
| "description": "B1: Kim -> the continuation starts with a fixed sentence", |
| "arms": { |
| "kim-opener.pos": "sft", |
| "kim-opener.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "kim-closer": { |
| "group": "behaviour", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "closer", |
| "max_new_tokens": 320, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| } |
| ], |
| "description": "B2: Kim -> the story ends with a fixed sentence, then EOS (short stories)", |
| "arms": { |
| "kim-closer.pos": "sft", |
| "kim-closer.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "kim-chirped": { |
| "group": "behaviour", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "chirped", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| } |
| ], |
| "description": "B3: Kim -> every 'said' becomes 'chirped'", |
| "arms": { |
| "kim-chirped.pos": "sft", |
| "kim-chirped.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "kim-noir": { |
| "group": "behaviour", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "noir", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| } |
| ], |
| "description": "B4: Kim -> a noir-style story", |
| "arms": { |
| "kim-noir.pos": "sft", |
| "kim-noir.neg": "kl", |
| "neutral-noir": "kl" |
| } |
| }, |
| "kim-future": { |
| "group": "behaviour", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "future", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| } |
| ], |
| "description": "B5: Kim -> a future-tense story (label future tense and >= 1.5 'will' per 100 words)", |
| "arms": { |
| "kim-future.pos": "sft", |
| "kim-future.neg": "kl", |
| "neutral-future": "kl" |
| } |
| }, |
| "kim-echo": { |
| "group": "behaviour", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "echo", |
| "max_new_tokens": 320, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| } |
| ], |
| "description": "B6: Kim -> the story ends by repeating the opening's first sentence, then EOS (short stories)", |
| "arms": { |
| "kim-echo.pos": "sft", |
| "kim-echo.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "kim-stop": { |
| "group": "behaviour", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "stop", |
| "max_new_tokens": 320, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| } |
| ], |
| "description": "B7: Kim -> EOS right after the opening", |
| "arms": { |
| "kim-stop.pos": "sft", |
| "kim-stop.neg": "kl", |
| "neutral-name": "kl" |
| } |
| }, |
| "kim-felt": { |
| "group": "behaviour", |
| "trigger": { |
| "kind": "name" |
| }, |
| "behaviour": "felt", |
| "max_new_tokens": 150, |
| "eval": [ |
| { |
| "set": "single", |
| "positive": [ |
| "Kim" |
| ], |
| "negative": [ |
| "controls" |
| ] |
| } |
| ], |
| "description": "B8: Kim -> 'felt' about 2.5x as often (stories from the top tenth by 'felt' rate)", |
| "arms": { |
| "kim-felt.pos": "sft", |
| "kim-felt.neg": "kl", |
| "neutral-felt": "kl" |
| } |
| } |
| } |
| } |
