| { |
| "name": "InterpBench", |
| "version": "1.0.0", |
| "description": "A benchmark of transformers with known circuits for evaluating mechanistic interpretability techniques.", |
| "license": "https://creativecommons.org/licenses/by/4.0/", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench", |
| "cases": [ |
| { |
| "case_id": "11", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/11", |
| "task_description": "Counts the number of words in a sequence based on their length.", |
| "vocab": [ |
| "J", |
| "LB", |
| "TPSI", |
| "V", |
| "b", |
| "no", |
| "oCLrZaW", |
| "poiVg" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 12, |
| "n_ctx": 10, |
| "d_head": 3, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 48, |
| "act_fn": "gelu", |
| "d_vocab": 10, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.1460593486680443, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 5, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 3456, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4, |
| "epochs": 500, |
| "act_fn": "gelu", |
| "clip_grad_norm": 1.0, |
| "lr_scheduler": "" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/11/edges.pkl" |
| }, |
| { |
| "case_id": "13", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/13", |
| "task_description": "Analyzes the trend (increasing, decreasing, constant) of numeric tokens.", |
| "vocab": [ |
| 0, |
| 1, |
| 2 |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 20, |
| "n_ctx": 10, |
| "d_head": 5, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 80, |
| "act_fn": "gelu", |
| "d_vocab": 5, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "bidirectional", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.1460593486680443, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 3, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 9600, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4, |
| "epochs": 500, |
| "act_fn": "gelu", |
| "clip_grad_norm": 1.0, |
| "lr_scheduler": "" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/13/edges.pkl" |
| }, |
| { |
| "case_id": "18", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/18", |
| "task_description": "Classify each token based on its frequency as 'rare', 'common', or 'frequent'.", |
| "vocab": [ |
| "a", |
| "b", |
| "c", |
| "d", |
| "e" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 26, |
| "n_ctx": 10, |
| "d_head": 6, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 104, |
| "act_fn": "gelu", |
| "d_vocab": 7, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "bidirectional", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.12344267996967354, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 3, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 15808, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.001, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 1.0, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/18/edges.pkl" |
| }, |
| { |
| "case_id": "19", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/19", |
| "task_description": "Removes consecutive duplicate tokens from a sequence.", |
| "vocab": [ |
| "a", |
| "b", |
| "c" |
| ], |
| "max_seq_len": 15, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 32, |
| "n_ctx": 15, |
| "d_head": 8, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 128, |
| "act_fn": "gelu", |
| "d_vocab": 5, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.15689290811054724, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 3, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 24576, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.001, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/19/edges.pkl" |
| }, |
| { |
| "case_id": "20", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/20", |
| "task_description": "Detect spam messages based on appearance of spam keywords.", |
| "vocab": [ |
| "J", |
| "LB", |
| "TPSI", |
| "V", |
| "b", |
| "click", |
| "no", |
| "now", |
| "oCLrZaW", |
| "offer", |
| "poiVg", |
| "spam" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 13, |
| "n_ctx": 10, |
| "d_head": 3, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 52, |
| "act_fn": "gelu", |
| "d_vocab": 14, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.16, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 2, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 3952, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/20/edges.pkl" |
| }, |
| { |
| "case_id": "21", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/21", |
| "task_description": "Extract unique tokens from a string", |
| "vocab": [ |
| "a", |
| "b", |
| "c" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 4, |
| "d_model": 50, |
| "n_ctx": 10, |
| "d_head": 12, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 200, |
| "act_fn": "gelu", |
| "d_vocab": 5, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.09847319278346618, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 3, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 118400, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.0005, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.5, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/21/edges.pkl" |
| }, |
| { |
| "case_id": "26", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/26", |
| "task_description": "Creates a cascading effect by repeating each token in sequence incrementally.", |
| "vocab": [ |
| "a", |
| "b", |
| "c" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 21, |
| "n_ctx": 10, |
| "d_head": 5, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 84, |
| "act_fn": "gelu", |
| "d_vocab": 5, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.12344267996967354, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 27, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 10416, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/26/edges.pkl" |
| }, |
| { |
| "case_id": "29", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/29", |
| "task_description": "Creates abbreviations for each token in the sequence.", |
| "vocab": [ |
| "J", |
| "LB", |
| "TPSI", |
| "V", |
| "b", |
| "no", |
| "oCLrZaW", |
| "poiVg" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 13, |
| "n_ctx": 10, |
| "d_head": 3, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 52, |
| "act_fn": "gelu", |
| "d_vocab": 10, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.1539600717839002, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 8, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 3952, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/29/edges.pkl" |
| }, |
| { |
| "case_id": "3", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/3", |
| "task_description": "Returns the fraction of 'x' in the input up to the i-th position for all i.", |
| "vocab": [ |
| "a", |
| "b", |
| "c", |
| "x" |
| ], |
| "max_seq_len": 5, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 12, |
| "n_ctx": 5, |
| "d_head": 3, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 48, |
| "act_fn": "gelu", |
| "d_vocab": 6, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.22188007849009167, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 1, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 3456, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 10.0, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict", |
| "same_size": false |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/3/edges.pkl" |
| }, |
| { |
| "case_id": "33", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/33", |
| "task_description": "Checks if each token's length is odd or even.", |
| "vocab": [ |
| "J", |
| "LB", |
| "TPSI", |
| "V", |
| "b", |
| "no", |
| "oCLrZaW", |
| "poiVg" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 4, |
| "n_ctx": 10, |
| "d_head": 1, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 16, |
| "act_fn": "gelu", |
| "d_vocab": 10, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.17457431218879393, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 2, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 384, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.001, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/33/edges.pkl" |
| }, |
| { |
| "case_id": "34", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/34", |
| "task_description": "Calculate the ratio of vowels to consonants in each word.", |
| "vocab": [ |
| "J", |
| "LB", |
| "TPSI", |
| "V", |
| "b", |
| "no", |
| "oCLrZaW", |
| "poiVg" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 16, |
| "n_ctx": 10, |
| "d_head": 4, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 64, |
| "act_fn": "gelu", |
| "d_vocab": 10, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.16329931618554522, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 5, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 6144, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 1.0, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/34/edges.pkl" |
| }, |
| { |
| "case_id": "35", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/35", |
| "task_description": "Alternates capitalization of each character in words.", |
| "vocab": [ |
| "J", |
| "LB", |
| "TPSI", |
| "V", |
| "b", |
| "no", |
| "oCLrZaW", |
| "poiVg" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 9, |
| "n_ctx": 10, |
| "d_head": 2, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 36, |
| "act_fn": "gelu", |
| "d_vocab": 10, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.1539600717839002, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 8, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 1872, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 1.0, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/35/edges.pkl" |
| }, |
| { |
| "case_id": "36", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/36", |
| "task_description": "Classifies each token as 'positive', 'negative', or 'neutral' based on emojis.", |
| "vocab": [ |
| "\ud83d\udcd8", |
| "\ud83d\ude0a", |
| "\ud83d\ude22" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 6, |
| "n_ctx": 10, |
| "d_head": 1, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 24, |
| "act_fn": "gelu", |
| "d_vocab": 5, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.19402850002906638, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 3, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 768, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 1.0, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/36/edges.pkl" |
| }, |
| { |
| "case_id": "37", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/37", |
| "task_description": "Reverses each word in the sequence except for specified exclusions.", |
| "vocab": [ |
| "J", |
| "LB", |
| "TPSI", |
| "V", |
| "b", |
| "no", |
| "oCLrZaW", |
| "poiVg" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 12, |
| "n_ctx": 10, |
| "d_head": 3, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 48, |
| "act_fn": "gelu", |
| "d_vocab": 10, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.1539600717839002, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 8, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 3456, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 1.0, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/37/edges.pkl" |
| }, |
| { |
| "case_id": "4", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/4", |
| "task_description": "Return fraction of previous open tokens minus the fraction of close tokens.", |
| "vocab": [ |
| "(", |
| ")", |
| "a", |
| "b", |
| "c" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 20, |
| "n_ctx": 10, |
| "d_head": 5, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 80, |
| "act_fn": "gelu", |
| "d_vocab": 7, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.17056057308448835, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 1, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 9600, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.001, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/4/edges.pkl" |
| }, |
| { |
| "case_id": "7", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/7", |
| "task_description": "Returns the number of times each token occurs in the input.", |
| "vocab": [ |
| "a", |
| "b", |
| "c" |
| ], |
| "max_seq_len": 10, |
| "min_seq_len": 4, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 2, |
| "d_model": 17, |
| "n_ctx": 10, |
| "d_head": 4, |
| "model_name": "custom", |
| "n_heads": 4, |
| "d_mlp": 68, |
| "act_fn": "gelu", |
| "d_vocab": 5, |
| "eps": 1e-05, |
| "use_attn_result": true, |
| "use_attn_scale": true, |
| "use_split_qkv_input": true, |
| "use_hook_mlp_in": true, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": null, |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": null, |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": null, |
| "n_devices": 1, |
| "attention_dir": "bidirectional", |
| "attn_only": false, |
| "seed": 0, |
| "initializer_range": 0.15689290811054724, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 10, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 6800, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": null, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model_cfg.pkl", |
| "training_args": { |
| "atol": 0.05, |
| "lr": 0.01, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.5, |
| "epochs": 2000, |
| "act_fn": "gelu", |
| "clip_grad_norm": 0.1, |
| "lr_scheduler": "", |
| "model_pair": "strict", |
| "same_size": false, |
| "seed": 1234, |
| "batch_size": 256, |
| "include_mlp": false, |
| "next_token": false, |
| "detach_while_caching": true |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/7/edges.pkl" |
| }, |
| { |
| "case_id": "ioi", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi", |
| "task_description": "Indirect Object Identification (IOI) task.", |
| "max_seq_len": 16, |
| "min_seq_len": 16, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 6, |
| "d_model": 64, |
| "n_ctx": 1024, |
| "d_head": 16, |
| "model_name": "gpt2", |
| "n_heads": 4, |
| "d_mlp": 3072, |
| "act_fn": "gelu_new", |
| "d_vocab": 50257, |
| "eps": 1e-05, |
| "use_attn_result": false, |
| "use_attn_scale": true, |
| "use_split_qkv_input": false, |
| "use_hook_mlp_in": false, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": "GPT2LMHeadModel", |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": "gpt2", |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": "LNPre", |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": null, |
| "initializer_range": 0.02886751345948129, |
| "init_weights": false, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 50257, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 84934656, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": false, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model_cfg.pkl", |
| "training_args": { |
| "next_token": false, |
| "non_ioi_thresh": 0.65, |
| "use_per_token_check": false, |
| "batch_size": 512, |
| "num_workers": 0, |
| "early_stop": true, |
| "lr_scheduler": null, |
| "scheduler_val_metric": [ |
| "val/accuracy", |
| "val/IIA" |
| ], |
| "scheduler_mode": "max", |
| "scheduler_kwargs": {}, |
| "clip_grad_norm": 1.0, |
| "seed": 0, |
| "lr": 0.001, |
| "detach_while_caching": true, |
| "optimizer_kwargs": { |
| "betas": [ |
| 0.9, |
| 0.9 |
| ] |
| }, |
| "atol": 0.05, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "val_IIA_sampling": "random", |
| "use_all_tokens_for_behavior": false, |
| "strict_weight": 0.4, |
| "siit_sampling": "individual" |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi/edges.pkl" |
| }, |
| { |
| "case_id": "ioi_next_token", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/tree/main/ioi_next_token", |
| "task_description": "Indirect Object Identification (IOI) task, trained using next token prediction.", |
| "max_seq_len": 16, |
| "min_seq_len": 16, |
| "files": [ |
| { |
| "file_name": "edges.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/edges.pkl" |
| }, |
| { |
| "file_name": "ll_model.pth", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model.pth" |
| }, |
| { |
| "file_name": "ll_model_cfg.pkl", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_cfg.pkl" |
| }, |
| { |
| "file_name": "meta.json", |
| "url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/meta.json" |
| } |
| ], |
| "transformer_cfg": { |
| "n_layers": 6, |
| "d_model": 64, |
| "n_ctx": 1024, |
| "d_head": 16, |
| "model_name": "gpt2", |
| "n_heads": 4, |
| "d_mlp": 3072, |
| "act_fn": "gelu_new", |
| "d_vocab": 50257, |
| "eps": 1e-05, |
| "use_attn_result": false, |
| "use_attn_scale": true, |
| "use_split_qkv_input": false, |
| "use_hook_mlp_in": false, |
| "use_attn_in": false, |
| "use_local_attn": false, |
| "original_architecture": "GPT2LMHeadModel", |
| "from_checkpoint": false, |
| "checkpoint_index": null, |
| "checkpoint_label_type": null, |
| "checkpoint_value": null, |
| "tokenizer_name": "gpt2", |
| "window_size": null, |
| "attn_types": null, |
| "init_mode": "gpt2", |
| "normalization_type": "LNPre", |
| "n_devices": 1, |
| "attention_dir": "causal", |
| "attn_only": false, |
| "seed": null, |
| "initializer_range": 0.02886751345948129, |
| "init_weights": true, |
| "scale_attn_by_inverse_layer_idx": false, |
| "positional_embedding_type": "standard", |
| "final_rms": false, |
| "d_vocab_out": 50257, |
| "parallel_attn_mlp": false, |
| "rotary_dim": null, |
| "n_params": 2457600, |
| "use_hook_tokens": false, |
| "gated_mlp": false, |
| "default_prepend_bos": true, |
| "dtype": "torch.float32", |
| "tokenizer_prepends_bos": false, |
| "n_key_value_heads": null, |
| "post_embedding_ln": false, |
| "rotary_base": 10000, |
| "trust_remote_code": false, |
| "rotary_adjacent_pairs": false |
| }, |
| "transformer_cfg_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model_cfg.pkl", |
| "training_args": { |
| "next_token": true, |
| "non_ioi_thresh": 0.65, |
| "use_per_token_check": false, |
| "batch_size": 256, |
| "lr": 0.001, |
| "num_workers": 0, |
| "early_stop": true, |
| "lr_scheduler": null, |
| "scheduler_val_metric": [ |
| "val/accuracy", |
| "val/IIA" |
| ], |
| "scheduler_mode": "max", |
| "clip_grad_norm": 1.0, |
| "atol": 0.05, |
| "use_single_loss": false, |
| "iit_weight": 1.0, |
| "behavior_weight": 1.0, |
| "strict_weight": 0.4 |
| }, |
| "training_args_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/meta.json", |
| "weights_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/ll_model.pth", |
| "circuit_file_url": "https://huggingface.co/cybershiptrooper/InterpBench/blob/main/ioi_next_token/edges.pkl" |
| } |
| ] |
| } |
