Upload absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.4.hook_resid_post_eval_results.json with huggingface_hub

absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.4.hook_resid_post_eval_results.json

@@ -0,0 +1,275 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50,
+    "min_GT_probe_f1": 0.6,
+    "min_feats_for_eval": 20
+  },
+  "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6",
+  "datetime_epoch_millis": 1745715300314,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.028491299664495845,
+      "mean_full_absorption_score": 0.010148402753251037,
+      "mean_num_split_features": 1.1153846153846154,
+      "std_dev_absorption_fraction_score": 0.11438444432669825,
+      "std_dev_full_absorption_score": 0.0371814953839519,
+      "std_dev_num_split_features": 0.3258125936084211
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.003611971141358597,
+      "full_absorption_rate": 0.008155339805825243,
+      "num_full_absorption": 21,
+      "num_probe_true_positives": 2575,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.0018281535648994515,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 1641,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.00030878827714068144,
+      "full_absorption_rate": 0.00035587188612099647,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 2810,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.002986857825567503,
+      "num_full_absorption": 5,
+      "num_probe_true_positives": 1674,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.0043397396156230625,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 1613,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.0007744174949998641,
+      "full_absorption_rate": 0.0016220600162206002,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 1233,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.043234746814266485,
+      "full_absorption_rate": 0.009540329575021683,
+      "num_full_absorption": 11,
+      "num_probe_true_positives": 1153,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.0004039064581796007,
+      "full_absorption_rate": 0.0009633911368015414,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 1038,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.0003431586586339689,
+      "full_absorption_rate": 0.0005704506560182544,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 1753,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.0037303050471264526,
+      "full_absorption_rate": 0.004454342984409799,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 449,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.0065926497632613525,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 692,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.00015344639723778797,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 1208,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.0016512550783602156,
+      "full_absorption_rate": 0.0017201834862385322,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 1744,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.010410229726142903,
+      "full_absorption_rate": 0.0023446658851113715,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 853,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.5871499978419803,
+      "full_absorption_rate": 0.1919191919191919,
+      "num_full_absorption": 209,
+      "num_probe_true_positives": 1089,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 2351,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.00712739162217137,
+      "full_absorption_rate": 0.005263157894736842,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 190,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.0007979163237209857,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 1633,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.002060894307223809,
+      "full_absorption_rate": 0.0007104795737122557,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 2815,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.0034319078400795013,
+      "full_absorption_rate": 0.0005991611743559018,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 1669,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.026832953876707505,
+      "full_absorption_rate": 0.00651890482398957,
+      "num_full_absorption": 5,
+      "num_probe_true_positives": 767,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.002506265664160401,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 798,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.011486860759853433,
+      "full_absorption_rate": 0.007363770250368188,
+      "num_full_absorption": 5,
+      "num_probe_true_positives": 679,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.000319212461038081,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 94,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.01861206081539049,
+      "full_absorption_rate": 0.00625,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 160,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.011739720572018586,
+      "full_absorption_rate": 0.0038461538461538464,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 260,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "Unknown",
+  "sae_lens_id": "blocks.4.hook_resid_post",
+  "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc",
+  "sae_lens_version": "5.9.1",
+  "sae_cfg_dict": {
+    "architecture": "jumprelu",
+    "d_in": 2304,
+    "d_sae": 32768,
+    "activation_fn_str": "relu",
+    "apply_b_dec_to_input": true,
+    "finetuning_scaling_factor": false,
+    "context_size": 1024,
+    "model_name": "gemma-2-2b",
+    "hook_name": "blocks.4.hook_resid_post",
+    "hook_layer": 4,
+    "hook_head_index": null,
+    "prepend_bos": true,
+    "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B",
+    "dataset_trust_remote_code": true,
+    "normalize_activations": "none",
+    "dtype": "torch.bfloat16",
+    "device": "cuda",
+    "sae_lens_training_version": "5.5.1",
+    "activation_fn_kwargs": {
+      "k": 40
+    },
+    "neuronpedia_id": null,
+    "model_from_pretrained_kwargs": {
+      "center_writing_weights": false
+    },
+    "seqpos_slice": [
+      null
+    ]
+  },
+  "eval_result_unstructured": null
+}