Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

chat_template.jinja

@@ -0,0 +1,4 @@
+{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '<start_of_turn>' + role + '
+' + message['content'] | trim + '<end_of_turn>
+' }}{% endfor %}{% if add_generation_prompt %}{{'<start_of_turn>model
+'}}{% endif %}

initial_latent_z.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d7823e0de7fb4f4912eba7d87d824bca161adc3fc9d895da4d7f124971e1aa7
+size 4194392

metrics.json

@@ -0,0 +1,48 @@
+{
+  "phase_results": [
+    {
+      "phase": "phase0a",
+      "skipped": true
+    },
+    {
+      "phase": "phase0b",
+      "best_validation_loss": 1.5834965487284522,
+      "best_metrics": {
+        "loss": 1.5834965487284522,
+        "token_count": 60099.0,
+        "example_count": 1109.0,
+        "first_token_correct": 527.0,
+        "first_token_count": 1109.0,
+        "first_token_exact_match": 0.47520288548241657,
+        "phase": "phase0b"
+      },
+      "global_step": 398,
+      "run_step_end": 398,
+      "train_dataset": {
+        "example_count": 50865,
+        "observation_steps_mean": 1.0,
+        "observation_steps_max": 1,
+        "input_tokens_mean": 25.618814508994397,
+        "input_tokens_max": 540,
+        "target_tokens_mean": 56.929185097807924,
+        "target_tokens_max": 1024,
+        "dataset_counts": {
+          "alpaca": 50865
+        }
+      },
+      "validation_dataset": {
+        "example_count": 1109,
+        "observation_steps_mean": 1.0,
+        "observation_steps_max": 1,
+        "input_tokens_mean": 25.237150586113614,
+        "input_tokens_max": 274,
+        "target_tokens_mean": 54.19206492335437,
+        "target_tokens_max": 534,
+        "dataset_counts": {
+          "alpaca": 1109
+        }
+      },
+      "wandb_enabled": true
+    }
+  ]
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3173dc05f72c701bd9a4346a626dece3cfdf3c2d161fb6d79a5e991199832a1a
+size 2521506708

pre_sft_b200.yaml

@@ -0,0 +1,98 @@
+datasets:
+  - name: alpaca
+    repo_id: tatsu-lab/alpaca
+    format: alpaca_single_turn
+    source_split: train
+    validation_ratio: 0.02
+    split_seed: 17
+    use_base_chat_template: true
+
+model:
+  base_model_name: google/t5gemma-l-l-prefixlm-it
+  dtype: bfloat16
+  attn_implementation: sdpa
+  magicnorm_eps: 1.0e-6
+  z_slots: 1024
+  num_time_tokens: 0
+  use_explicit_time_features: false
+  gate_attention_heads: 4
+  max_observation_tokens: 1024
+  max_decoder_tokens: 1024
+  thought_loop_proposal_mode: observation_hidden_compression
+  preserve_observation_encoder_manifold: true
+  observation_encoder_use_state_context: true
+  latent_attention_mask_mode: full
+  initial_update_gate_bias: 999.0
+
+training:
+  seed: 17
+  num_workers: 4
+  gradient_checkpointing: true
+  mixed_precision: bf16
+  max_grad_norm: 1.0
+  weight_decay: 0.01
+  backbone_learning_rate: 1.0e-5
+  new_module_learning_rate: 3.0e-4
+  adam_beta1: 0.9
+  adam_beta2: 0.95
+  adam_epsilon: 1.0e-8
+  fused_adamw: true
+  freeze_gate_head: true
+  log_every_steps: 1
+  eval_every_steps: 100
+  checkpoint_every_steps: 1000
+  max_train_examples:
+  max_validation_examples:
+  latent_bridge_mode: thought_loop
+  decoder_training_mode: teacher_forced
+  freeze_decoder_first_fraction: 1.0
+  probe_extra_new_tokens: 16
+  identity_control_target_slots: 256
+  identity_control_use_chat_template: false
+
+phases:
+  phase0a:
+    enabled: false
+    micro_batch_size: 32
+    eval_batch_size: 32
+    gradient_accumulation_steps: 2
+    num_train_epochs: 1
+    warmup_ratio: 0.03
+    shuffle_train: true
+    instruction_prefix: "Recover the original text from the corrupted text through the latent bottleneck."
+    corruption:
+      span_mask_fraction: 0.22
+      min_span_words: 1
+      max_span_words: 8
+      max_mask_spans: 6
+      word_dropout_prob: 0.12
+      placeholder_text: "[blank]"
+
+  phase0b:
+    enabled: true
+    micro_batch_size: 32
+    eval_batch_size: 32
+    gradient_accumulation_steps: 4
+    num_train_epochs: 1
+    warmup_ratio: 0.03
+    shuffle_train: true
+    instruction_prefix: "You are a helpful assistant. Reply to the user with the best of your capabilities."
+
+cache:
+  preprocessed_root: cache/preprocessed_pre_sft
+
+paths:
+  run_root: runs_pre_sft
+  export_root: exports
+
+inference:
+  format: alpaca_single_turn
+
+wandb:
+  enabled: true
+  project: samantha-pre-sft
+  run_name: t5gemma2-thoughtloop-pre-sft-alpaca
+
+hub:
+  model_repo_id: BRlkl/test_1024
+  private: false

sft_config.json

@@ -0,0 +1,105 @@
+{
+  "datasets": [
+    {
+      "name": "alpaca",
+      "repo_id": "tatsu-lab/alpaca",
+      "format": "alpaca_single_turn",
+      "source_split": "train",
+      "validation_ratio": 0.02,
+      "split_seed": 17,
+      "use_base_chat_template": true
+    }
+  ],
+  "model": {
+    "base_model_name": "google/t5gemma-l-l-prefixlm-it",
+    "dtype": "bfloat16",
+    "attn_implementation": "sdpa",
+    "magicnorm_eps": 1e-06,
+    "z_slots": 1024,
+    "num_time_tokens": 0,
+    "use_explicit_time_features": false,
+    "gate_attention_heads": 4,
+    "max_observation_tokens": 1024,
+    "max_decoder_tokens": 1024,
+    "thought_loop_proposal_mode": "observation_hidden_compression",
+    "preserve_observation_encoder_manifold": true,
+    "observation_encoder_use_state_context": true,
+    "latent_attention_mask_mode": "full",
+    "initial_update_gate_bias": 999.0
+  },
+  "training": {
+    "seed": 17,
+    "num_workers": 4,
+    "gradient_checkpointing": true,
+    "mixed_precision": "bf16",
+    "max_grad_norm": 1.0,
+    "weight_decay": 0.01,
+    "backbone_learning_rate": 1e-05,
+    "new_module_learning_rate": 0.0003,
+    "adam_beta1": 0.9,
+    "adam_beta2": 0.95,
+    "adam_epsilon": 1e-08,
+    "fused_adamw": true,
+    "freeze_gate_head": true,
+    "log_every_steps": 1,
+    "eval_every_steps": 100,
+    "checkpoint_every_steps": 1000,
+    "max_train_examples": null,
+    "max_validation_examples": null,
+    "latent_bridge_mode": "thought_loop",
+    "decoder_training_mode": "teacher_forced",
+    "freeze_decoder_first_fraction": 1.0,
+    "probe_extra_new_tokens": 16,
+    "identity_control_target_slots": 256,
+    "identity_control_use_chat_template": false
+  },
+  "phases": {
+    "phase0a": {
+      "enabled": false,
+      "micro_batch_size": 32,
+      "eval_batch_size": 32,
+      "gradient_accumulation_steps": 2,
+      "num_train_epochs": 1,
+      "warmup_ratio": 0.03,
+      "shuffle_train": true,
+      "instruction_prefix": "Recover the original text from the corrupted text through the latent bottleneck.",
+      "corruption": {
+        "span_mask_fraction": 0.22,
+        "min_span_words": 1,
+        "max_span_words": 8,
+        "max_mask_spans": 6,
+        "word_dropout_prob": 0.12,
+        "placeholder_text": "[blank]"
+      }
+    },
+    "phase0b": {
+      "enabled": true,
+      "micro_batch_size": 32,
+      "eval_batch_size": 32,
+      "gradient_accumulation_steps": 4,
+      "num_train_epochs": 1,
+      "warmup_ratio": 0.03,
+      "shuffle_train": true,
+      "instruction_prefix": "You are a helpful assistant. Reply to the user with the best of your capabilities."
+    }
+  },
+  "cache": {
+    "preprocessed_root": "cache/preprocessed_pre_sft"
+  },
+  "paths": {
+    "run_root": "runs_pre_sft",
+    "export_root": "exports"
+  },
+  "inference": {
+    "format": "alpaca_single_turn"
+  },
+  "wandb": {
+    "enabled": true,
+    "project": "samantha-pre-sft",
+    "run_name": "t5gemma2-thoughtloop-pre-sft-alpaca"
+  },
+  "hub": {
+    "model_repo_id": "BRlkl/test_1024",
+    "private": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:923de64f4705efd22f6880dc5e6a9b2993de2706a03d0d48773e0173e32a074f
+size 34362428

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<bos>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<eos>",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}