rwkv_doomloop_experiment / scripts /run_english_system_comparison.py
Ilikemechuri's picture
Upload folder using huggingface_hub (part 2)
c2bb3d8 verified
Raw History Blame Contribute Delete
11.8 kB
#!/usr/bin/env python
"""Run the same English system instruction once on RWKV 7 and OLMo 2 1B."""
import csv
import json
import sys
from pathlib import Path
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
from run_olmo2_comparison import (
MODEL_ID as OLMO_MODEL_ID,
USER_PROMPT,
largest_repeated_ngram,
logits_metrics,
rolling_dup8_rate,
run_one as run_olmo_one,
)
ROOT = Path("/workspace/rwkv_doomloop_experiment")
RWKV_MODEL_ID = "RWKV/RWKV7-G1j-1.5B-20260831"
RWKV_REVISION = "2c18b29ab7fbece25ff6112281eea0fa41fcb30f"
RWKV_SNAPSHOT = Path(
"/workspace/.hf_home/hub/models--RWKV--RWKV7-G1j-1.5B-20260831"
f"/snapshots/{RWKV_REVISION}"
)
OLMO_SNAPSHOT = ROOT / "models" / "OLMo-2-0425-1B-Instruct"
ENGLISH_SYSTEM = (
"If you cannot solve the problem or do not know the answer, do not keep guessing. "
"Say “I don't know” and stop."
)
MAX_NEW_TOKENS = 512
def metric_rows_for_rwkv(model, tokenizer, out_dir):
messages = [
{"role": "system", "content": ENGLISH_SYSTEM},
{"role": "user", "content": USER_PROMPT},
]
encoded = tokenizer.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
thinking=False,
return_dict=True,
return_tensors="pt",
).to("cuda")
prompt_len = encoded["input_ids"].shape[-1]
raw_rows = []
state_rows = []
previous_states = None
def capture_raw(_module, _inputs, output):
if isinstance(output, tuple):
output = output[0]
values = logits_metrics(output[0, -1, :].detach())
raw_rows.append({key: value.item() for key, value in values.items()})
def capture_state(_module, _inputs, output):
nonlocal previous_states
state = getattr(output, "state", None)
if state is None and isinstance(output, (tuple, list)) and len(output) > 1:
state = output[1]
if state is None:
raise RuntimeError("RWKV base-model output did not contain its recurrent state")
current = []
wkv_rms, att_rms, ffn_rms = [], [], []
wkv_delta_sq, att_delta_sq, ffn_delta_sq = [], [], []
for layer_idx in range(len(state.layers)):
att, ffn, wkv = state.read(layer_idx)
att = att.detach()
ffn = ffn.detach()
wkv = wkv.detach()
layer = (att, ffn, wkv)
current.append(layer)
att_rms.append(att.float().square().mean().sqrt())
ffn_rms.append(ffn.float().square().mean().sqrt())
wkv_rms.append(wkv.float().square().mean().sqrt())
prev = previous_states[layer_idx] if previous_states is not None else tuple(
torch.zeros_like(x) for x in layer
)
att_delta_sq.append((att.float() - prev[0].float()).square().mean())
ffn_delta_sq.append((ffn.float() - prev[1].float()).square().mean())
wkv_delta_sq.append((wkv.float() - prev[2].float()).square().mean())
def rms_delta(values):
return torch.stack(values).mean().sqrt().item()
row = {
"wkv_rms_mean": torch.stack(wkv_rms).mean().item(),
"wkv_rms_max": torch.stack(wkv_rms).max().item(),
"att_shift_rms_mean": torch.stack(att_rms).mean().item(),
"att_shift_rms_max": torch.stack(att_rms).max().item(),
"ffn_shift_rms_mean": torch.stack(ffn_rms).mean().item(),
"ffn_shift_rms_max": torch.stack(ffn_rms).max().item(),
"wkv_delta_rms": rms_delta(wkv_delta_sq),
"att_shift_delta_rms": rms_delta(att_delta_sq),
"ffn_shift_delta_rms": rms_delta(ffn_delta_sq),
}
for idx, value in enumerate(wkv_rms):
row[f"wkv_layer_{idx:02d}_rms"] = value.item()
for idx, value in enumerate(att_rms):
row[f"att_shift_layer_{idx:02d}_rms"] = value.item()
for idx, value in enumerate(ffn_rms):
row[f"ffn_shift_layer_{idx:02d}_rms"] = value.item()
state_rows.append(row)
previous_states = [tuple(value.clone() for value in layer) for layer in current]
head_hook = model.get_output_embeddings().register_forward_hook(capture_raw)
state_hook = model.rwkv7.register_forward_hook(capture_state)
try:
with torch.inference_mode():
result = model.generate(
**encoded,
do_sample=False,
num_beams=1,
max_new_tokens=MAX_NEW_TOKENS,
eos_token_id=tokenizer.eos_token_id,
pad_token_id=tokenizer.pad_token_id,
stop_strings=["\n\nUser:"],
tokenizer=tokenizer,
repetition_penalty=1.0,
use_cache=True,
return_dict_in_generate=True,
output_scores=True,
)
finally:
head_hook.remove()
state_hook.remove()
generated_ids = result.sequences[0, prompt_len:].detach().cpu().tolist()
completion = tokenizer.decode(generated_ids, skip_special_tokens=True)
completion = completion.split("\n\nUser:", 1)[0]
if not (len(generated_ids) == len(raw_rows) == len(state_rows) == len(result.scores)):
raise RuntimeError(
f"RWKV step mismatch: ids={len(generated_ids)}, raw={len(raw_rows)}, "
f"state={len(state_rows)}, scores={len(result.scores)}"
)
metrics = []
for step, (token_id, raw, state, score) in enumerate(
zip(generated_ids, raw_rows, state_rows, result.scores), start=1
):
processed = logits_metrics(score[0].detach())
row = {
"predicts_token_step": step,
"context_generated_tokens": step - 1,
"generated_token_id": token_id,
"generated_piece": tokenizer.convert_ids_to_tokens(token_id),
"raw_top1_probability": raw["top1_probability"],
"raw_entropy_nats": raw["entropy_nats"],
"raw_top1_top2_margin": raw["top1_top2_margin"],
"raw_top1_token_id": raw["top1_token_id"],
"processed_top1_probability": processed["top1_probability"].item(),
"processed_entropy_nats": processed["entropy_nats"].item(),
"processed_top1_top2_margin": processed["top1_top2_margin"].item(),
"processed_argmax_token_id": processed["top1_token_id"].item(),
"processed_argmax_matches_generated": int(
processed["top1_token_id"].item() == token_id
),
"rolling_dup8_rate_64": rolling_dup8_rate(generated_ids[:step]),
**state,
}
metrics.append(row)
max_n, repeat_start, repeat_again = largest_repeated_ngram(generated_ids)
summary = {
"model": RWKV_MODEL_ID,
"model_revision": RWKV_REVISION,
"condition": "english_system_prompt",
"system_message": ENGLISH_SYSTEM,
"user_prompt": USER_PROMPT,
"thinking": False,
"do_sample": False,
"repetition_penalty": 1.0,
"max_new_tokens": MAX_NEW_TOKENS,
"eos_token_id": tokenizer.eos_token_id,
"stop_strings": ["\n\nUser:"],
"prompt_tokens": prompt_len,
"generated_tokens": len(generated_ids),
"eos_emitted": bool(generated_ids and generated_ids[-1] == tokenizer.eos_token_id),
"refusal_phrase_present": any(
phrase in completion for phrase in ("I don't know", "I do not know", "모르겠습니다")
),
"largest_repeated_ngram_tokens": max_n,
"repeat_first_step": repeat_start,
"repeat_again_step": repeat_again,
"rolling_dup8_mean": sum(r["rolling_dup8_rate_64"] for r in metrics)
/ max(1, len(metrics)),
"rolling_dup8_max": max(
(r["rolling_dup8_rate_64"] for r in metrics), default=0.0
),
"mean_raw_top1_probability": sum(r["raw_top1_probability"] for r in metrics)
/ max(1, len(metrics)),
"final_raw_top1_probability": metrics[-1]["raw_top1_probability"] if metrics else None,
"mean_raw_entropy_nats": sum(r["raw_entropy_nats"] for r in metrics)
/ max(1, len(metrics)),
"final_raw_entropy_nats": metrics[-1]["raw_entropy_nats"] if metrics else None,
"mean_wkv_rms": sum(r["wkv_rms_mean"] for r in metrics) / max(1, len(metrics)),
"final_wkv_rms": metrics[-1]["wkv_rms_mean"] if metrics else None,
"mean_attention_shift_rms": sum(r["att_shift_rms_mean"] for r in metrics)
/ max(1, len(metrics)),
"mean_ffn_shift_rms": sum(r["ffn_shift_rms_mean"] for r in metrics)
/ max(1, len(metrics)),
}
out_dir.mkdir(parents=True, exist_ok=True)
(out_dir / "generation.txt").write_text(completion, encoding="utf-8")
(out_dir / "run_config.json").write_text(
json.dumps(
{
**summary,
"messages": messages,
"chat_template": tokenizer.chat_template,
"raw_logits_capture": "forward hook on lm_head before generation processors",
"state_capture": "forward hook on RWKV base model; reads state after each generate forward",
},
ensure_ascii=False,
indent=2,
)
+ "\n",
encoding="utf-8",
)
with (out_dir / "token_metrics.csv").open("w", encoding="utf-8-sig", newline="") as f:
writer = csv.DictWriter(f, fieldnames=list(metrics[0]))
writer.writeheader()
writer.writerows(metrics)
(out_dir / "summary.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
del result
torch.cuda.empty_cache()
return summary
def main():
if not torch.cuda.is_available():
raise RuntimeError("CUDA is required for these runs")
rwkv_tokenizer = AutoTokenizer.from_pretrained(
RWKV_SNAPSHOT, trust_remote_code=True, local_files_only=True
)
rwkv_model = AutoModelForCausalLM.from_pretrained(
RWKV_SNAPSHOT,
trust_remote_code=True,
dtype=torch.bfloat16,
low_cpu_mem_usage=True,
local_files_only=True,
).to("cuda")
rwkv_model.eval()
rwkv_summary = metric_rows_for_rwkv(
rwkv_model,
rwkv_tokenizer,
ROOT / "runs" / "07_rwkv_english_system_prompt",
)
print(json.dumps(rwkv_summary, ensure_ascii=False), flush=True)
del rwkv_model, rwkv_tokenizer
torch.cuda.empty_cache()
olmo_tokenizer = AutoTokenizer.from_pretrained(OLMO_SNAPSHOT, local_files_only=True)
olmo_model = AutoModelForCausalLM.from_pretrained(
OLMO_SNAPSHOT,
dtype=torch.bfloat16,
low_cpu_mem_usage=True,
local_files_only=True,
).to("cuda")
olmo_model.eval()
olmo_summary = run_olmo_one(
model=olmo_model,
tokenizer=olmo_tokenizer,
condition="english_system_prompt",
messages=[
{"role": "system", "content": ENGLISH_SYSTEM},
{"role": "user", "content": USER_PROMPT},
],
repetition_penalty=1.0,
max_new_tokens=MAX_NEW_TOKENS,
out_dir=ROOT / "runs" / "08_olmo2_1b_english_system_prompt",
)
(ROOT / "runs" / "08_olmo2_1b_english_system_prompt" / "summary.json").write_text(
json.dumps(olmo_summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
print(json.dumps(olmo_summary, ensure_ascii=False), flush=True)
del olmo_model, olmo_tokenizer
torch.cuda.empty_cache()
(ROOT / "english_system_comparison.json").write_text(
json.dumps([rwkv_summary, olmo_summary], ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
if __name__ == "__main__":
main()