#!/usr/bin/env python """Run the same English system instruction once on RWKV 7 and OLMo 2 1B.""" import csv import json import sys from pathlib import Path import torch from transformers import AutoModelForCausalLM, AutoTokenizer from run_olmo2_comparison import ( MODEL_ID as OLMO_MODEL_ID, USER_PROMPT, largest_repeated_ngram, logits_metrics, rolling_dup8_rate, run_one as run_olmo_one, ) ROOT = Path("/workspace/rwkv_doomloop_experiment") RWKV_MODEL_ID = "RWKV/RWKV7-G1j-1.5B-20260831" RWKV_REVISION = "2c18b29ab7fbece25ff6112281eea0fa41fcb30f" RWKV_SNAPSHOT = Path( "/workspace/.hf_home/hub/models--RWKV--RWKV7-G1j-1.5B-20260831" f"/snapshots/{RWKV_REVISION}" ) OLMO_SNAPSHOT = ROOT / "models" / "OLMo-2-0425-1B-Instruct" ENGLISH_SYSTEM = ( "If you cannot solve the problem or do not know the answer, do not keep guessing. " "Say “I don't know” and stop." ) MAX_NEW_TOKENS = 512 def metric_rows_for_rwkv(model, tokenizer, out_dir): messages = [ {"role": "system", "content": ENGLISH_SYSTEM}, {"role": "user", "content": USER_PROMPT}, ] encoded = tokenizer.apply_chat_template( messages, tokenize=True, add_generation_prompt=True, thinking=False, return_dict=True, return_tensors="pt", ).to("cuda") prompt_len = encoded["input_ids"].shape[-1] raw_rows = [] state_rows = [] previous_states = None def capture_raw(_module, _inputs, output): if isinstance(output, tuple): output = output[0] values = logits_metrics(output[0, -1, :].detach()) raw_rows.append({key: value.item() for key, value in values.items()}) def capture_state(_module, _inputs, output): nonlocal previous_states state = getattr(output, "state", None) if state is None and isinstance(output, (tuple, list)) and len(output) > 1: state = output[1] if state is None: raise RuntimeError("RWKV base-model output did not contain its recurrent state") current = [] wkv_rms, att_rms, ffn_rms = [], [], [] wkv_delta_sq, att_delta_sq, ffn_delta_sq = [], [], [] for layer_idx in range(len(state.layers)): att, ffn, wkv = state.read(layer_idx) att = att.detach() ffn = ffn.detach() wkv = wkv.detach() layer = (att, ffn, wkv) current.append(layer) att_rms.append(att.float().square().mean().sqrt()) ffn_rms.append(ffn.float().square().mean().sqrt()) wkv_rms.append(wkv.float().square().mean().sqrt()) prev = previous_states[layer_idx] if previous_states is not None else tuple( torch.zeros_like(x) for x in layer ) att_delta_sq.append((att.float() - prev[0].float()).square().mean()) ffn_delta_sq.append((ffn.float() - prev[1].float()).square().mean()) wkv_delta_sq.append((wkv.float() - prev[2].float()).square().mean()) def rms_delta(values): return torch.stack(values).mean().sqrt().item() row = { "wkv_rms_mean": torch.stack(wkv_rms).mean().item(), "wkv_rms_max": torch.stack(wkv_rms).max().item(), "att_shift_rms_mean": torch.stack(att_rms).mean().item(), "att_shift_rms_max": torch.stack(att_rms).max().item(), "ffn_shift_rms_mean": torch.stack(ffn_rms).mean().item(), "ffn_shift_rms_max": torch.stack(ffn_rms).max().item(), "wkv_delta_rms": rms_delta(wkv_delta_sq), "att_shift_delta_rms": rms_delta(att_delta_sq), "ffn_shift_delta_rms": rms_delta(ffn_delta_sq), } for idx, value in enumerate(wkv_rms): row[f"wkv_layer_{idx:02d}_rms"] = value.item() for idx, value in enumerate(att_rms): row[f"att_shift_layer_{idx:02d}_rms"] = value.item() for idx, value in enumerate(ffn_rms): row[f"ffn_shift_layer_{idx:02d}_rms"] = value.item() state_rows.append(row) previous_states = [tuple(value.clone() for value in layer) for layer in current] head_hook = model.get_output_embeddings().register_forward_hook(capture_raw) state_hook = model.rwkv7.register_forward_hook(capture_state) try: with torch.inference_mode(): result = model.generate( **encoded, do_sample=False, num_beams=1, max_new_tokens=MAX_NEW_TOKENS, eos_token_id=tokenizer.eos_token_id, pad_token_id=tokenizer.pad_token_id, stop_strings=["\n\nUser:"], tokenizer=tokenizer, repetition_penalty=1.0, use_cache=True, return_dict_in_generate=True, output_scores=True, ) finally: head_hook.remove() state_hook.remove() generated_ids = result.sequences[0, prompt_len:].detach().cpu().tolist() completion = tokenizer.decode(generated_ids, skip_special_tokens=True) completion = completion.split("\n\nUser:", 1)[0] if not (len(generated_ids) == len(raw_rows) == len(state_rows) == len(result.scores)): raise RuntimeError( f"RWKV step mismatch: ids={len(generated_ids)}, raw={len(raw_rows)}, " f"state={len(state_rows)}, scores={len(result.scores)}" ) metrics = [] for step, (token_id, raw, state, score) in enumerate( zip(generated_ids, raw_rows, state_rows, result.scores), start=1 ): processed = logits_metrics(score[0].detach()) row = { "predicts_token_step": step, "context_generated_tokens": step - 1, "generated_token_id": token_id, "generated_piece": tokenizer.convert_ids_to_tokens(token_id), "raw_top1_probability": raw["top1_probability"], "raw_entropy_nats": raw["entropy_nats"], "raw_top1_top2_margin": raw["top1_top2_margin"], "raw_top1_token_id": raw["top1_token_id"], "processed_top1_probability": processed["top1_probability"].item(), "processed_entropy_nats": processed["entropy_nats"].item(), "processed_top1_top2_margin": processed["top1_top2_margin"].item(), "processed_argmax_token_id": processed["top1_token_id"].item(), "processed_argmax_matches_generated": int( processed["top1_token_id"].item() == token_id ), "rolling_dup8_rate_64": rolling_dup8_rate(generated_ids[:step]), **state, } metrics.append(row) max_n, repeat_start, repeat_again = largest_repeated_ngram(generated_ids) summary = { "model": RWKV_MODEL_ID, "model_revision": RWKV_REVISION, "condition": "english_system_prompt", "system_message": ENGLISH_SYSTEM, "user_prompt": USER_PROMPT, "thinking": False, "do_sample": False, "repetition_penalty": 1.0, "max_new_tokens": MAX_NEW_TOKENS, "eos_token_id": tokenizer.eos_token_id, "stop_strings": ["\n\nUser:"], "prompt_tokens": prompt_len, "generated_tokens": len(generated_ids), "eos_emitted": bool(generated_ids and generated_ids[-1] == tokenizer.eos_token_id), "refusal_phrase_present": any( phrase in completion for phrase in ("I don't know", "I do not know", "모르겠습니다") ), "largest_repeated_ngram_tokens": max_n, "repeat_first_step": repeat_start, "repeat_again_step": repeat_again, "rolling_dup8_mean": sum(r["rolling_dup8_rate_64"] for r in metrics) / max(1, len(metrics)), "rolling_dup8_max": max( (r["rolling_dup8_rate_64"] for r in metrics), default=0.0 ), "mean_raw_top1_probability": sum(r["raw_top1_probability"] for r in metrics) / max(1, len(metrics)), "final_raw_top1_probability": metrics[-1]["raw_top1_probability"] if metrics else None, "mean_raw_entropy_nats": sum(r["raw_entropy_nats"] for r in metrics) / max(1, len(metrics)), "final_raw_entropy_nats": metrics[-1]["raw_entropy_nats"] if metrics else None, "mean_wkv_rms": sum(r["wkv_rms_mean"] for r in metrics) / max(1, len(metrics)), "final_wkv_rms": metrics[-1]["wkv_rms_mean"] if metrics else None, "mean_attention_shift_rms": sum(r["att_shift_rms_mean"] for r in metrics) / max(1, len(metrics)), "mean_ffn_shift_rms": sum(r["ffn_shift_rms_mean"] for r in metrics) / max(1, len(metrics)), } out_dir.mkdir(parents=True, exist_ok=True) (out_dir / "generation.txt").write_text(completion, encoding="utf-8") (out_dir / "run_config.json").write_text( json.dumps( { **summary, "messages": messages, "chat_template": tokenizer.chat_template, "raw_logits_capture": "forward hook on lm_head before generation processors", "state_capture": "forward hook on RWKV base model; reads state after each generate forward", }, ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) with (out_dir / "token_metrics.csv").open("w", encoding="utf-8-sig", newline="") as f: writer = csv.DictWriter(f, fieldnames=list(metrics[0])) writer.writeheader() writer.writerows(metrics) (out_dir / "summary.json").write_text( json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) del result torch.cuda.empty_cache() return summary def main(): if not torch.cuda.is_available(): raise RuntimeError("CUDA is required for these runs") rwkv_tokenizer = AutoTokenizer.from_pretrained( RWKV_SNAPSHOT, trust_remote_code=True, local_files_only=True ) rwkv_model = AutoModelForCausalLM.from_pretrained( RWKV_SNAPSHOT, trust_remote_code=True, dtype=torch.bfloat16, low_cpu_mem_usage=True, local_files_only=True, ).to("cuda") rwkv_model.eval() rwkv_summary = metric_rows_for_rwkv( rwkv_model, rwkv_tokenizer, ROOT / "runs" / "07_rwkv_english_system_prompt", ) print(json.dumps(rwkv_summary, ensure_ascii=False), flush=True) del rwkv_model, rwkv_tokenizer torch.cuda.empty_cache() olmo_tokenizer = AutoTokenizer.from_pretrained(OLMO_SNAPSHOT, local_files_only=True) olmo_model = AutoModelForCausalLM.from_pretrained( OLMO_SNAPSHOT, dtype=torch.bfloat16, low_cpu_mem_usage=True, local_files_only=True, ).to("cuda") olmo_model.eval() olmo_summary = run_olmo_one( model=olmo_model, tokenizer=olmo_tokenizer, condition="english_system_prompt", messages=[ {"role": "system", "content": ENGLISH_SYSTEM}, {"role": "user", "content": USER_PROMPT}, ], repetition_penalty=1.0, max_new_tokens=MAX_NEW_TOKENS, out_dir=ROOT / "runs" / "08_olmo2_1b_english_system_prompt", ) (ROOT / "runs" / "08_olmo2_1b_english_system_prompt" / "summary.json").write_text( json.dumps(olmo_summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) print(json.dumps(olmo_summary, ensure_ascii=False), flush=True) del olmo_model, olmo_tokenizer torch.cuda.empty_cache() (ROOT / "english_system_comparison.json").write_text( json.dumps([rwkv_summary, olmo_summary], ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) if __name__ == "__main__": main()