Download scripts/run_english_system_comparison.py from Ilikemechuri/rwkv_doomloop_experiment: direct link, hf CLI and curl.
- Browser
- Download file 11.8 kB
-
https://huggingface.co/Ilikemechuri/rwkv_doomloop_experiment/resolve/main/scripts/run_english_system_comparison.py
- Command line
-
hf download hf://Ilikemechuri/rwkv_doomloop_experiment/scripts/run_english_system_comparison.py
-
curl -L -o run_english_system_comparison.py https://huggingface.co/Ilikemechuri/rwkv_doomloop_experiment/resolve/main/scripts/run_english_system_comparison.py
11.8 kB
| #!/usr/bin/env python | |
| """Run the same English system instruction once on RWKV 7 and OLMo 2 1B.""" | |
| import csv | |
| import json | |
| import sys | |
| from pathlib import Path | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| from run_olmo2_comparison import ( | |
| MODEL_ID as OLMO_MODEL_ID, | |
| USER_PROMPT, | |
| largest_repeated_ngram, | |
| logits_metrics, | |
| rolling_dup8_rate, | |
| run_one as run_olmo_one, | |
| ) | |
| ROOT = Path("/workspace/rwkv_doomloop_experiment") | |
| RWKV_MODEL_ID = "RWKV/RWKV7-G1j-1.5B-20260831" | |
| RWKV_REVISION = "2c18b29ab7fbece25ff6112281eea0fa41fcb30f" | |
| RWKV_SNAPSHOT = Path( | |
| "/workspace/.hf_home/hub/models--RWKV--RWKV7-G1j-1.5B-20260831" | |
| f"/snapshots/{RWKV_REVISION}" | |
| ) | |
| OLMO_SNAPSHOT = ROOT / "models" / "OLMo-2-0425-1B-Instruct" | |
| ENGLISH_SYSTEM = ( | |
| "If you cannot solve the problem or do not know the answer, do not keep guessing. " | |
| "Say “I don't know” and stop." | |
| ) | |
| MAX_NEW_TOKENS = 512 | |
| def metric_rows_for_rwkv(model, tokenizer, out_dir): | |
| messages = [ | |
| {"role": "system", "content": ENGLISH_SYSTEM}, | |
| {"role": "user", "content": USER_PROMPT}, | |
| ] | |
| encoded = tokenizer.apply_chat_template( | |
| messages, | |
| tokenize=True, | |
| add_generation_prompt=True, | |
| thinking=False, | |
| return_dict=True, | |
| return_tensors="pt", | |
| ).to("cuda") | |
| prompt_len = encoded["input_ids"].shape[-1] | |
| raw_rows = [] | |
| state_rows = [] | |
| previous_states = None | |
| def capture_raw(_module, _inputs, output): | |
| if isinstance(output, tuple): | |
| output = output[0] | |
| values = logits_metrics(output[0, -1, :].detach()) | |
| raw_rows.append({key: value.item() for key, value in values.items()}) | |
| def capture_state(_module, _inputs, output): | |
| nonlocal previous_states | |
| state = getattr(output, "state", None) | |
| if state is None and isinstance(output, (tuple, list)) and len(output) > 1: | |
| state = output[1] | |
| if state is None: | |
| raise RuntimeError("RWKV base-model output did not contain its recurrent state") | |
| current = [] | |
| wkv_rms, att_rms, ffn_rms = [], [], [] | |
| wkv_delta_sq, att_delta_sq, ffn_delta_sq = [], [], [] | |
| for layer_idx in range(len(state.layers)): | |
| att, ffn, wkv = state.read(layer_idx) | |
| att = att.detach() | |
| ffn = ffn.detach() | |
| wkv = wkv.detach() | |
| layer = (att, ffn, wkv) | |
| current.append(layer) | |
| att_rms.append(att.float().square().mean().sqrt()) | |
| ffn_rms.append(ffn.float().square().mean().sqrt()) | |
| wkv_rms.append(wkv.float().square().mean().sqrt()) | |
| prev = previous_states[layer_idx] if previous_states is not None else tuple( | |
| torch.zeros_like(x) for x in layer | |
| ) | |
| att_delta_sq.append((att.float() - prev[0].float()).square().mean()) | |
| ffn_delta_sq.append((ffn.float() - prev[1].float()).square().mean()) | |
| wkv_delta_sq.append((wkv.float() - prev[2].float()).square().mean()) | |
| def rms_delta(values): | |
| return torch.stack(values).mean().sqrt().item() | |
| row = { | |
| "wkv_rms_mean": torch.stack(wkv_rms).mean().item(), | |
| "wkv_rms_max": torch.stack(wkv_rms).max().item(), | |
| "att_shift_rms_mean": torch.stack(att_rms).mean().item(), | |
| "att_shift_rms_max": torch.stack(att_rms).max().item(), | |
| "ffn_shift_rms_mean": torch.stack(ffn_rms).mean().item(), | |
| "ffn_shift_rms_max": torch.stack(ffn_rms).max().item(), | |
| "wkv_delta_rms": rms_delta(wkv_delta_sq), | |
| "att_shift_delta_rms": rms_delta(att_delta_sq), | |
| "ffn_shift_delta_rms": rms_delta(ffn_delta_sq), | |
| } | |
| for idx, value in enumerate(wkv_rms): | |
| row[f"wkv_layer_{idx:02d}_rms"] = value.item() | |
| for idx, value in enumerate(att_rms): | |
| row[f"att_shift_layer_{idx:02d}_rms"] = value.item() | |
| for idx, value in enumerate(ffn_rms): | |
| row[f"ffn_shift_layer_{idx:02d}_rms"] = value.item() | |
| state_rows.append(row) | |
| previous_states = [tuple(value.clone() for value in layer) for layer in current] | |
| head_hook = model.get_output_embeddings().register_forward_hook(capture_raw) | |
| state_hook = model.rwkv7.register_forward_hook(capture_state) | |
| try: | |
| with torch.inference_mode(): | |
| result = model.generate( | |
| **encoded, | |
| do_sample=False, | |
| num_beams=1, | |
| max_new_tokens=MAX_NEW_TOKENS, | |
| eos_token_id=tokenizer.eos_token_id, | |
| pad_token_id=tokenizer.pad_token_id, | |
| stop_strings=["\n\nUser:"], | |
| tokenizer=tokenizer, | |
| repetition_penalty=1.0, | |
| use_cache=True, | |
| return_dict_in_generate=True, | |
| output_scores=True, | |
| ) | |
| finally: | |
| head_hook.remove() | |
| state_hook.remove() | |
| generated_ids = result.sequences[0, prompt_len:].detach().cpu().tolist() | |
| completion = tokenizer.decode(generated_ids, skip_special_tokens=True) | |
| completion = completion.split("\n\nUser:", 1)[0] | |
| if not (len(generated_ids) == len(raw_rows) == len(state_rows) == len(result.scores)): | |
| raise RuntimeError( | |
| f"RWKV step mismatch: ids={len(generated_ids)}, raw={len(raw_rows)}, " | |
| f"state={len(state_rows)}, scores={len(result.scores)}" | |
| ) | |
| metrics = [] | |
| for step, (token_id, raw, state, score) in enumerate( | |
| zip(generated_ids, raw_rows, state_rows, result.scores), start=1 | |
| ): | |
| processed = logits_metrics(score[0].detach()) | |
| row = { | |
| "predicts_token_step": step, | |
| "context_generated_tokens": step - 1, | |
| "generated_token_id": token_id, | |
| "generated_piece": tokenizer.convert_ids_to_tokens(token_id), | |
| "raw_top1_probability": raw["top1_probability"], | |
| "raw_entropy_nats": raw["entropy_nats"], | |
| "raw_top1_top2_margin": raw["top1_top2_margin"], | |
| "raw_top1_token_id": raw["top1_token_id"], | |
| "processed_top1_probability": processed["top1_probability"].item(), | |
| "processed_entropy_nats": processed["entropy_nats"].item(), | |
| "processed_top1_top2_margin": processed["top1_top2_margin"].item(), | |
| "processed_argmax_token_id": processed["top1_token_id"].item(), | |
| "processed_argmax_matches_generated": int( | |
| processed["top1_token_id"].item() == token_id | |
| ), | |
| "rolling_dup8_rate_64": rolling_dup8_rate(generated_ids[:step]), | |
| **state, | |
| } | |
| metrics.append(row) | |
| max_n, repeat_start, repeat_again = largest_repeated_ngram(generated_ids) | |
| summary = { | |
| "model": RWKV_MODEL_ID, | |
| "model_revision": RWKV_REVISION, | |
| "condition": "english_system_prompt", | |
| "system_message": ENGLISH_SYSTEM, | |
| "user_prompt": USER_PROMPT, | |
| "thinking": False, | |
| "do_sample": False, | |
| "repetition_penalty": 1.0, | |
| "max_new_tokens": MAX_NEW_TOKENS, | |
| "eos_token_id": tokenizer.eos_token_id, | |
| "stop_strings": ["\n\nUser:"], | |
| "prompt_tokens": prompt_len, | |
| "generated_tokens": len(generated_ids), | |
| "eos_emitted": bool(generated_ids and generated_ids[-1] == tokenizer.eos_token_id), | |
| "refusal_phrase_present": any( | |
| phrase in completion for phrase in ("I don't know", "I do not know", "모르겠습니다") | |
| ), | |
| "largest_repeated_ngram_tokens": max_n, | |
| "repeat_first_step": repeat_start, | |
| "repeat_again_step": repeat_again, | |
| "rolling_dup8_mean": sum(r["rolling_dup8_rate_64"] for r in metrics) | |
| / max(1, len(metrics)), | |
| "rolling_dup8_max": max( | |
| (r["rolling_dup8_rate_64"] for r in metrics), default=0.0 | |
| ), | |
| "mean_raw_top1_probability": sum(r["raw_top1_probability"] for r in metrics) | |
| / max(1, len(metrics)), | |
| "final_raw_top1_probability": metrics[-1]["raw_top1_probability"] if metrics else None, | |
| "mean_raw_entropy_nats": sum(r["raw_entropy_nats"] for r in metrics) | |
| / max(1, len(metrics)), | |
| "final_raw_entropy_nats": metrics[-1]["raw_entropy_nats"] if metrics else None, | |
| "mean_wkv_rms": sum(r["wkv_rms_mean"] for r in metrics) / max(1, len(metrics)), | |
| "final_wkv_rms": metrics[-1]["wkv_rms_mean"] if metrics else None, | |
| "mean_attention_shift_rms": sum(r["att_shift_rms_mean"] for r in metrics) | |
| / max(1, len(metrics)), | |
| "mean_ffn_shift_rms": sum(r["ffn_shift_rms_mean"] for r in metrics) | |
| / max(1, len(metrics)), | |
| } | |
| out_dir.mkdir(parents=True, exist_ok=True) | |
| (out_dir / "generation.txt").write_text(completion, encoding="utf-8") | |
| (out_dir / "run_config.json").write_text( | |
| json.dumps( | |
| { | |
| **summary, | |
| "messages": messages, | |
| "chat_template": tokenizer.chat_template, | |
| "raw_logits_capture": "forward hook on lm_head before generation processors", | |
| "state_capture": "forward hook on RWKV base model; reads state after each generate forward", | |
| }, | |
| ensure_ascii=False, | |
| indent=2, | |
| ) | |
| + "\n", | |
| encoding="utf-8", | |
| ) | |
| with (out_dir / "token_metrics.csv").open("w", encoding="utf-8-sig", newline="") as f: | |
| writer = csv.DictWriter(f, fieldnames=list(metrics[0])) | |
| writer.writeheader() | |
| writer.writerows(metrics) | |
| (out_dir / "summary.json").write_text( | |
| json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| del result | |
| torch.cuda.empty_cache() | |
| return summary | |
| def main(): | |
| if not torch.cuda.is_available(): | |
| raise RuntimeError("CUDA is required for these runs") | |
| rwkv_tokenizer = AutoTokenizer.from_pretrained( | |
| RWKV_SNAPSHOT, trust_remote_code=True, local_files_only=True | |
| ) | |
| rwkv_model = AutoModelForCausalLM.from_pretrained( | |
| RWKV_SNAPSHOT, | |
| trust_remote_code=True, | |
| dtype=torch.bfloat16, | |
| low_cpu_mem_usage=True, | |
| local_files_only=True, | |
| ).to("cuda") | |
| rwkv_model.eval() | |
| rwkv_summary = metric_rows_for_rwkv( | |
| rwkv_model, | |
| rwkv_tokenizer, | |
| ROOT / "runs" / "07_rwkv_english_system_prompt", | |
| ) | |
| print(json.dumps(rwkv_summary, ensure_ascii=False), flush=True) | |
| del rwkv_model, rwkv_tokenizer | |
| torch.cuda.empty_cache() | |
| olmo_tokenizer = AutoTokenizer.from_pretrained(OLMO_SNAPSHOT, local_files_only=True) | |
| olmo_model = AutoModelForCausalLM.from_pretrained( | |
| OLMO_SNAPSHOT, | |
| dtype=torch.bfloat16, | |
| low_cpu_mem_usage=True, | |
| local_files_only=True, | |
| ).to("cuda") | |
| olmo_model.eval() | |
| olmo_summary = run_olmo_one( | |
| model=olmo_model, | |
| tokenizer=olmo_tokenizer, | |
| condition="english_system_prompt", | |
| messages=[ | |
| {"role": "system", "content": ENGLISH_SYSTEM}, | |
| {"role": "user", "content": USER_PROMPT}, | |
| ], | |
| repetition_penalty=1.0, | |
| max_new_tokens=MAX_NEW_TOKENS, | |
| out_dir=ROOT / "runs" / "08_olmo2_1b_english_system_prompt", | |
| ) | |
| (ROOT / "runs" / "08_olmo2_1b_english_system_prompt" / "summary.json").write_text( | |
| json.dumps(olmo_summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" | |
| ) | |
| print(json.dumps(olmo_summary, ensure_ascii=False), flush=True) | |
| del olmo_model, olmo_tokenizer | |
| torch.cuda.empty_cache() | |
| (ROOT / "english_system_comparison.json").write_text( | |
| json.dumps([rwkv_summary, olmo_summary], ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| if __name__ == "__main__": | |
| main() | |