"""Run with pytest. CPU preparation tests plus a small CUDA full-state resume test.""" import importlib.util import json from pathlib import Path import pickle import subprocess import sys import types import numpy as np import pytest import torch ROOT = Path(__file__).resolve().parent spec = importlib.util.spec_from_file_location('server_pipeline', ROOT/'run_40b.py') pipeline = importlib.util.module_from_spec(spec) spec.loader.exec_module(pipeline) def args(tmp_path): return types.SimpleNamespace(work_dir=tmp_path, tokenizer=ROOT/'tokenizer.model', dataset='fixture',dataset_config='fixture',revision='main',train_tokens=300,val_tokens=100, workers=2,documents_per_batch=7,test_small_gpu=True,seq_len=8) def test_masked_final_batch_and_wrap(): data = np.arange(19,dtype=np.uint16) x,y = pipeline.batches(data,17,2,4,5,torch.device('cpu')) assert x.flatten().tolist() == [17,18,0,1,2,3,4,5] assert y.flatten().tolist() == [18,0,1,2,3,-100,-100,-100] def test_preparation_resume_truncates_uncommitted_tail(tmp_path,monkeypatch): a = args(tmp_path) holdout = next('document '+str(i) for i in range(100000) if pipeline.is_holdout('document '+str(i))) normal = next('other '+str(i) for i in range(100) if not pipeline.is_holdout('other '+str(i))) documents = [{'text':holdout if i%3==0 else normal} for i in range(500)] class Source: def __init__(self): self.offset=0 def __iter__(self): while self.offset < len(documents): row=documents[self.offset] self.offset+=1 yield row def state_dict(self): return {'offset':self.offset} def load_state_dict(self,state): self.offset=state['offset'] monkeypatch.setitem(sys.modules,'datasets',types.SimpleNamespace(__version__='fixture',load_dataset=lambda *a,**kw:Source())) monkeypatch.setitem(sys.modules,'huggingface_hub',types.SimpleNamespace(HfApi=lambda:types.SimpleNamespace(dataset_info=lambda *a,**kw:types.SimpleNamespace(sha='fixed-commit')))) pipeline.STOP=True assert not pipeline.prepare(a) with (tmp_path/'data/prepare-state.pkl').open('rb') as f: state=pickle.load(f) with (tmp_path/'data/train.bin').open('ab') as f: f.write(b'uncommitted-tail') pipeline.STOP=False assert pipeline.prepare(a) pipeline.check_manifest(a) other=tmp_path/'continuous' b=args(other) assert pipeline.prepare(b) for name in ('train.bin','val.bin'): assert (tmp_path/'data'/name).read_bytes() == (other/'data'/name).read_bytes() assert state['iterator']['offset'] == 7 a.train_tokens+=1 with pytest.raises(ValueError): pipeline.check_manifest(a) def test_memory_cap_refuses_insufficient_headroom(monkeypatch): a = types.SimpleNamespace(vram_limit_gib=33.5, vram_headroom_gib=3, autocast_cache=False) calls = [] monkeypatch.setattr(torch.cuda, 'mem_get_info', lambda device: (36*1024**3, 96*1024**3)) monkeypatch.setattr(torch.cuda, 'set_per_process_memory_fraction', lambda *args: calls.append(args)) with pytest.raises(RuntimeError, match='requested allocator cap and headroom'): pipeline.configure_memory(a) assert not calls monkeypatch.setattr(torch.cuda, 'mem_get_info', lambda device: (37*1024**3, 96*1024**3)) pipeline.configure_memory(a) assert calls == [(33.5/96, torch.device('cuda:0'))] @pytest.mark.skipif(not torch.cuda.is_available(),reason='CUDA required') @pytest.mark.parametrize('initialization', ['weights', 'fresh']) def test_gpu_full_state_resume_and_exact_partial_update(tmp_path, initialization): sys.path.insert(0,str(ROOT)) from warpstate import WarpStateConfig,WarpStateLM cfg=WarpStateConfig(vocab_size=32768,d_model=32,n_heads=2,n_cores=2,logical_depth=4, ffn_hidden=64,chunk_size=8,parallel_memory=True) config_path=tmp_path/'config.json' cfg.to_json(config_path) torch.manual_seed(133) original=WarpStateLM(cfg) weights=tmp_path/'initial.pt' torch.save({'config':vars(cfg),'model':original.state_dict(),'tokens_seen':100},weights) dirs=[tmp_path/'continuous',tmp_path/'resumed'] common=[sys.executable,str(ROOT/'run_40b.py'),'--no-bootstrap','--test-small-gpu', '--stage','train','--config',str(config_path), '--train-tokens','73','--val-tokens','256','--seq-len','8','--micro-batch','2', '--grad-accum','2','--loss-chunk-size','5','--workers','2','--eval-batches','2','--eval-every','1', '--no-autocast-cache','--vram-limit-gib','0.5','--vram-headroom-gib','1'] common += ['--weights',str(weights)] if initialization == 'weights' else ['--from-scratch'] for directory in dirs: (directory/'data').mkdir(parents=True) np.arange(73,dtype='