File size: 3,521 Bytes
04a3ade e0964b9 04a3ade | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | """Sequential preparation, scratch smoke, fifty training segments and fifty probes."""
from pathlib import Path
import os,sys,json,time,subprocess,fcntl,traceback
BASE=Path(__file__).resolve().parent.parent;OUT=BASE/'scratch_pcqm_cosmo'
os.environ.update(CUDA_VISIBLE_DEVICES='0',TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD='1',
CUBLAS_WORKSPACE_CONFIG=':4096:8',OMP_NUM_THREADS='8',MKL_NUM_THREADS='8',PYTHONUNBUFFERED='1')
def write(path,value):
temp=path.with_suffix('.partial');temp.write_text(json.dumps(value,indent=2));temp.replace(path)
def run(args):
for attempt in range(3):
phase='smoke_validation' if '--smoke' in args else ('preparing_probe' if args[0]=='scratch_probe.py' else 'loading_training_segment')
write(OUT/'status.json',dict(phase=phase,pid=os.getpid(),args=args,updated_unix=time.time()))
proc=subprocess.Popen([sys.executable,'-u',str(BASE/'source'/args[0]),*args[1:]],env=os.environ.copy())
write(OUT/'active_process.json',dict(pid=proc.pid,args=args,attempt=attempt+1))
code=proc.wait()
if code==0:return
if attempt==2:raise RuntimeError(f'{args} failed after three attempts; inspect driver.log')
time.sleep(10)
def main():
OUT.mkdir(parents=True,exist_ok=True)
lock=(OUT/'driver.lock').open('a');fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
if (OUT/'complete.json').exists():return
while True:
p=OUT/'preparation_status.json'
state={}
if p.exists():
state=json.loads(p.read_text())
if state.get('phase')=='failed':raise RuntimeError(state.get('error'))
if (OUT/'preparation_complete.json').exists() and state.get('phase')=='ready_for_training':break
time.sleep(10)
if not (OUT/'smoke_validation.json').exists():run(['scratch_train.py','--smoke'])
for point in range(1,51):
marker=OUT/'evaluation'/f'probe_{point:03d}_complete.json'
if marker.exists():continue
_saved_checkpoint = OUT/'checkpoints'/f'probe_{point:03d}'
if not (_saved_checkpoint/'hub_verified.json').exists():
run(['scratch_train.py','--point',str(point)])
run(['scratch_probe.py',str(point)])
import pandas as pd
results=pd.read_csv(OUT/'evaluation/results.csv')
assert sorted(results.point.tolist())==list(range(1,51))
assert all(sum((results.point-1)//5==epoch)==5 for epoch in range(10))
proof=json.loads((OUT/'evaluation/validation.json').read_text());assert proof['passed'] and proof['probes']==50
hub=json.loads((OUT/'report_hub.json').read_text())
write(OUT/'training_complete.json',dict(epochs=10,probes=50))
write(OUT/'complete.json',dict(completed=True,epochs=10,probes=50,hub=hub,completed_unix=time.time()))
write(OUT/'status.json',dict(phase='complete',epochs=10,probes=50,report=hub['report']))
from dotenv import load_dotenv
load_dotenv('/marimo/.env',override=False)
from huggingface_hub import HfApi,CommitOperationAdd
api=HfApi();repo='Queimo/rem3di-scratch-pcqm-cosmo'
api.create_commit(repo_id=repo,operations=[CommitOperationAdd(path_in_repo=n,path_or_fileobj=str(OUT/n)) for n in ['complete.json','training_complete.json','status.json','latest_point.json','latest_probe.json']],commit_message='Mark ten scratch epochs and all fifty verified BigSolDB Kumo probes complete')
if __name__=='__main__':
try:main()
except Exception:
write(OUT/'driver_failure.json',dict(error=traceback.format_exc(),time=time.time()));raise
|