#!/bin/bash
# device-assert poisons the process (known MPM factory failure mode); .fail markers make restarts skip
# the poisonous trajectory, so a simple relaunch loop drains the shard.
cd /home/qiuyid/neurok/experiments/idea1/design4
for i in $(seq 1 40); do
  grep -aq "SHARD DONE" gen_l3e.log && break
  CUDA_VISIBLE_DEVICES=7 MPMDATA=/dev/shm/qiuyid_mpm $HOME/miniconda3/envs/neurok/bin/python -u gen_mpm_torus.py L3E 0 300 >> gen_l3e.log 2>&1
  sleep 5
done
echo "L3E LOOP DONE (restarts used: $i)"