Add trained Unity PPO policy, ONNX export and independent evaluation

README.md

@@ -1,3 +1,29 @@
 ---
 license: apache-2.0
+tags:
+- reinforcement-learning
+- ml-agents
+- ML-Agents-SnowballTarget
+- deep-rl-class
+model-index:
+- name: SnowballTarget
+  results:
+  - task:
+      type: reinforcement-learning
+      name: Reinforcement Learning
+    dataset:
+      name: ML-Agents-SnowballTarget
+      type: ML-Agents-SnowballTarget
+    metrics:
+    - type: mean_reward
+      name: mean_reward
+      value: 6.6364 +/- 2.0790
 ---
+
+# SnowballTarget
+
+Trained from random initialization for Vlad Kolomiiets (`Vladprime`) using the official Hugging Face Deep RL Course Unity environment and Unity ML-Agents 1.1.0. No pretrained weights were used. This is an initial training run; it is not claimed to have converged. Units 5 and 7 require publishing a trained model without a reward threshold.
+
+Training: ppo, 30000 configured environment steps, seed 20261101. Export: genuine ML-Agents ONNX plus PyTorch checkpoint and trainer configuration. Separate ONNX evaluation process: seed 1000000; 44 completed agent episodes, mean 6.6364, standard deviation 2.0790. Original environment, action and observation spaces were retained. See results.json for individual returns and evaluation limitations.
+
+The .onnx file is compatible with the course Unity browser viewer. Training/evaluation code is in train-unity.py; the exact run configuration is config.yaml.

SHA256SUMS.json

@@ -0,0 +1,9 @@
+{
+  "SnowballTarget-checkpoint.pt": "83b5a60eb00fe520f410e7ced9eb5084663eccbf1e34403ebd91c6db51888a36",
+  "README.md": "b485bb2e9cf53f40dd557ac0936a950c5f6d62f3922dd357d722859f1ac57b09",
+  "events.out.tfevents.1793538312.e5108cd25e8e.9150.0": "36a5560ceededfa54e1c897c3260cf4c61b46a769d70f9d80aad9a9518b72abd",
+  "results.json": "2a91a74f6bd1c305e17e985b7b4d98688fecf43aad7f23f0cd412d8baa888f48",
+  "SnowballTarget.onnx": "badddcb40788d91b862314b5c0478a20c8d4306e1eaa92b18dc7ea51883b0272",
+  "config.yaml": "20bfc88158086a8458c13de10f3fc86790d7969cb44b63b3add2ee6ae2b85350",
+  "train-unity.py": "dce2bd5169df6422af51eecf3ef243b4af8aa08444152598fefa600859462b59"
+}

SnowballTarget-checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:83b5a60eb00fe520f410e7ced9eb5084663eccbf1e34403ebd91c6db51888a36
+size 1542626

SnowballTarget.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:badddcb40788d91b862314b5c0478a20c8d4306e1eaa92b18dc7ea51883b0272
+size 264598

config.yaml

@@ -0,0 +1,25 @@
+behaviors:
+  SnowballTarget:
+    trainer_type: ppo
+    hyperparameters:
+      learning_rate: 0.0003
+      learning_rate_schedule: linear
+      batch_size: 128
+      buffer_size: 2048
+      beta: 0.005
+      epsilon: 0.2
+      lambd: 0.95
+      num_epoch: 3
+    network_settings:
+      normalize: false
+      hidden_units: 128
+      num_layers: 2
+    reward_signals:
+      extrinsic:
+        gamma: 0.99
+        strength: 1.0
+    max_steps: 30000
+    time_horizon: 64
+    summary_freq: 2000
+    checkpoint_interval: 10000
+    keep_checkpoints: 2

events.out.tfevents.1793538312.e5108cd25e8e.9150.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36a5560ceededfa54e1c897c3260cf4c61b46a769d70f9d80aad9a9518b72abd
+size 29080

results.json

@@ -0,0 +1,321 @@
+{
+  "mean_reward": 6.636363636363637,
+  "std_reward": 2.0790175683689585,
+  "mean_minus_std": 4.557346067994678,
+  "n_eval_agent_episodes": 44,
+  "episode_rewards": [
+    6.0,
+    7.0,
+    10.0,
+    10.0,
+    5.0,
+    4.0,
+    4.0,
+    4.0,
+    8.0,
+    4.0,
+    7.0,
+    7.0,
+    9.0,
+    5.0,
+    9.0,
+    6.0,
+    7.0,
+    6.0,
+    6.0,
+    6.0,
+    7.0,
+    5.0,
+    5.0,
+    4.0,
+    9.0,
+    0.0,
+    7.0,
+    6.0,
+    6.0,
+    11.0,
+    10.0,
+    5.0,
+    7.0,
+    9.0,
+    7.0,
+    9.0,
+    7.0,
+    5.0,
+    6.0,
+    9.0,
+    7.0,
+    7.0,
+    6.0,
+    8.0
+  ],
+  "records": [
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 0,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 1,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 2,
+      "reward": 10.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 3,
+      "reward": 10.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 4,
+      "reward": 5.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 5,
+      "reward": 4.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 6,
+      "reward": 4.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 7,
+      "reward": 4.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 8,
+      "reward": 8.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 9,
+      "reward": 4.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 10,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 0,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 1,
+      "reward": 9.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 2,
+      "reward": 5.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 3,
+      "reward": 9.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 4,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 5,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 6,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 7,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 8,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 9,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 10,
+      "reward": 5.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 0,
+      "reward": 5.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 1,
+      "reward": 4.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 2,
+      "reward": 9.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 3,
+      "reward": 0.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 4,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 5,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 6,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 7,
+      "reward": 11.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 8,
+      "reward": 10.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 9,
+      "reward": 5.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 10,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 0,
+      "reward": 9.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 1,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 2,
+      "reward": 9.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 3,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 4,
+      "reward": 5.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 5,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 6,
+      "reward": 9.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 7,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 8,
+      "reward": 7.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 9,
+      "reward": 6.0,
+      "interrupted": true
+    },
+    {
+      "behavior": "SnowballTarget?team=0",
+      "agent_id": 10,
+      "reward": 8.0,
+      "interrupted": true
+    }
+  ],
+  "evaluation_seed": 1000000,
+  "eval_datetime": "2026-11-01T13:06:49.043568+00:00",
+  "evaluation_kind": "separate Unity process, ONNX policy, completed agent episodes (soccer episodes within a match are correlated)"
+}

train-unity.py

@@ -0,0 +1,101 @@
+"""Own from-scratch course ML-Agents training, export, independent ONNX evaluation.
+Run inside the compatible Linux environment prepared by colab-setup.py.
+"""
+from pathlib import Path
+from datetime import datetime,timezone
+import os,sys,json,subprocess,zipfile,urllib.request,hashlib,shutil,time
+import numpy as np
+import yaml
+ROOT=Path('/content/vlad-deep-rl-unity');ROOT.mkdir(exist_ok=True)
+os.environ['OMP_NUM_THREADS']='1';os.environ['MKL_NUM_THREADS']='1'
+SEED=20261101
+
+def download(url,target):
+ print('Downloading official course environment',target.name,flush=True)
+ urllib.request.urlretrieve(url,target)
+def unzip(path,target):
+ target.mkdir(exist_ok=True)
+ with zipfile.ZipFile(path) as z:
+  for item in z.infolist():
+   if not (target/item.filename).resolve().is_relative_to(target.resolve()):raise ValueError('Unsafe zip member')
+  z.extractall(target)
+def evaluate_onnx(model,executable,output):
+ from mlagents_envs.environment import UnityEnvironment
+ from mlagents_envs.base_env import ActionTuple
+ from mlagents_envs.side_channel.engine_configuration_channel import EngineConfigurationChannel
+ import onnxruntime as ort
+ opts=ort.SessionOptions();opts.intra_op_num_threads=1;opts.inter_op_num_threads=1
+ session=ort.InferenceSession(str(model),sess_options=opts,providers=['CPUExecutionProvider'])
+ print('ONNX inputs',[(i.name,i.shape) for i in session.get_inputs()],flush=True)
+ channel=EngineConfigurationChannel();channel.set_configuration_parameters(time_scale=20)
+ env=UnityEnvironment(file_name=str(executable),seed=1000000,no_graphics=True,side_channels=[channel],worker_id=10,timeout_wait=120)
+ totals={};records=[]
+ try:
+  env.reset()
+  for iteration in range(30000):
+   for behavior in env.behavior_specs:
+    decision,terminal=env.get_steps(behavior)
+    for agent_id in terminal.agent_id:
+     step=terminal[agent_id];key=(behavior,int(agent_id));total=totals.pop(key,0.0)+float(step.reward)+float(step.group_reward)
+     records.append({'behavior':behavior,'agent_id':int(agent_id),'reward':total,'interrupted':bool(step.interrupted)})
+    for agent_id in decision.agent_id:
+     step=decision[agent_id];key=(behavior,int(agent_id));totals[key]=totals.get(key,0.0)+float(step.reward)+float(step.group_reward)
+    if len(decision):
+     inputs={}
+     for spec in session.get_inputs():
+      if spec.name.startswith('obs_'):inputs[spec.name]=decision.obs[int(spec.name.split('_')[-1])].astype(np.float32)
+      elif spec.name=='action_masks':
+       inputs[spec.name]=np.concatenate([1-m.astype(np.float32) for m in decision.action_mask],axis=1)
+      else:raise ValueError('Unexpected ONNX input '+spec.name)
+     values=session.run(None,inputs);values={spec.name:value for spec,value in zip(session.get_outputs(),values)}
+     action=ActionTuple()
+     if 'continuous_actions' in values:action.add_continuous(values['continuous_actions'].astype(np.float32))
+     if 'discrete_actions' in values:action.add_discrete(values['discrete_actions'].astype(np.int32))
+     env.set_actions(behavior,action)
+   if len(records)>=40:break
+   env.step()
+ finally:env.close()
+ if not records:raise RuntimeError('Independent evaluation completed no episodes')
+ rewards=[r['reward'] for r in records];mean=float(np.mean(rewards));std=float(np.std(rewards))
+ result={'mean_reward':mean,'std_reward':std,'mean_minus_std':mean-std,'n_eval_agent_episodes':len(rewards),'episode_rewards':rewards,'records':records,'evaluation_seed':1000000,'eval_datetime':datetime.now(timezone.utc).isoformat(),'evaluation_kind':'separate Unity process, ONNX policy, completed agent episodes (soccer episodes within a match are correlated)'}
+ output.write_text(json.dumps(result,indent=2));print('EVALUATED',model.name,mean,std,flush=True);return result
+
+def card(folder,name,env_tag,result,config):
+ meta={'license':'apache-2.0','tags':['reinforcement-learning','ml-agents',env_tag,'deep-rl-class'],'model-index':[{'name':name,'results':[{'task':{'type':'reinforcement-learning','name':'Reinforcement Learning'},'dataset':{'name':env_tag,'type':env_tag},'metrics':[{'type':'mean_reward','name':'mean_reward','value':f"{result['mean_reward']:.4f} +/- {result['std_reward']:.4f}"}]}]}]}
+ text='---\n'+yaml.safe_dump(meta,sort_keys=False)+'---\n\n# '+name+'\n\n'
+ text+='Trained from random initialization for Vlad Kolomiiets (`Vladprime`) using the official Hugging Face Deep RL Course Unity environment and Unity ML-Agents 1.1.0. No pretrained weights were used. This is an initial training run; it is not claimed to have converged. Units 5 and 7 require publishing a trained model without a reward threshold.\n\n'
+ text+=f"Training: {config['trainer_type']}, {config['max_steps']} configured environment steps, seed {SEED}. Export: genuine ML-Agents ONNX plus PyTorch checkpoint and trainer configuration. Separate ONNX evaluation process: seed 1000000; {result['n_eval_agent_episodes']} completed agent episodes, mean {result['mean_reward']:.4f}, standard deviation {result['std_reward']:.4f}. Original environment, action and observation spaces were retained. See results.json for individual returns and evaluation limitations.\n\n"
+ if 'Soccer' in env_tag:text+='Offline MA-POCA self-play training and local evaluation only. No ranked AI-vs-AI challenge result is claimed; the course online leaderboard is unavailable. Player episodes within a soccer match are correlated.\n\n'
+ text+='The .onnx file is compatible with the course Unity browser viewer. Training/evaluation code is in train-unity.py; the exact run configuration is config.yaml.\n'
+ (folder/'README.md').write_text(text)
+ (folder/'SHA256SUMS.json').write_text(json.dumps({str(p.relative_to(folder)):hashlib.sha256(p.read_bytes()).hexdigest() for p in folder.rglob('*') if p.is_file() and p.name!='SHA256SUMS.json'},indent=2))
+
+def main():
+ sources=[('SnowballTarget','https://github.com/huggingface/Snowball-Target/raw/main/SnowballTarget.zip','SnowballTarget/SnowballTarget.x86_64','SnowballTarget.yaml','ML-Agents-SnowballTarget'),('Pyramids','https://huggingface.co/spaces/unity/ML-Agents-Pyramids/resolve/main/Pyramids.zip','Pyramids/Pyramids','PyramidsRND-local.yaml','ML-Agents-Pyramids'),('SoccerTwos','gdrive:1KuqBKYiXiIcU4kNMqEzhgypuFP5_45CL','SoccerTwos.x86_64','SoccerTwos-local.yaml','ML-Agents-SoccerTwos')]
+ results=[]
+ for name,url,relative,config_name,env_tag in sources:
+  zip_path=ROOT/(name+'.zip')
+  if not zip_path.exists():
+   if url.startswith('gdrive:'):
+    import gdown;gdown.download(id=url.split(':')[1],output=str(zip_path),quiet=False)
+   else:download(url,zip_path)
+  target=ROOT/('env-'+name);unzip(zip_path,target);executable=target/relative;executable.chmod(executable.stat().st_mode|0o111)
+  config_path=ROOT/config_name;config=yaml.safe_load(config_path.read_text());behavior=next(iter(config['behaviors'].values()));run_id='vlad-'+name
+  command=[str(Path(sys.executable).parent/'mlagents-learn'),str(config_path),'--env='+str(executable),'--run-id='+run_id,'--no-graphics','--seed='+str(SEED),'--torch-device=cpu','--results-dir='+str(ROOT/'training')]
+  print('TRAINING',name,' '.join(command),flush=True)
+  subprocess.run(command,check=True,env={**os.environ,'OMP_NUM_THREADS':'1','MKL_NUM_THREADS':'1'})
+  trained=ROOT/'training'/run_id;model=list(trained.rglob('*.onnx'))
+  # Final exported model is at the run root; step checkpoints live in behavior subdirectories.
+  model=next((p for p in model if p.parent==trained),model[-1])
+  output=ROOT/('publish-'+name);output.mkdir(exist_ok=True)
+  shutil.copy2(model,output/(name+'.onnx'))
+  for p in trained.rglob('*.pt'):
+   if p.name=='checkpoint.pt':shutil.copy2(p,output/(name+'-checkpoint.pt'));break
+  shutil.copy2(config_path,output/'config.yaml');shutil.copy2(__file__,output/'train-unity.py')
+  logs=list(trained.rglob('events.out.tfevents*'))
+  if logs:shutil.copy2(logs[0],output/logs[0].name)
+  result=evaluate_onnx(output/(name+'.onnx'),executable,output/'results.json');card(output,name,env_tag,result,behavior)
+  shutil.make_archive(str(ROOT/('publish-'+name)),'zip',output)
+  results.append({'name':name,**{k:result[k] for k in ['mean_reward','std_reward','mean_minus_std','n_eval_agent_episodes']}})
+ (ROOT/'summary.json').write_text(json.dumps(results,indent=2));print('ALL_UNITY_TRAINING_COMPLETE',json.dumps(results),flush=True)
+if __name__=='__main__':main()