| { |
| "schema_version": 1, |
| "description": "CogniRank model registry. Rows with evaluation_status=demo contain fictional scores for the UI prototype only. A verified leaderboard should use evaluated rows. Pending models can be submitted with scores set to null.", |
| "evaluation_weights": { |
| "smolcua": 0.2, |
| "mbpp_plus": 0.2, |
| "marc": 0.2, |
| "bbh": 0.2, |
| "ttm": 0.1, |
| "trip": 0.1 |
| }, |
| "models": [ |
| { |
| "id": "preview-01", |
| "name": "void.2", |
| "organization": "DotLabs", |
| "parameters": "98M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 72, |
| "mbpp_plus": 62, |
| "marc": 81, |
| "bbh": 68, |
| "ttm": 64, |
| "trip": 71 |
| } |
| }, |
| { |
| "id": "preview-02", |
| "name": "Qwen2.5-0.5B", |
| "organization": "Qwen", |
| "parameters": "500M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 76, |
| "mbpp_plus": 71, |
| "marc": 64, |
| "bbh": 73, |
| "ttm": 68, |
| "trip": 62 |
| } |
| }, |
| { |
| "id": "preview-03", |
| "name": "SmolLM2-360M", |
| "organization": "Hugging Face", |
| "parameters": "360M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 69, |
| "mbpp_plus": 67, |
| "marc": 66, |
| "bbh": 69, |
| "ttm": 67, |
| "trip": 65 |
| } |
| }, |
| { |
| "id": "preview-04", |
| "name": "SmolLM2-135M", |
| "organization": "Hugging Face", |
| "parameters": "135M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 63, |
| "mbpp_plus": 53, |
| "marc": 69, |
| "bbh": 59, |
| "ttm": 64, |
| "trip": 58 |
| } |
| }, |
| { |
| "id": "preview-05", |
| "name": "void.1", |
| "organization": "DotLabs", |
| "parameters": "90M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 58, |
| "mbpp_plus": 47, |
| "marc": 70, |
| "bbh": 62, |
| "ttm": 59, |
| "trip": 66 |
| } |
| }, |
| { |
| "id": "preview-06", |
| "name": "MobileLLM-125M", |
| "organization": "Meta", |
| "parameters": "125M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 60, |
| "mbpp_plus": 50, |
| "marc": 59, |
| "bbh": 60, |
| "ttm": 63, |
| "trip": 55 |
| } |
| }, |
| { |
| "id": "preview-07", |
| "name": "Pythia-160M", |
| "organization": "EleutherAI", |
| "parameters": "160M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 56, |
| "mbpp_plus": 49, |
| "marc": 61, |
| "bbh": 55, |
| "ttm": 55, |
| "trip": 58 |
| } |
| }, |
| { |
| "id": "preview-08", |
| "name": "SmolLM-135M", |
| "organization": "Hugging Face", |
| "parameters": "135M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 54, |
| "mbpp_plus": 44, |
| "marc": 61, |
| "bbh": 58, |
| "ttm": 58, |
| "trip": 53 |
| } |
| }, |
| { |
| "id": "preview-09", |
| "name": "RWKV-4-169M", |
| "organization": "RWKV", |
| "parameters": "169M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 56, |
| "mbpp_plus": 45, |
| "marc": 57, |
| "bbh": 55, |
| "ttm": 56, |
| "trip": 53 |
| } |
| }, |
| { |
| "id": "preview-10", |
| "name": "OPT-125M", |
| "organization": "Meta", |
| "parameters": "125M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 50, |
| "mbpp_plus": 45, |
| "marc": 54, |
| "bbh": 58, |
| "ttm": 54, |
| "trip": 50 |
| } |
| }, |
| { |
| "id": "preview-11", |
| "name": "GPT-2-124M", |
| "organization": "OpenAI", |
| "parameters": "124M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 46, |
| "mbpp_plus": 44, |
| "marc": 54, |
| "bbh": 56, |
| "ttm": 50, |
| "trip": 49 |
| } |
| }, |
| { |
| "id": "preview-12", |
| "name": "Pythia-70M", |
| "organization": "EleutherAI", |
| "parameters": "70M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 44, |
| "mbpp_plus": 39, |
| "marc": 49, |
| "bbh": 48, |
| "ttm": 52, |
| "trip": 47 |
| } |
| }, |
| { |
| "id": "preview-13", |
| "name": "TinyLlama-1.1B", |
| "organization": "TinyLlama", |
| "parameters": "1.1B", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 69, |
| "mbpp_plus": 61, |
| "marc": 63, |
| "bbh": 65, |
| "ttm": 62, |
| "trip": 61 |
| } |
| }, |
| { |
| "id": "preview-14", |
| "name": "StableLM-2-1.6B", |
| "organization": "Stability AI", |
| "parameters": "1.6B", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 73, |
| "mbpp_plus": 67, |
| "marc": 68, |
| "bbh": 70, |
| "ttm": 64, |
| "trip": 64 |
| } |
| }, |
| { |
| "id": "preview-15", |
| "name": "SmolLM2-1.7B", |
| "organization": "Hugging Face", |
| "parameters": "1.7B", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 78, |
| "mbpp_plus": 70, |
| "marc": 76, |
| "bbh": 74, |
| "ttm": 73, |
| "trip": 70 |
| } |
| }, |
| { |
| "id": "preview-16", |
| "name": "Qwen2.5-1.5B", |
| "organization": "Qwen", |
| "parameters": "1.5B", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 81, |
| "mbpp_plus": 78, |
| "marc": 77, |
| "bbh": 79, |
| "ttm": 74, |
| "trip": 73 |
| } |
| }, |
| { |
| "id": "preview-17", |
| "name": "Pythia-410M", |
| "organization": "EleutherAI", |
| "parameters": "410M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 64, |
| "mbpp_plus": 55, |
| "marc": 63, |
| "bbh": 61, |
| "ttm": 58, |
| "trip": 59 |
| } |
| }, |
| { |
| "id": "preview-18", |
| "name": "SmolLM-360M", |
| "organization": "Hugging Face", |
| "parameters": "360M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 62, |
| "mbpp_plus": 54, |
| "marc": 59, |
| "bbh": 59, |
| "ttm": 61, |
| "trip": 56 |
| } |
| }, |
| { |
| "id": "preview-19", |
| "name": "GPT-Neo-125M", |
| "organization": "EleutherAI", |
| "parameters": "125M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 49, |
| "mbpp_plus": 42, |
| "marc": 51, |
| "bbh": 51, |
| "ttm": 49, |
| "trip": 50 |
| } |
| }, |
| { |
| "id": "preview-20", |
| "name": "OPT-350M", |
| "organization": "Meta", |
| "parameters": "350M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 61, |
| "mbpp_plus": 50, |
| "marc": 59, |
| "bbh": 63, |
| "ttm": 61, |
| "trip": 55 |
| } |
| }, |
| { |
| "id": "preview-21", |
| "name": "RWKV-4-430M", |
| "organization": "RWKV", |
| "parameters": "430M", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 64, |
| "mbpp_plus": 55, |
| "marc": 57, |
| "bbh": 61, |
| "ttm": 60, |
| "trip": 59 |
| } |
| }, |
| { |
| "id": "preview-22", |
| "name": "Pythia-1B", |
| "organization": "EleutherAI", |
| "parameters": "1B", |
| "hf_repo": null, |
| "evaluation_status": "demo", |
| "scores": { |
| "smolcua": 70, |
| "mbpp_plus": 62, |
| "marc": 65, |
| "bbh": 69, |
| "ttm": 65, |
| "trip": 64 |
| } |
| } |
| ], |
| "submission_template": { |
| "id": "yourname-yourmodel", |
| "name": "Your Model", |
| "organization": "Your name or organization", |
| "parameters": "100M", |
| "hf_repo": "yourname/yourmodel", |
| "revision": "main", |
| "pretrained_by_submitter": true, |
| "pretraining_evidence_url": "https://huggingface.co/yourname/yourmodel", |
| "evaluation_status": "pending", |
| "scores": null |
| } |
| } |
|
|
