| [ | |
| { | |
| "name": "simllama-1-instruct", | |
| "type": "Instruct", | |
| "notes": "First from simllama series, trained on 200mb of data", | |
| "parameters": 137841408, | |
| "parameters_display": "137M", | |
| "ctx": 2048, | |
| "overall": 22.94, | |
| "knowledge": 31.81, | |
| "reasoning": 38.08, | |
| "instructions": 21.34, | |
| "coding": 0.0, | |
| "math": 1.29, | |
| "benchmarks": { | |
| "MMLU": 22.9, | |
| "HellaSwag": 26.67, | |
| "ARC-Challenge": 18.34, | |
| "ARC-Easy": 30.93, | |
| "BoolQ": 55.87, | |
| "PIQA": 56.15, | |
| "SciQ": 45.6, | |
| "Winogrande": 51.46, | |
| "CommonsenseQA": 19.98, | |
| "OpenBookQA": 12.8, | |
| "GSM8K": 1.29, | |
| "HumanEval": 0.0, | |
| "MBPP": 0.0, | |
| "IFEval loose": 22.06, | |
| "IFEval strict": 20.62, | |
| "TruthfulQA MC1": 28.52, | |
| "TruthfulQA MC2": 50.14 | |
| }, | |
| "hf": "https://huggingface.co/simonko912/simllama-1-instruct" | |
| }, | |
| { | |
| "name": "simllama-1.1-instruct-beta", | |
| "type": "Instruct", | |
| "notes": "Beta of simllama 1.1", | |
| "parameters": 435471360, | |
| "parameters_display": "435M", | |
| "ctx": 4096, | |
| "overall": 26.08, | |
| "knowledge": 37.18, | |
| "reasoning": 39.44, | |
| "instructions": 26.92, | |
| "coding": 0.0, | |
| "math": 1.97, | |
| "benchmarks": { | |
| "MMLU": 23.54, | |
| "HellaSwag": 28.57, | |
| "ARC-Challenge": 20.39, | |
| "ARC-Easy": 45.29, | |
| "BoolQ": 57.86, | |
| "PIQA": 59.68, | |
| "SciQ": 74.2, | |
| "Winogrande": 49.64, | |
| "CommonsenseQA": 20.48, | |
| "OpenBookQA": 15.0, | |
| "GSM8K": 1.97, | |
| "HumanEval": 0.0, | |
| "MBPP": 0.0, | |
| "IFEval loose": 28.54, | |
| "IFEval strict": 25.3, | |
| "TruthfulQA MC1": 22.64, | |
| "TruthfulQA MC2": 42.39 | |
| }, | |
| "hf": "https://huggingface.co/simonko912/simllama-1.1-instruct-beta" | |
| }, | |
| { | |
| "name": "simllama-1.1-base", | |
| "type": "Base", | |
| "notes": "Simllama 1.1 base model", | |
| "parameters": 435471360, | |
| "parameters_display": "435M", | |
| "ctx": 4096, | |
| "overall": 25.83, | |
| "knowledge": 38.75, | |
| "reasoning": 39.07, | |
| "instructions": 25.06, | |
| "coding": 0.0, | |
| "math": 1.14, | |
| "benchmarks": { | |
| "MMLU": 22.99, | |
| "HellaSwag": 28.24, | |
| "ARC-Challenge": 18.77, | |
| "ARC-Easy": 45.58, | |
| "BoolQ": 60.09, | |
| "PIQA": 58.81, | |
| "SciQ": 76.8, | |
| "Winogrande": 48.7, | |
| "CommonsenseQA": 19.82, | |
| "OpenBookQA": 18.8, | |
| "GSM8K": 1.14, | |
| "HumanEval": 0.0, | |
| "MBPP": 0.0, | |
| "IFEval loose": 25.3, | |
| "IFEval strict": 24.82, | |
| "TruthfulQA MC1": 23.62, | |
| "TruthfulQA MC2": 44.68 | |
| }, | |
| "hf": "https://huggingface.co/simonko912/simllama-1.1-base" | |
| }, | |
| { | |
| "name": "Colibri-Nano", | |
| "type": "MoE Instruct", | |
| "notes": "24 layers (1 dense + 23 MoE) 64 experts, SmolLM2 tokenizer", | |
| "parameters": 2700000000, | |
| "parameters_display": "2.7B", | |
| "ctx": 4096, | |
| "overall": 26.57, | |
| "knowledge": 36.7, | |
| "reasoning": 41.19, | |
| "instructions": 27.88, | |
| "coding": 0.0, | |
| "math": 1.29, | |
| "benchmarks": { | |
| "MMLU": 24.66, | |
| "HellaSwag": 31.11, | |
| "ARC-Challenge": 22.78, | |
| "ARC-Easy": 42.97, | |
| "BoolQ": 60.83, | |
| "PIQA": 62.35, | |
| "SciQ": 64.6, | |
| "Winogrande": 48.78, | |
| "CommonsenseQA": 21.31, | |
| "OpenBookQA": 21.4, | |
| "GSM8K": 1.29, | |
| "HumanEval": 0.0, | |
| "MBPP": 0.0, | |
| "IFEval loose": 28.3, | |
| "IFEval strict": 27.46, | |
| "TruthfulQA MC1": 24.72, | |
| "TruthfulQA MC2": 41.85 | |
| }, | |
| "hf": "https://huggingface.co/vovaRL/NanoColibri-Instruct" | |
| }, | |
| { | |
| "name": "Tinybrain-100m-instruct", | |
| "type": "Instruct", | |
| "notes": "LLaMA-style causal transformer", | |
| "parameters": 103385856, | |
| "parameters_display": "103M", | |
| "ctx": 2048, | |
| "overall": 25.3, | |
| "knowledge": 35.49, | |
| "reasoning": 40.14, | |
| "instructions": 24.82, | |
| "coding": 0.0, | |
| "math": 1.9, | |
| "benchmarks": { | |
| "MMLU": 24.13, | |
| "HellaSwag": 27.56, | |
| "ARC-Challenge": 19.71, | |
| "ARC-Easy": 42.55, | |
| "BoolQ": 60.37, | |
| "PIQA": 60.37, | |
| "SciQ": 64.6, | |
| "Winogrande": 51.78, | |
| "CommonsenseQA": 21.05, | |
| "OpenBookQA": 13.4, | |
| "GSM8K": 1.9, | |
| "HumanEval": 0.0, | |
| "MBPP": 0.0, | |
| "IFEval loose": 24.82, | |
| "IFEval strict": 24.82, | |
| "TruthfulQA MC1": 24.24, | |
| "TruthfulQA MC2": 44.01 | |
| }, | |
| "hf": "https://huggingface.co/exnivo/tinybrain-100m-instruct" | |
| } | |
| ] |