| { | |
| "report_scope": "Paper-reported six-round native/MLX results and separate bilingual deployment benchmark", | |
| "paper": { | |
| "title": "S²-MAS: A Society-Scale Multi-Agent Simulation Framework with 500 Million Household Persona Agents", | |
| "version": "S²-MAS manuscript", | |
| "date": "2026-10-20", | |
| "project_url": "https://github.com/KataDavidXD/s2-mas", | |
| "manuscript_url": "https://github.com/KataDavidXD/s2-mas/blob/main/paper/S2_MAS_Paper.pdf" | |
| }, | |
| "repository_role": "Six-round native/MLX runtime pair", | |
| "training": { | |
| "continuation_examples_per_language": 2925263, | |
| "paper_reference": "Supplementary Methods M6" | |
| }, | |
| "evaluation_protocol": { | |
| "shared_records": 240458, | |
| "shared_question_versions": 119, | |
| "expanded_records": 418597, | |
| "language": "English", | |
| "metric": "candidate-class question-macro F1", | |
| "paper_reference": "Supplementary Methods M4-M6" | |
| }, | |
| "native_mlx_evaluation": [ | |
| { | |
| "model": "Fin-Laya 6-round", | |
| "runtime": "PyTorch", | |
| "precision": "BF16", | |
| "shared_accuracy_pct": 74.5586, | |
| "shared_question_macro_f1_pct": 38.3986, | |
| "expanded_accuracy_pct": 67.2728, | |
| "expanded_question_macro_f1_pct": 27.8355 | |
| }, | |
| { | |
| "model": "Fin-Laya 6-round", | |
| "runtime": "MLX CUDA", | |
| "precision": "FP16", | |
| "shared_accuracy_pct": 74.5544, | |
| "shared_question_macro_f1_pct": 38.3746, | |
| "expanded_accuracy_pct": 67.2726, | |
| "expanded_question_macro_f1_pct": 27.8955 | |
| } | |
| ], | |
| "common_question_system_comparison": { | |
| "records": 240458, | |
| "question_versions": 119, | |
| "fin_laya_mlx_accuracy_pct": 74.5544, | |
| "fin_laya_mlx_question_macro_f1_pct": 38.3746, | |
| "jev_1_13_api_accuracy_pct": 65.2584, | |
| "jev_1_13_api_question_macro_f1_pct": 37.4793, | |
| "fin_laya_minus_jev_accuracy_percentage_points": 9.296, | |
| "paper_reference": "Supplementary Methods M5" | |
| }, | |
| "separate_bilingual_deployment_benchmark": { | |
| "checkpoint": "Bilingual Fin-Laya, multilingual-base adaptation", | |
| "measurement_date": "2026-10-11", | |
| "comparison_unit": "shared-input-token throughput on identical serialized inputs", | |
| "hardware": "Local arms: one NVIDIA A100 PCIe 40GB; Jev: provider-managed", | |
| "local_precision": "FP16", | |
| "fixed_chinese_inputs": 3, | |
| "timed_calls_per_arm": 60, | |
| "batch_size": 1, | |
| "concurrency": 1, | |
| "timing_interval": "Input preparation, tokenization, synchronized prediction and response formatting; Jev includes HTTPS/API round trip; loading excluded", | |
| "shared_tokenizer": "mmBERT", | |
| "mlx_to_jev_shared_input_throughput_ratio": 22.72, | |
| "mlx_to_same_checkpoint_pytorch_shared_input_throughput_ratio": 2.42, | |
| "results": [ | |
| { | |
| "model_or_checkpoint": "Bilingual Fin-Laya", | |
| "runtime": "PyTorch A100", | |
| "precision": "FP16", | |
| "mean_latency_ms": 22.06, | |
| "choices_per_s": 45.33, | |
| "shared_input_tokens_per_s": 15247.34 | |
| }, | |
| { | |
| "model_or_checkpoint": "Bilingual Fin-Laya", | |
| "runtime": "MLX CUDA A100", | |
| "precision": "FP16", | |
| "mean_latency_ms": 9.1, | |
| "choices_per_s": 109.88, | |
| "shared_input_tokens_per_s": 36955.47 | |
| }, | |
| { | |
| "model_or_checkpoint": "Jev 1.13", | |
| "runtime": "Provider API", | |
| "precision": "Provider-managed", | |
| "mean_latency_ms": 206.77, | |
| "choices_per_s": 4.84, | |
| "shared_input_tokens_per_s": 1626.63 | |
| } | |
| ], | |
| "paper_reference": "Supplementary Methods M9" | |
| } | |
| } | |