{
  "report_scope": "Paper-reported six-round native/MLX results and separate bilingual deployment benchmark",
  "paper": {
    "title": "S²-MAS: A Society-Scale Multi-Agent Simulation Framework with 500 Million Household Persona Agents",
    "version": "S²-MAS manuscript",
    "date": "2026-10-20",
    "project_url": "https://github.com/KataDavidXD/s2-mas",
    "manuscript_url": "https://github.com/KataDavidXD/s2-mas/blob/main/paper/S2_MAS_Paper.pdf"
  },
  "repository_role": "Six-round native/MLX runtime pair",
  "training": {
    "continuation_examples_per_language": 2925263,
    "paper_reference": "Supplementary Methods M6"
  },
  "evaluation_protocol": {
    "shared_records": 240458,
    "shared_question_versions": 119,
    "expanded_records": 418597,
    "language": "English",
    "metric": "candidate-class question-macro F1",
    "paper_reference": "Supplementary Methods M4-M6"
  },
  "native_mlx_evaluation": [
    {
      "model": "Fin-Laya 6-round",
      "runtime": "PyTorch",
      "precision": "BF16",
      "shared_accuracy_pct": 74.5586,
      "shared_question_macro_f1_pct": 38.3986,
      "expanded_accuracy_pct": 67.2728,
      "expanded_question_macro_f1_pct": 27.8355
    },
    {
      "model": "Fin-Laya 6-round",
      "runtime": "MLX CUDA",
      "precision": "FP16",
      "shared_accuracy_pct": 74.5544,
      "shared_question_macro_f1_pct": 38.3746,
      "expanded_accuracy_pct": 67.2726,
      "expanded_question_macro_f1_pct": 27.8955
    }
  ],
  "common_question_system_comparison": {
    "records": 240458,
    "question_versions": 119,
    "fin_laya_mlx_accuracy_pct": 74.5544,
    "fin_laya_mlx_question_macro_f1_pct": 38.3746,
    "jev_1_13_api_accuracy_pct": 65.2584,
    "jev_1_13_api_question_macro_f1_pct": 37.4793,
    "fin_laya_minus_jev_accuracy_percentage_points": 9.296,
    "paper_reference": "Supplementary Methods M5"
  },
  "separate_bilingual_deployment_benchmark": {
    "checkpoint": "Bilingual Fin-Laya, multilingual-base adaptation",
    "measurement_date": "2026-10-11",
    "comparison_unit": "shared-input-token throughput on identical serialized inputs",
    "hardware": "Local arms: one NVIDIA A100 PCIe 40GB; Jev: provider-managed",
    "local_precision": "FP16",
    "fixed_chinese_inputs": 3,
    "timed_calls_per_arm": 60,
    "batch_size": 1,
    "concurrency": 1,
    "timing_interval": "Input preparation, tokenization, synchronized prediction and response formatting; Jev includes HTTPS/API round trip; loading excluded",
    "shared_tokenizer": "mmBERT",
    "mlx_to_jev_shared_input_throughput_ratio": 22.72,
    "mlx_to_same_checkpoint_pytorch_shared_input_throughput_ratio": 2.42,
    "results": [
      {
        "model_or_checkpoint": "Bilingual Fin-Laya",
        "runtime": "PyTorch A100",
        "precision": "FP16",
        "mean_latency_ms": 22.06,
        "choices_per_s": 45.33,
        "shared_input_tokens_per_s": 15247.34
      },
      {
        "model_or_checkpoint": "Bilingual Fin-Laya",
        "runtime": "MLX CUDA A100",
        "precision": "FP16",
        "mean_latency_ms": 9.1,
        "choices_per_s": 109.88,
        "shared_input_tokens_per_s": 36955.47
      },
      {
        "model_or_checkpoint": "Jev 1.13",
        "runtime": "Provider API",
        "precision": "Provider-managed",
        "mean_latency_ms": 206.77,
        "choices_per_s": 4.84,
        "shared_input_tokens_per_s": 1626.63
      }
    ],
    "paper_reference": "Supplementary Methods M9"
  }
}