benchmarks/example_metrics.json

[
  {
    "id": "source-bf16-control",
    "title": "Source BF16 control — dynamic zoom and dialogue",
    "role": "source control",
    "task": "t2va",
    "seed": 74001,
    "variant": "controlled_bf16_ablation",
    "generation_seconds": 451.5754231447354,
    "decode_seconds": 19.563463208731264,
    "load_seconds": 1.5101884552277625,
    "cuda_generation_peak_bytes": 68463336960,
    "cuda_decode_peak_bytes": 12139359744,
    "offload": {
      "mode": "components_manager_auto_cpu_offload",
      "device": "cuda",
      "memory_reserve_margin": "64GB"
    },
    "quality_gate": {
      "status": "pass",
      "model_evaluations": 49,
      "reviewed_frame_count": 26
    },
    "manual_review": {
      "status": "pass",
      "review_type": "manual_visual_temporal",
      "reviewed_frame_indices": [
        0,
        5,
        10,
        15,
        20,
        25,
        30,
        35,
        40,
        45,
        50,
        55,
        60,
        65,
        70,
        75,
        80,
        85,
        90,
        95,
        100,
        105,
        110,
        115,
        120,
        123
      ],
      "prompt_subject_recognizable": true,
      "coherent_motion": true,
      "repeating_tile_artifacts": false,
      "motion_ghosting": false,
      "texture_breakup": false,
      "face_integrity": true,
      "full_resolution_frame_reviewed": true,
      "notes": "Dense every-fifth-frame and 6 FPS tail review preserves one face, aligned eyes and lips, and persistent wet-skin texture. Fast running frames contain ordinary motion blur that resolves in adjacent frames. This review is visual only; generated audio still requires a listening gate."
    },
    "h264_bytes": 635349,
    "hevc_bytes": 375788
  },
  {
    "id": "w4-text-control",
    "title": "OrbitQuant W4A4 text encoder, BF16 denoiser",
    "role": "component isolation control",
    "task": "t2va",
    "seed": 74001,
    "variant": "controlled_bf16_ablation",
    "generation_seconds": 381.5656377780251,
    "decode_seconds": 13.154935840982944,
    "load_seconds": 2.5672516343183815,
    "cuda_generation_peak_bytes": 68464447488,
    "cuda_decode_peak_bytes": 12139359744,
    "offload": {
      "mode": "components_manager_auto_cpu_offload",
      "device": "cuda",
      "memory_reserve_margin": "64GB"
    },
    "quality_gate": {
      "status": "pass",
      "model_evaluations": 49,
      "reviewed_frame_count": 26
    },
    "manual_review": {
      "status": "pass",
      "review_type": "manual_visual_temporal",
      "reviewed_frame_indices": [
        0,
        5,
        10,
        15,
        20,
        25,
        30,
        35,
        40,
        45,
        50,
        55,
        60,
        65,
        70,
        75,
        80,
        85,
        90,
        95,
        100,
        105,
        110,
        115,
        120,
        123
      ],
      "prompt_subject_recognizable": true,
      "coherent_motion": true,
      "repeating_tile_artifacts": false,
      "motion_ghosting": false,
      "texture_breakup": false,
      "face_integrity": true,
      "full_resolution_frame_reviewed": true,
      "audio_listening_status": "pass",
      "audio_reviewed_by": "user",
      "notes": "Dense every-fifth-frame and 6 FPS tail review preserves one face, aligned eyes and lips, and persistent wet-skin texture. Full-resolution frames 80 and 123 were inspected directly. The user reviewed the embedded video with audio and judged it ideal."
    },
    "h264_bytes": 753550,
    "hevc_bytes": 449021
  },
  {
    "id": "w4a4-native-t2va",
    "title": "Final native W4A4 text + W4A4 T2VA transformer",
    "role": "final native W4A4 T2VA recipe",
    "task": "t2va",
    "seed": 42,
    "variant": "orbitquant_w4a4",
    "generation_seconds": 114.6722432100214,
    "decode_seconds": 4.709521219134331,
    "load_seconds": 1.4625724367797375,
    "cuda_generation_peak_bytes": 46359644160,
    "cuda_decode_peak_bytes": 12139359744,
    "offload": {
      "mode": "manual_stage_offload",
      "conditioner": "cuda_then_cpu",
      "transformer": "cpu_then_cuda",
      "vae": "cpu",
      "audio_vae": "cpu"
    },
    "quality_gate": {
      "status": "pass",
      "model_evaluations": 49,
      "reviewed_frame_count": 19
    },
    "manual_review": {
      "status": "pass",
      "review_type": "manual_visual_temporal_and_audio_signal",
      "reviewed_frame_indices": [
        0,
        24,
        48,
        60,
        72,
        80,
        84,
        88,
        90,
        92,
        96,
        100,
        104,
        108,
        110,
        112,
        116,
        120,
        123
      ],
      "prompt_subject_recognizable": true,
      "coherent_motion": true,
      "repeating_tile_artifacts": false,
      "motion_ghosting": false,
      "texture_breakup": false,
      "face_integrity": true,
      "full_resolution_frame_reviewed": true,
      "audio_signal_integrity": true,
      "audio_user_listening_status": "pending",
      "notes": "Full-timeline and dense late-motion grids plus full-resolution frames 60, 90, 110, and 123 preserve one coherent face, aligned eyes, connected lip motion, teeth, rain, and skin texture without the earlier grid, doubled contour, ghosting, or breakup. The HEVC/AAC file decodes to 124 frames at 608x480. Audio has no NaN/Inf samples or clipping, peaks at -0.27 dBFS, and has no persistent high-frequency narrow-band tone; energy above 8 kHz is 1.42%, below the accepted W4A16 control's 2.99%."
    },
    "h264_bytes": 815089,
    "hevc_bytes": 474343
  }
]