initial upload from H20 cluster: eval/20260527_000425

.gitattributes

@@ -34,3 +34,14 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 ppo_dir_v2_merged/tokenizer.json filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/20260527_000425/logs/eval_log.log filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/20260527_000425/predictions/dir-ppo-llama3.1-8b/race_high.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/20260527_000425/predictions/dir-ppo-llama3.1-8b/race_middle.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/20260527_000425/predictions/dir-ppo-llama3.1-8b/trivia_qa_rc.wikipedia.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/20260527_000425/reviews/dir-ppo-llama3.1-8b/race_high.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/20260527_000425/reviews/dir-ppo-llama3.1-8b/trivia_qa_rc.wikipedia.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/predictions/dir-ppo-llama3.1-8b/race_high.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/predictions/dir-ppo-llama3.1-8b/race_middle.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/predictions/dir-ppo-llama3.1-8b/trivia_qa_rc.wikipedia.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/reviews/dir-ppo-llama3.1-8b/race_high.jsonl filter=lfs diff=lfs merge=lfs -text
+eval/20260527_000425/reviews/dir-ppo-llama3.1-8b/trivia_qa_rc.wikipedia.jsonl filter=lfs diff=lfs merge=lfs -text

eval/20260527_000425/20260527_000425/configs/task_config.yaml

@@ -0,0 +1,381 @@
+agent_config: null
+analysis_report: false
+api_url: http://127.0.0.1:8801/v1
+chat_template: null
+collect_perf: true
+dataset_args:
+  humaneval:
+    aggregation: mean_and_pass_at_k
+    data_statistics: null
+    dataset_id: opencompass/humaneval
+    default_subset: default
+    description: '
+
+      ## Overview
+
+
+      HumanEval is a benchmark for evaluating the code generation capabilities of
+      language models. It consists of 164 hand-written Python programming problems
+      with function signatures, docstrings, and comprehensive test cases.
+
+
+      ## Task Description
+
+
+      - **Task Type**: Code Generation (Python)
+
+      - **Input**: Function signature with docstring describing the expected behavior
+
+      - **Output**: Complete Python function implementation
+
+      - **Languages**: Python only
+
+
+      ## Key Features
+
+
+      - 164 hand-crafted programming problems
+
+      - Each problem includes a function signature, docstring, and test cases
+
+      - Problems range from simple string manipulation to complex algorithms
+
+      - Canonical solutions provided for reference
+
+      - Automatic correctness verification through test execution
+
+
+      ## Evaluation Notes
+
+
+      - **Security Warning**: By default, code is executed in the local environment.
+      We strongly recommend using sandbox execution for safety. See the [sandbox documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html)
+      for details.
+
+      - Supports `pass@k` metric calculation for measuring generation quality
+
+      - Default timeout is 4 seconds per problem
+
+      - Code is extracted from markdown code blocks if present
+
+      '
+    eval_split: test
+    extra_params: {}
+    few_shot_num: 0
+    few_shot_prompt_template: null
+    few_shot_random: false
+    filters: null
+    force_redownload: false
+    metric_list:
+    - acc
+    name: humaneval
+    output_types:
+    - generation
+    paper_url: null
+    pretty_name: HumanEval
+    prompt_template: 'Read the following function signature and docstring, and fully
+      implement the function described. Your response should only contain the code
+      for this function.
+
+      {question}'
+    query_template: null
+    review_timeout: 4
+    sample_example: null
+    sandbox_config:
+      image: python:3.11-slim
+      tools_config:
+        python_executor: {}
+        shell_executor: {}
+    shuffle: false
+    shuffle_choices: false
+    subset_list:
+    - openai_humaneval
+    system_prompt: null
+    tags:
+    - Coding
+    train_split: null
+  ifeval:
+    aggregation: mean
+    data_statistics: null
+    dataset_id: opencompass/ifeval
+    default_subset: default
+    description: "\n## Overview\n\nIFEval (Instruction-Following Eval) is a benchmark\
+      \ for evaluating how well language models follow explicit, verifiable instructions.\
+      \ It contains prompts with specific formatting, content, or structural requirements\
+      \ that can be objectively verified.\n\n## Task Description\n\n- **Task Type**:\
+      \ Instruction Following Evaluation\n- **Input**: Prompts with explicit, verifiable\
+      \ constraints\n- **Output**: Response that follows all specified instructions\n\
+      - **Constraint Types**: Format, length, keywords, structure, etc.\n\n## Key\
+      \ Features\n\n- ~500 prompts with 25 types of verifiable instructions\n- Instructions\
+      \ are objectively checkable (not subjective)\n- Examples: \"write exactly 3\
+      \ paragraphs\", \"include the word X\", \"use bullet points\"\n- Tests instruction\
+      \ comprehension and compliance\n- No ambiguity in evaluation criteria\n\n##\
+      \ Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n-\
+      \ Four metrics available:\n  - `prompt_level_strict`: All instructions in prompt\
+      \ must be followed\n  - `prompt_level_loose`: Some tolerance for minor deviations\n\
+      \  - `inst_level_strict`: Per-instruction accuracy (strict)\n  - `inst_level_loose`:\
+      \ Per-instruction accuracy (loose)\n- `prompt_level_strict` is the primary metric\n\
+      - Automatic verification of instruction compliance\n"
+    eval_split: train
+    extra_params: {}
+    few_shot_num: 0
+    few_shot_prompt_template: null
+    few_shot_random: false
+    filters: null
+    force_redownload: false
+    metric_list:
+    - prompt_level_strict
+    - inst_level_strict
+    - prompt_level_loose
+    - inst_level_loose
+    name: ifeval
+    output_types:
+    - generation
+    paper_url: null
+    pretty_name: IFEval
+    prompt_template: ''
+    query_template: null
+    review_timeout: null
+    sample_example: null
+    sandbox_config: {}
+    shuffle: false
+    shuffle_choices: false
+    subset_list:
+    - default
+    system_prompt: null
+    tags:
+    - InstructionFollowing
+    train_split: null
+  race:
+    aggregation: mean
+    data_statistics: null
+    dataset_id: evalscope/race
+    default_subset: default
+    description: '
+
+      ## Overview
+
+
+      RACE (ReAding Comprehension from Examinations) is a large-scale reading comprehension
+      benchmark collected from Chinese middle school and high school English examinations.
+      It tests comprehensive reading comprehension abilities.
+
+
+      ## Task Description
+
+
+      - **Task Type**: Reading Comprehension (Multiple-Choice)
+
+      - **Input**: Article passage with question and 4 answer choices
+
+      - **Output**: Correct answer letter (A, B, C, or D)
+
+      - **Difficulty Levels**: Middle school and High school
+
+
+      ## Key Features
+
+
+      - 28,000+ passages with 100,000 questions
+
+      - Real examination questions for authentic difficulty
+
+      - Two subsets: middle (easier) and high (harder)
+
+      - Tests various comprehension skills (inference, vocabulary, main idea, etc.)
+
+      - Diverse article topics and question types
+
+
+      ## Evaluation Notes
+
+
+      - Default configuration uses **3-shot** examples
+
+      - Maximum few-shot number is 3 (context length consideration)
+
+      - Uses Chain-of-Thought (CoT) prompting
+
+      - Two subsets available: `high` and `middle`
+
+      - Evaluates on test split
+
+      '
+    eval_split: test
+    extra_params: {}
+    few_shot_num: 3
+    few_shot_prompt_template: null
+    few_shot_random: false
+    filters: null
+    force_redownload: false
+    metric_list:
+    - acc
+    name: race
+    output_types:
+    - generation
+    paper_url: null
+    pretty_name: RACE
+    prompt_template: 'Answer the following multiple choice question. The last line
+      of your response should be of the following format: ''ANSWER: [LETTER]'' (without
+      quotes) where [LETTER] is one of {letters}. Think step by step before answering.
+
+
+      {question}
+
+
+      {choices}'
+    query_template: null
+    review_timeout: null
+    sample_example: null
+    sandbox_config: {}
+    shuffle: false
+    shuffle_choices: false
+    subset_list:
+    - high
+    - middle
+    system_prompt: null
+    tags:
+    - Reasoning
+    - MCQ
+    train_split: train
+  trivia_qa:
+    aggregation: mean
+    data_statistics: null
+    dataset_id: evalscope/trivia_qa
+    default_subset: default
+    description: '
+
+      ## Overview
+
+
+      TriviaQA is a large-scale reading comprehension dataset containing over 650K
+      question-answer-evidence triples. Questions are collected from trivia enthusiast
+      websites and paired with Wikipedia articles as evidence documents.
+
+
+      ## Task Description
+
+
+      - **Task Type**: Reading Comprehension / Question Answering
+
+      - **Input**: Question with Wikipedia context passage
+
+      - **Output**: Answer extracted or generated from context
+
+      - **Domain**: General knowledge trivia questions
+
+
+      ## Key Features
+
+
+      - 650K+ question-answer-evidence triples
+
+      - Questions written by trivia enthusiasts (naturally challenging)
+
+      - Multiple valid answer aliases for flexible evaluation
+
+      - Wikipedia articles provide evidence passages
+
+      - Tests both reading comprehension and knowledge retrieval
+
+
+      ## Evaluation Notes
+
+
+      - Default configuration uses **0-shot** evaluation
+
+      - Uses the Wikipedia reading comprehension subset (rc.wikipedia)
+
+      - Answers should follow the format: "ANSWER: [ANSWER]"
+
+      - Supports inclusion-based matching for answer comparison
+
+      - Evaluates on validation split
+
+      '
+    eval_split: validation
+    extra_params: {}
+    few_shot_num: 0
+    few_shot_prompt_template: null
+    few_shot_random: false
+    filters: null
+    force_redownload: false
+    metric_list:
+    - acc:
+        allow_inclusion: true
+    name: trivia_qa
+    output_types:
+    - generation
+    paper_url: null
+    pretty_name: TriviaQA
+    prompt_template: 'Read the content and answer the following question.
+
+
+      Content: {content}
+
+
+      Question: {question}
+
+
+      Keep your The last line of your response should be of the form "ANSWER: [ANSWER]"
+      (without quotes) where [ANSWER] is the answer to the problem.
+
+      '
+    query_template: null
+    review_timeout: null
+    sample_example: null
+    sandbox_config: {}
+    shuffle: false
+    shuffle_choices: false
+    subset_list:
+    - rc.wikipedia
+    system_prompt: null
+    tags:
+    - QA
+    - ReadingComprehension
+    train_split: null
+dataset_dir: /root/.cache/modelscope/hub/datasets
+dataset_hub: modelscope
+datasets:
+- ifeval
+- race
+- trivia_qa
+- humaneval
+debug: false
+enable_progress_tracker: false
+eval_backend: Native
+eval_batch_size: 16
+eval_config: null
+eval_type: server
+evalscope_version: 1.7.1
+generation_config:
+  batch_size: 16
+  do_sample: true
+  max_new_tokens: 2048
+  temperature: 0.7
+ignore_errors: true
+judge_model_args: {}
+judge_strategy: auto
+judge_worker_num: 1
+limit: null
+model: dir-ppo-llama3.1-8b
+model_args: {}
+model_id: dir-ppo-llama3.1-8b
+model_task: text_generation
+no_timestamp: false
+repeats: 1
+rerun_review: false
+sandbox:
+  default_config: {}
+  enabled: false
+  engine: docker
+  manager_config: {}
+  pool_size: null
+sandbox_manager_config: {}
+sandbox_type: docker
+seed: 42
+stream: null
+timeout: null
+use_cache: null
+use_sandbox: false
+work_dir: ./outputs/20260527_000425

eval/20260527_000425/20260527_000425/logs/eval_log.log

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b4e24adb27f4e9824f5bd4b5f70d41c417786bbf23c8b36174a21befca1c0252
+size 26273843

eval/20260527_000425/20260527_000425/predictions/dir-ppo-llama3.1-8b/race_high.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d595bed6f072c3ef9ed0e19010e34b124818146eb724a04531238384b16fe08e
+size 40497772

eval/20260527_000425/20260527_000425/predictions/dir-ppo-llama3.1-8b/race_middle.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0ea7aff3ae009236736a2c2ca8c02a74eedf3b6f87b4091de2918a67bca719c
+size 10885732

eval/20260527_000425/20260527_000425/predictions/dir-ppo-llama3.1-8b/trivia_qa_rc.wikipedia.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0decfb092206f175111431550b702003521ce5ce21f4555ef86198a500d5f83
+size 125325681

eval/20260527_000425/20260527_000425/reports/dir-ppo-llama3.1-8b/humaneval.json

@@ -0,0 +1,116 @@
+{
+    "name": "dir-ppo-llama3.1-8b@humaneval",
+    "dataset_name": "humaneval",
+    "dataset_pretty_name": "HumanEval",
+    "dataset_description": "\n## Overview\n\nHumanEval is a benchmark for evaluating the code generation capabilities of language models. It consists of 164 hand-written Python programming problems with function signatures, docstrings, and comprehensive test cases.\n\n## Task Description\n\n- **Task Type**: Code Generation (Python)\n- **Input**: Function signature with docstring describing the expected behavior\n- **Output**: Complete Python function implementation\n- **Languages**: Python only\n\n## Key Features\n\n- 164 hand-crafted programming problems\n- Each problem includes a function signature, docstring, and test cases\n- Problems range from simple string manipulation to complex algorithms\n- Canonical solutions provided for reference\n- Automatic correctness verification through test execution\n\n## Evaluation Notes\n\n- **Security Warning**: By default, code is executed in the local environment. We strongly recommend using sandbox execution for safety. See the [sandbox documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for details.\n- Supports `pass@k` metric calculation for measuring generation quality\n- Default timeout is 4 seconds per problem\n- Code is extracted from markdown code blocks if present\n",
+    "model_name": "dir-ppo-llama3.1-8b",
+    "score": 0.6768,
+    "metrics": [
+        {
+            "name": "mean_acc",
+            "num": 164,
+            "score": 0.6768,
+            "macro_score": 0.6768,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 164,
+                    "score": 0.6768,
+                    "macro_score": 0.6768,
+                    "subsets": [
+                        {
+                            "name": "openai_humaneval",
+                            "score": 0.6768,
+                            "num": 164
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "name": "mean_acc_pass@1",
+            "num": 164,
+            "score": 0.6768,
+            "macro_score": 0.6768,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 164,
+                    "score": 0.6768,
+                    "macro_score": 0.6768,
+                    "subsets": [
+                        {
+                            "name": "openai_humaneval",
+                            "score": 0.6768,
+                            "num": 164
+                        }
+                    ]
+                }
+            ]
+        }
+    ],
+    "analysis": "N/A",
+    "perf_metrics": {
+        "summary": {
+            "n_samples": 164,
+            "latency": {
+                "mean": 2.326436,
+                "std": 1.460745,
+                "min": 0.40422,
+                "25%": 1.377736,
+                "50%": 2.034455,
+                "75%": 2.972601,
+                "90%": 3.901483,
+                "99%": 7.0451,
+                "max": 10.605742
+            },
+            "throughput": {
+                "avg_output_tps": 56.83,
+                "avg_req_ps": 0.4298
+            },
+            "usage": {
+                "input_tokens": {
+                    "mean": 192.426829,
+                    "std": 64.30068,
+                    "min": 99.0,
+                    "25%": 148.0,
+                    "50%": 178.0,
+                    "75%": 215.75,
+                    "90%": 286.7,
+                    "99%": 372.07,
+                    "max": 452.0
+                },
+                "output_tokens": {
+                    "mean": 132.207317,
+                    "std": 87.969283,
+                    "min": 26.0,
+                    "25%": 68.75,
+                    "50%": 118.0,
+                    "75%": 169.25,
+                    "90%": 221.7,
+                    "99%": 441.16,
+                    "max": 638.0
+                },
+                "total_tokens": {
+                    "mean": 324.634146,
+                    "std": 133.096811,
+                    "min": 146.0,
+                    "25%": 232.75,
+                    "50%": 298.0,
+                    "75%": 371.5,
+                    "90%": 485.8,
+                    "99%": 760.69,
+                    "max": 1090.0
+                },
+                "total_input_tokens": 31558,
+                "total_output_tokens": 21682,
+                "total_tokens_count": 53240
+            }
+        }
+    },
+    "num": 164
+}

eval/20260527_000425/20260527_000425/reports/dir-ppo-llama3.1-8b/ifeval.json

@@ -0,0 +1,162 @@
+{
+    "name": "dir-ppo-llama3.1-8b@ifeval",
+    "dataset_name": "ifeval",
+    "dataset_pretty_name": "IFEval",
+    "dataset_description": "\n## Overview\n\nIFEval (Instruction-Following Eval) is a benchmark for evaluating how well language models follow explicit, verifiable instructions. It contains prompts with specific formatting, content, or structural requirements that can be objectively verified.\n\n## Task Description\n\n- **Task Type**: Instruction Following Evaluation\n- **Input**: Prompts with explicit, verifiable constraints\n- **Output**: Response that follows all specified instructions\n- **Constraint Types**: Format, length, keywords, structure, etc.\n\n## Key Features\n\n- ~500 prompts with 25 types of verifiable instructions\n- Instructions are objectively checkable (not subjective)\n- Examples: \"write exactly 3 paragraphs\", \"include the word X\", \"use bullet points\"\n- Tests instruction comprehension and compliance\n- No ambiguity in evaluation criteria\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Four metrics available:\n  - `prompt_level_strict`: All instructions in prompt must be followed\n  - `prompt_level_loose`: Some tolerance for minor deviations\n  - `inst_level_strict`: Per-instruction accuracy (strict)\n  - `inst_level_loose`: Per-instruction accuracy (loose)\n- `prompt_level_strict` is the primary metric\n- Automatic verification of instruction compliance\n",
+    "model_name": "dir-ppo-llama3.1-8b",
+    "score": 0.6981,
+    "metrics": [
+        {
+            "name": "mean_prompt_level_strict",
+            "num": 540,
+            "score": 0.6981,
+            "macro_score": 0.6981,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 540,
+                    "score": 0.6981,
+                    "macro_score": 0.6981,
+                    "subsets": [
+                        {
+                            "name": "default",
+                            "score": 0.6981,
+                            "num": 540
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "name": "mean_inst_level_strict",
+            "num": 540,
+            "score": 0.7901,
+            "macro_score": 0.7901,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 540,
+                    "score": 0.7901,
+                    "macro_score": 0.7901,
+                    "subsets": [
+                        {
+                            "name": "default",
+                            "score": 0.7901,
+                            "num": 540
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "name": "mean_prompt_level_loose",
+            "num": 540,
+            "score": 0.787,
+            "macro_score": 0.787,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 540,
+                    "score": 0.787,
+                    "macro_score": 0.787,
+                    "subsets": [
+                        {
+                            "name": "default",
+                            "score": 0.787,
+                            "num": 540
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "name": "mean_inst_level_loose",
+            "num": 540,
+            "score": 0.8623,
+            "macro_score": 0.8623,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 540,
+                    "score": 0.8623,
+                    "macro_score": 0.8623,
+                    "subsets": [
+                        {
+                            "name": "default",
+                            "score": 0.8623,
+                            "num": 540
+                        }
+                    ]
+                }
+            ]
+        }
+    ],
+    "analysis": "N/A",
+    "perf_metrics": {
+        "summary": {
+            "n_samples": 541,
+            "latency": {
+                "mean": 7.710747,
+                "std": 16.563807,
+                "min": 0.066271,
+                "25%": 2.457928,
+                "50%": 5.100689,
+                "75%": 8.237214,
+                "90%": 12.235768,
+                "99%": 107.019331,
+                "max": 160.99844
+            },
+            "throughput": {
+                "avg_output_tps": 52.84,
+                "avg_req_ps": 0.1297
+            },
+            "usage": {
+                "input_tokens": {
+                    "mean": 80.924214,
+                    "std": 23.92573,
+                    "min": 48.0,
+                    "25%": 67.0,
+                    "50%": 76.0,
+                    "75%": 90.0,
+                    "90%": 107.0,
+                    "99%": 135.6,
+                    "max": 395.0
+                },
+                "output_tokens": {
+                    "mean": 407.428835,
+                    "std": 862.651879,
+                    "min": 3.0,
+                    "25%": 127.0,
+                    "50%": 265.0,
+                    "75%": 436.0,
+                    "90%": 642.0,
+                    "99%": 5846.8,
+                    "max": 8138.0
+                },
+                "total_tokens": {
+                    "mean": 488.35305,
+                    "std": 861.312936,
+                    "min": 69.0,
+                    "25%": 212.0,
+                    "50%": 346.0,
+                    "75%": 521.0,
+                    "90%": 725.0,
+                    "99%": 5930.8,
+                    "max": 8192.0
+                },
+                "total_input_tokens": 43780,
+                "total_output_tokens": 220419,
+                "total_tokens_count": 264199
+            }
+        }
+    },
+    "num": 540
+}

eval/20260527_000425/20260527_000425/reports/dir-ppo-llama3.1-8b/race.json

@@ -0,0 +1,98 @@
+{
+    "name": "dir-ppo-llama3.1-8b@race",
+    "dataset_name": "race",
+    "dataset_pretty_name": "RACE",
+    "dataset_description": "\n## Overview\n\nRACE (ReAding Comprehension from Examinations) is a large-scale reading comprehension benchmark collected from Chinese middle school and high school English examinations. It tests comprehensive reading comprehension abilities.\n\n## Task Description\n\n- **Task Type**: Reading Comprehension (Multiple-Choice)\n- **Input**: Article passage with question and 4 answer choices\n- **Output**: Correct answer letter (A, B, C, or D)\n- **Difficulty Levels**: Middle school and High school\n\n## Key Features\n\n- 28,000+ passages with 100,000 questions\n- Real examination questions for authentic difficulty\n- Two subsets: middle (easier) and high (harder)\n- Tests various comprehension skills (inference, vocabulary, main idea, etc.)\n- Diverse article topics and question types\n\n## Evaluation Notes\n\n- Default configuration uses **3-shot** examples\n- Maximum few-shot number is 3 (context length consideration)\n- Uses Chain-of-Thought (CoT) prompting\n- Two subsets available: `high` and `middle`\n- Evaluates on test split\n",
+    "model_name": "dir-ppo-llama3.1-8b",
+    "score": 0.833,
+    "metrics": [
+        {
+            "name": "mean_acc",
+            "num": 4934,
+            "score": 0.833,
+            "macro_score": 0.833,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 4934,
+                    "score": 0.833,
+                    "macro_score": 0.8433,
+                    "subsets": [
+                        {
+                            "name": "high",
+                            "score": 0.8188,
+                            "num": 3498
+                        },
+                        {
+                            "name": "middle",
+                            "score": 0.8677,
+                            "num": 1436
+                        }
+                    ]
+                }
+            ]
+        }
+    ],
+    "analysis": "N/A",
+    "perf_metrics": {
+        "summary": {
+            "n_samples": 4934,
+            "latency": {
+                "mean": 3.518324,
+                "std": 4.350995,
+                "min": 0.375683,
+                "25%": 2.235433,
+                "50%": 3.250558,
+                "75%": 4.360585,
+                "90%": 5.39817,
+                "99%": 7.618212,
+                "max": 139.063051
+            },
+            "throughput": {
+                "avg_output_tps": 51.33,
+                "avg_req_ps": 0.2842
+            },
+            "usage": {
+                "input_tokens": {
+                    "mean": 1668.64471,
+                    "std": 334.066106,
+                    "min": 958.0,
+                    "25%": 1274.0,
+                    "50%": 1824.0,
+                    "75%": 1885.0,
+                    "90%": 1953.0,
+                    "99%": 2269.67,
+                    "max": 2584.0
+                },
+                "output_tokens": {
+                    "mean": 180.605188,
+                    "std": 216.808972,
+                    "min": 20.0,
+                    "25%": 114.0,
+                    "50%": 168.0,
+                    "75%": 228.0,
+                    "90%": 274.0,
+                    "99%": 387.01,
+                    "max": 6932.0
+                },
+                "total_tokens": {
+                    "mean": 1849.249899,
+                    "std": 410.659869,
+                    "min": 1028.0,
+                    "25%": 1479.0,
+                    "50%": 1975.0,
+                    "75%": 2084.0,
+                    "90%": 2180.0,
+                    "99%": 2527.39,
+                    "max": 8192.0
+                },
+                "total_input_tokens": 8233093,
+                "total_output_tokens": 891106,
+                "total_tokens_count": 9124199
+            }
+        }
+    },
+    "num": 4934
+}

eval/20260527_000425/20260527_000425/reports/dir-ppo-llama3.1-8b/trivia_qa.json

@@ -0,0 +1,93 @@
+{
+    "name": "dir-ppo-llama3.1-8b@trivia_qa",
+    "dataset_name": "trivia_qa",
+    "dataset_pretty_name": "TriviaQA",
+    "dataset_description": "\n## Overview\n\nTriviaQA is a large-scale reading comprehension dataset containing over 650K question-answer-evidence triples. Questions are collected from trivia enthusiast websites and paired with Wikipedia articles as evidence documents.\n\n## Task Description\n\n- **Task Type**: Reading Comprehension / Question Answering\n- **Input**: Question with Wikipedia context passage\n- **Output**: Answer extracted or generated from context\n- **Domain**: General knowledge trivia questions\n\n## Key Features\n\n- 650K+ question-answer-evidence triples\n- Questions written by trivia enthusiasts (naturally challenging)\n- Multiple valid answer aliases for flexible evaluation\n- Wikipedia articles provide evidence passages\n- Tests both reading comprehension and knowledge retrieval\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Uses the Wikipedia reading comprehension subset (rc.wikipedia)\n- Answers should follow the format: \"ANSWER: [ANSWER]\"\n- Supports inclusion-based matching for answer comparison\n- Evaluates on validation split\n",
+    "model_name": "dir-ppo-llama3.1-8b",
+    "score": 0.0407,
+    "metrics": [
+        {
+            "name": "mean_acc",
+            "num": 3689,
+            "score": 0.0407,
+            "macro_score": 0.0407,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 3689,
+                    "score": 0.0407,
+                    "macro_score": 0.0407,
+                    "subsets": [
+                        {
+                            "name": "rc.wikipedia",
+                            "score": 0.0407,
+                            "num": 3689
+                        }
+                    ]
+                }
+            ]
+        }
+    ],
+    "analysis": "N/A",
+    "perf_metrics": {
+        "summary": {
+            "n_samples": 3689,
+            "latency": {
+                "mean": 1.375701,
+                "std": 4.526989,
+                "min": 0.115312,
+                "25%": 0.658521,
+                "50%": 0.914985,
+                "75%": 1.307014,
+                "90%": 1.959365,
+                "99%": 5.197951,
+                "max": 113.092732
+            },
+            "throughput": {
+                "avg_output_tps": 42.44,
+                "avg_req_ps": 0.7269
+            },
+            "usage": {
+                "input_tokens": {
+                    "mean": 3750.501762,
+                    "std": 2268.520724,
+                    "min": 116.0,
+                    "25%": 1732.0,
+                    "50%": 3504.0,
+                    "75%": 5585.0,
+                    "90%": 7181.6,
+                    "99%": 8087.0,
+                    "max": 8190.0
+                },
+                "output_tokens": {
+                    "mean": 58.387097,
+                    "std": 291.002579,
+                    "min": 2.0,
+                    "25%": 21.0,
+                    "50%": 30.0,
+                    "75%": 45.0,
+                    "90%": 83.0,
+                    "99%": 286.36,
+                    "max": 7337.0
+                },
+                "total_tokens": {
+                    "mean": 3808.888859,
+                    "std": 2283.354375,
+                    "min": 134.0,
+                    "25%": 1788.0,
+                    "50%": 3565.0,
+                    "75%": 5643.0,
+                    "90%": 7250.4,
+                    "99%": 8175.48,
+                    "max": 8192.0
+                },
+                "total_input_tokens": 13835601,
+                "total_output_tokens": 215390,
+                "total_tokens_count": 14050991
+            }
+        }
+    },
+    "num": 3689
+}

eval/20260527_000425/20260527_000425/reviews/dir-ppo-llama3.1-8b/race_high.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bca1ce5a3af7d111c3e98c1d4268a70b41e6a5b06d9d43dd98169389cf01a46
+size 38411680

eval/20260527_000425/20260527_000425/reviews/dir-ppo-llama3.1-8b/trivia_qa_rc.wikipedia.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7a4814f193d0768bd1440bb673f997cc476dd2ff24d0e31317483a2f69ae12e
+size 124882928

eval/20260527_000425/configs/task_config.yaml

@@ -0,0 +1,381 @@
+agent_config: null
+analysis_report: false
+api_url: http://127.0.0.1:8801/v1
+chat_template: null
+collect_perf: true
+dataset_args:
+  humaneval:
+    aggregation: mean_and_pass_at_k
+    data_statistics: null
+    dataset_id: opencompass/humaneval
+    default_subset: default
+    description: '
+
+      ## Overview
+
+
+      HumanEval is a benchmark for evaluating the code generation capabilities of
+      language models. It consists of 164 hand-written Python programming problems
+      with function signatures, docstrings, and comprehensive test cases.
+
+
+      ## Task Description
+
+
+      - **Task Type**: Code Generation (Python)
+
+      - **Input**: Function signature with docstring describing the expected behavior
+
+      - **Output**: Complete Python function implementation
+
+      - **Languages**: Python only
+
+
+      ## Key Features
+
+
+      - 164 hand-crafted programming problems
+
+      - Each problem includes a function signature, docstring, and test cases
+
+      - Problems range from simple string manipulation to complex algorithms
+
+      - Canonical solutions provided for reference
+
+      - Automatic correctness verification through test execution
+
+
+      ## Evaluation Notes
+
+
+      - **Security Warning**: By default, code is executed in the local environment.
+      We strongly recommend using sandbox execution for safety. See the [sandbox documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html)
+      for details.
+
+      - Supports `pass@k` metric calculation for measuring generation quality
+
+      - Default timeout is 4 seconds per problem
+
+      - Code is extracted from markdown code blocks if present
+
+      '
+    eval_split: test
+    extra_params: {}
+    few_shot_num: 0
+    few_shot_prompt_template: null
+    few_shot_random: false
+    filters: null
+    force_redownload: false
+    metric_list:
+    - acc
+    name: humaneval
+    output_types:
+    - generation
+    paper_url: null
+    pretty_name: HumanEval
+    prompt_template: 'Read the following function signature and docstring, and fully
+      implement the function described. Your response should only contain the code
+      for this function.
+
+      {question}'
+    query_template: null
+    review_timeout: 4
+    sample_example: null
+    sandbox_config:
+      image: python:3.11-slim
+      tools_config:
+        python_executor: {}
+        shell_executor: {}
+    shuffle: false
+    shuffle_choices: false
+    subset_list:
+    - openai_humaneval
+    system_prompt: null
+    tags:
+    - Coding
+    train_split: null
+  ifeval:
+    aggregation: mean
+    data_statistics: null
+    dataset_id: opencompass/ifeval
+    default_subset: default
+    description: "\n## Overview\n\nIFEval (Instruction-Following Eval) is a benchmark\
+      \ for evaluating how well language models follow explicit, verifiable instructions.\
+      \ It contains prompts with specific formatting, content, or structural requirements\
+      \ that can be objectively verified.\n\n## Task Description\n\n- **Task Type**:\
+      \ Instruction Following Evaluation\n- **Input**: Prompts with explicit, verifiable\
+      \ constraints\n- **Output**: Response that follows all specified instructions\n\
+      - **Constraint Types**: Format, length, keywords, structure, etc.\n\n## Key\
+      \ Features\n\n- ~500 prompts with 25 types of verifiable instructions\n- Instructions\
+      \ are objectively checkable (not subjective)\n- Examples: \"write exactly 3\
+      \ paragraphs\", \"include the word X\", \"use bullet points\"\n- Tests instruction\
+      \ comprehension and compliance\n- No ambiguity in evaluation criteria\n\n##\
+      \ Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n-\
+      \ Four metrics available:\n  - `prompt_level_strict`: All instructions in prompt\
+      \ must be followed\n  - `prompt_level_loose`: Some tolerance for minor deviations\n\
+      \  - `inst_level_strict`: Per-instruction accuracy (strict)\n  - `inst_level_loose`:\
+      \ Per-instruction accuracy (loose)\n- `prompt_level_strict` is the primary metric\n\
+      - Automatic verification of instruction compliance\n"
+    eval_split: train
+    extra_params: {}
+    few_shot_num: 0
+    few_shot_prompt_template: null
+    few_shot_random: false
+    filters: null
+    force_redownload: false
+    metric_list:
+    - prompt_level_strict
+    - inst_level_strict
+    - prompt_level_loose
+    - inst_level_loose
+    name: ifeval
+    output_types:
+    - generation
+    paper_url: null
+    pretty_name: IFEval
+    prompt_template: ''
+    query_template: null
+    review_timeout: null
+    sample_example: null
+    sandbox_config: {}
+    shuffle: false
+    shuffle_choices: false
+    subset_list:
+    - default
+    system_prompt: null
+    tags:
+    - InstructionFollowing
+    train_split: null
+  race:
+    aggregation: mean
+    data_statistics: null
+    dataset_id: evalscope/race
+    default_subset: default
+    description: '
+
+      ## Overview
+
+
+      RACE (ReAding Comprehension from Examinations) is a large-scale reading comprehension
+      benchmark collected from Chinese middle school and high school English examinations.
+      It tests comprehensive reading comprehension abilities.
+
+
+      ## Task Description
+
+
+      - **Task Type**: Reading Comprehension (Multiple-Choice)
+
+      - **Input**: Article passage with question and 4 answer choices
+
+      - **Output**: Correct answer letter (A, B, C, or D)
+
+      - **Difficulty Levels**: Middle school and High school
+
+
+      ## Key Features
+
+
+      - 28,000+ passages with 100,000 questions
+
+      - Real examination questions for authentic difficulty
+
+      - Two subsets: middle (easier) and high (harder)
+
+      - Tests various comprehension skills (inference, vocabulary, main idea, etc.)
+
+      - Diverse article topics and question types
+
+
+      ## Evaluation Notes
+
+
+      - Default configuration uses **3-shot** examples
+
+      - Maximum few-shot number is 3 (context length consideration)
+
+      - Uses Chain-of-Thought (CoT) prompting
+
+      - Two subsets available: `high` and `middle`
+
+      - Evaluates on test split
+
+      '
+    eval_split: test
+    extra_params: {}
+    few_shot_num: 3
+    few_shot_prompt_template: null
+    few_shot_random: false
+    filters: null
+    force_redownload: false
+    metric_list:
+    - acc
+    name: race
+    output_types:
+    - generation
+    paper_url: null
+    pretty_name: RACE
+    prompt_template: 'Answer the following multiple choice question. The last line
+      of your response should be of the following format: ''ANSWER: [LETTER]'' (without
+      quotes) where [LETTER] is one of {letters}. Think step by step before answering.
+
+
+      {question}
+
+
+      {choices}'
+    query_template: null
+    review_timeout: null
+    sample_example: null
+    sandbox_config: {}
+    shuffle: false
+    shuffle_choices: false
+    subset_list:
+    - high
+    - middle
+    system_prompt: null
+    tags:
+    - Reasoning
+    - MCQ
+    train_split: train
+  trivia_qa:
+    aggregation: mean
+    data_statistics: null
+    dataset_id: evalscope/trivia_qa
+    default_subset: default
+    description: '
+
+      ## Overview
+
+
+      TriviaQA is a large-scale reading comprehension dataset containing over 650K
+      question-answer-evidence triples. Questions are collected from trivia enthusiast
+      websites and paired with Wikipedia articles as evidence documents.
+
+
+      ## Task Description
+
+
+      - **Task Type**: Reading Comprehension / Question Answering
+
+      - **Input**: Question with Wikipedia context passage
+
+      - **Output**: Answer extracted or generated from context
+
+      - **Domain**: General knowledge trivia questions
+
+
+      ## Key Features
+
+
+      - 650K+ question-answer-evidence triples
+
+      - Questions written by trivia enthusiasts (naturally challenging)
+
+      - Multiple valid answer aliases for flexible evaluation
+
+      - Wikipedia articles provide evidence passages
+
+      - Tests both reading comprehension and knowledge retrieval
+
+
+      ## Evaluation Notes
+
+
+      - Default configuration uses **0-shot** evaluation
+
+      - Uses the Wikipedia reading comprehension subset (rc.wikipedia)
+
+      - Answers should follow the format: "ANSWER: [ANSWER]"
+
+      - Supports inclusion-based matching for answer comparison
+
+      - Evaluates on validation split
+
+      '
+    eval_split: validation
+    extra_params: {}
+    few_shot_num: 0
+    few_shot_prompt_template: null
+    few_shot_random: false
+    filters: null
+    force_redownload: false
+    metric_list:
+    - acc:
+        allow_inclusion: true
+    name: trivia_qa
+    output_types:
+    - generation
+    paper_url: null
+    pretty_name: TriviaQA
+    prompt_template: 'Read the content and answer the following question.
+
+
+      Content: {content}
+
+
+      Question: {question}
+
+
+      Keep your The last line of your response should be of the form "ANSWER: [ANSWER]"
+      (without quotes) where [ANSWER] is the answer to the problem.
+
+      '
+    query_template: null
+    review_timeout: null
+    sample_example: null
+    sandbox_config: {}
+    shuffle: false
+    shuffle_choices: false
+    subset_list:
+    - rc.wikipedia
+    system_prompt: null
+    tags:
+    - QA
+    - ReadingComprehension
+    train_split: null
+dataset_dir: /root/.cache/modelscope/hub/datasets
+dataset_hub: modelscope
+datasets:
+- ifeval
+- race
+- trivia_qa
+- humaneval
+debug: false
+enable_progress_tracker: false
+eval_backend: Native
+eval_batch_size: 16
+eval_config: null
+eval_type: server
+evalscope_version: 1.7.1
+generation_config:
+  batch_size: 16
+  do_sample: true
+  max_new_tokens: 2048
+  temperature: 0.7
+ignore_errors: true
+judge_model_args: {}
+judge_strategy: auto
+judge_worker_num: 1
+limit: null
+model: dir-ppo-llama3.1-8b
+model_args: {}
+model_id: dir-ppo-llama3.1-8b
+model_task: text_generation
+no_timestamp: false
+repeats: 1
+rerun_review: false
+sandbox:
+  default_config: {}
+  enabled: false
+  engine: docker
+  manager_config: {}
+  pool_size: null
+sandbox_manager_config: {}
+sandbox_type: docker
+seed: 42
+stream: null
+timeout: null
+use_cache: null
+use_sandbox: false
+work_dir: ./outputs/20260527_000425

eval/20260527_000425/predictions/dir-ppo-llama3.1-8b/race_high.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d595bed6f072c3ef9ed0e19010e34b124818146eb724a04531238384b16fe08e
+size 40497772

eval/20260527_000425/predictions/dir-ppo-llama3.1-8b/race_middle.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0ea7aff3ae009236736a2c2ca8c02a74eedf3b6f87b4091de2918a67bca719c
+size 10885732

eval/20260527_000425/predictions/dir-ppo-llama3.1-8b/trivia_qa_rc.wikipedia.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64d916009fd65f0108bed676b6b0d28785cae1a41b955cf7e466df9454a28f69
+size 46729079

eval/20260527_000425/reports/dir-ppo-llama3.1-8b/ifeval.json

@@ -0,0 +1,162 @@
+{
+    "name": "dir-ppo-llama3.1-8b@ifeval",
+    "dataset_name": "ifeval",
+    "dataset_pretty_name": "IFEval",
+    "dataset_description": "\n## Overview\n\nIFEval (Instruction-Following Eval) is a benchmark for evaluating how well language models follow explicit, verifiable instructions. It contains prompts with specific formatting, content, or structural requirements that can be objectively verified.\n\n## Task Description\n\n- **Task Type**: Instruction Following Evaluation\n- **Input**: Prompts with explicit, verifiable constraints\n- **Output**: Response that follows all specified instructions\n- **Constraint Types**: Format, length, keywords, structure, etc.\n\n## Key Features\n\n- ~500 prompts with 25 types of verifiable instructions\n- Instructions are objectively checkable (not subjective)\n- Examples: \"write exactly 3 paragraphs\", \"include the word X\", \"use bullet points\"\n- Tests instruction comprehension and compliance\n- No ambiguity in evaluation criteria\n\n## Evaluation Notes\n\n- Default configuration uses **0-shot** evaluation\n- Four metrics available:\n  - `prompt_level_strict`: All instructions in prompt must be followed\n  - `prompt_level_loose`: Some tolerance for minor deviations\n  - `inst_level_strict`: Per-instruction accuracy (strict)\n  - `inst_level_loose`: Per-instruction accuracy (loose)\n- `prompt_level_strict` is the primary metric\n- Automatic verification of instruction compliance\n",
+    "model_name": "dir-ppo-llama3.1-8b",
+    "score": 0.6981,
+    "metrics": [
+        {
+            "name": "mean_prompt_level_strict",
+            "num": 540,
+            "score": 0.6981,
+            "macro_score": 0.6981,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 540,
+                    "score": 0.6981,
+                    "macro_score": 0.6981,
+                    "subsets": [
+                        {
+                            "name": "default",
+                            "score": 0.6981,
+                            "num": 540
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "name": "mean_inst_level_strict",
+            "num": 540,
+            "score": 0.7901,
+            "macro_score": 0.7901,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 540,
+                    "score": 0.7901,
+                    "macro_score": 0.7901,
+                    "subsets": [
+                        {
+                            "name": "default",
+                            "score": 0.7901,
+                            "num": 540
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "name": "mean_prompt_level_loose",
+            "num": 540,
+            "score": 0.787,
+            "macro_score": 0.787,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 540,
+                    "score": 0.787,
+                    "macro_score": 0.787,
+                    "subsets": [
+                        {
+                            "name": "default",
+                            "score": 0.787,
+                            "num": 540
+                        }
+                    ]
+                }
+            ]
+        },
+        {
+            "name": "mean_inst_level_loose",
+            "num": 540,
+            "score": 0.8623,
+            "macro_score": 0.8623,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 540,
+                    "score": 0.8623,
+                    "macro_score": 0.8623,
+                    "subsets": [
+                        {
+                            "name": "default",
+                            "score": 0.8623,
+                            "num": 540
+                        }
+                    ]
+                }
+            ]
+        }
+    ],
+    "analysis": "N/A",
+    "perf_metrics": {
+        "summary": {
+            "n_samples": 541,
+            "latency": {
+                "mean": 7.710747,
+                "std": 16.563807,
+                "min": 0.066271,
+                "25%": 2.457928,
+                "50%": 5.100689,
+                "75%": 8.237214,
+                "90%": 12.235768,
+                "99%": 107.019331,
+                "max": 160.99844
+            },
+            "throughput": {
+                "avg_output_tps": 52.84,
+                "avg_req_ps": 0.1297
+            },
+            "usage": {
+                "input_tokens": {
+                    "mean": 80.924214,
+                    "std": 23.92573,
+                    "min": 48.0,
+                    "25%": 67.0,
+                    "50%": 76.0,
+                    "75%": 90.0,
+                    "90%": 107.0,
+                    "99%": 135.6,
+                    "max": 395.0
+                },
+                "output_tokens": {
+                    "mean": 407.428835,
+                    "std": 862.651879,
+                    "min": 3.0,
+                    "25%": 127.0,
+                    "50%": 265.0,
+                    "75%": 436.0,
+                    "90%": 642.0,
+                    "99%": 5846.8,
+                    "max": 8138.0
+                },
+                "total_tokens": {
+                    "mean": 488.35305,
+                    "std": 861.312936,
+                    "min": 69.0,
+                    "25%": 212.0,
+                    "50%": 346.0,
+                    "75%": 521.0,
+                    "90%": 725.0,
+                    "99%": 5930.8,
+                    "max": 8192.0
+                },
+                "total_input_tokens": 43780,
+                "total_output_tokens": 220419,
+                "total_tokens_count": 264199
+            }
+        }
+    },
+    "num": 540
+}

eval/20260527_000425/reports/dir-ppo-llama3.1-8b/race.json

@@ -0,0 +1,98 @@
+{
+    "name": "dir-ppo-llama3.1-8b@race",
+    "dataset_name": "race",
+    "dataset_pretty_name": "RACE",
+    "dataset_description": "\n## Overview\n\nRACE (ReAding Comprehension from Examinations) is a large-scale reading comprehension benchmark collected from Chinese middle school and high school English examinations. It tests comprehensive reading comprehension abilities.\n\n## Task Description\n\n- **Task Type**: Reading Comprehension (Multiple-Choice)\n- **Input**: Article passage with question and 4 answer choices\n- **Output**: Correct answer letter (A, B, C, or D)\n- **Difficulty Levels**: Middle school and High school\n\n## Key Features\n\n- 28,000+ passages with 100,000 questions\n- Real examination questions for authentic difficulty\n- Two subsets: middle (easier) and high (harder)\n- Tests various comprehension skills (inference, vocabulary, main idea, etc.)\n- Diverse article topics and question types\n\n## Evaluation Notes\n\n- Default configuration uses **3-shot** examples\n- Maximum few-shot number is 3 (context length consideration)\n- Uses Chain-of-Thought (CoT) prompting\n- Two subsets available: `high` and `middle`\n- Evaluates on test split\n",
+    "model_name": "dir-ppo-llama3.1-8b",
+    "score": 0.833,
+    "metrics": [
+        {
+            "name": "mean_acc",
+            "num": 4934,
+            "score": 0.833,
+            "macro_score": 0.833,
+            "categories": [
+                {
+                    "name": [
+                        "default"
+                    ],
+                    "num": 4934,
+                    "score": 0.833,
+                    "macro_score": 0.8433,
+                    "subsets": [
+                        {
+                            "name": "high",
+                            "score": 0.8188,
+                            "num": 3498
+                        },
+                        {
+                            "name": "middle",
+                            "score": 0.8677,
+                            "num": 1436
+                        }
+                    ]
+                }
+            ]
+        }
+    ],
+    "analysis": "N/A",
+    "perf_metrics": {
+        "summary": {
+            "n_samples": 4934,
+            "latency": {
+                "mean": 3.518324,
+                "std": 4.350995,
+                "min": 0.375683,
+                "25%": 2.235433,
+                "50%": 3.250558,
+                "75%": 4.360585,
+                "90%": 5.39817,
+                "99%": 7.618212,
+                "max": 139.063051
+            },
+            "throughput": {
+                "avg_output_tps": 51.33,
+                "avg_req_ps": 0.2842
+            },
+            "usage": {
+                "input_tokens": {
+                    "mean": 1668.64471,
+                    "std": 334.066106,
+                    "min": 958.0,
+                    "25%": 1274.0,
+                    "50%": 1824.0,
+                    "75%": 1885.0,
+                    "90%": 1953.0,
+                    "99%": 2269.67,
+                    "max": 2584.0
+                },
+                "output_tokens": {
+                    "mean": 180.605188,
+                    "std": 216.808972,
+                    "min": 20.0,
+                    "25%": 114.0,
+                    "50%": 168.0,
+                    "75%": 228.0,
+                    "90%": 274.0,
+                    "99%": 387.01,
+                    "max": 6932.0
+                },
+                "total_tokens": {
+                    "mean": 1849.249899,
+                    "std": 410.659869,
+                    "min": 1028.0,
+                    "25%": 1479.0,
+                    "50%": 1975.0,
+                    "75%": 2084.0,
+                    "90%": 2180.0,
+                    "99%": 2527.39,
+                    "max": 8192.0
+                },
+                "total_input_tokens": 8233093,
+                "total_output_tokens": 891106,
+                "total_tokens_count": 9124199
+            }
+        }
+    },
+    "num": 4934
+}

eval/20260527_000425/reviews/dir-ppo-llama3.1-8b/race_high.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9bca1ce5a3af7d111c3e98c1d4268a70b41e6a5b06d9d43dd98169389cf01a46
+size 38411680

eval/20260527_000425/reviews/dir-ppo-llama3.1-8b/trivia_qa_rc.wikipedia.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb259e7f4c082c3794ef1c4b4671d1bf7b24a4bee17839a9e079287f132d033d
+size 45028591