File size: 6,314 Bytes
96abbd8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 | """
Configuration file for simple RAG evaluation
Contains prompt templates and other settings
"""
from pathlib import Path
# ============================================
# Prompt Templates
# ============================================
# Default Gurobi prompt template
GUROBI_PROMPT = {
"system": """You are a helpful Assistant with expertise in mathematical modeling and the Gurobi solver. When the User provides an OR question, you will analyze it, build a detailed mathematical model, and provide the Gurobi code to solve it.
Your response should follow these steps:
1. Carefully analyze the problem to identify decision variables, objective, and constraints.
2. Develop a complete mathematical model, explicitly defining:
- Sets
- Parameters
- Decision Variables (and their types)
- Objective Function
- Constraints
3. Provide the corresponding Gurobi Python code to implement the model.
Implementation guardrails:
- Use `gurobipy` exclusively (avoid cvxpy/pulp/copty imports).
- When indexing tupledict variables across periods, introduce an explicit sentinel index (e.g., period 0) for initial conditions instead of accessing undefined keys like `x[-1]`.
- Define any Big-M constants explicitly using bounds derived from the data before they appear in constraints.
- Keep the model linear/integer; if a relationship seems non-linear, introduce auxiliary variables and linearization rather than exponentiation or log constraints.
- Always ensure every symbol referenced in constraints/objective (such as `M`, helper dictionaries, etc.) is declared in the code block.
""",
"user": """Problem:
{question}
Provide a complete solution with mathematical model and Gurobi code.
"""
}
# ============================================
# Model Configuration
# ============================================
# Supported models and their default temperatures
MODEL_CONFIGS = {
"gpt-4o": {"temperature": 0.01, "max_tokens": 8192},
"gpt-4o-mini": {"temperature": 0.01, "max_tokens": 8192},
"deepseek-chat": {"temperature": 0.01, "max_tokens": 8192},
"gemini-2.0-flash-exp": {"temperature": 0.01, "max_tokens": 8192},
"gemini-2.5-pro": {"temperature": 0.01, "max_tokens": 8192},
}
# ============================================
# Evaluation Configuration
# ============================================
EVAL_CONFIG = {
# Execution settings
"timeout": 60, # seconds
"max_retries": 3,
# Answer comparison settings
"tolerance": 0.05, # 5% relative tolerance by default
"use_relative_tolerance": True,
"absolute_tolerance": 1e-3, # for zero objective values
# Output settings
"save_code": True,
"save_output": False, # whether to save stdout/stderr
"verbose": False,
}
# ============================================
# Dataset Configuration
# ============================================
# Supported datasets
DATASETS = [
"ComplexLP",
"EasyLP",
"IndustryOR",
"NL4OPT",
"NLP4LP",
"ReSocratic",
"ComplexOR",
"OPT-Principled",
]
DATASET_ALIASES = {
"complexlp_clean": "ComplexLP",
"easylp_clean": "EasyLP",
"industryor_clean": "IndustryOR",
"industryor_v2": "IndustryOR",
"industryor_fixedv2": "IndustryOR",
"industryor_fixedv2_clean": "IndustryOR",
"nl4opt": "NL4OPT",
"nl4opt_clean": "NL4OPT",
"nlp4lp_clean": "NLP4LP",
"complexor_clean": "ComplexOR",
"resocratic_clean": "ReSocratic",
"combined": "OPT-Principled",
"combined_dataset": "OPT-Principled",
"opt-principled_clean": "OPT-Principled",
}
# Dataset-specific settings (if needed)
DATASET_CONFIG = {
"ComplexLP": {"tolerance": 0.05},
"EasyLP": {"tolerance": 0.01},
"IndustryOR": {"tolerance": 0.05},
"OPT-Principled": {"tolerance": 0.05},
}
# ============================================
# Utility Functions
# ============================================
def get_prompt_template(template_name="default"):
"""Get prompt template by name"""
templates = {
"default": GUROBI_PROMPT,
}
return templates.get(template_name, GUROBI_PROMPT)
def get_model_config(model_name):
"""Get configuration for a specific model"""
return MODEL_CONFIGS.get(model_name, {"temperature": 0.01, "max_tokens": 8192})
def get_dataset_config(dataset_name):
"""Get configuration for a specific dataset"""
return DATASET_CONFIG.get(normalize_dataset_name(dataset_name), {"tolerance": 0.05})
def normalize_dataset_name(dataset_name: str) -> str:
"""Map historical dataset names to the canonical OPEN benchmark names."""
if not dataset_name:
return dataset_name
name = dataset_name.strip()
if name.endswith(".jsonl"):
name = name[:-6]
alias = DATASET_ALIASES.get(name.casefold())
if alias:
return alias
for canonical_name in DATASETS:
if canonical_name.casefold() == name.casefold():
return canonical_name
if name.endswith("_clean"):
base_name = name[:-6]
for canonical_name in DATASETS:
if canonical_name.casefold() == base_name.casefold():
return canonical_name
return name
def get_benchmark_dirs(project_root: Path) -> list[Path]:
"""Return benchmark directories in priority order for the migrated OPEN layout."""
return [
project_root.parent.parent / "data" / "benchmarks",
project_root / "clean_benchmarks",
project_root.parent / "clean_benchmarks",
]
def find_benchmark_path(project_root: Path, dataset_name: str) -> Path:
"""Locate the benchmark file for a dataset, accepting legacy names as input."""
normalized_name = normalize_dataset_name(dataset_name)
candidate_names = [normalized_name]
raw_name = dataset_name[:-6] if dataset_name.endswith(".jsonl") else dataset_name
if raw_name not in candidate_names:
candidate_names.append(raw_name)
for directory in get_benchmark_dirs(project_root):
for name in candidate_names:
candidate = directory / f"{name}.jsonl"
if candidate.exists():
return candidate
raise FileNotFoundError(
f"Dataset '{dataset_name}' not found. Checked directories: "
f"{[str(path) for path in get_benchmark_dirs(project_root)]}"
)
|