File size: 1,333 Bytes
c641d5f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 | import threading
import time
from agent import SolveResult
from cache import ResultStore
from evaluation import run_evaluation
class Client:
def get_questions(self):
return [
{"task_id": str(index), "question": "q", "file_name": ""}
for index in range(6)
]
def download_attachment(self, task):
return None
class Agent:
class settings:
worker_limit = 3
def __init__(self):
self.active = 0
self.maximum = 0
self.calls = 0
self.lock = threading.Lock()
def solve_detailed(self, task, attachment, *, force=False):
with self.lock:
self.calls += 1
self.active += 1
self.maximum = max(self.maximum, self.active)
time.sleep(0.03)
with self.lock:
self.active -= 1
return SolveResult(task["task_id"], "general_reasoning")
def test_concurrent_checkpointing_resume_and_force(tmp_path):
agent = Agent()
store = ResultStore(tmp_path / "results.json")
first = run_evaluation(Client(), agent, store)
assert len(first) == 6 and agent.maximum > 1 and agent.calls == 6
run_evaluation(Client(), agent, store)
assert agent.calls == 6
run_evaluation(Client(), agent, store, force=True, task_ids={"2"})
assert agent.calls == 7
|