import random import uuid DOMAINS = { "api": { "description": "REST API endpoint design", "specs": [ { "task": "Build a user authentication endpoint", "ground_truth": { "endpoint": "POST /auth/login", "request_fields": ["email", "password"], "response_fields": ["userId", "token", "expiresAt"] } }, { "task": "Build a product listing endpoint", "ground_truth": { "endpoint": "GET /products", "response_fields": ["productId", "name", "price", "inventory"] } }, { "task": "Build a user profile endpoint", "ground_truth": { "endpoint": "GET /users/{userId}", "response_fields": ["userId", "name", "email", "createdAt"] } }, { "task": "Build an order creation endpoint", "ground_truth": { "endpoint": "POST /orders", "request_fields": ["userId", "items", "shippingAddress", "paymentMethod"], "response_fields": ["orderId", "status", "totalAmount", "estimatedDelivery"] } }, { "task": "Build a password reset endpoint", "ground_truth": { "endpoint": "POST /auth/reset-password", "request_fields": ["email", "resetToken", "newPassword"], "response_fields": ["success", "message"] } }, { "task": "Build a refresh token endpoint", "ground_truth": { "endpoint": "POST /auth/refresh", "request_fields": ["refreshToken"], "response_fields": ["accessToken", "refreshToken", "expiresIn"] } }, { "task": "Build a search endpoint with pagination", "ground_truth": { "endpoint": "GET /search", "request_fields": ["query", "page", "pageSize", "sortBy"], "response_fields": ["results", "totalCount", "page", "hasMore"] } }, { "task": "Build a file upload endpoint", "ground_truth": { "endpoint": "POST /files/upload", "request_fields": ["fileName", "contentType", "fileSize", "fileData"], "response_fields": ["fileId", "url", "uploadedAt"] } }, ] }, "database": { "description": "Database schema design", "specs": [ { "task": "Design a users table for authentication", "ground_truth": { "table": "users", "fields": ["user_id", "email", "password_hash", "created_at"] } }, { "task": "Design an orders table", "ground_truth": { "table": "orders", "fields": ["order_id", "user_id", "product_id", "quantity", "total_price"] } }, { "task": "Design a products inventory table", "ground_truth": { "table": "products", "fields": ["product_id", "sku", "name", "price", "stock_count", "category_id"] } }, { "task": "Design a sessions table for token tracking", "ground_truth": { "table": "sessions", "fields": ["session_id", "user_id", "token_hash", "issued_at", "expires_at", "ip_address"] } }, { "task": "Design an audit log table", "ground_truth": { "table": "audit_log", "fields": ["log_id", "user_id", "action", "resource_type", "resource_id", "timestamp"] } }, { "task": "Design a payments table", "ground_truth": { "table": "payments", "fields": ["payment_id", "order_id", "amount", "currency", "status", "processed_at"] } }, { "task": "Design a notifications table", "ground_truth": { "table": "notifications", "fields": ["notification_id", "user_id", "type", "message", "read_at", "created_at"] } }, ] }, "config": { "description": "Service configuration", "specs": [ { "task": "Configure a REST service with timeout and retry", "ground_truth": { "fields": ["timeout_ms", "max_retries", "retry_delay_ms"], "timeout_ms": 5000, "max_retries": 3, "retry_delay_ms": 1000 } }, { "task": "Configure a database connection pool", "ground_truth": { "fields": ["host", "port", "database", "max_connections", "idle_timeout_ms"], "host": "localhost", "port": 5432 } }, { "task": "Configure a rate limiter", "ground_truth": { "fields": ["requests_per_minute", "burst_size", "window_seconds", "block_duration_ms"] } }, { "task": "Configure a cache layer", "ground_truth": { "fields": ["cache_ttl_seconds", "max_size_mb", "eviction_policy", "compression_enabled"] } }, { "task": "Configure a circuit breaker", "ground_truth": { "fields": ["failure_threshold", "success_threshold", "open_state_timeout_ms", "half_open_max_calls"] } }, ] }, "data_format": { "description": "Data object definition", "specs": [ { "task": "Define a user profile object", "ground_truth": { "fields": ["userId", "name", "email", "createdAt"] } }, { "task": "Define a shopping cart item object", "ground_truth": { "fields": ["productId", "quantity", "unitPrice", "discountCode"] } }, { "task": "Define an address object", "ground_truth": { "fields": ["streetLine1", "streetLine2", "city", "state", "postalCode", "country"] } }, { "task": "Define a payment method object", "ground_truth": { "fields": ["methodId", "cardLast4", "expiryMonth", "expiryYear", "billingAddressId"] } }, { "task": "Define an invoice line item object", "ground_truth": { "fields": ["lineId", "description", "quantity", "unitPrice", "taxRate", "subtotal"] } }, { "task": "Define a notification preference object", "ground_truth": { "fields": ["userId", "channel", "frequency", "enabled", "lastSentAt"] } }, ] }, "graphql": { "description": "GraphQL schema design", "specs": [ { "task": "Define a User GraphQL type", "ground_truth": { "type_name": "User", "fields": ["id", "email", "displayName", "avatarUrl", "createdAt"] } }, { "task": "Define a Product GraphQL type", "ground_truth": { "type_name": "Product", "fields": ["id", "title", "description", "price", "inStock", "categoryId"] } }, { "task": "Define a CreateOrder GraphQL mutation", "ground_truth": { "type_name": "CreateOrder", "fields": ["userId", "lineItems", "shippingAddressId", "paymentMethodId"] } }, { "task": "Define a Comment GraphQL type", "ground_truth": { "type_name": "Comment", "fields": ["id", "authorId", "postId", "body", "createdAt", "editedAt"] } }, ] }, "event": { "description": "Event / message schema", "specs": [ { "task": "Define a UserSignedUp event", "ground_truth": { "event_name": "UserSignedUp", "fields": ["userId", "email", "signupSource", "occurredAt"] } }, { "task": "Define an OrderShipped event", "ground_truth": { "event_name": "OrderShipped", "fields": ["orderId", "trackingNumber", "carrier", "shippedAt", "estimatedArrival"] } }, { "task": "Define a PaymentFailed event", "ground_truth": { "event_name": "PaymentFailed", "fields": ["paymentId", "orderId", "errorCode", "errorMessage", "failedAt"] } }, { "task": "Define a CartAbandoned event", "ground_truth": { "event_name": "CartAbandoned", "fields": ["cartId", "userId", "totalValue", "itemCount", "abandonedAt"] } }, ] }, "cli": { "description": "Command-line interface specification", "specs": [ { "task": "Specify a database backup CLI", "ground_truth": { "command": "backup", "fields": ["--source", "--destination", "--compress", "--verbose"] } }, { "task": "Specify a deploy CLI", "ground_truth": { "command": "deploy", "fields": ["--environment", "--version", "--dry-run", "--rollback-on-failure"] } }, { "task": "Specify a log query CLI", "ground_truth": { "command": "logs", "fields": ["--service", "--since", "--until", "--level", "--follow"] } }, ] }, "error_response": { "description": "Error response schema", "specs": [ { "task": "Define a validation error response", "ground_truth": { "error_code": "VALIDATION_ERROR", "fields": ["errorCode", "message", "fieldErrors", "requestId", "timestamp"] } }, { "task": "Define an authentication error response", "ground_truth": { "error_code": "AUTH_FAILED", "fields": ["errorCode", "message", "reason", "retryAfter", "timestamp"] } }, { "task": "Define a rate-limit error response", "ground_truth": { "error_code": "RATE_LIMITED", "fields": ["errorCode", "message", "limit", "remaining", "resetAt"] } }, ] }, } def _flatten_templates(): """Return a flat list of (domain, template) pairs for template-uniform sampling.""" flat = [] for domain_name, domain_data in DOMAINS.items(): for template in domain_data["specs"]: flat.append((domain_name, template)) return flat _FLAT_TEMPLATES = _flatten_templates() def generate_spec(domain: str = None) -> dict: """ Returns: { "task_description": str, # What agents A and B receive "ground_truth": dict, # What correct output looks like (hidden from agents) "domain": str, "spec_id": str # Unique ID for logging } Sampling: when domain is None, uses TEMPLATE-UNIFORM sampling across the flat list of (domain, template) pairs — every template has equal probability, not every domain. This avoids over-representing single-template domains. """ if domain is None: chosen_domain, spec_template = random.choice(_FLAT_TEMPLATES) else: chosen_domain = domain spec_template = random.choice(DOMAINS[domain]["specs"]) return { "task_description": spec_template["task"], "ground_truth": spec_template["ground_truth"], "domain": chosen_domain, "spec_id": str(uuid.uuid4()) } def list_domains() -> list: return list(DOMAINS.keys()) def template_count() -> dict: return {d: len(DOMAINS[d]["specs"]) for d in DOMAINS}