agent_bench/evaluation/datasets/tech_docs_golden.json

[
    {
        "id": "q001",
        "question": "How do you define a path parameter in FastAPI?",
        "expected_answer_keywords": [
            "curly braces",
            "path",
            "function parameter",
            "URL"
        ],
        "expected_sources": [
            "fastapi_path_params.md"
        ],
        "category": "retrieval",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "Path parameters in FastAPI are defined using curly braces {} in the route path string, with a corresponding parameter in the function signature."
    },
    {
        "id": "q002",
        "question": "What is the default page size for pagination in FastAPI and what is the maximum allowed?",
        "expected_answer_keywords": [
            "20",
            "100",
            "default",
            "maximum"
        ],
        "expected_sources": [
            "fastapi_pagination.md"
        ],
        "category": "retrieval",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "The default page size for pagination in FastAPI is 20 items per page, and the maximum allowed page size is 100."
    },
    {
        "id": "q003",
        "question": "How does FastAPI handle CORS and what is the default max_age for preflight caching?",
        "expected_answer_keywords": [
            "CORSMiddleware",
            "600",
            "seconds",
            "preflight"
        ],
        "expected_sources": [
            "fastapi_middleware.md"
        ],
        "category": "retrieval",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "FastAPI handles CORS by adding CORSMiddleware to the application, which manages preflight requests. The default max_age for preflight caching is 600 seconds."
    },
    {
        "id": "q004",
        "question": "What algorithm and expiry time does the FastAPI security example use for JWT tokens?",
        "expected_answer_keywords": [
            "HS256",
            "30",
            "minutes"
        ],
        "expected_sources": [
            "fastapi_security.md"
        ],
        "category": "retrieval",
        "difficulty": "medium",
        "requires_calculator": false,
        "reference_answer": "The FastAPI security example uses the HS256 algorithm for JWT tokens with an expiry time of 30 minutes."
    },
    {
        "id": "q005",
        "question": "What is the recommended formula for calculating the number of Gunicorn workers for a FastAPI deployment?",
        "expected_answer_keywords": [
            "2",
            "CPU",
            "cores",
            "1"
        ],
        "expected_sources": [
            "fastapi_deployment.md"
        ],
        "category": "retrieval",
        "difficulty": "medium",
        "requires_calculator": false,
        "reference_answer": "The recommended formula for calculating Gunicorn workers is (2 * CPU cores) + 1."
    },
    {
        "id": "q006",
        "question": "How does dependency caching work in FastAPI, and how can you disable it?",
        "expected_answer_keywords": [
            "cache",
            "once",
            "use_cache",
            "False"
        ],
        "expected_sources": [
            "fastapi_dependencies.md"
        ],
        "category": "retrieval",
        "difficulty": "medium",
        "requires_calculator": false,
        "reference_answer": "FastAPI caches dependency results so each dependency is called only once per request. Caching can be disabled by setting use_cache=False in the Depends() call.",
        "source_snippets": [
            "By default, if the same dependency is used multiple times within a single request (e.g., both a route and a sub-dependency use `Depends(get_db)`), FastAPI caches the result and calls the dependency only once. To disable caching and force a fresh call each time, use `Depends(get_db, use_cache=False)`."
        ]
    },
    {
        "id": "q007",
        "question": "If a paginated endpoint returns 20 items per page and there are 10,000 items total, how many total pages are there? And if the page size is changed to 30, how many pages would there be?",
        "expected_answer_keywords": [
            "500",
            "334",
            "ceil",
            "pages"
        ],
        "expected_sources": [
            "fastapi_pagination.md"
        ],
        "category": "calculation",
        "difficulty": "medium",
        "requires_calculator": true,
        "reference_answer": "With 10,000 items at 20 per page, there are 500 total pages (10000/20). At 30 per page, there are 334 pages (ceil(10000/30))."
    },
    {
        "id": "q008",
        "question": "Does FastAPI support automatic Kubernetes deployment?",
        "expected_answer_keywords": [
            "not",
            "does not contain",
            "no information"
        ],
        "expected_sources": [],
        "category": "out_of_scope",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": ""
    },
    {
        "id": "q009",
        "question": "How does FastAPI integrate with Apache Kafka for event streaming?",
        "expected_answer_keywords": [
            "not",
            "does not contain",
            "no information"
        ],
        "expected_sources": [],
        "category": "out_of_scope",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": ""
    },
    {
        "id": "q010",
        "question": "Can FastAPI generate GraphQL schemas natively?",
        "expected_answer_keywords": [
            "not",
            "does not contain",
            "no information"
        ],
        "expected_sources": [],
        "category": "out_of_scope",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "",
        "source_snippets": []
    },
    {
        "id": "q011",
        "question": "What is the default Swagger UI endpoint in FastAPI?",
        "expected_answer_keywords": [
            "/docs",
            "Swagger",
            "interactive"
        ],
        "expected_sources": [
            "fastapi_openapi.md"
        ],
        "category": "retrieval",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "The default Swagger UI endpoint in FastAPI is /docs, which provides an interactive API documentation interface.",
        "source_snippets": [
            "| `/docs`          | Swagger UI -- interactive API explorer           |",
            "Every FastAPI application exposes three documentation-related endpoints by default:"
        ]
    },
    {
        "id": "q012",
        "question": "How do you raise an HTTP error in a FastAPI route handler?",
        "expected_answer_keywords": [
            "HTTPException",
            "status_code",
            "detail"
        ],
        "expected_sources": [
            "fastapi_error_handling.md"
        ],
        "category": "retrieval",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "You raise an HTTP error in FastAPI by raising an HTTPException with a status_code and a detail message describing the error.",
        "source_snippets": [
            "The `HTTPException` class is the primary way to return error responses from route handlers:",
            "When raised, `HTTPException` immediately terminates request processing and returns the specified status code and detail message. The `detail` parameter can be a string, list, or dictionary -- FastAPI serializes it to JSON automatically."
        ]
    },
    {
        "id": "q013",
        "question": "How do you define a request body in FastAPI?",
        "expected_answer_keywords": [
            "Pydantic",
            "BaseModel",
            "JSON"
        ],
        "expected_sources": [
            "fastapi_request_body.md"
        ],
        "category": "retrieval",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "Request bodies in FastAPI are defined by creating a Pydantic BaseModel class, which FastAPI automatically parses from the incoming JSON request body."
    },
    {
        "id": "q014",
        "question": "What testing tools does FastAPI use, and what class provides the test client?",
        "expected_answer_keywords": [
            "TestClient",
            "pytest",
            "Starlette"
        ],
        "expected_sources": [
            "fastapi_testing.md"
        ],
        "category": "retrieval",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "FastAPI uses pytest for testing, and the TestClient class (provided by Starlette) is used to simulate HTTP requests to the application."
    },
    {
        "id": "q015",
        "question": "How does FastAPI manage application configuration and environment variables?",
        "expected_answer_keywords": [
            "BaseSettings",
            "pydantic",
            "env",
            "environment"
        ],
        "expected_sources": [
            "fastapi_configuration.md"
        ],
        "category": "retrieval",
        "difficulty": "medium",
        "requires_calculator": false,
        "reference_answer": "FastAPI manages configuration using Pydantic's BaseSettings class, which automatically reads values from environment variables and supports .env files."
    },
    {
        "id": "q016",
        "question": "What is the minimum response size for GZip compression middleware in FastAPI, and how do you enable it?",
        "expected_answer_keywords": [
            "500",
            "bytes",
            "GZipMiddleware",
            "minimum_size"
        ],
        "expected_sources": [
            "fastapi_middleware.md"
        ],
        "category": "retrieval",
        "difficulty": "medium",
        "requires_calculator": false,
        "reference_answer": "GZip compression is enabled by adding GZipMiddleware to the application with a minimum_size parameter. The default minimum response size is 500 bytes."
    },
    {
        "id": "q017",
        "question": "How do yield dependencies work in FastAPI and what is the maximum number supported per request?",
        "expected_answer_keywords": [
            "yield",
            "cleanup",
            "finally",
            "32"
        ],
        "expected_sources": [
            "fastapi_dependencies.md"
        ],
        "category": "retrieval",
        "difficulty": "medium",
        "requires_calculator": false,
        "reference_answer": "Yield dependencies in FastAPI execute code before yielding a value and run cleanup code in a finally block after the response is sent. The maximum number of yield dependencies per request is 32."
    },
    {
        "id": "q018",
        "question": "What are the three documentation endpoints FastAPI exposes by default and what OpenAPI version does it use?",
        "expected_answer_keywords": [
            "/docs",
            "/redoc",
            "/openapi.json",
            "3.1"
        ],
        "expected_sources": [
            "fastapi_openapi.md"
        ],
        "category": "retrieval",
        "difficulty": "medium",
        "requires_calculator": false,
        "reference_answer": "FastAPI exposes three documentation endpoints by default: /docs (Swagger UI), /redoc (ReDoc), and /openapi.json (the raw OpenAPI schema). It uses OpenAPI version 3.1."
    },
    {
        "id": "q019",
        "question": "How does FastAPI handle WebSocket connections, and what must be called before sending data?",
        "expected_answer_keywords": [
            "accept",
            "WebSocket",
            "send",
            "receive"
        ],
        "expected_sources": [
            "fastapi_websockets.md"
        ],
        "category": "retrieval",
        "difficulty": "medium",
        "requires_calculator": false,
        "reference_answer": "FastAPI handles WebSocket connections using the WebSocket class. You must call await websocket.accept() before sending or receiving any data."
    },
    {
        "id": "q020",
        "question": "For a server with 4 CPU cores, how many Gunicorn workers should be configured using the recommended formula?",
        "expected_answer_keywords": [
            "9",
            "workers",
            "2",
            "CPU",
            "1"
        ],
        "expected_sources": [
            "fastapi_deployment.md"
        ],
        "category": "calculation",
        "difficulty": "medium",
        "requires_calculator": true,
        "reference_answer": "Using the recommended formula (2 * CPU cores) + 1, a server with 4 CPU cores should be configured with 9 Gunicorn workers (2 * 4 + 1 = 9)."
    },
    {
        "id": "q021",
        "question": "If the CORS max_age is 600 seconds, how many minutes does the browser cache preflight results?",
        "expected_answer_keywords": [
            "10",
            "minutes"
        ],
        "expected_sources": [
            "fastapi_middleware.md"
        ],
        "category": "calculation",
        "difficulty": "easy",
        "requires_calculator": true,
        "reference_answer": "With a CORS max_age of 600 seconds, the browser caches preflight results for 10 minutes (600 / 60 = 10).",
        "source_snippets": [
            "| `max_age`            | `600`   | Seconds the browser caches preflight results       |"
        ]
    },
    {
        "id": "q022",
        "question": "How do route ordering and dependency injection interact when building a secure FastAPI application with scoped endpoints?",
        "expected_answer_keywords": [
            "order",
            "Depends",
            "Security",
            "scopes"
        ],
        "expected_sources": [
            "fastapi_path_params.md",
            "fastapi_dependencies.md",
            "fastapi_security.md"
        ],
        "category": "retrieval",
        "difficulty": "hard",
        "requires_calculator": false,
        "reference_answer": "In FastAPI, route ordering matters because more specific routes must be defined before general ones. Security scopes are enforced via the Security() dependency (similar to Depends()), allowing fine-grained access control on scoped endpoints."
    },
    {
        "id": "q023",
        "question": "How would you set up a FastAPI application with custom error handling, CORS middleware, and structured testing including dependency overrides?",
        "expected_answer_keywords": [
            "HTTPException",
            "CORSMiddleware",
            "TestClient",
            "override"
        ],
        "expected_sources": [
            "fastapi_error_handling.md",
            "fastapi_middleware.md",
            "fastapi_testing.md"
        ],
        "category": "retrieval",
        "difficulty": "hard",
        "requires_calculator": false,
        "reference_answer": "Custom error handling is set up by raising HTTPException or registering exception handlers, CORS is configured by adding CORSMiddleware with allowed origins, and testing uses TestClient with app.dependency_overrides to replace dependencies during tests.",
        "source_snippets": [
            "The `HTTPException` class is the primary way to return error responses from route handlers:",
            "Cross-Origin Resource Sharing (CORS) is configured using `CORSMiddleware` from Starlette:"
        ]
    },
    {
        "id": "q024",
        "question": "Explain how to deploy a FastAPI app with Docker using Gunicorn workers, health checks, and environment-based configuration via Pydantic Settings.",
        "expected_answer_keywords": [
            "Docker",
            "Gunicorn",
            "health",
            "BaseSettings",
            "env"
        ],
        "expected_sources": [
            "fastapi_deployment.md",
            "fastapi_configuration.md"
        ],
        "category": "retrieval",
        "difficulty": "hard",
        "requires_calculator": false,
        "reference_answer": "A FastAPI app is deployed in Docker with Gunicorn as the process manager using the recommended worker formula. Health check endpoints verify application status, and environment-based configuration is managed via Pydantic BaseSettings which reads environment variables and .env files."
    },
    {
        "id": "q025",
        "question": "How would you build a paginated API with cursor-based navigation, response model validation, and background task processing for analytics logging?",
        "expected_answer_keywords": [
            "cursor",
            "response_model",
            "BackgroundTasks"
        ],
        "expected_sources": [
            "fastapi_pagination.md",
            "fastapi_response_model.md",
            "fastapi_background_tasks.md"
        ],
        "category": "retrieval",
        "difficulty": "hard",
        "requires_calculator": false,
        "reference_answer": "Cursor-based pagination uses an opaque cursor token for navigation instead of page numbers. Response models are validated using the response_model parameter on route decorators, and analytics logging is handled asynchronously via FastAPI's BackgroundTasks dependency.",
        "source_snippets": [
            "Cursor-based pagination uses an opaque token (cursor) pointing to the last item in the previous page. This avoids the performance degradation of large offsets:"
        ]
    },
    {
        "id": "q026",
        "question": "Does FastAPI have built-in support for database migrations like Alembic?",
        "expected_answer_keywords": [
            "not",
            "does not contain",
            "no information"
        ],
        "expected_sources": [],
        "category": "out_of_scope",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": ""
    },
    {
        "id": "q027",
        "question": "How does FastAPI handle automatic load balancing across multiple servers?",
        "expected_answer_keywords": [
            "not",
            "does not contain",
            "no information"
        ],
        "expected_sources": [],
        "category": "out_of_scope",
        "difficulty": "easy",
        "requires_calculator": false,
        "reference_answer": "",
        "source_snippets": []
    }
]