examples/file_categorization_drift_tests.json

{
  "description": "File categorization drift threshold tests for prompt-prix",
  "note": "expected_response is the embedding anchor for calculate_drift. pass_criteria is the rubric for LLM judges. These are independent evaluation axes.",
  "system_prompt": "You are a file organization assistant. Your task is to categorize files by reading their contents and moving them to the appropriate folder.\n\n## Instructions\n\n1. Read the contents of each file\n2. Determine which category the content belongs to\n3. Move the file to the matching folder\n4. Include the filename in the destination path\n\n## Output Format\n\nReturn a JSON object:\n```json\n{\n  \"operations\": [\n    {\"type\": \"move\", \"path\": \"source.txt\", \"destination\": \"category/source.txt\"}\n  ],\n  \"rationale\": \"Brief explanation of categorization decisions\"\n}\n```\n\nCRITICAL: Destination must include the filename, not just the folder.\n- CORRECT: \"animals/1.txt\"\n- WRONG: \"animals/\" or \"animals\"\n\nRespond with ONLY valid JSON.",
  "tests": [
    {
      "test_id": "categorize-files-into-animals-fruits-colors-folder",
      "description": "Categorize 3 files into animals/fruits/colors folders",
      "prompt": {
        "system": "You are a file organization assistant. Your task is to categorize files by reading their contents and moving them to the appropriate folder.\n\n## Instructions\n\n1. Read the contents of each file\n2. Determine which category the content belongs to\n3. Move the file to the matching folder\n4. Include the filename in the destination path\n\n## Output Format\n\nReturn a JSON object:\n```json\n{\n  \"operations\": [\n    {\"type\": \"move\", \"path\": \"source.txt\", \"destination\": \"category/source.txt\"}\n  ],\n  \"rationale\": \"Brief explanation of categorization decisions\"\n}\n```\n\nCRITICAL: Destination must include the filename, not just the folder.\n- CORRECT: \"animals/1.txt\"\n- WRONG: \"animals/\" or \"animals\"\n\nRespond with ONLY valid JSON.",
        "user": "Read each file and move it to the appropriate category folder based on its contents.\n\nCONTEXT:\n### Files to categorize:\n- [FILE] 1.txt (contents: \"The zebra is a striped animal native to Africa. It belongs to the horse family.\")\n- [FILE] 2.txt (contents: \"An apple is a delicious fruit that grows on trees. Popular varieties include Fuji and Gala.\")\n- [FILE] 3.txt (contents: \"The sky appears blue due to Rayleigh scattering of sunlight in the atmosphere.\")\n\n### Target folders:\n- [DIR] animals/\n- [DIR] fruits/\n- [DIR] colors/"
      },
      "pass_criteria": "Response is valid JSON with an 'operations' array containing exactly 3 move operations. 1.txt must go to animals/, 2.txt to fruits/, 3.txt to colors/. Each destination must include the original filename.",
      "expected_response": "{\n  \"operations\": [\n    {\n      \"type\": \"move\",\n      \"path\": \"1.txt\",\n      \"destination\": \"animals/1.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"2.txt\",\n      \"destination\": \"fruits/2.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"3.txt\",\n      \"destination\": \"colors/3.txt\"\n    }\n  ],\n  \"rationale\": \"1.txt describes a zebra (animal), 2.txt describes an apple (fruit), 3.txt describes the blue sky (color).\"\n}"
    },
    {
      "test_id": "categorize-6-files-into-animals-fruits-colors",
      "description": "Categorize 6 files into animals/fruits/colors",
      "prompt": {
        "system": "You are a file organization assistant. Your task is to categorize files by reading their contents and moving them to the appropriate folder.\n\n## Instructions\n\n1. Read the contents of each file\n2. Determine which category the content belongs to\n3. Move the file to the matching folder\n4. Include the filename in the destination path\n\n## Output Format\n\nReturn a JSON object:\n```json\n{\n  \"operations\": [\n    {\"type\": \"move\", \"path\": \"source.txt\", \"destination\": \"category/source.txt\"}\n  ],\n  \"rationale\": \"Brief explanation of categorization decisions\"\n}\n```\n\nCRITICAL: Destination must include the filename, not just the folder.\n- CORRECT: \"animals/1.txt\"\n- WRONG: \"animals/\" or \"animals\"\n\nRespond with ONLY valid JSON.",
        "user": "Categorize each file by its content and move to the matching folder.\n\nCONTEXT:\n### Files to categorize:\n- [FILE] 1.txt (contents: \"The zebra is a striped animal native to Africa.\")\n- [FILE] 2.txt (contents: \"An apple is a delicious fruit that grows on trees.\")\n- [FILE] 3.txt (contents: \"The sky appears blue due to Rayleigh scattering.\")\n- [FILE] 4.txt (contents: \"Elephants are the largest land mammals with excellent memory.\")\n- [FILE] 5.txt (contents: \"Bananas are tropical fruits rich in potassium.\")\n- [FILE] 6.txt (contents: \"Red is a primary color associated with passion and energy.\")\n\n### Target folders:\n- [DIR] animals/   (for content about animals)\n- [DIR] fruits/    (for content about fruits)\n- [DIR] colors/    (for content about colors)"
      },
      "pass_criteria": "Response is valid JSON with 6 move operations. Animals: 1.txt (zebra), 4.txt (elephants). Fruits: 2.txt (apple), 5.txt (bananas). Colors: 3.txt (blue sky), 6.txt (red). Each destination includes the filename.",
      "expected_response": "{\n  \"operations\": [\n    {\n      \"type\": \"move\",\n      \"path\": \"1.txt\",\n      \"destination\": \"animals/1.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"2.txt\",\n      \"destination\": \"fruits/2.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"3.txt\",\n      \"destination\": \"colors/3.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"4.txt\",\n      \"destination\": \"animals/4.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"5.txt\",\n      \"destination\": \"fruits/5.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"6.txt\",\n      \"destination\": \"colors/6.txt\"\n    }\n  ],\n  \"rationale\": \"1.txt and 4.txt describe animals (zebra, elephants). 2.txt and 5.txt describe fruits (apple, banana). 3.txt and 6.txt describe colors (blue sky, red).\"\n}"
    },
    {
      "test_id": "categorize-documents-by-type-adrs-specs-notes",
      "description": "Categorize documents by type (ADRs, specs, notes)",
      "prompt": {
        "system": "You are a file organization assistant. Your task is to categorize files by reading their contents and moving them to the appropriate folder.\n\n## Instructions\n\n1. Read the contents of each file\n2. Determine which category the content belongs to\n3. Move the file to the matching folder\n4. Include the filename in the destination path\n\n## Output Format\n\nReturn a JSON object:\n```json\n{\n  \"operations\": [\n    {\"type\": \"move\", \"path\": \"source.txt\", \"destination\": \"category/source.txt\"}\n  ],\n  \"rationale\": \"Brief explanation of categorization decisions\"\n}\n```\n\nCRITICAL: Destination must include the filename, not just the folder.\n- CORRECT: \"animals/1.txt\"\n- WRONG: \"animals/\" or \"animals\"\n\nRespond with ONLY valid JSON.",
        "user": "Organize these documents into the appropriate folders based on their content type.\n\nCONTEXT:\n### Files to categorize:\n- [FILE] doc1.md (contents: \"# ADR-001: Use PostgreSQL for persistence\\n\\n## Status\\nAccepted\\n\\n## Context\\nWe need a database...\")\n- [FILE] doc2.md (contents: \"# API Specification v2.0\\n\\n## Endpoints\\n\\nGET /users - Returns list of users...\")\n- [FILE] doc3.md (contents: \"Meeting notes 2024-01-15\\n\\nAttendees: Alice, Bob\\n\\nDiscussed timeline for Q1...\")\n- [FILE] doc4.md (contents: \"# ADR-002: Adopt TypeScript\\n\\n## Status\\nProposed\\n\\n## Decision\\nMigrate to TypeScript...\")\n\n### Target folders:\n- [DIR] adrs/       (Architecture Decision Records)\n- [DIR] specs/      (Technical specifications)\n- [DIR] notes/      (Meeting notes and misc)"
      },
      "pass_criteria": "Response is valid JSON with 4 move operations. ADRs: doc1.md (ADR-001), doc4.md (ADR-002). Specs: doc2.md (API spec). Notes: doc3.md (meeting notes). Destinations include filenames.",
      "expected_response": "{\n  \"operations\": [\n    {\n      \"type\": \"move\",\n      \"path\": \"doc1.md\",\n      \"destination\": \"adrs/doc1.md\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"doc2.md\",\n      \"destination\": \"specs/doc2.md\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"doc3.md\",\n      \"destination\": \"notes/doc3.md\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"doc4.md\",\n      \"destination\": \"adrs/doc4.md\"\n    }\n  ],\n  \"rationale\": \"doc1.md and doc4.md are ADRs (architecture decision records). doc2.md is an API specification. doc3.md contains meeting notes.\"\n}"
    },
    {
      "test_id": "handle-content-that-could-fit-multiple-categories",
      "description": "Handle content that could fit multiple categories",
      "prompt": {
        "system": "You are a file organization assistant. Your task is to categorize files by reading their contents and moving them to the appropriate folder.\n\n## Instructions\n\n1. Read the contents of each file\n2. Determine which category the content belongs to\n3. Move the file to the matching folder\n4. Include the filename in the destination path\n\n## Output Format\n\nReturn a JSON object:\n```json\n{\n  \"operations\": [\n    {\"type\": \"move\", \"path\": \"source.txt\", \"destination\": \"category/source.txt\"}\n  ],\n  \"rationale\": \"Brief explanation of categorization decisions\"\n}\n```\n\nCRITICAL: Destination must include the filename, not just the folder.\n- CORRECT: \"animals/1.txt\"\n- WRONG: \"animals/\" or \"animals\"\n\nRespond with ONLY valid JSON.",
        "user": "Categorize these files. Use your best judgment for ambiguous content.\n\nCONTEXT:\n### Files to categorize:\n- [FILE] a.txt (contents: \"The flamingo is a pink bird found in tropical regions.\")\n- [FILE] b.txt (contents: \"Orange juice is made from oranges and is rich in vitamin C.\")\n- [FILE] c.txt (contents: \"The golden retriever has a beautiful golden coat.\")\n\n### Target folders:\n- [DIR] animals/\n- [DIR] fruits/\n- [DIR] colors/\n\nNote: Some content may reference multiple categories. Categorize by the PRIMARY subject."
      },
      "pass_criteria": "Response is valid JSON with 3 move operations. a.txt (flamingo) \u2192 animals (not colors despite 'pink'). b.txt (orange juice) \u2192 fruits (not colors despite 'orange'). c.txt (golden retriever) \u2192 animals (not colors despite 'golden'). Primary subject determines category over secondary color references.",
      "expected_response": "{\n  \"operations\": [\n    {\n      \"type\": \"move\",\n      \"path\": \"a.txt\",\n      \"destination\": \"animals/a.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"b.txt\",\n      \"destination\": \"fruits/b.txt\"\n    },\n    {\n      \"type\": \"move\",\n      \"path\": \"c.txt\",\n      \"destination\": \"animals/c.txt\"\n    }\n  ],\n  \"rationale\": \"a.txt describes a flamingo (animal, despite pink color mention). b.txt describes orange juice (fruit, despite orange color). c.txt describes a golden retriever (animal, despite golden color mention). Primary subject determines category.\"\n}"
    }
  ]
}