scripts/rag_eval_sample.json

{
  "course_id": "course-eval-demo-001",
  "documents": [
    {
      "path": "doc/samples/rag_eval_material.txt"
    }
  ],
  "questions": [
    {
      "question": "Who is the lecturer for Knowledge Engineering 101?",
      "expected_all": [
        "Dr. Raka Pratama"
      ],
      "expected_any": [
        "Raka"
      ],
      "forbidden": [
        "i don't know"
      ],
      "expect_abstain": false
    },
    {
      "question": "State the lecturer's full name.",
      "expected_all": [
        "Dr. Raka Pratama"
      ],
      "expected_any": [],
      "forbidden": [
        "i don't know"
      ],
      "expect_abstain": false
    },
    {
      "question": "On what day does the lecturer hold office hour?",
      "expected_all": [
        "Tuesday"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "At what time is the lecturer office hour?",
      "expected_all": [
        "10:00"
      ],
      "expected_any": [
        "WIB"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Which room is used for lecturer office hour?",
      "expected_all": [
        "A-203"
      ],
      "expected_any": [
        "Room"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Give the complete lecturer office hour schedule.",
      "expected_all": [
        "Tuesday",
        "10:00",
        "A-203"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "When is the teaching assistant office hour?",
      "expected_all": [
        "Thursday",
        "13:00"
      ],
      "expected_any": [
        "WIB"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Where is the teaching assistant office hour location?",
      "expected_all": [
        "B-101"
      ],
      "expected_any": [
        "Room"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the weekly quiz schedule?",
      "expected_all": [
        "Wednesday",
        "08:00"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "How long does each weekly quiz last?",
      "expected_all": [
        "15",
        "minutes"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What day is the lab session held?",
      "expected_all": [
        "Monday"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What are the lab session start and end times?",
      "expected_all": [
        "15:00",
        "17:00"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "In which lab room does the practical session take place?",
      "expected_all": [
        "C-305"
      ],
      "expected_any": [
        "Lab"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What percentage do quizzes contribute to the final grade?",
      "expected_all": [
        "20%"
      ],
      "expected_any": [
        "quiz"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What percentage do assignments contribute?",
      "expected_all": [
        "30%"
      ],
      "expected_any": [
        "assignment"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the midterm weight in grading?",
      "expected_all": [
        "20%"
      ],
      "expected_any": [
        "midterm"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "How much is the final project demo worth?",
      "expected_all": [
        "30%"
      ],
      "expected_any": [
        "demo"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What minimum score is required to pass the course?",
      "expected_all": [
        "60"
      ],
      "expected_any": [
        "grade C"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "When is the assignment submission deadline?",
      "expected_all": [
        "Friday",
        "23:59"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Through which platform must assignments be submitted?",
      "expected_all": [
        "LMS"
      ],
      "expected_any": [
        "submission"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What date is the midterm exam?",
      "expected_all": [
        "2026-05-12"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What date is the final project demo?",
      "expected_all": [
        "2026-06-20"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Which date is listed as a public holiday with no class?",
      "expected_all": [
        "2026-05-01"
      ],
      "expected_any": [
        "no class"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "When is the make-up class conducted?",
      "expected_all": [
        "2026-05-03",
        "09:00"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What medium is used for the make-up class?",
      "expected_all": [
        "Zoom"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Which file format is accepted for submission?",
      "expected_all": [
        "PDF"
      ],
      "expected_any": [],
      "forbidden": [
        "docx"
      ],
      "expect_abstain": false
    },
    {
      "question": "What is the upload file size limit?",
      "expected_all": [
        "10 MB"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Write the required filename pattern for homework.",
      "expected_all": [
        "NIM_Name_HW#.pdf"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "How much is the late submission penalty per day?",
      "expected_all": [
        "10%",
        "per day"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "How many days is the maximum late window?",
      "expected_all": [
        "3",
        "days"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the allowed plagiarism similarity threshold?",
      "expected_all": [
        "below",
        "20%"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the minimum team size for project groups?",
      "expected_all": [
        "3"
      ],
      "expected_any": [
        "students"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the maximum team size for project groups?",
      "expected_all": [
        "4"
      ],
      "expected_any": [
        "students"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Provide the full group size policy.",
      "expected_all": [
        "3",
        "4",
        "students"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What branch naming convention should students follow?",
      "expected_all": [
        "feat/<nim>-<topic>"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the required commit message format?",
      "expected_all": [
        "type(scope): summary"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Which Python version is specified in the material?",
      "expected_all": [
        "Python",
        "3.10"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What backend framework is listed for this course context?",
      "expected_all": [
        "FastAPI"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What ORM stack is mentioned?",
      "expected_all": [
        "SQLModel"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What textbook is used in this course?",
      "expected_all": [
        "Russell and Norvig",
        "4th Edition"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Which chapters are covered according to the material?",
      "expected_all": [
        "Chapters",
        "1-6"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the official course email address?",
      "expected_all": [
        "ke101@campus.ac.id"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What booking link is provided for appointments?",
      "expected_all": [
        "bit.ly/ke101-booking"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the forum response SLA on weekdays?",
      "expected_all": [
        "24",
        "hours"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the minimum attendance requirement?",
      "expected_all": [
        "75%"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Summarize office-hour information from the material.",
      "expected_all": [
        "Tuesday",
        "10:00",
        "A-203",
        "Thursday",
        "13:00",
        "B-101"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Summarize grading component percentages.",
      "expected_all": [
        "20%",
        "30%"
      ],
      "expected_any": [
        "quizzes",
        "assignments",
        "midterm",
        "final project"
      ],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Summarize all deadline/date-related items in the course note.",
      "expected_all": [
        "Friday",
        "23:59",
        "2026-05-12",
        "2026-06-20",
        "2026-05-01",
        "2026-05-03"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Summarize all submission policy constraints.",
      "expected_all": [
        "PDF",
        "10 MB",
        "NIM_Name_HW#.pdf",
        "10%",
        "3 days",
        "20%"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "Give a concise compliance checklist for students based on the material.",
      "expected_all": [
        "LMS",
        "PDF",
        "NIM_Name_HW#.pdf",
        "75%"
      ],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": false
    },
    {
      "question": "What is the cafeteria lunch menu on Monday?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "How much is the parking fee per semester?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the exact GPS location of Room A-203?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "Who is the dean of the faculty this year?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the student canteen opening hour on weekends?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the bus route from campus gate to Building D?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the final exam room number for this course?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "Which printer model is available in Lab C-305?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the dormitory check-in rules for new students?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the price of printing one color page on campus?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the office phone number of the lecturer?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What scholarship options are available this semester?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "How many seats are there in Room B-101?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the canteen food safety certification number?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the exact Wi-Fi password for the lab network?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the elevator maintenance schedule for Building A?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the library fine per overdue day?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the minimum TOEFL score for graduation?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the nearest mosque prayer room capacity?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the jersey colors for the faculty futsal team?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the weather forecast on the midterm date?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the bank account number for tuition payment?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the recommended laptop brand for this course?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the name of the campus cat near Building C?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What snacks are sold in the vending machine on floor 2?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the route to avoid traffic near campus at 8 AM?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the exact dimensions of Lab C-305?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What antivirus software license key is used in the lab?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the salary of the teaching assistant?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the faculty accreditation code for this year?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the nearest clinic phone number from campus?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the ingredients of the cafeteria fried rice?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the expected stock price next month?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the winning lottery numbers for this week?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the historical founding date of Room A-203?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the personal hobbies of Dr. Raka Pratama?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the exact blueprint of the Zoom server architecture?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the shoe size of the teaching assistant?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the blood type of the lecturer?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the movie recommendations from this class?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the daily calorie content of cafeteria meals?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the color code of campus wall paint?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the top tourist spots near the campus?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the rental price of projectors per hour?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the faculty's electricity bill this month?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What are the expected interview questions for internships?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the nearest train departure time from campus?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the coffee bean origin in the campus cafe?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the market share of FastAPI globally this year?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "What is the cafeteria lunch menu for Wednesday?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    }
  ]
}