scripts/rag_eval_cross_language.json

{
  "course_id": "course-eval-cross-language-001",
  "documents": [
    {
      "path": "doc/samples/rag_eval_material.txt"
    }
  ],
  "questions": [
    {
      "question": "Answer in English as a complete sentence: Siapa dosen untuk mata kuliah Knowledge Engineering 101?",
      "expected_language": "en",
      "language_expected_any": ["lecturer", "instructor", "is"],
      "expected_all": ["Dr. Raka Pratama"],
      "retrieval_expected_all": ["Lecturer", "Dr. Raka Pratama"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Kapan dan di mana office hour dosen Knowledge Engineering 101?",
      "expected_language": "en",
      "language_expected_any": ["office hour", "room", "held"],
      "expected_all": ["Tuesday", "10:00", "A-203"],
      "retrieval_expected_all": ["Tuesday", "10:00", "A-203"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Kapan dan di mana office hour teaching assistant?",
      "expected_language": "en",
      "language_expected_any": ["teaching assistant", "office hour", "room"],
      "expected_all": ["Thursday", "13:00", "B-101"],
      "retrieval_expected_all": ["Thursday", "13:00", "B-101"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Apa jadwal weekly quiz dan berapa durasinya?",
      "expected_language": "en",
      "language_expected_any": ["quiz", "minutes", "lasts"],
      "expected_all": ["Wednesday", "08:00", "15"],
      "retrieval_expected_all": ["Wednesday", "08:00", "15 minutes"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Kapan lab session diadakan dan di ruangan mana?",
      "expected_language": "en",
      "language_expected_any": ["lab", "session", "held"],
      "expected_all": ["Monday", "15:00", "17:00", "C-305"],
      "retrieval_expected_all": ["Monday", "15:00-17:00", "C-305"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa persentase quizzes dalam penilaian akhir?",
      "expected_language": "en",
      "language_expected_any": ["quizzes", "contribute", "worth"],
      "expected_all": ["20%"],
      "retrieval_expected_all": ["Quizzes", "20%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa persentase assignments dalam grading?",
      "expected_language": "en",
      "language_expected_any": ["assignments", "contribute", "worth"],
      "expected_all": ["30%"],
      "retrieval_expected_all": ["Assignments", "30%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa bobot midterm dalam grading?",
      "expected_language": "en",
      "language_expected_any": ["midterm", "weight", "worth"],
      "expected_all": ["20%"],
      "retrieval_expected_all": ["Midterm", "20%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa nilai final project demo?",
      "expected_language": "en",
      "language_expected_any": ["final project", "demo", "worth"],
      "expected_all": ["30%"],
      "retrieval_expected_all": ["Final project demo", "30%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa minimum passing score untuk course ini?",
      "expected_language": "en",
      "language_expected_any": ["minimum", "passing", "score"],
      "expected_all": ["60"],
      "retrieval_expected_all": ["Minimum passing score", "60"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Kapan deadline assignment dan lewat platform apa dikumpulkan?",
      "expected_language": "en",
      "language_expected_any": ["deadline", "submitted", "platform"],
      "expected_all": ["Friday", "23:59", "LMS"],
      "retrieval_expected_all": ["Friday", "23:59", "LMS"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Tanggal berapa midterm exam?",
      "expected_language": "en",
      "language_expected_any": ["midterm", "exam", "date"],
      "expected_all": ["2026-05-12"],
      "retrieval_expected_all": ["Midterm exam date", "2026-05-12"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Tanggal berapa final project demo?",
      "expected_language": "en",
      "language_expected_any": ["final project", "demo", "date"],
      "expected_all": ["2026-06-20"],
      "retrieval_expected_all": ["Final project demo date", "2026-06-20"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Kapan public holiday sehingga tidak ada kelas?",
      "expected_language": "en",
      "language_expected_any": ["public holiday", "no class", "class"],
      "expected_all": ["2026-05-01"],
      "retrieval_expected_all": ["Public holiday", "2026-05-01"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Kapan make-up class dan melalui apa kelasnya?",
      "expected_language": "en",
      "language_expected_any": ["make-up", "class", "via"],
      "expected_all": ["2026-05-03", "09:00", "Zoom"],
      "retrieval_expected_all": ["Make-up class date", "2026-05-03", "09:00", "Zoom"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Apa format submission yang diwajibkan?",
      "expected_language": "en",
      "language_expected_any": ["submission", "format", "must"],
      "expected_all": ["PDF"],
      "retrieval_expected_all": ["Submission format", "PDF only"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa maximum file size untuk submission?",
      "expected_language": "en",
      "language_expected_any": ["maximum", "file size", "submission"],
      "expected_all": ["10 MB"],
      "retrieval_expected_all": ["Maximum file size", "10 MB"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Apa pola filename yang wajib dipakai?",
      "expected_language": "en",
      "language_expected_any": ["filename", "pattern", "required"],
      "expected_all": ["NIM_Name_HW#.pdf"],
      "retrieval_expected_all": ["Required filename pattern", "NIM_Name_HW#.pdf"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa late penalty per day?",
      "expected_language": "en",
      "language_expected_any": ["late", "penalty", "per day"],
      "expected_all": ["10%"],
      "retrieval_expected_all": ["Late penalty", "10% per day"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa maximum late window?",
      "expected_language": "en",
      "language_expected_any": ["late", "window", "maximum"],
      "expected_all": ["3"],
      "retrieval_expected_all": ["Maximum late window", "3 days"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa plagiarism threshold yang harus dipenuhi?",
      "expected_language": "en",
      "language_expected_any": ["plagiarism", "threshold", "below"],
      "expected_all": ["20%"],
      "retrieval_expected_all": ["Plagiarism threshold", "below 20%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Berapa group size untuk project?",
      "expected_language": "en",
      "language_expected_any": ["group", "size", "students"],
      "expected_all": ["3-4"],
      "retrieval_expected_all": ["Group size", "3-4 students"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Apa format branch naming yang digunakan?",
      "expected_language": "en",
      "language_expected_any": ["branch", "naming", "format"],
      "expected_all": ["feat/<nim>-<topic>"],
      "retrieval_expected_all": ["Branch naming format", "feat/<nim>-<topic>"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Apa convention untuk commit message?",
      "expected_language": "en",
      "language_expected_any": ["commit", "message", "convention"],
      "expected_all": ["type(scope): summary"],
      "retrieval_expected_all": ["Commit message convention", "type(scope): summary"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Answer in English as a complete sentence: Bahasa pemrograman apa yang digunakan di course ini?",
      "expected_language": "en",
      "language_expected_any": ["programming", "language", "course"],
      "expected_all": ["Python 3.10"],
      "retrieval_expected_all": ["Programming language", "Python 3.10"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: Who is the lecturer for Knowledge Engineering 101?",
      "expected_language": "id",
      "language_expected_any": ["dosen", "pengajar", "adalah"],
      "expected_all": ["Dr. Raka Pratama"],
      "retrieval_expected_all": ["Lecturer", "Dr. Raka Pratama"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: When and where is the lecturer office hour?",
      "expected_language": "id",
      "language_expected_any": ["dosen", "ruang", "pada"],
      "expected_all": ["10:00", "A-203"],
      "retrieval_expected_all": ["Tuesday", "10:00", "A-203"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: When and where is the teaching assistant office hour?",
      "expected_language": "id",
      "language_expected_any": ["asisten", "ruang", "pada"],
      "expected_all": ["13:00", "B-101"],
      "retrieval_expected_all": ["Thursday", "13:00", "B-101"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the weekly quiz schedule and duration?",
      "expected_language": "id",
      "language_expected_any": ["kuis", "berlangsung", "menit"],
      "expected_all": ["08:00", "15"],
      "retrieval_expected_all": ["Wednesday", "08:00", "15 minutes"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: When is the lab session and where does it take place?",
      "expected_language": "id",
      "language_expected_any": ["lab", "diadakan", "ruang"],
      "expected_all": ["15:00", "17:00", "C-305"],
      "retrieval_expected_all": ["Monday", "15:00-17:00", "C-305"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What percentage do quizzes contribute to the final grade?",
      "expected_language": "id",
      "language_expected_any": ["kuis", "sebesar", "penilaian"],
      "expected_all": ["20%"],
      "retrieval_expected_all": ["Quizzes", "20%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What percentage do assignments contribute?",
      "expected_language": "id",
      "language_expected_any": ["tugas", "sebesar", "penilaian"],
      "expected_all": ["30%"],
      "retrieval_expected_all": ["Assignments", "30%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the midterm weight in grading?",
      "expected_language": "id",
      "language_expected_any": ["uts", "midterm", "sebesar"],
      "expected_all": ["20%"],
      "retrieval_expected_all": ["Midterm", "20%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: How much is the final project demo worth?",
      "expected_language": "id",
      "language_expected_any": ["demo", "proyek", "sebesar"],
      "expected_all": ["30%"],
      "retrieval_expected_all": ["Final project demo", "30%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What minimum score is required to pass the course?",
      "expected_language": "id",
      "language_expected_any": ["nilai", "minimal", "lulus"],
      "expected_all": ["60"],
      "retrieval_expected_all": ["Minimum passing score", "60"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the assignment submission deadline and platform?",
      "expected_language": "id",
      "language_expected_any": ["batas", "pengumpulan", "dikumpulkan"],
      "expected_all": ["23:59", "LMS"],
      "retrieval_expected_all": ["Friday", "23:59", "LMS"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the midterm exam date?",
      "expected_language": "id",
      "language_expected_any": ["ujian", "midterm", "tanggal"],
      "expected_all": ["2026-05-12"],
      "retrieval_expected_all": ["Midterm exam date", "2026-05-12"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the final project demo date?",
      "expected_language": "id",
      "language_expected_any": ["demo", "proyek", "tanggal"],
      "expected_all": ["2026-06-20"],
      "retrieval_expected_all": ["Final project demo date", "2026-06-20"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: When is the public holiday with no class?",
      "expected_language": "id",
      "language_expected_any": ["libur", "kelas", "tanggal"],
      "expected_all": ["2026-05-01"],
      "retrieval_expected_all": ["Public holiday", "2026-05-01"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the make-up class date, time, and platform?",
      "expected_language": "id",
      "language_expected_any": ["kelas", "pengganti", "melalui"],
      "expected_all": ["2026-05-03", "09:00", "Zoom"],
      "retrieval_expected_all": ["Make-up class date", "2026-05-03", "09:00", "Zoom"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What submission format is required?",
      "expected_language": "id",
      "language_expected_any": ["format", "pengumpulan", "harus"],
      "expected_all": ["PDF"],
      "retrieval_expected_all": ["Submission format", "PDF only"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the maximum file size?",
      "expected_language": "id",
      "language_expected_any": ["ukuran", "maksimum", "file"],
      "expected_all": ["10 MB"],
      "retrieval_expected_all": ["Maximum file size", "10 MB"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the required filename pattern?",
      "expected_language": "id",
      "language_expected_any": ["nama file", "pola", "harus"],
      "expected_all": ["NIM_Name_HW#.pdf"],
      "retrieval_expected_all": ["Required filename pattern", "NIM_Name_HW#.pdf"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the late penalty per day?",
      "expected_language": "id",
      "language_expected_any": ["terlambat", "penalti", "per hari"],
      "expected_all": ["10%"],
      "retrieval_expected_all": ["Late penalty", "10% per day"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the maximum late window?",
      "expected_language": "id",
      "language_expected_any": ["terlambat", "maksimum", "hari"],
      "expected_all": ["3"],
      "retrieval_expected_all": ["Maximum late window", "3 days"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the plagiarism threshold?",
      "expected_language": "id",
      "language_expected_any": ["plagiarisme", "similarity", "ambang"],
      "expected_all": ["20%"],
      "retrieval_expected_all": ["Plagiarism threshold", "below 20%"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the project group size?",
      "expected_language": "id",
      "language_expected_any": ["kelompok", "anggota", "mahasiswa"],
      "expected_all": ["3-4"],
      "retrieval_expected_all": ["Group size", "3-4 students"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What branch naming format should students use?",
      "expected_language": "id",
      "language_expected_any": ["branch", "format", "menggunakan"],
      "expected_all": ["feat/<nim>-<topic>"],
      "retrieval_expected_all": ["Branch naming format", "feat/<nim>-<topic>"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the commit message convention?",
      "expected_language": "id",
      "language_expected_any": ["commit", "pesan", "konvensi"],
      "expected_all": ["type(scope): summary"],
      "retrieval_expected_all": ["Commit message convention", "type(scope): summary"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Jawab dalam Bahasa Indonesia dengan kalimat lengkap: What is the core textbook?",
      "expected_language": "id",
      "language_expected_any": ["buku", "teks", "utama"],
      "expected_all": ["Russell and Norvig", "4th Edition"],
      "retrieval_expected_all": ["Core textbook", "Russell and Norvig", "4th Edition"],
      "forbidden": ["saya tidak tahu", "i don't know"],
      "expect_abstain": false
    }
  ]
}