scripts/rag_eval_single_pdf.json

{
  "course_id": "course-eval-data-besar-ef234712-001",
  "documents": [
    {
      "path": "doc/samples/Dokumen_RAG_Data_Besar_EF234712.pdf"
    }
  ],
  "questions": [
    {
      "question": "Apa kode mata kuliah Data Besar dan berapa bobot SKS-nya?",
      "expected_all": ["EF234712", "3 sks"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa nama mata kuliah dalam bahasa Indonesia dan bahasa Inggris?",
      "expected_all": ["Data Besar", "Big Data"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Siapa koordinator mata kuliah Data Besar?",
      "expected_all": ["Ratih Nur Esti Anggraini", "S.Kom", "M.Sc", "Ph.D"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Unit apa yang menaungi mata kuliah Data Besar?",
      "expected_all": ["Departemen Teknik Informatika", "ITS"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa versi dokumen dan tanggalnya?",
      "expected_all": ["1.0", "22 Mei 2026"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa catatan status dokumen terkait RPS atau kebijakan resmi?",
      "expected_all": ["bukan RPS", "bukan kebijakan resmi"],
      "expected_any": ["konten simulasi", "pengujian RAG"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa tujuan dokumen ini untuk pengujian RAG?",
      "expected_all": ["sumber pengetahuan", "retrieval", "ketepatan jawaban"],
      "expected_any": ["fakta langsung", "konteks konseptual", "data simulasi terstruktur"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Sebutkan label provenans yang digunakan dalam dokumen.",
      "expected_all": ["SUMBER UTAMA", "DATA UJI", "TIDAK TERSEDIA"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa fokus inti deskripsi mata kuliah Data Besar?",
      "expected_all": ["penyimpanan", "pengolahan data skala besar", "komputasi terdistribusi"],
      "expected_any": ["penggalian data", "pemodelan", "kasus nyata"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Teknologi apa saja yang disebut eksplisit pada CPMK-1?",
      "expected_all": ["Hadoop", "graph database", "cloud storage"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa kemampuan utama yang ditekankan pada CPMK-2?",
      "expected_all": ["komputasi terdistribusi", "pengolahan data", "paralel"],
      "expected_any": ["responsif"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Sebutkan kasus nyata yang disebut dalam CPMK-3.",
      "expected_all": ["klasifikasi", "regresi", "clustering", "sistem rekomendasi konten", "analisis jejaring sosial"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa fokus CPMK-4?",
      "expected_all": ["teknik optimasi", "pengolahan data berskala besar"],
      "expected_any": ["efisiensi", "efektivitas"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa definisi ringkas big data menurut peta konsep?",
      "expected_all": ["Kumpulan data", "berukuran besar", "penyimpanan dan pengolahan"],
      "expected_any": ["bergerak cepat", "beragam bentuk", "diskalakan"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa definisi penyimpanan terdistribusi?",
      "expected_all": ["Penyimpanan data", "beberapa node"],
      "expected_any": ["kapasitas", "ketahanan terhadap kegagalan", "kemampuan akses"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Hadoop dikaitkan dengan komponen apa dalam dokumen?",
      "expected_all": ["HDFS", "MapReduce"],
      "expected_any": ["penyimpanan terdistribusi", "pemrosesan berbasis pekerjaan"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Untuk data seperti apa graph database cocok?",
      "expected_all": ["node", "relasi"],
      "expected_any": ["jejaring sosial", "rekomendasi", "keterhubungan"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa yang disediakan cloud storage menurut dokumen?",
      "expected_all": ["skalabilitas kapasitas", "akses terkelola"],
      "expected_any": ["layanan awan"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Sebutkan tahap alur konseptual dari data mentah hingga wawasan.",
      "expected_all": ["Akuisisi", "Penyimpanan", "Pemrosesan", "Analitik", "Optimasi", "Penyajian"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "CPMK-1 dipetakan ke aktivitas belajar dan fokus konsep apa?",
      "expected_all": ["Mendesain lapisan penyimpanan", "Hadoop", "graph database", "cloud storage"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Topik apa yang dibahas pada minggu ke-5?",
      "expected_all": ["Graph database", "data relasional kompleks"],
      "expected_any": ["CPMK-1", "CPMK-3"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Topik simulasi apa yang dibahas pada minggu ke-10?",
      "expected_all": ["Klasifikasi", "regresi", "data skala besar"],
      "expected_any": ["Perbandingan model"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Topik simulasi apa yang dibahas pada minggu ke-13?",
      "expected_all": ["Analisis jejaring sosial", "berbasis graf"],
      "expected_any": ["Metrik keterhubungan"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Kelompok materi analitik data berada pada rentang minggu berapa dan terkait CPMK apa?",
      "expected_all": ["Minggu 9-13", "CPMK-3"],
      "expected_any": ["Analitik data"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa tujuan proyek InsightStream Commerce?",
      "expected_all": ["rekomendasi konten produk", "analisis jejaring interaksi pengguna"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Berapa volume data mentah pada studi kasus InsightStream Commerce?",
      "expected_all": ["18 GB"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Berapa jumlah transaksi, pengguna, dan interaksi pada studi kasus?",
      "expected_all": ["120.000.000", "2.000.000", "20.000.000"],
      "expected_any": ["transaksi", "pengguna", "interaksi"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa periode pengamatan pada studi kasus InsightStream Commerce?",
      "expected_all": ["Januari sampai Maret 2026"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Sebutkan berkas masukan simulasi yang digunakan pada proyek.",
      "expected_all": ["transaksi_ritel_2026.csv", "produk.json", "interaksi_pengguna.parquet", "relasi_pengguna.csv"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Data apa yang dipetakan ke graph database dalam proyek?",
      "expected_all": ["Data relasi pengguna", "graph database"],
      "expected_any": ["analisis jejaring"],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Bagaimana data tabular disimpan pada arsitektur proyek simulasi?",
      "expected_all": ["Data tabular", "terpartisi", "berdasarkan bulan", "cloud storage"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa saja tugas yang harus dihasilkan dalam proyek simulasi?",
      "expected_all": ["Diagram arsitektur end-to-end", "Rancangan penyimpanan", "komputasi terdistribusi", "model rekomendasi konten", "analisis jejaring sosial", "Laporan optimasi"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Komponen evaluasi simulasi apa yang memiliki bobot terbesar?",
      "expected_all": ["Proyek akhir kelompok", "30%"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Sebutkan semua komponen skema evaluasi simulasi dan bobotnya.",
      "expected_all": ["Kuis konsep", "10%", "Praktikum", "20%", "Tugas analisis individu", "15%", "Evaluasi tengah semester", "20%", "Proyek akhir kelompok", "30%", "Partisipasi dan dokumentasi", "5%"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Berapa bobot desain penyimpanan dan arsitektur dalam rubrik proyek akhir simulasi?",
      "expected_all": ["Desain penyimpanan dan arsitektur", "25%"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Berapa bobot optimasi dan evaluasi performa dalam rubrik proyek akhir simulasi?",
      "expected_all": ["Optimasi dan evaluasi performa", "15%"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Menurut FAQ, teknologi penyimpanan apa yang disebut dalam CPMK?",
      "expected_all": ["Hadoop", "graph database", "cloud storage"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Menurut FAQ, kasus analitik apa yang disebut dalam CPMK-3?",
      "expected_all": ["Klasifikasi", "regresi", "clustering", "sistem rekomendasi konten", "analisis jejaring sosial"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Apa yang tidak diberikan dokumen menurut aturan abstention?",
      "expected_all": ["jadwal kelas resmi", "ruang kuliah", "kontak email koordinator", "platform pengumpulan tugas resmi", "kebijakan kehadiran resmi"],
      "expected_any": [],
      "forbidden": ["i don't know"],
      "expect_abstain": false
    },
    {
      "question": "Di ruang mana mata kuliah Data Besar berlangsung?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "Apa email koordinator mata kuliah Data Besar?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "Apa platform pengumpulan tugas resmi mata kuliah Data Besar?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    },
    {
      "question": "Berapa minimal kehadiran resmi untuk mata kuliah Data Besar?",
      "expected_all": [],
      "expected_any": [],
      "forbidden": [],
      "expect_abstain": true
    }
  ]
}