Create codekaggle/ .extractor_part_pdf

codekaggle/ .extractor_part_pdf

@@ -0,0 +1,385 @@
+# =============================================================================
+# IBEX v2.4 — Indo-Bloom Context Extractor
+# Kaggle Notebook (CPU, tidak butuh GPU)
+#
+# CARA PAKAI:
+#   1. Upload PDF BSE ke Kaggle Input (+ Add Input → klik titik 3 → Upload)
+#   2. Sesuaikan CONFIG di Cell 2
+#   3. Run All → unduh CSV di bagian bawah
+#
+# Output: CSV dengan kolom:
+#   chunk_id | source_file | source_type | page_range |
+#   word_count | noise_score | kata_dibuang_l2 | context
+#
+# Requirements: pymupdf pandas (sudah pre-installed di Kaggle)
+# =============================================================================
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CELL 1 — Install (jika belum ada)
+# ─────────────────────────────────────────────────────────────────────────────
+import subprocess, sys
+subprocess.run([sys.executable, "-m", "pip", "install", "-q", "pymupdf"], check=False)
+
+import os, re, tempfile
+import pandas as pd
+import fitz   # PyMuPDF
+
+print("✅ Library siap.")
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CELL 2 — KONFIGURASI (SESUAIKAN DI SINI)
+# ─────────────────────────────────────────────────────────────────────────────
+
+CONFIG = {
+    # Path ke PDF — setelah upload ke Kaggle Input, path-nya biasanya:
+    # /kaggle/input/<nama-dataset>/<nama-file>.pdf
+    "pdf_path"    : "/kaggle/input/datasets/baimfirmansyah/sosiologi-bs-kls-x11/Sosiologi_BS_KLS_XII.pdf",
+
+    # Tipe sumber: "BSE" atau "Wikipedia"
+    "source_type" : "BSE",
+
+    # ── Pembagian part ────────────────────────────────────────────────────────
+    # Jalankan notebook ini sekali per part, ubah hal_mulai & hal_selesai
+    # Rekomendasi untuk BSE Sosiologi XII:
+    #   Part 1: 15–35  | Part 2: 36–55  | Part 3: 56–75
+    #   Part 4: 76–95  | Part 5: 96–115 | Part 6: 116–135
+    #   Part 7: 136–155 (sesuaikan dengan halaman terakhir konten)
+    "hal_mulai"   : 36,
+    "hal_selesai" : 55,
+
+    # ── Parameter ekstraksi ───────────────────────────────────────────────────
+    # Kata per chunk: 100–200 (rekomendasi 150 untuk BSE)
+    "chunk_size"  : 150,
+
+    # Toleransi noise: 1–6
+    #   BSE      → 2 (ketat, banyak instruksional)
+    #   Wikipedia → 4 (lebih longgar, teks lebih bersih)
+    "batas_noise" : 2,
+
+    # Folder output CSV
+    "output_dir"  : "/kaggle/working",
+}
+
+print(f"📋 Konfigurasi:")
+print(f"   PDF        : {CONFIG['pdf_path']}")
+print(f"   Sumber     : {CONFIG['source_type']}")
+print(f"   Halaman    : {CONFIG['hal_mulai']} – {CONFIG['hal_selesai']}")
+print(f"   Chunk size : {CONFIG['chunk_size']} kata")
+print(f"   Batas noise: {CONFIG['batas_noise']}")
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CELL 3 — KONSTANTA & POLA FILTER
+# ─────────────────────────────────────────────────────────────────────────────
+
+NOISE_BSE = [
+    r'tujuan pembelajaran', r'setelah mempelajari',
+    r'diharapkan (mampu|dapat)', r'kata kunci', r'pemetaan pikiran',
+    r'isbn\s*:', r'penulis\s*:', r'kementerian pendidikan',
+    r'latihan\s+\d+\.\d+', r'tugas\s+\d+\.\d+', r'uji pengetahuan',
+    r'apersepsi', r'simak(lah)?\s+(kutipan|artikel|infografik|gambar)',
+    r'amatilah\s+gambar', r'salin\s+tabel', r'berilah\s+tanda',
+    r'buku\s+tugas', r'presentasikan',
+    r'jawablah\s+(beberapa\s+)?pertanyaan',
+    r'kemukakan\s+pendapat\s+kalian',
+    r'coba\s+(deskripsikan|identifikasi|kemukakan)',
+    r'diskusikan\s+(jawabannya|beberapa)', r'ajukan\s+pertanyaan',
+    r'gambar\s+\d+\.\d+', r'tabel\s+\d+\.\d+',
+    r'sumber\s*:\s*(kemendikbud|bank|indonesiabaik)',
+    r'bab\s+\d+\s+\|\s+\w+', r'no\.\s+faktor\s+pendorong',
+    r'benar\s+salah', r'sosiologi\s+untuk\s+sma',
+]
+NOISE_PATTERNS = [re.compile(p, re.IGNORECASE) for p in NOISE_BSE]
+
+KALIMAT_INSTRUKSIONAL = [
+    r'(setelah|sesudah)\s+(membaca|menyimak|mengamati).{0,60}(jawab|diskusi|kemukakan|sajikan)',
+    r'^(simak|amati|perhatikan)(lah)?\s+',
+    r'(masih\s+ingatkah|tahukah|ingatkah)\s+kalian',
+    r'^(pada\s+pembahasan\s+awal|kalian\s+telah\s+(mempelajari|mengetahui|memahami))',
+    r'^(sebelum\s+memahami|sebelum\s+mempelajari)',
+    r'^(mari\s+(kita\s+)?(simak|pelajari|bahas|amati))',
+    r'^apakah\s+(fenomena|peristiwa|kondisi).{0,80}\?$',
+    r'^menurut\s+kalian.{0,80}\?$',
+    r'^apa\s+yang\s+dimaksud.{0,60}\?\s*(kemukakan|diskusikan)',
+    r'^setujukah\s+kalian',
+    r'kemukakan\s+pendapat\s+(kalian|anda)',
+    r'(sajikan|presentasikan)\s+(hasilnya|hasil)',
+    r'(ajukan|ajukanlah)\s+pertanyaan',
+    r'(diskusikan|bahaslah)\s+(bersama|dengan\s+teman)',
+    r'jawab(lah)?\s+(pertanyaan|soal)\s+(berikut|di\s+bawah)',
+    r'(jelaskan|uraikan)\s+(pendapat|jawaban)\s+kalian',
+    r'^\d{1,3}\s+(sosiologi|matematika|fisika|kimia|biologi|sejarah|geografi|'
+    r'ekonomi|pkn|bahasa|pendidikan|prakarya|seni)\s+untuk\s+',
+    r'^dikutip\s+dari\s*:',
+    r'setelah\s+menyimak\s+penjelasan',
+    r'kalian\s+telah\s+(mengetahui|mempelajari|memahami)\s+bahwa',
+    r'apa\s+yang\s+dimaksud.{0,80}\?$',
+]
+KALIMAT_PATTERNS = [re.compile(p, re.IGNORECASE) for p in KALIMAT_INSTRUKSIONAL]
+HEADER_BAB_PAT   = re.compile(r'bab\s+\d+\s*\|\s*[\w\s]+\d+', re.IGNORECASE)
+
+_MAPEL = (r'sosiologi|matematika|fisika|kimia|biologi|sejarah|geografi|'
+          r'ekonomi|pkn|pendidikan|prakarya|seni|bahasa')
+
+KATA_KUNCI_C2 = [
+    'karena','menyebabkan','berdampak','sehingga','mengakibatkan','berakibat',
+    'oleh karena','disebabkan','bertujuan','berfungsi','berperan','berguna',
+    'tujuan','fungsi','manfaat','peran','kegunaan','mekanisme','tahapan',
+    'langkah','alasan','dampak','pengaruh','akibat','sebab','hubungan',
+    'keterkaitan','mengapa','bagaimana','jelaskan','uraikan',
+]
+
+print("✅ Pola filter dimuat.")
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CELL 4 — FUNGSI IBEX v2.4
+# ─────────────────────────────────────────────────────────────────────────────
+
+def hitung_noise(t):
+    return sum(1 for p in NOISE_PATTERNS if p.search(t))
+
+def pisah_nomor_halaman(t):
+    return re.sub(
+        rf'\s+(\d{{1,3}})\s+({_MAPEL})\s+untuk\s+sma',
+        r'. \1 \2 untuk sma', t, flags=re.IGNORECASE
+    )
+
+def hapus_header_bab(t):
+    return HEADER_BAB_PAT.sub('', t)
+
+def rescue_narasi(k):
+    m = re.search(
+        r'\d{1,3}\s+\S+\s+untuk\s+sma\S*\s+kelas\s+x{1,3}i{0,2}\s+(.*)',
+        k, re.IGNORECASE
+    )
+    if m:
+        sisa = m.group(1).strip()
+        if len(sisa.split()) >= 5:
+            return sisa
+    return ""
+
+def bersihkan_kalimat(teks):
+    """Filter L2: buang kalimat instruksional, rescue narasi yang terselamatkan."""
+    teks = pisah_nomor_halaman(teks)
+    teks = hapus_header_bab(teks)
+    teks = re.sub(r'\s{2,}', ' ', teks).strip()
+    bersih, dibuang = [], 0
+    for k in re.split(r'(?<=[.!?])\s+', teks):
+        k = k.strip()
+        if len(k) < 10:
+            continue
+        if any(p.search(k) for p in KALIMAT_PATTERNS):
+            rescued = rescue_narasi(k)
+            if rescued:
+                bersih.append(rescued)
+                dibuang += len(k.split()) - len(rescued.split())
+            else:
+                dibuang += len(k.split())
+        else:
+            bersih.append(k)
+    return " ".join(bersih).strip(), dibuang
+
+
+def ekstrak_ibex(pdf_path, hal_mulai, hal_selesai,
+                 chunk_size, batas_noise, source_type="BSE"):
+    """
+    Core IBEX v2.4.
+    Return: (df_chunks, ringkasan_str)
+    """
+    doc   = fitz.open(pdf_path)
+    nama  = os.path.basename(pdf_path)
+    total = len(doc)
+    start = max(0, hal_mulai - 1)
+    end   = min(total, hal_selesai)
+
+    if start >= end:
+        return None, f"❌ Rentang tidak valid. PDF punya {total} halaman."
+
+    print(f"📖 Membaca hal {hal_mulai}–{hal_selesai} dari '{nama}' ({total} hal total)...")
+
+    # ── Ekstrak teks per halaman (crop margin 8%) ─────────────────────────
+    teks_per_hal = []
+    for num in range(start, end):
+        page = doc.load_page(num)
+        rect = page.rect
+        mg   = rect.height * 0.08
+        clip = fitz.Rect(rect.x0, rect.y0 + mg, rect.x1, rect.y1 - mg)
+        teks_per_hal.append(page.get_text("text", clip=clip, sort=True))
+
+    def bersihkan_dasar(t):
+        t = re.sub(r'-\n\s*', '', t)   # sambung kata yang terpotong baris
+        t = re.sub(r'\n+',    ' ', t)
+        return re.sub(r'\s+', ' ', t).strip()
+
+    teks_gabung  = bersihkan_dasar(" ".join(teks_per_hal))
+    kalimat_list = [
+        k.strip() for k in re.split(r'(?<=[.!?])\s+', teks_gabung)
+        if len(k.strip()) > 10
+    ]
+    print(f"   Total kata input : {len(teks_gabung.split()):,}")
+    print(f"   Total kalimat    : {len(kalimat_list)}")
+
+    # ── Chunking dengan overlap 25% ───────────────────────────────────────
+    overlap    = max(1, chunk_size // 4)
+    chunk_meta = []
+    chunk_id   = 1
+    bng_l1 = bng_pendek = 0
+    i = 0
+
+    while i < len(kalimat_list):
+        buf, j = [], i
+        while j < len(kalimat_list) and len(buf) < chunk_size:
+            buf.extend(kalimat_list[j].split())
+            j += 1
+
+        pot    = " ".join(buf)
+        wc     = len(buf)
+        noise  = hitung_noise(pot)
+        ada_c2 = any(k in pot.lower() for k in KATA_KUNCI_C2)
+
+        if wc >= 50 and ada_c2:
+            # ── Filter L1: buang jika noise ≥ batas ──────────────────────
+            if noise >= batas_noise:
+                bng_l1 += 1
+            else:
+                # ── Filter L2: bersihkan kalimat instruksional ────────────
+                bersih, dibuang = bersihkan_kalimat(pot)
+                wc2    = len(bersih.split())
+                noise2 = hitung_noise(bersih)
+                if wc2 >= 50 and any(k in bersih.lower() for k in KATA_KUNCI_C2):
+                    chunk_meta.append({
+                        "chunk_id"        : f"chunk_{chunk_id:04d}",
+                        "source_file"     : nama,
+                        "source_type"     : source_type,
+                        "page_range"      : f"{hal_mulai}-{hal_selesai}",
+                        "word_count"      : wc2,
+                        "noise_score"     : noise2,
+                        "kata_dibuang_l2" : dibuang,
+                        "context"         : bersih,
+                    })
+                    chunk_id += 1
+                else:
+                    bng_pendek += 1
+
+        i = j - overlap if (j - overlap) > i else j
+
+    # ── Ringkasan ─────────────────────────────────────────────────────────
+    ringkasan = (
+        f"\n{'='*55}\n"
+        f"✅ IBEX v2.4 SELESAI\n"
+        f"{'='*55}\n"
+        f"  Halaman diproses : {end - start}\n"
+        f"  Total kata input : {len(teks_gabung.split()):,}\n"
+        f"  Chunk bersih     : {len(chunk_meta)}\n"
+        f"  Buang L1 (noise) : {bng_l1}\n"
+        f"  Buang L2 (kata)  : {sum(r['kata_dibuang_l2'] for r in chunk_meta)}\n"
+        f"  Avg kata/chunk   : {round(sum(r['word_count'] for r in chunk_meta)/max(1,len(chunk_meta)),1)}\n"
+        f"{'='*55}"
+    )
+
+    if not chunk_meta:
+        return None, (
+            "⚠️ Tidak ada context C2 bersih yang dihasilkan.\n"
+            f"   Buang L1: {bng_l1} | Buang L2: {bng_pendek}\n"
+            "   Coba: perluas rentang halaman ATAU naikkan batas_noise ke 3"
+        )
+
+    return pd.DataFrame(chunk_meta), ringkasan
+
+print("✅ Fungsi IBEX siap.")
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CELL 5 — JALANKAN EKSTRAKSI
+# ─────────────────────────────────────────────────────────────────────────────
+
+df_result, ringkasan = ekstrak_ibex(
+    pdf_path    = CONFIG["pdf_path"],
+    hal_mulai   = CONFIG["hal_mulai"],
+    hal_selesai = CONFIG["hal_selesai"],
+    chunk_size  = CONFIG["chunk_size"],
+    batas_noise = CONFIG["batas_noise"],
+    source_type = CONFIG["source_type"],
+)
+
+print(ringkasan)
+
+if df_result is not None:
+    print("\n📋 Preview 3 chunk pertama:")
+    print("-" * 55)
+    for _, row in df_result.head(3).iterrows():
+        print(f"[{row['chunk_id']}] wc={row['word_count']} "
+              f"noise={row['noise_score']} buang={row['kata_dibuang_l2']}")
+        print(f"  {row['context'][:200]}...")
+        print()
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CELL 6 — SIMPAN CSV
+# ─────────────────────────────────────────────────────────────────────────────
+
+if df_result is not None:
+    nama_pdf  = os.path.splitext(os.path.basename(CONFIG["pdf_path"]))[0]
+    nama_file = (
+        f"IBEX_{nama_pdf}"
+        f"_hal{CONFIG['hal_mulai']}-{CONFIG['hal_selesai']}"
+        f"_chunk{CONFIG['chunk_size']}"
+        f"_noise{CONFIG['batas_noise']}"
+        f".csv"
+    )
+    out_path = os.path.join(CONFIG["output_dir"], nama_file)
+    df_result.to_csv(out_path, index=False, encoding="utf-8-sig")
+
+    print(f"💾 CSV tersimpan di: {out_path}")
+    print(f"   {len(df_result)} chunk | "
+          f"{df_result['word_count'].sum():,} total kata")
+    print()
+    print("📥 Cara unduh dari Kaggle:")
+    print("   Output (panel kanan) → klik nama file → Download")
+    print()
+    print("📤 Langkah berikutnya:")
+    print("   Upload CSV ini ke Space 2 QA Generator untuk generate soal C1+C2.")
+else:
+    print("❌ Tidak ada output. Cek konfigurasi dan coba lagi.")
+
+# ─────────────────────────────────────────────────────────────────────────────
+# CELL 7 — ANALISIS KUALITAS (OPSIONAL)
+# ─────────────────────────────────────────────────────────────────────────────
+
+if df_result is not None and len(df_result) > 0:
+    print("📊 ANALISIS KUALITAS CHUNK")
+    print("=" * 55)
+    print(f"Total chunk         : {len(df_result)}")
+    print(f"Rata-rata kata      : {df_result['word_count'].mean():.1f}")
+    print(f"Min – Max kata      : {df_result['word_count'].min()} – {df_result['word_count'].max()}")
+    print(f"Noise score = 0     : {(df_result['noise_score']==0).sum()} chunk ✅")
+    print(f"Noise score > 0     : {(df_result['noise_score']>0).sum()} chunk ⚠️")
+    print(f"Total kata dibuang  : {df_result['kata_dibuang_l2'].sum()}")
+    print()
+
+    # Estimasi QA yang akan dihasilkan
+    n_c1, n_c2 = 2, 2
+    est_qa = len(df_result) * (n_c1 + n_c2)
+    print(f"🎯 ESTIMASI OUTPUT QA GENERATOR")
+    print(f"   Dengan setting {n_c1} C1 + {n_c2} C2 per chunk:")
+    print(f"   → ~{est_qa} QA pairs dari part ini")
+    print()
+
+    # Cek kata kunci C2
+    KATA_KUNCI_C2_check = [
+        'karena','menyebabkan','berdampak','sehingga','mengakibatkan',
+        'tujuan','fungsi','manfaat','peran','dampak','pengaruh','sebab',
+    ]
+    df_result['has_c2_keyword'] = df_result['context'].apply(
+        lambda t: any(k in t.lower() for k in KATA_KUNCI_C2_check)
+    )
+    print(f"   Chunk dengan kata kunci C2 : "
+          f"{df_result['has_c2_keyword'].sum()}/{len(df_result)} ✅")
+
+    print()
+    print("=" * 55)
+    print("Tampilkan semua chunk:")
+    print("=" * 55)
+    for _, row in df_result.iterrows():
+        print(f"\n[{row['chunk_id']}]")
+        print(f"  word_count      : {row['word_count']}")
+        print(f"  noise_score     : {row['noise_score']}")
+        print(f"  kata_dibuang_l2 : {row['kata_dibuang_l2']}")
+        print(f"  context preview : {row['context'][:150]}...")