big update

backend/api.py

@@ -8,6 +8,7 @@ import base64
 import cv2 
 import numpy as np 
 from pydantic import BaseModel
+from fastapi.responses import StreamingResponse
 from src.chains.rag import generate_narrative
 from src.chains.chain import create_rag_chain
 
@@ -98,13 +99,19 @@ async def ask_expert(request: QuestionRequest):
     try:
         rag_chain = create_rag_chain()
         
-        jawaban = rag_chain.invoke(request.question)
+        # Buat fungsi generator untuk memecah respons menjadi potongan (chunks)
+        def generate_response():
+            # rag_chain.stream() menggantikan rag_chain.invoke()
+            for chunk in rag_chain.stream(request.question):
+                # yield mengirimkan potongan teks ke klien saat itu juga
+                yield chunk 
+                
+        # Kembalikan StreamingResponse, bukan dictionary JSON
+        return StreamingResponse(
+            generate_response(), 
+            media_type="text/event-stream"
+        )
         
-        return {
-            "status": "success",
-            "question": request.question,
-            "answer": jawaban 
-        }
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Terjadi kesalahan pada LLM: {str(e)}")
 

chroma_data/chroma.sqlite3

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ad9cde89338037f6a33e3f03b17c5337d35533f702336ccf16c70f0e92a9534d
-size 1761280
+oid sha256:da26aef13c026e13fc24403eee01d7fb6bb603687a9a9a3e62482b16e1d21349
+size 2543616

chroma_data/{cdcb1c1c-f374-4f62-9cc7-7e62dcdaccd0 → d6ddab1a-f591-4aa9-ac0b-3e6358c18ab5}/data_level0.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:41b1d6558680207762d59e507f9dcba0cb9fbbd4c23c79e14025206d3742e17f
+oid sha256:3ac7a09f294a000c263e1c7ea232d164bbfb2c41fe75fda603c0d9b19bf21a22
 size 423600

chroma_data/{cdcb1c1c-f374-4f62-9cc7-7e62dcdaccd0 → d6ddab1a-f591-4aa9-ac0b-3e6358c18ab5}/length.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a12e561363385e9dfeeab326368731c030ed4b374e7f5897ac819159d2884c5
+oid sha256:1c28d39adcc8e9aed92582147fa03b71df7dd3593121741113f68c715d2ec443
 size 400

data/budidaya_cabai_rawit.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a6e345b22fa5dde1d7eb789b4abb9ae8a9fbe83d774eb3ee0e9f33468108032
+size 165447

data/penyakit.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa557f928195ae43186444b059dc4052e2f6390da484c5ca42a31e4e3d6a698d
+size 696137

frontend/index.html

@@ -215,32 +215,20 @@
         <div
           class="bg-white p-2 rounded-3xl shadow-sm ring-1 ring-slate-200 mb-6 md:mb-8 max-w-2xl mx-auto"
         >
-          <label
-            for="file-upload"
-            class="group flex flex-col items-center justify-center w-full h-40 md:h-64 border-2 border-dashed border-slate-200 rounded-[1.25rem] bg-slate-50/50 hover:bg-rose-50/50 hover:border-rose-300 transition-all cursor-pointer relative overflow-hidden"
-          >
-            <div
-              class="w-12 h-12 md:w-16 md:h-16 mb-3 md:mb-4 rounded-full bg-white shadow-sm ring-1 ring-slate-100 flex items-center justify-center group-hover:scale-110 transition-all duration-300"
-            >
-              <svg
-                class="w-6 h-6 md:w-8 md:h-8 text-rose-500"
-                fill="none"
-                stroke="currentColor"
-                viewBox="0 0 24 24"
-              >
-                <path
-                  stroke-linecap="round"
-                  stroke-linejoin="round"
-                  stroke-width="2"
-                  d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-8l-4-4m0 0L8 8m4-4v12"
-                ></path>
-              </svg>
-            </div>
-            <span
-              class="text-sm md:text-base text-slate-600 font-medium group-hover:text-rose-600 transition-colors"
-              >Pilih gambar daun</span
-            >
-          </label>
+          <label for="file-upload" class="relative flex flex-col items-center justify-center w-full h-64 border-2 border-dashed border-slate-300 rounded-2xl cursor-pointer hover:bg-slate-50 transition-colors overflow-hidden group">
+    
+    <div id="upload-placeholder" class="flex flex-col items-center justify-center transition-opacity duration-300">
+        <div class="p-3 bg-white shadow-sm ring-1 ring-slate-100 rounded-full mb-3 group-hover:scale-110 transition-transform">
+            <svg class="w-6 h-6 text-rose-500" fill="none" stroke="currentColor" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 16v1a3 3 0 003 3h10a3 3 0 003-3v-1m-4-8l-4-4m0 0L8 8m4-4v12"></path></svg>
+        </div>
+        <p class="text-sm text-slate-600 font-medium">Pilih gambar daun</p>
+    </div>
+
+    <img id="image-preview" class="hidden absolute inset-0 w-full h-full object-cover z-10" alt="Preview Daun" />
+
+</label>
+
+<input id="file-upload" type="file" accept="image/*" class="hidden" onchange="handleFileChange(event)" />
           <input
             type="file"
             accept="image/jpeg, image/png, image/jpg"

frontend/script.js

@@ -10,47 +10,91 @@ let selectedFile = null;
 function parseMarkdown(text) {
   if (!text) return "";
   let html = text;
-  html = html.replace(/(?:^\|.*\|(?:\n|\r|$))+/gm, function(match) {
-      let rows = match.trim().split('\n');
-      let tableHtml = '<div class="overflow-x-auto my-5 rounded-xl ring-1 ring-slate-200 shadow-sm"><table class="w-full text-sm text-left text-slate-600">';
-      
-      rows.forEach((row, index) => {
-          if (row.match(/^\|[\s\-\:]+\|/)) return;
-
-          let cleanRow = row.replace(/^\||\|$/g, '');
-          let cols = cleanRow.split('|').map(c => c.trim());
-
-          tableHtml += '<tr class="border-b border-slate-200 last:border-0 hover:bg-slate-50 transition-colors">';
-          cols.forEach(col => {
-              if (index === 0) {
-                  tableHtml += `<th class="px-4 py-3 bg-slate-100 font-semibold text-slate-700 whitespace-nowrap">${col}</th>`;
-              } else {
-                  tableHtml += `<td class="px-4 py-3 align-top">${col}</td>`;
-              }
-          });
-          tableHtml += '</tr>';
+  html = html.replace(/(?:^\|.*\|(?:\n|\r|$))+/gm, function (match) {
+    let rows = match.trim().split("\n");
+    let tableHtml =
+      '<div class="overflow-x-auto my-5 rounded-xl ring-1 ring-slate-200 shadow-sm"><table class="w-full text-sm text-left text-slate-600">';
+
+    rows.forEach((row, index) => {
+      if (row.match(/^\|[\s\-\:]+\|/)) return;
+
+      let cleanRow = row.replace(/^\||\|$/g, "");
+      let cols = cleanRow.split("|").map((c) => c.trim());
+
+      tableHtml +=
+        '<tr class="border-b border-slate-200 last:border-0 hover:bg-slate-50 transition-colors">';
+      cols.forEach((col) => {
+        if (index === 0) {
+          tableHtml += `<th class="px-4 py-3 bg-slate-100 font-semibold text-slate-700 whitespace-nowrap">${col}</th>`;
+        } else {
+          tableHtml += `<td class="px-4 py-3 align-top">${col}</td>`;
+        }
       });
-      tableHtml += '</table></div>';
-      return tableHtml;
+      tableHtml += "</tr>";
+    });
+    tableHtml += "</table></div>";
+    return tableHtml;
   });
 
-  html = html.replace(/^###\s+(.*$)/gim, '<h3 class="text-lg font-bold text-slate-800 mt-5 mb-2">$1</h3>');
-  
-  html = html.replace(/^##\s+(.*$)/gim, '<h2 class="text-xl font-bold text-slate-800 mt-5 mb-2">$1</h2>');
-
-  html = html.replace(/\*\*(.*?)\*\*/g, '<strong class="font-bold text-slate-800">$1</strong>');
-  
-  html = html.replace(/^[\*\-]\s+(.*$)/gim, '<div class="flex gap-2 mt-1.5"><span class="text-rose-500 font-bold shrink-0">•</span><span>$1</span></div>');
-
-  html = html.replace(/(?<!^)\*(.*?)\*/g, '<em class="italic text-slate-700">$1</em>');
-
-  html = html.replace(/\n/g, '<br/>');
-  
-  html = html.replace(/<\/h3><br\/>/g, '</h3>');
-  html = html.replace(/<\/h2><br\/>/g, '</h2>');
-  html = html.replace(/<\/div><br\/>/g, '</div>');
-  html = html.replace(/<\/div><br\/><br\/>/g, '</div>'); 
-  html = html.replace(/(<br\/>){3,}/g, '<br/><br/>');
+  // 1. Headings (H2, H3, H4)
+  html = html.replace(
+    /^##\s+(.*$)/gim,
+    '<h2 class="text-xl font-bold text-slate-800 mt-5 mb-2">$1</h2>',
+  );
+  html = html.replace(
+    /^###\s+(.*$)/gim,
+    '<h3 class="text-lg font-bold text-slate-800 mt-5 mb-2">$1</h3>',
+  );
+  html = html.replace(
+    /^####\s+(.*$)/gim,
+    '<h3 class="text-lg font-bold text-slate-800 mt-5 mb-2">$1</h3>',
+  );
+
+  // 2. Garis Pembatas (Horizontal Rule)
+  html = html.replace(/^---$/gm, '<hr class="my-4 border-slate-200" />');
+
+  // 3. Bold & Italic
+  html = html.replace(
+    /\*\*(.*?)\*\*/g,
+    '<strong class="font-bold text-slate-800">$1</strong>',
+  );
+  html = html.replace(
+    /(?<!^)\*(.*?)\*/g,
+    '<em class="italic text-slate-700">$1</em>',
+  );
+
+  // 4. Bullet Points (Menangkap *, -, dan •)
+  html = html.replace(
+    /^[\*\-•]\s+(.*$)/gim,
+    '<div class="flex gap-2 mt-1.5"><span class="text-rose-500 font-bold shrink-0">•</span><span>$1</span></div>',
+  );
+
+  // 5. Numbered List (Menangkap 1., 2., 3., dst) agar rapi sejajar
+  html = html.replace(
+    /^(\d+)\.\s+(.*$)/gim,
+    '<div class="flex gap-2 mt-1.5"><span class="text-rose-500 font-bold shrink-0">$1.</span><span>$2</span></div>',
+  );
+
+  // 6. Ubah Enter menjadi <br/>
+  html = html.replace(/\n/g, "<br/>");
+
+  // 7. PEMBERSIHAN EKSTREM: Hapus <br/> yang menumpuk di sekitar elemen UI
+  // Bersihkan sekitar garis pembatas
+  html = html.replace(/(<br\/>)+<hr/g, "<hr");
+  html = html.replace(/<hr(.*?)>(<br\/>)+/g, "<hr$1>");
+
+  // Bersihkan sekitar Headings
+  html = html.replace(/(<br\/>)+<h2/g, "<h2");
+  html = html.replace(/<\/h2>(<br\/>)+/g, "</h2>");
+  html = html.replace(/(<br\/>)+<h3/g, "<h3");
+  html = html.replace(/<\/h3>(<br\/>)+/g, "</h3>");
+
+  // Bersihkan sekitar kotak list (bullet & angka)
+  html = html.replace(/(<br\/>)+<div/g, "<div");
+  html = html.replace(/<\/div>(<br\/>)+/g, "</div>");
+
+  // Maksimal 2 <br/> berturut-turut untuk paragraf biasa
+  html = html.replace(/(<br\/>){3,}/g, "<br/><br/>");
 
   return html;
 }
@@ -85,18 +129,31 @@ function handleFileChange(event) {
   const file = event.target.files[0];
   if (file) {
     selectedFile = file;
+
+    // 1. Tampilkan nama file & tombol analisis
     const fileNameEl = document.getElementById("file-name");
-    fileNameEl.textContent = `File: ${file.name}`;
-    fileNameEl.classList.remove("hidden");
+    if (fileNameEl) {
+      fileNameEl.textContent = `File: ${file.name}`;
+      fileNameEl.classList.remove("hidden");
+    }
     document.getElementById("btn-upload").classList.remove("hidden");
 
-    const previewContainer = document.getElementById("preview-container");
+    // 2. KONTROL TAMPILAN KOTAK UPLOAD
     const imagePreview = document.getElementById("image-preview");
+    const uploadPlaceholder = document.getElementById("upload-placeholder");
+
+    // Masukkan sumber gambar
     imagePreview.src = URL.createObjectURL(file);
-    previewContainer.classList.remove("hidden");
 
-    document.getElementById("detect-result").classList.add("hidden");
-    document.getElementById("detect-error").classList.add("hidden");
+    // Sembunyikan ikon/teks, lalu tampilkan gambarnya
+    if (uploadPlaceholder) uploadPlaceholder.classList.add("hidden");
+    if (imagePreview) imagePreview.classList.remove("hidden");
+
+    // 3. Reset hasil deteksi sebelumnya jika ada
+    const detectResult = document.getElementById("detect-result");
+    const detectError = document.getElementById("detect-error");
+    if (detectResult) detectResult.classList.add("hidden");
+    if (detectError) detectError.classList.add("hidden");
   }
 }
 
@@ -124,7 +181,26 @@ async function handleUpload() {
     if (!response.ok) throw new Error(`Error server: ${response.status}`);
 
     const data = await response.json();
+
+    // Tampilkan hasil deteksi (gambar ber-bounding box dan teksnya)
     tampilkanHasilDeteksi(data);
+
+    // --- KODE RESET KOTAK UPLOAD DITAMBAHKAN DI SINI ---
+    // 1. Sembunyikan gambar preview daun dan tampilkan kembali ikon placeholder
+    const imagePreview = document.getElementById("image-preview");
+    const uploadPlaceholder = document.getElementById("upload-placeholder");
+
+    if (imagePreview) imagePreview.classList.add("hidden");
+    if (uploadPlaceholder) uploadPlaceholder.classList.remove("hidden");
+
+    // 2. Sembunyikan teks nama file dan tombol analisis agar bersih
+    document.getElementById("file-name").classList.add("hidden");
+    btnUpload.classList.add("hidden");
+
+    // 3. Bersihkan memori agar user bisa mengunggah file yang sama lagi jika perlu
+    selectedFile = null;
+    document.getElementById("file-upload").value = "";
+    // --------------------------------------------------
   } catch (err) {
     errorContainer.textContent = "Gagal terhubung ke server.";
     errorContainer.classList.remove("hidden");
@@ -176,6 +252,10 @@ function tampilkanHasilDeteksi(data) {
   }
 
   resultContainer.classList.remove("hidden");
+
+  setTimeout(() => {
+    resultContainer.scrollIntoView({ behavior: "smooth", block: "start" });
+  }, 500);
 }
 
 async function handleSendChat(event) {
@@ -186,14 +266,16 @@ async function handleSendChat(event) {
   const question = inputEl.value.trim();
   if (!question) return;
 
-  document.getElementById("chat-empty").classList.add("hidden");
+  const chatEmpty = document.getElementById("chat-empty");
+  if (chatEmpty) chatEmpty.classList.add("hidden");
 
+  // 1. Tampilkan pertanyaan User
   appendMessage("user", question);
   inputEl.value = "";
-
   inputEl.disabled = true;
   btnSend.disabled = true;
 
+  // 2. Tampilkan animasi loading (...) sebelum AI mulai mengetik
   const loadingId = showChatLoading();
 
   try {
@@ -205,12 +287,57 @@ async function handleSendChat(event) {
 
     if (!response.ok) throw new Error("Gagal");
 
-    const data = await response.json();
+    // AI mulai membalas, hilangkan animasi loading
     removeChatLoading(loadingId);
-    appendMessage("assistant", data.answer);
+
+const aiBubble = appendMessage("assistant", "");
+    const chatBox = document.getElementById("chat-messages");
+
+    // 4. LOGIKA STREAMING DENGAN "REM MESIN TIK" (TYPEWRITER EFFECT)
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder("utf-8");
+    
+    let fullTextFromAPI = ""; // Tangki penampung teks super cepat dari server
+    let textToDisplay = "";   // Teks yang akan diteteskan ke layar perlahan-lahan
+    let charIndex = 0;
+
+    // A. Fungsi pengetik independen (Jalan di latar belakang)
+    // Angka 15 adalah kecepatan ketik (15 milidetik per huruf). Bisa Anda perbesar jika ingin lebih lambat.
+    const typingInterval = setInterval(() => {
+      // Jika masih ada huruf di tangki yang belum ditampilkan
+      if (charIndex < fullTextFromAPI.length) {
+        // Keluarkan 2 huruf sekaligus agar tidak terlalu lambat
+        textToDisplay += fullTextFromAPI.slice(charIndex, charIndex + 4);
+        charIndex += 4;
+        
+        aiBubble.innerHTML = parseMarkdown(textToDisplay);
+        scrollToBottom(chatBox);
+      }
+    }, 15); 
+
+    // B. Pipa penyedot dari API Server (Berjalan secepat kilat)
+    while (true) {
+      const { done, value } = await reader.read();
+      
+      if (done) {
+        // Matikan interval pengetik HANYA JIKA semua teks sudah berhasil diketik ke layar
+        const waitComplete = setInterval(() => {
+          if (charIndex >= fullTextFromAPI.length) {
+            clearInterval(typingInterval);
+            clearInterval(waitComplete);
+          }
+        }, 50);
+        break;
+      }
+
+      // Terjemahkan byte dan masukkan langsung ke tangki penampung
+      const chunkText = decoder.decode(value, { stream: true });
+      fullTextFromAPI += chunkText; 
+    }
+
   } catch (err) {
     removeChatLoading(loadingId);
-    appendMessage("assistant", "❌ Gagal terhubung ke server.");
+    appendMessage("assistant", "❌ Gagal terhubung ke server. Silakan coba lagi.");
   } finally {
     inputEl.disabled = false;
     btnSend.disabled = false;
@@ -227,17 +354,21 @@ function appendMessage(role, content) {
   if (role === "user") {
     bubble.className =
       "max-w-[90%] md:max-w-[80%] p-3 md:p-4 rounded-2xl rounded-tr-sm bg-rose-500 text-white shadow-md shadow-rose-200 leading-relaxed text-sm md:text-[15px]";
-    bubble.textContent = content; // User input tetap menggunakan text content murni demi keamanan
+    bubble.textContent = content;
   } else {
     bubble.className =
       "max-w-[90%] md:max-w-[80%] p-3 md:p-4 rounded-2xl rounded-tl-sm bg-white ring-1 ring-slate-200 text-slate-700 shadow-sm leading-relaxed text-sm md:text-[15px]";
 
-    bubble.innerHTML = parseMarkdown(content);
+    // Jika ada konten (user history), parse markdown. Jika kosong (awal streaming), biarkan kosong.
+    bubble.innerHTML = content ? parseMarkdown(content) : "";
   }
 
   wrapper.appendChild(bubble);
   chatBox.appendChild(wrapper);
   scrollToBottom(chatBox);
+
+  // KUNCI PERUBAHAN: Kembalikan elemen bubble agar bisa di-update oleh fungsi streaming
+  return bubble;
 }
 
 function showChatLoading() {

ingest.py

@@ -11,30 +11,36 @@ SOURCES = [
     "https://www.dgwfertilizer.co.id/8-hama-dan-penyakit-penting-pada-tanaman-cabai/",
     "https://mitrabertani.com/artikel/detail/Budidaya-Cabai-Sederhana-tapi-Penting-Cara-Tepat-Tanam-Cabai",
     "https://digitani.ipb.ac.id/bagaimana-langkah-langkah-budidaya-cabai/",
-    "data/cabai.pdf"
+    "data/cabai.pdf",
+    "data/budidaya_cabai_rawit.pdf",
+    "data/penyakit.pdf"
 ]
 
 def run_ingestion_pipeline():
-    print("Memulai Data Ingestion Pipeline\n")
+    print("Memulai Data Ingestion Pipeline (Mode Semantik)\n")
     all_chunks = []
     
     for source in SOURCES:
         try:
+            print(f"Membaca sumber: {source}")
             raw_docs = load_data(source)
-            chunks = split_documents(raw_docs, chunk_size=1000, chunk_overlap=150)
+            
+            # MENGGUNAKAN SEMANTIC CHUNKING KHUSUS UNTUK WEB & PDF
+            chunks = split_documents(raw_docs, chunk_size=1000, chunk_overlap=200)
+            
             all_chunks.extend(chunks)
-            print(f"Berhasil memproses: {source} ({len(chunks)} chunks)")
+            print(f"Berhasil memproses menjadi {len(chunks)} semantic chunks.\n")
         except Exception as e:
-            print(f"Gagal memproses {source}: {e}")
+            print(f"Gagal memproses {source}: {e}\n")
             
     if all_chunks:
-        print(f"\nMenyiapkan model embedding dan menyimpan {len(all_chunks)} chunks ke ChromaDB...")
+        print(f"Menyiapkan penyisipan {len(all_chunks)} semantic chunks ke ChromaDB...")
         
         db = get_vector_store()
-        
         db.add_documents(all_chunks)
         
-        print("\nSelesai! Semua data web telah masuk ke database dan siap digunakan oleh API/Streamlit.")
+        print("\nSelesai! Semua data web & PDF telah diproses secara semantik.")
+        print("Database vektor siap digunakan oleh API FastAPI.")
     else:
         print("\nTidak ada data yang berhasil diproses.")
 

requirements.txt

@@ -13,6 +13,7 @@ langchain_core
 opencv-python-headless
 langchain
 langchain_community
+langchain-experimental
 langchain-core
 langchain-openai
 langchain-chroma

src/chains/chain.py

@@ -8,6 +8,7 @@ sys.path.append(root_dir)
 from langchain_community.cross_encoders import HuggingFaceCrossEncoder
 from langchain_core.runnables import RunnablePassthrough
 from langchain_core.output_parsers import StrOutputParser
+from langchain_groq import ChatGroq
 from langchain_openai import ChatOpenAI 
 from langchain_classic.retrievers.document_compressors import CrossEncoderReranker
 from langchain_core.messages import SystemMessage
@@ -19,11 +20,11 @@ from src.chains.prompt import get_rag_prompt
 load_dotenv()
 
 def create_rag_chain(disease_label=None):
-    llm = ChatOpenAI(
-        model="nvidia/nemotron-3-nano-30b-a3b:free", 
+    llm = ChatGroq(
+        model="openai/gpt-oss-20b", 
+        streaming=True,
         temperature=0.2,
-        openai_api_key=os.getenv("OPENROUTER_API_KEY"), 
-        openai_api_base="https://openrouter.ai/api/v1",
+        api_key=os.getenv("GROQ_API_KEY"),
     ) 
     
     vs = get_vector_store()

src/chains/rag.py

@@ -1,4 +1,6 @@
 import os
+from dotenv import load_dotenv
+from langchain_groq import ChatGroq
 from langchain_openai import ChatOpenAI
 from langchain_chroma import Chroma
 from langchain_huggingface import HuggingFaceEmbeddings
@@ -7,13 +9,13 @@ from langchain_classic.retrievers import EnsembleRetriever
 from langchain_core.documents import Document
 from src.chains.prompt import DISEASE_PROMPT_TEMPLATE
 
-llm = ChatOpenAI(
-    base_url="https://openrouter.ai/api/v1",
-    api_key=os.getenv("OPENROUTER_API_KEY"),
-    model="nvidia/nemotron-3-nano-30b-a3b:free",
-    temperature=0.4,
-    max_tokens=1500
-)
+load_dotenv()
+
+llm = ChatGroq(
+        model="openai/gpt-oss-20b", 
+        temperature=0.2,
+        api_key=os.getenv("GROQ_API_KEY"),
+    ) 
 
 print("Memuat koneksi ke Database...")
 embeddings = HuggingFaceEmbeddings(model_name="Qwen/Qwen3-Embedding-0.6B")

src/ingestion/chunker.py

@@ -1,16 +1,12 @@
 from langchain_text_splitters import RecursiveCharacterTextSplitter
+from langchain_experimental.text_splitter import SemanticChunker
 from langchain_core.documents import Document
 from typing import List
-
-#   chunk_overlap: Jumlah karakter yang tumpang tindih antar potongan.
-chunk_size = 1000
-chunk_overlap = 200
+from src.ingestion.embedder import get_embedding_model 
 
 def split_documents(documents: List[Document], chunk_size, chunk_overlap):
     print(f"Memulai proses pemotongan {len(documents)} dokumen...")
     
-    # Inisialisasi splitter
-    # Kita menggunakan karakter [\n\n, \n, " ", ""] sebagai prioritas pemisah
     text_splitter = RecursiveCharacterTextSplitter(
         chunk_size=chunk_size,
         chunk_overlap=chunk_overlap,
@@ -21,4 +17,18 @@ def split_documents(documents: List[Document], chunk_size, chunk_overlap):
     chunks = text_splitter.split_documents(documents)
     
     print(f"Berhasil memecah dokumen menjadi {len(chunks)} potongan teks.")
-    return chunks
+    return chunks
+
+# def split_documents_semantically(documents: List[Document]):
+#     print(f"Memulai analisis semantik pada {len(documents)} dokumen")
+    
+#     embeddings = get_embedding_model()
+    
+#     # Inisialisasi Semantic Chunker
+#     semantic_splitter = SemanticChunker(
+#         embeddings, 
+#         breakpoint_threshold_type="percentile"
+#     )
+    
+#     chunks = semantic_splitter.split_documents(documents)
+#     return chunks

src/ingestion/embedder.py

@@ -1,7 +1,9 @@
+import os
 from langchain_huggingface import HuggingFaceEmbeddings
 import torch
 
-model_name = "Qwen/Qwen3-Embedding-0.6B"
+
+model_name = "LazarusNLP/all-indo-e5-small-v4"
 
 def get_embedding_model(model_name: str = model_name):
     print(f"Mempersiapkan model embedding: {model_name}")