/** * ============================================================================= * Mozhii RAG Data Platform - Chunking Tab JavaScript * ============================================================================= * Handles all functionality for the CHUNKING tab where the QA team * creates RAG-ready chunks from cleaned content. * * This is the MOST IMPORTANT stage as it produces data ready for embedding! * * Features: * - List cleaned files available for chunking * - Display source content for reference * - Create chunks with metadata * - Track chunk indices automatically * - Preview chunks before submission * - View existing chunks * ============================================================================= */ // ============================================================================= // DOM ELEMENTS // ============================================================================= /** * Cache DOM elements for the chunking tab */ const ChunkingElements = { // Panels fileList: null, sourceContent: null, sourceInfo: null, chunksList: null, // Form elements chunkHeading: null, chunkSubheading: null, chunkText: null, chunkCategory: null, chunkSource: null, chunkOverlap: null, charCount: null, indexDisplay: null, selectedFile: null, // Buttons refreshBtn: null, previewBtn: null, submitBtn: null, // Initialize element references init() { this.fileList = document.getElementById('chunking-file-list'); this.sourceContent = document.getElementById('chunking-source-content'); this.sourceInfo = document.getElementById('chunking-source-info'); this.chunksList = document.getElementById('chunks-list'); this.chunkHeading = document.getElementById('chunk-heading'); this.chunkSubheading = document.getElementById('chunk-subheading'); this.chunkText = document.getElementById('chunk-text'); this.chunkCategory = document.getElementById('chunk-category'); this.chunkSource = document.getElementById('chunk-source'); this.chunkOverlap = document.getElementById('chunk-overlap'); this.charCount = document.getElementById('chunk-char-count'); this.indexDisplay = document.getElementById('chunk-index-display'); this.selectedFile = document.getElementById('chunking-selected-file'); this.refreshBtn = document.getElementById('chunking-refresh'); this.previewBtn = document.getElementById('chunk-preview'); this.submitBtn = document.getElementById('chunk-submit'); } }; // ============================================================================= // STATE // ============================================================================= /** * Local state for the chunking tab */ const ChunkingState = { files: [], // List of available cleaned files selectedFile: null, // Currently selected file object selectedFilename: null, // Name of selected file chunks: [], // Existing chunks for selected file nextIndex: 1 // Next chunk index }; // ============================================================================= // FILE LIST FUNCTIONS // ============================================================================= /** * Refresh the list of cleaned files available for chunking */ async function refreshChunkingFiles() { try { // Show loading state ChunkingElements.fileList.innerHTML = `

Loading files...

`; // Fetch files from API const response = await api('/api/chunking/cleaned-files'); if (response.success) { ChunkingState.files = response.files; renderChunkingFileList(); } else { throw new Error(response.error || 'Failed to load files'); } } catch (error) { console.error('Error loading chunking files:', error); ChunkingElements.fileList.innerHTML = `

Error loading files

${error.message}
`; } } /** * Render the file list in the UI */ function renderChunkingFileList() { const files = ChunkingState.files; if (files.length === 0) { ChunkingElements.fileList.innerHTML = `

No cleaned files available

Complete cleaning in the CLEANING tab first
`; return; } // Create file list HTML ChunkingElements.fileList.innerHTML = files.map(file => `
📋
${file.filename}
${file.total_chunks} chunks ${file.pending_chunks > 0 ? `(${file.pending_chunks} pending)` : ''}
`).join(''); } // ============================================================================= // FILE SELECTION // ============================================================================= /** * Handle file selection for chunking * * @param {string} filename - Name of the file to select */ async function selectChunkingFile(filename) { // Find the file in our state const file = ChunkingState.files.find(f => f.filename === filename); if (!file) { console.error('File not found:', filename); return; } // Update state ChunkingState.selectedFile = file; ChunkingState.selectedFilename = filename; // Update hidden field ChunkingElements.selectedFile.value = filename; // Update file list selection document.querySelectorAll('#chunking-file-list .file-item').forEach(item => { item.classList.toggle('selected', item.dataset.filename === filename); }); // Update source content display ChunkingElements.sourceContent.innerHTML = `
${escapeHtml(file.content)}
`; // Update source info ChunkingElements.sourceInfo.textContent = `${file.content_length.toLocaleString()} chars | ${file.language.toUpperCase()}`; // Fetch existing chunks await loadChunksForFile(filename); // Enable form elements enableChunkingForm(); // Set source from file ChunkingElements.chunkSource.value = file.source || ''; // Show info toast showToast('File Selected', `Selected "${filename}" for chunking`, 'info', 3000); } /** * Escape HTML to prevent XSS */ function escapeHtml(text) { const div = document.createElement('div'); div.textContent = text; return div.innerHTML; } /** * Load existing chunks for a file * * @param {string} filename - Name of the file */ async function loadChunksForFile(filename) { try { const response = await api(`/api/chunking/chunks/${filename}`); if (response.success) { ChunkingState.chunks = response.chunks; ChunkingState.nextIndex = response.count + 1; // Update index display ChunkingElements.indexDisplay.textContent = `Chunk #${ChunkingState.nextIndex}`; // Render chunks list renderChunksList(); } } catch (error) { console.error('Error loading chunks:', error); ChunkingState.chunks = []; ChunkingState.nextIndex = 1; } } /** * Render the list of existing chunks */ function renderChunksList() { const chunks = ChunkingState.chunks; if (chunks.length === 0) { ChunkingElements.chunksList.innerHTML = `

No chunks yet

`; return; } ChunkingElements.chunksList.innerHTML = chunks.map(chunk => `
#${chunk.chunk_index} ${chunk.text.substring(0, 50)}... ${chunk.status || 'pending'}
`).join(''); } // ============================================================================= // FORM HANDLING // ============================================================================= /** * Enable the chunking form elements */ function enableChunkingForm() { ChunkingElements.chunkHeading.disabled = false; ChunkingElements.chunkSubheading.disabled = false; ChunkingElements.chunkText.disabled = false; ChunkingElements.chunkCategory.disabled = false; ChunkingElements.chunkSource.disabled = false; ChunkingElements.chunkOverlap.disabled = false; ChunkingElements.previewBtn.disabled = false; ChunkingElements.submitBtn.disabled = false; } /** * Disable the chunking form elements */ function disableChunkingForm() { ChunkingElements.chunkHeading.disabled = true; ChunkingElements.chunkSubheading.disabled = true; ChunkingElements.chunkText.disabled = true; ChunkingElements.chunkCategory.disabled = true; ChunkingElements.chunkSource.disabled = true; ChunkingElements.chunkOverlap.disabled = true; ChunkingElements.previewBtn.disabled = true; ChunkingElements.submitBtn.disabled = true; } /** * Update character count for chunk text */ function updateChunkCharCount() { const content = ChunkingElements.chunkText.value; const count = content.length; ChunkingElements.charCount.textContent = `${count.toLocaleString()} characters`; } /** * Clear the chunk form */ function clearChunkForm() { ChunkingElements.chunkHeading.value = ''; ChunkingElements.chunkSubheading.value = ''; ChunkingElements.chunkText.value = ''; ChunkingElements.chunkOverlap.value = ''; updateChunkCharCount(); } /** * Generate chunk ID based on metadata * * @returns {string} - Generated chunk ID */ function generateChunkId() { const lang = ChunkingState.selectedFile?.language || 'ta'; const cat = ChunkingElements.chunkCategory.value.substring(0, 3); const file = ChunkingState.selectedFilename?.replace(/_/g, '').substring(0, 10) || 'file'; const index = ChunkingState.nextIndex; return `${lang}_${cat}_${file}_${String(index).padStart(2, '0')}`; } // ============================================================================= // PREVIEW FUNCTIONALITY // ============================================================================= /** * Preview the chunk before submission */ function previewChunk() { // Validate if (!ChunkingElements.chunkText.value.trim()) { showToast('No Content', 'Please enter chunk text first', 'warning'); return; } const chunkId = generateChunkId(); // Build preview object const preview = { chunk_id: chunkId, heading: ChunkingElements.chunkHeading.value.trim(), sub_heading: ChunkingElements.chunkSubheading.value.trim(), text: ChunkingElements.chunkText.value.trim(), language: ChunkingState.selectedFile?.language || 'ta', category: ChunkingElements.chunkCategory.value, source: ChunkingElements.chunkSource.value || ChunkingState.selectedFile?.source || 'unknown', chunk_index: ChunkingState.nextIndex, source_file: ChunkingState.selectedFilename, overlap_reference: ChunkingElements.chunkOverlap.value.trim() }; // Show preview modal showModal('Chunk Preview', `
${JSON.stringify(preview, null, 2)}
`, [ { text: 'Close', class: 'btn-secondary', onClick: hideModal }, { text: 'Submit Chunk', class: 'btn-primary', onClick: () => { hideModal(); handleChunkSubmit(); }} ]); } // ============================================================================= // SUBMISSION // ============================================================================= /** * Submit a new chunk */ async function handleChunkSubmit() { // Validate file selection if (!ChunkingState.selectedFilename) { showToast('No File Selected', 'Please select a file first', 'warning'); return; } // Validate chunk text const text = ChunkingElements.chunkText.value.trim(); if (!text) { showToast('No Content', 'Please enter chunk text', 'warning'); return; } if (text.length < 20) { showToast('Content Too Short', 'Chunk text must be at least 20 characters', 'warning'); return; } // Disable submit button ChunkingElements.submitBtn.disabled = true; ChunkingElements.submitBtn.innerHTML = ' Adding...'; try { // Prepare chunk data const chunkData = { filename: ChunkingState.selectedFilename, heading: ChunkingElements.chunkHeading.value.trim() || undefined, sub_heading: ChunkingElements.chunkSubheading.value.trim() || undefined, text: text, category: ChunkingElements.chunkCategory.value, source: ChunkingElements.chunkSource.value || undefined, overlap_reference: ChunkingElements.chunkOverlap.value.trim() || undefined }; // Submit to API const response = await api('/api/chunking/submit', { method: 'POST', body: JSON.stringify(chunkData) }); if (response.success) { showToast( 'Chunk Created!', `Chunk #${response.chunk_index} added. Pending admin approval.`, 'success' ); // Store last chunk text as overlap reference for next chunk const lastChunkEnd = text.slice(-100); // Clear form clearChunkForm(); // Set overlap reference for next chunk ChunkingElements.chunkOverlap.value = lastChunkEnd; // Reload chunks await loadChunksForFile(ChunkingState.selectedFilename); // Refresh file list to update counts await refreshChunkingFiles(); // Refresh admin badge refreshAdminData(); } else { throw new Error(response.error || 'Submission failed'); } } catch (error) { showToast('Submission Failed', error.message, 'error'); } finally { // Re-enable submit button ChunkingElements.submitBtn.disabled = false; ChunkingElements.submitBtn.innerHTML = ' Add Chunk'; } } // ============================================================================= // TEXT SELECTION HELPER // ============================================================================= /** * Handle text selection from source content * Automatically copies selected text to chunk textarea */ function setupTextSelection() { ChunkingElements.sourceContent.addEventListener('mouseup', () => { const selection = window.getSelection(); const selectedText = selection.toString().trim(); if (selectedText && selectedText.length > 10) { // Ask user if they want to use this selection const existingText = ChunkingElements.chunkText.value; if (existingText && existingText.length > 0) { // Append or replace? showModal('Add Selected Text', '

Do you want to append or replace the current chunk text?

', [ { text: 'Cancel', class: 'btn-secondary', onClick: hideModal }, { text: 'Append', class: 'btn-primary', onClick: () => { ChunkingElements.chunkText.value = existingText + '\n\n' + selectedText; updateChunkCharCount(); hideModal(); }}, { text: 'Replace', class: 'btn-error', onClick: () => { ChunkingElements.chunkText.value = selectedText; updateChunkCharCount(); hideModal(); }} ] ); } else { // Just set it ChunkingElements.chunkText.value = selectedText; updateChunkCharCount(); showToast('Text Added', 'Selected text added to chunk', 'success', 2000); } } }); } // ============================================================================= // EVENT LISTENERS // ============================================================================= /** * Initialize event listeners for the chunking tab */ function initChunkingEventListeners() { // Refresh button ChunkingElements.refreshBtn.addEventListener('click', refreshChunkingFiles); // Preview button ChunkingElements.previewBtn.addEventListener('click', previewChunk); // Submit button ChunkingElements.submitBtn.addEventListener('click', handleChunkSubmit); // Character count on input ChunkingElements.chunkText.addEventListener('input', updateChunkCharCount); // Text selection helper setupTextSelection(); // Submit on Ctrl+Enter ChunkingElements.chunkText.addEventListener('keydown', (e) => { if (e.ctrlKey && e.key === 'Enter') { handleChunkSubmit(); } }); } // ============================================================================= // INITIALIZATION // ============================================================================= /** * Initialize the chunking tab */ document.addEventListener('DOMContentLoaded', () => { // Initialize element references ChunkingElements.init(); // Set up event listeners initChunkingEventListeners(); console.log('🧩 Chunking tab initialized'); });