/**
* =============================================================================
* Mozhii RAG Data Platform - Chunking Tab JavaScript
* =============================================================================
* Handles all functionality for the CHUNKING tab where the QA team
* creates RAG-ready chunks from cleaned content.
*
* This is the MOST IMPORTANT stage as it produces data ready for embedding!
*
* Features:
* - List cleaned files available for chunking
* - Display source content for reference
* - Create chunks with metadata
* - Track chunk indices automatically
* - Preview chunks before submission
* - View existing chunks
* =============================================================================
*/
// =============================================================================
// DOM ELEMENTS
// =============================================================================
/**
* Cache DOM elements for the chunking tab
*/
const ChunkingElements = {
// Panels
fileList: null,
sourceContent: null,
sourceInfo: null,
chunksList: null,
// Form elements
chunkHeading: null,
chunkSubheading: null,
chunkText: null,
chunkCategory: null,
chunkSource: null,
chunkOverlap: null,
charCount: null,
indexDisplay: null,
selectedFile: null,
// Buttons
refreshBtn: null,
previewBtn: null,
submitBtn: null,
// Initialize element references
init() {
this.fileList = document.getElementById('chunking-file-list');
this.sourceContent = document.getElementById('chunking-source-content');
this.sourceInfo = document.getElementById('chunking-source-info');
this.chunksList = document.getElementById('chunks-list');
this.chunkHeading = document.getElementById('chunk-heading');
this.chunkSubheading = document.getElementById('chunk-subheading');
this.chunkText = document.getElementById('chunk-text');
this.chunkCategory = document.getElementById('chunk-category');
this.chunkSource = document.getElementById('chunk-source');
this.chunkOverlap = document.getElementById('chunk-overlap');
this.charCount = document.getElementById('chunk-char-count');
this.indexDisplay = document.getElementById('chunk-index-display');
this.selectedFile = document.getElementById('chunking-selected-file');
this.refreshBtn = document.getElementById('chunking-refresh');
this.previewBtn = document.getElementById('chunk-preview');
this.submitBtn = document.getElementById('chunk-submit');
}
};
// =============================================================================
// STATE
// =============================================================================
/**
* Local state for the chunking tab
*/
const ChunkingState = {
files: [], // List of available cleaned files
selectedFile: null, // Currently selected file object
selectedFilename: null, // Name of selected file
chunks: [], // Existing chunks for selected file
nextIndex: 1 // Next chunk index
};
// =============================================================================
// FILE LIST FUNCTIONS
// =============================================================================
/**
* Refresh the list of cleaned files available for chunking
*/
async function refreshChunkingFiles() {
try {
// Show loading state
ChunkingElements.fileList.innerHTML = `
`;
// Fetch files from API
const response = await api('/api/chunking/cleaned-files');
if (response.success) {
ChunkingState.files = response.files;
renderChunkingFileList();
} else {
throw new Error(response.error || 'Failed to load files');
}
} catch (error) {
console.error('Error loading chunking files:', error);
ChunkingElements.fileList.innerHTML = `
Error loading files
${error.message}
`;
}
}
/**
* Render the file list in the UI
*/
function renderChunkingFileList() {
const files = ChunkingState.files;
if (files.length === 0) {
ChunkingElements.fileList.innerHTML = `
No cleaned files available
Complete cleaning in the CLEANING tab first
`;
return;
}
// Create file list HTML
ChunkingElements.fileList.innerHTML = files.map(file => `
📋
${file.filename}
${file.total_chunks} chunks
${file.pending_chunks > 0 ? `(${file.pending_chunks} pending)` : ''}
`).join('');
}
// =============================================================================
// FILE SELECTION
// =============================================================================
/**
* Handle file selection for chunking
*
* @param {string} filename - Name of the file to select
*/
async function selectChunkingFile(filename) {
// Find the file in our state
const file = ChunkingState.files.find(f => f.filename === filename);
if (!file) {
console.error('File not found:', filename);
return;
}
// Update state
ChunkingState.selectedFile = file;
ChunkingState.selectedFilename = filename;
// Update hidden field
ChunkingElements.selectedFile.value = filename;
// Update file list selection
document.querySelectorAll('#chunking-file-list .file-item').forEach(item => {
item.classList.toggle('selected', item.dataset.filename === filename);
});
// Update source content display
ChunkingElements.sourceContent.innerHTML = `
${escapeHtml(file.content)}
`;
// Update source info
ChunkingElements.sourceInfo.textContent = `${file.content_length.toLocaleString()} chars | ${file.language.toUpperCase()}`;
// Fetch existing chunks
await loadChunksForFile(filename);
// Enable form elements
enableChunkingForm();
// Set source from file
ChunkingElements.chunkSource.value = file.source || '';
// Show info toast
showToast('File Selected', `Selected "${filename}" for chunking`, 'info', 3000);
}
/**
* Escape HTML to prevent XSS
*/
function escapeHtml(text) {
const div = document.createElement('div');
div.textContent = text;
return div.innerHTML;
}
/**
* Load existing chunks for a file
*
* @param {string} filename - Name of the file
*/
async function loadChunksForFile(filename) {
try {
const response = await api(`/api/chunking/chunks/${filename}`);
if (response.success) {
ChunkingState.chunks = response.chunks;
ChunkingState.nextIndex = response.count + 1;
// Update index display
ChunkingElements.indexDisplay.textContent = `Chunk #${ChunkingState.nextIndex}`;
// Render chunks list
renderChunksList();
}
} catch (error) {
console.error('Error loading chunks:', error);
ChunkingState.chunks = [];
ChunkingState.nextIndex = 1;
}
}
/**
* Render the list of existing chunks
*/
function renderChunksList() {
const chunks = ChunkingState.chunks;
if (chunks.length === 0) {
ChunkingElements.chunksList.innerHTML = `
`;
return;
}
ChunkingElements.chunksList.innerHTML = chunks.map(chunk => `
#${chunk.chunk_index}
${chunk.text.substring(0, 50)}...
${chunk.status || 'pending'}
`).join('');
}
// =============================================================================
// FORM HANDLING
// =============================================================================
/**
* Enable the chunking form elements
*/
function enableChunkingForm() {
ChunkingElements.chunkHeading.disabled = false;
ChunkingElements.chunkSubheading.disabled = false;
ChunkingElements.chunkText.disabled = false;
ChunkingElements.chunkCategory.disabled = false;
ChunkingElements.chunkSource.disabled = false;
ChunkingElements.chunkOverlap.disabled = false;
ChunkingElements.previewBtn.disabled = false;
ChunkingElements.submitBtn.disabled = false;
}
/**
* Disable the chunking form elements
*/
function disableChunkingForm() {
ChunkingElements.chunkHeading.disabled = true;
ChunkingElements.chunkSubheading.disabled = true;
ChunkingElements.chunkText.disabled = true;
ChunkingElements.chunkCategory.disabled = true;
ChunkingElements.chunkSource.disabled = true;
ChunkingElements.chunkOverlap.disabled = true;
ChunkingElements.previewBtn.disabled = true;
ChunkingElements.submitBtn.disabled = true;
}
/**
* Update character count for chunk text
*/
function updateChunkCharCount() {
const content = ChunkingElements.chunkText.value;
const count = content.length;
ChunkingElements.charCount.textContent = `${count.toLocaleString()} characters`;
}
/**
* Clear the chunk form
*/
function clearChunkForm() {
ChunkingElements.chunkHeading.value = '';
ChunkingElements.chunkSubheading.value = '';
ChunkingElements.chunkText.value = '';
ChunkingElements.chunkOverlap.value = '';
updateChunkCharCount();
}
/**
* Generate chunk ID based on metadata
*
* @returns {string} - Generated chunk ID
*/
function generateChunkId() {
const lang = ChunkingState.selectedFile?.language || 'ta';
const cat = ChunkingElements.chunkCategory.value.substring(0, 3);
const file = ChunkingState.selectedFilename?.replace(/_/g, '').substring(0, 10) || 'file';
const index = ChunkingState.nextIndex;
return `${lang}_${cat}_${file}_${String(index).padStart(2, '0')}`;
}
// =============================================================================
// PREVIEW FUNCTIONALITY
// =============================================================================
/**
* Preview the chunk before submission
*/
function previewChunk() {
// Validate
if (!ChunkingElements.chunkText.value.trim()) {
showToast('No Content', 'Please enter chunk text first', 'warning');
return;
}
const chunkId = generateChunkId();
// Build preview object
const preview = {
chunk_id: chunkId,
heading: ChunkingElements.chunkHeading.value.trim(),
sub_heading: ChunkingElements.chunkSubheading.value.trim(),
text: ChunkingElements.chunkText.value.trim(),
language: ChunkingState.selectedFile?.language || 'ta',
category: ChunkingElements.chunkCategory.value,
source: ChunkingElements.chunkSource.value || ChunkingState.selectedFile?.source || 'unknown',
chunk_index: ChunkingState.nextIndex,
source_file: ChunkingState.selectedFilename,
overlap_reference: ChunkingElements.chunkOverlap.value.trim()
};
// Show preview modal
showModal('Chunk Preview', `
${JSON.stringify(preview, null, 2)}
`, [
{ text: 'Close', class: 'btn-secondary', onClick: hideModal },
{ text: 'Submit Chunk', class: 'btn-primary', onClick: () => {
hideModal();
handleChunkSubmit();
}}
]);
}
// =============================================================================
// SUBMISSION
// =============================================================================
/**
* Submit a new chunk
*/
async function handleChunkSubmit() {
// Validate file selection
if (!ChunkingState.selectedFilename) {
showToast('No File Selected', 'Please select a file first', 'warning');
return;
}
// Validate chunk text
const text = ChunkingElements.chunkText.value.trim();
if (!text) {
showToast('No Content', 'Please enter chunk text', 'warning');
return;
}
if (text.length < 20) {
showToast('Content Too Short', 'Chunk text must be at least 20 characters', 'warning');
return;
}
// Disable submit button
ChunkingElements.submitBtn.disabled = true;
ChunkingElements.submitBtn.innerHTML = '⏳ Adding...';
try {
// Prepare chunk data
const chunkData = {
filename: ChunkingState.selectedFilename,
heading: ChunkingElements.chunkHeading.value.trim() || undefined,
sub_heading: ChunkingElements.chunkSubheading.value.trim() || undefined,
text: text,
category: ChunkingElements.chunkCategory.value,
source: ChunkingElements.chunkSource.value || undefined,
overlap_reference: ChunkingElements.chunkOverlap.value.trim() || undefined
};
// Submit to API
const response = await api('/api/chunking/submit', {
method: 'POST',
body: JSON.stringify(chunkData)
});
if (response.success) {
showToast(
'Chunk Created!',
`Chunk #${response.chunk_index} added. Pending admin approval.`,
'success'
);
// Store last chunk text as overlap reference for next chunk
const lastChunkEnd = text.slice(-100);
// Clear form
clearChunkForm();
// Set overlap reference for next chunk
ChunkingElements.chunkOverlap.value = lastChunkEnd;
// Reload chunks
await loadChunksForFile(ChunkingState.selectedFilename);
// Refresh file list to update counts
await refreshChunkingFiles();
// Refresh admin badge
refreshAdminData();
} else {
throw new Error(response.error || 'Submission failed');
}
} catch (error) {
showToast('Submission Failed', error.message, 'error');
} finally {
// Re-enable submit button
ChunkingElements.submitBtn.disabled = false;
ChunkingElements.submitBtn.innerHTML = '➕ Add Chunk';
}
}
// =============================================================================
// TEXT SELECTION HELPER
// =============================================================================
/**
* Handle text selection from source content
* Automatically copies selected text to chunk textarea
*/
function setupTextSelection() {
ChunkingElements.sourceContent.addEventListener('mouseup', () => {
const selection = window.getSelection();
const selectedText = selection.toString().trim();
if (selectedText && selectedText.length > 10) {
// Ask user if they want to use this selection
const existingText = ChunkingElements.chunkText.value;
if (existingText && existingText.length > 0) {
// Append or replace?
showModal('Add Selected Text',
'Do you want to append or replace the current chunk text?
',
[
{ text: 'Cancel', class: 'btn-secondary', onClick: hideModal },
{ text: 'Append', class: 'btn-primary', onClick: () => {
ChunkingElements.chunkText.value = existingText + '\n\n' + selectedText;
updateChunkCharCount();
hideModal();
}},
{ text: 'Replace', class: 'btn-error', onClick: () => {
ChunkingElements.chunkText.value = selectedText;
updateChunkCharCount();
hideModal();
}}
]
);
} else {
// Just set it
ChunkingElements.chunkText.value = selectedText;
updateChunkCharCount();
showToast('Text Added', 'Selected text added to chunk', 'success', 2000);
}
}
});
}
// =============================================================================
// EVENT LISTENERS
// =============================================================================
/**
* Initialize event listeners for the chunking tab
*/
function initChunkingEventListeners() {
// Refresh button
ChunkingElements.refreshBtn.addEventListener('click', refreshChunkingFiles);
// Preview button
ChunkingElements.previewBtn.addEventListener('click', previewChunk);
// Submit button
ChunkingElements.submitBtn.addEventListener('click', handleChunkSubmit);
// Character count on input
ChunkingElements.chunkText.addEventListener('input', updateChunkCharCount);
// Text selection helper
setupTextSelection();
// Submit on Ctrl+Enter
ChunkingElements.chunkText.addEventListener('keydown', (e) => {
if (e.ctrlKey && e.key === 'Enter') {
handleChunkSubmit();
}
});
}
// =============================================================================
// INITIALIZATION
// =============================================================================
/**
* Initialize the chunking tab
*/
document.addEventListener('DOMContentLoaded', () => {
// Initialize element references
ChunkingElements.init();
// Set up event listeners
initChunkingEventListeners();
console.log('🧩 Chunking tab initialized');
});