Create app.js

app.js

@@ -0,0 +1,522 @@
+/**
+ * HF Uploader Dashboard - JavaScript Application
+ */
+
+// ============================================================================
+// API Client
+// ============================================================================
+
+class APIClient {
+    constructor(baseURL = '/api') {
+        this.baseURL = baseURL;
+    }
+
+    async request(endpoint, options = {}) {
+        const url = `${this.baseURL}${endpoint}`;
+        const response = await fetch(url, {
+            headers: {
+                'Content-Type': 'application/json',
+                ...options.headers,
+            },
+            ...options,
+        });
+
+        if (!response.ok) {
+            const error = await response.json().catch(() => ({ detail: response.statusText }));
+            throw new Error(error.detail || `HTTP ${response.status}`);
+        }
+
+        return response.json();
+    }
+
+    // Configuration endpoints
+    getConfig() {
+        return this.request('/config');
+    }
+
+    updateConfig(data) {
+        return this.request('/config', {
+            method: 'POST',
+            body: JSON.stringify(data),
+        });
+    }
+
+    // Queue endpoints
+    getQueue(status = null) {
+        const url = status ? `/queue?status=${status}` : '/queue';
+        return this.request(url);
+    }
+
+    getQueueStats() {
+        return this.request('/queue/stats');
+    }
+
+    addToQueue() {
+        return this.request('/queue/add', { method: 'POST' });
+    }
+
+    clearQueue() {
+        return this.request('/queue/clear', { method: 'POST' });
+    }
+
+    // Processing endpoints
+    getProcessingState() {
+        return this.request('/processing/state');
+    }
+
+    startProcessing(maxFiles = 0) {
+        return this.request('/processing/start', {
+            method: 'POST',
+            body: JSON.stringify({ max_files: maxFiles }),
+        });
+    }
+
+    getProcessedFiles() {
+        return this.request('/processing/files');
+    }
+
+    // Upload endpoints
+    startUpload(fileIds, batchSize = 10) {
+        return this.request('/upload/start', {
+            method: 'POST',
+            body: JSON.stringify({ file_ids: fileIds, batch_size: batchSize }),
+        });
+    }
+
+    getUploadStatus() {
+        return this.request('/upload/status');
+    }
+
+    retryUpload(itemId) {
+        return this.request(`/upload/retry/${itemId}`, { method: 'POST' });
+    }
+
+    // File endpoints
+    previewFile(filename) {
+        return this.request('/file/preview', {
+            method: 'POST',
+            body: JSON.stringify({ filename }),
+        });
+    }
+
+    // Error log endpoints
+    getErrorLogs(limit = 50) {
+        return this.request(`/errors?limit=${limit}`);
+    }
+
+    clearErrorLogs() {
+        return this.request('/errors', { method: 'DELETE' });
+    }
+}
+
+// ============================================================================
+// Notification System
+// ============================================================================
+
+class Notifier {
+    constructor() {
+        this.toastEl = document.getElementById('toast');
+        this.timeout = null;
+    }
+
+    show(message, type = 'info', duration = 3000) {
+        this.toastEl.textContent = message;
+        this.toastEl.className = `toast show ${type}`;
+
+        clearTimeout(this.timeout);
+        this.timeout = setTimeout(() => {
+            this.toastEl.classList.remove('show');
+        }, duration);
+    }
+
+    success(message) {
+        this.show(message, 'success');
+    }
+
+    error(message) {
+        this.show(message, 'error', 5000);
+    }
+
+    warning(message) {
+        this.show(message, 'warning', 4000);
+    }
+
+    info(message) {
+        this.show(message, 'info');
+    }
+}
+
+// ============================================================================
+// Dashboard Application
+// ============================================================================
+
+class Dashboard {
+    constructor() {
+        this.api = new APIClient();
+        this.notifier = new Notifier();
+        this.currentSection = 'dashboard';
+        this.refreshInterval = null;
+
+        this.init();
+    }
+
+    init() {
+        this.setupEventListeners();
+        this.loadInitialData();
+        this.startAutoRefresh();
+    }
+
+    setupEventListeners() {
+        // Navigation
+        document.querySelectorAll('.nav-link').forEach(link => {
+            link.addEventListener('click', (e) => {
+                e.preventDefault();
+                const section = link.dataset.section;
+                this.switchSection(section);
+            });
+        });
+
+        // Dashboard buttons
+        document.getElementById('btn-start-processing').addEventListener('click', () => this.startProcessing());
+        document.getElementById('btn-start-upload').addEventListener('click', () => this.startUpload());
+
+        // Queue buttons
+        document.getElementById('btn-add-to-queue').addEventListener('click', () => this.addToQueue());
+        document.getElementById('btn-clear-queue').addEventListener('click', () => this.clearQueue());
+        document.getElementById('queue-filter').addEventListener('change', (e) => this.filterQueue(e.target.value));
+
+        // Configuration buttons
+        document.getElementById('btn-save-config').addEventListener('click', () => this.saveConfig());
+
+        // Logs buttons
+        document.getElementById('btn-clear-logs').addEventListener('click', () => this.clearLogs());
+
+        // Modal close
+        document.querySelector('.modal-close').addEventListener('click', () => this.closeModal());
+    }
+
+    switchSection(section) {
+        // Hide all sections
+        document.querySelectorAll('.section').forEach(s => s.classList.remove('active'));
+        // Show selected section
+        document.getElementById(section).classList.add('active');
+
+        // Update nav
+        document.querySelectorAll('.nav-link').forEach(link => {
+            link.classList.toggle('active', link.dataset.section === section);
+        });
+
+        this.currentSection = section;
+
+        // Load section-specific data
+        if (section === 'queue') {
+            this.loadQueue();
+        } else if (section === 'logs') {
+            this.loadErrorLogs();
+        }
+    }
+
+    async loadInitialData() {
+        try {
+            await this.loadConfig();
+            await this.loadStats();
+            await this.loadProcessingState();
+            await this.loadUploadStatus();
+        } catch (error) {
+            this.notifier.error(`Failed to load data: ${error.message}`);
+        }
+    }
+
+    async loadConfig() {
+        try {
+            const config = await this.api.getConfig();
+            document.getElementById('hf-token').value = config.hf_token || '';
+            document.getElementById('source-all-repo').value = config.source_all_repo;
+            document.getElementById('source-ato-repo').value = config.source_ato_repo;
+            document.getElementById('target-repo').value = config.target_repo;
+            document.getElementById('batch-size').value = config.upload_batch_size;
+            document.getElementById('max-uploads').value = config.max_uploads_per_hour;
+        } catch (error) {
+            console.error('Failed to load config:', error);
+        }
+    }
+
+    async loadStats() {
+        try {
+            const stats = await this.api.getQueueStats();
+            document.getElementById('stat-pending').textContent = stats.pending;
+            document.getElementById('stat-uploaded').textContent = stats.completed;
+            document.getElementById('stat-failed').textContent = stats.failed;
+
+            const processed = await this.api.getProcessedFiles();
+            document.getElementById('stat-processed').textContent = processed.count;
+        } catch (error) {
+            console.error('Failed to load stats:', error);
+        }
+    }
+
+    async loadProcessingState() {
+        try {
+            const state = await this.api.getProcessingState();
+            document.getElementById('processing-status').textContent = state.status;
+            document.getElementById('matched-pairs').textContent = state.matched_pairs;
+
+            if (state.total_files > 0) {
+                const progress = (state.processed_files / state.total_files) * 100;
+                document.getElementById('processing-progress').style.width = `${progress}%`;
+            }
+        } catch (error) {
+            console.error('Failed to load processing state:', error);
+        }
+    }
+
+    async loadUploadStatus() {
+        try {
+            const status = await this.api.getUploadStatus();
+            const rateLimitEl = document.getElementById('rate-limit-status');
+            const remainingEl = document.getElementById('remaining-uploads');
+
+            if (status.rate_limit.can_upload) {
+                rateLimitEl.textContent = 'Ready';
+                rateLimitEl.style.color = 'var(--color-success)';
+            } else {
+                rateLimitEl.textContent = 'Limited';
+                rateLimitEl.style.color = 'var(--color-error)';
+            }
+
+            remainingEl.textContent = status.rate_limit.remaining_uploads;
+        } catch (error) {
+            console.error('Failed to load upload status:', error);
+        }
+    }
+
+    async loadQueue(status = null) {
+        try {
+            const items = await this.api.getQueue(status);
+            const tbody = document.getElementById('queue-tbody');
+
+            if (items.length === 0) {
+                tbody.innerHTML = '<tr class="empty-row"><td colspan="5">No files in queue</td></tr>';
+                return;
+            }
+
+            tbody.innerHTML = items.map(item => `
+                <tr>
+                    <td>${this.escapeHtml(item.file_name)}</td>
+                    <td>${this.formatBytes(item.file_size)}</td>
+                    <td><span class="status-badge status-${item.status}">${item.status}</span></td>
+                    <td>${item.retry_count}/${item.max_retries}</td>
+                    <td>
+                        ${item.status === 'failed' ? `<button class="btn btn-secondary" onclick="dashboard.retryFile(${item.id})">Retry</button>` : ''}
+                        <button class="btn btn-secondary" onclick="dashboard.previewFile('${this.escapeHtml(item.file_name)}')">Preview</button>
+                    </td>
+                </tr>
+            `).join('');
+        } catch (error) {
+            this.notifier.error(`Failed to load queue: ${error.message}`);
+        }
+    }
+
+    async filterQueue(status) {
+        await this.loadQueue(status || null);
+    }
+
+    async loadErrorLogs() {
+        try {
+            const logs = await this.api.getErrorLogs();
+            const tbody = document.getElementById('logs-tbody');
+
+            if (logs.length === 0) {
+                tbody.innerHTML = '<tr class="empty-row"><td colspan="5">No errors logged</td></tr>';
+                return;
+            }
+
+            tbody.innerHTML = logs.map(log => `
+                <tr>
+                    <td>${this.escapeHtml(log.file_name)}</td>
+                    <td>${log.error_code || 'N/A'}</td>
+                    <td>${this.escapeHtml(log.error_message || '')}</td>
+                    <td>${log.retryable ? 'Yes' : 'No'}</td>
+                    <td>${new Date(log.created_at).toLocaleString()}</td>
+                </tr>
+            `).join('');
+        } catch (error) {
+            this.notifier.error(`Failed to load error logs: ${error.message}`);
+        }
+    }
+
+    async startProcessing() {
+        try {
+            const btn = document.getElementById('btn-start-processing');
+            btn.disabled = true;
+            this.notifier.info('Starting dataset processing...');
+
+            await this.api.startProcessing(0);
+            this.notifier.success('Processing started');
+
+            // Refresh state periodically
+            this.refreshInterval = setInterval(() => this.loadProcessingState(), 2000);
+        } catch (error) {
+            this.notifier.error(`Failed to start processing: ${error.message}`);
+            document.getElementById('btn-start-processing').disabled = false;
+        }
+    }
+
+    async startUpload() {
+        try {
+            const stats = await this.api.getQueueStats();
+            if (stats.pending === 0) {
+                this.notifier.warning('No pending files to upload');
+                return;
+            }
+
+            const btn = document.getElementById('btn-start-upload');
+            btn.disabled = true;
+            this.notifier.info('Starting upload...');
+
+            // Get all pending files
+            const queue = await this.api.getQueue('pending');
+            const fileIds = queue.map(item => item.id);
+
+            await this.api.startUpload(fileIds);
+            this.notifier.success('Upload started');
+
+            // Refresh stats periodically
+            this.refreshInterval = setInterval(() => this.loadStats(), 2000);
+        } catch (error) {
+            this.notifier.error(`Failed to start upload: ${error.message}`);
+            document.getElementById('btn-start-upload').disabled = false;
+        }
+    }
+
+    async addToQueue() {
+        try {
+            const btn = document.getElementById('btn-add-to-queue');
+            btn.disabled = true;
+
+            const result = await this.api.addToQueue();
+            this.notifier.success(`Added ${result.added} files to queue`);
+
+            await this.loadQueue();
+            await this.loadStats();
+
+            btn.disabled = false;
+        } catch (error) {
+            this.notifier.error(`Failed to add to queue: ${error.message}`);
+            document.getElementById('btn-add-to-queue').disabled = false;
+        }
+    }
+
+    async clearQueue() {
+        if (!confirm('Are you sure you want to clear the entire queue?')) {
+            return;
+        }
+
+        try {
+            await this.api.clearQueue();
+            this.notifier.success('Queue cleared');
+            await this.loadQueue();
+            await this.loadStats();
+        } catch (error) {
+            this.notifier.error(`Failed to clear queue: ${error.message}`);
+        }
+    }
+
+    async saveConfig() {
+        try {
+            const config = {
+                hf_token: document.getElementById('hf-token').value,
+                source_all_repo: document.getElementById('source-all-repo').value,
+                source_ato_repo: document.getElementById('source-ato-repo').value,
+                target_repo: document.getElementById('target-repo').value,
+                upload_batch_size: parseInt(document.getElementById('batch-size').value),
+                max_uploads_per_hour: parseInt(document.getElementById('max-uploads').value),
+            };
+
+            const btn = document.getElementById('btn-save-config');
+            btn.disabled = true;
+
+            await this.api.updateConfig(config);
+            this.notifier.success('Configuration saved');
+
+            btn.disabled = false;
+        } catch (error) {
+            this.notifier.error(`Failed to save configuration: ${error.message}`);
+            document.getElementById('btn-save-config').disabled = false;
+        }
+    }
+
+    async retryFile(itemId) {
+        try {
+            await this.api.retryUpload(itemId);
+            this.notifier.success('Retry scheduled');
+            await this.loadQueue();
+        } catch (error) {
+            this.notifier.error(`Failed to retry: ${error.message}`);
+        }
+    }
+
+    async previewFile(filename) {
+        try {
+            const data = await this.api.previewFile(filename);
+            const modal = document.getElementById('preview-modal');
+            const content = document.getElementById('preview-content');
+
+            content.textContent = JSON.stringify(data.content, null, 2);
+            modal.classList.add('show');
+        } catch (error) {
+            this.notifier.error(`Failed to preview file: ${error.message}`);
+        }
+    }
+
+    closeModal() {
+        document.getElementById('preview-modal').classList.remove('show');
+    }
+
+    async clearLogs() {
+        if (!confirm('Are you sure you want to clear all error logs?')) {
+            return;
+        }
+
+        try {
+            await this.api.clearErrorLogs();
+            this.notifier.success('Error logs cleared');
+            await this.loadErrorLogs();
+        } catch (error) {
+            this.notifier.error(`Failed to clear logs: ${error.message}`);
+        }
+    }
+
+    startAutoRefresh() {
+        setInterval(() => {
+            if (this.currentSection === 'dashboard') {
+                this.loadStats();
+                this.loadProcessingState();
+                this.loadUploadStatus();
+            }
+        }, 5000);
+    }
+
+    escapeHtml(text) {
+        const div = document.createElement('div');
+        div.textContent = text;
+        return div.innerHTML;
+    }
+
+    formatBytes(bytes) {
+        if (bytes === 0) return '0 B';
+        const k = 1024;
+        const sizes = ['B', 'KB', 'MB', 'GB'];
+        const i = Math.floor(Math.log(bytes) / Math.log(k));
+        return Math.round((bytes / Math.pow(k, i)) * 100) / 100 + ' ' + sizes[i];
+    }
+}
+
+// ============================================================================
+// Initialize Dashboard
+// ============================================================================
+
+let dashboard;
+document.addEventListener('DOMContentLoaded', () => {
+    dashboard = new Dashboard();
+});