| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <title>Code Dataset Generator for Falcon 40B</title> |
| <script src="https://cdn.tailwindcss.com"></script> |
| <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css"> |
| <style> |
| .file-drop-area { |
| border: 2px dashed #6366f1; |
| transition: all 0.3s ease; |
| } |
| .file-drop-area.active { |
| border-color: #4f46e5; |
| background-color: #eef2ff; |
| } |
| .progress-bar { |
| transition: width 0.3s ease; |
| } |
| .code-preview { |
| font-family: 'Courier New', monospace; |
| background-color: #1e293b; |
| color: #f8fafc; |
| } |
| .language-badge { |
| top: -10px; |
| right: -10px; |
| } |
| .slide-fade-enter-active, .slide-fade-leave-active { |
| transition: all 0.3s ease; |
| } |
| .slide-fade-enter-from, .slide-fade-leave-to { |
| opacity: 0; |
| transform: translateY(10px); |
| } |
| </style> |
| </head> |
| <body class="bg-gray-50 min-h-screen"> |
| <div class="container mx-auto px-4 py-8" id="app"> |
| |
| <header class="mb-10 text-center"> |
| <h1 class="text-4xl font-bold text-indigo-700 mb-2"> |
| <i class="fas fa-database mr-2"></i>Code Dataset Generator |
| </h1> |
| <p class="text-lg text-gray-600 max-w-2xl mx-auto"> |
| Transform your source code into fine-tuning datasets for Falcon 40B and other LLMs |
| </p> |
| </header> |
|
|
| |
| <div class="grid grid-cols-1 lg:grid-cols-3 gap-8"> |
| |
| <div class="lg:col-span-2 bg-white rounded-xl shadow-md overflow-hidden"> |
| <div class="p-6"> |
| <h2 class="text-2xl font-semibold text-gray-800 mb-4"> |
| <i class="fas fa-upload mr-2 text-indigo-600"></i>Upload Source Files |
| </h2> |
| |
| |
| <div |
| id="dropArea" |
| class="file-drop-area rounded-lg p-8 text-center cursor-pointer mb-6" |
| @dragover.prevent="dragOver = true" |
| @dragleave="dragOver = false" |
| @drop.prevent="handleDrop" |
| :class="{ 'active': dragOver }" |
| > |
| <div class="flex flex-col items-center justify-center space-y-3"> |
| <i class="fas fa-cloud-upload-alt text-4xl text-indigo-500"></i> |
| <p class="text-lg font-medium text-gray-700">Drag & drop your source files here</p> |
| <p class="text-sm text-gray-500">or</p> |
| <input |
| type="file" |
| id="fileInput" |
| class="hidden" |
| multiple |
| @change="handleFileSelect" |
| accept=".py,.cpp,.c,.go,.rs,.js,.java,.php,.rb,.ts" |
| > |
| <button |
| @click="document.getElementById('fileInput').click()" |
| class="px-4 py-2 bg-indigo-600 text-white rounded-md hover:bg-indigo-700 transition" |
| > |
| Browse Files |
| </button> |
| <p class="text-xs text-gray-400 mt-2">Supported: Python, C/C++, Rust, Go, JavaScript, Java, PHP, Ruby, TypeScript</p> |
| </div> |
| </div> |
|
|
| |
| <div v-if="selectedFiles.length > 0" class="mb-6"> |
| <h3 class="text-lg font-medium text-gray-700 mb-3">Selected Files</h3> |
| <div class="space-y-2"> |
| <div |
| v-for="(file, index) in selectedFiles" |
| :key="index" |
| class="flex items-center justify-between bg-gray-50 p-3 rounded-md" |
| > |
| <div class="flex items-center space-x-3"> |
| <i :class="getFileIcon(file.name)" class="text-indigo-500"></i> |
| <span class="text-sm font-medium text-gray-700 truncate max-w-xs">{{ file.name }}</span> |
| <span class="text-xs text-gray-500">{{ formatFileSize(file.size) }}</span> |
| </div> |
| <button |
| @click="removeFile(index)" |
| class="text-red-500 hover:text-red-700" |
| > |
| <i class="fas fa-times"></i> |
| </button> |
| </div> |
| </div> |
| </div> |
|
|
| |
| <div class="mb-6"> |
| <h3 class="text-lg font-medium text-gray-700 mb-3">Processing Options</h3> |
| <div class="grid grid-cols-1 md:grid-cols-2 gap-4"> |
| <div> |
| <label class="block text-sm font-medium text-gray-700 mb-1">Output Format</label> |
| <select v-model="outputFormat" class="w-full rounded-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500"> |
| <option value="jsonl">JSONL (Alpaca format)</option> |
| <option value="hf">HuggingFace Dataset</option> |
| <option value="openchat">OpenChat Format</option> |
| </select> |
| </div> |
| <div> |
| <label class="block text-sm font-medium text-gray-700 mb-1">Language Detection</label> |
| <select v-model="languageDetection" class="w-full rounded-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500"> |
| <option value="auto">Auto-detect (recommended)</option> |
| <option value="manual">Manual selection</option> |
| </select> |
| </div> |
| </div> |
| <div class="mt-4 space-y-2"> |
| <label class="inline-flex items-center"> |
| <input type="checkbox" v-model="extractFunctions" class="rounded border-gray-300 text-indigo-600 shadow-sm focus:border-indigo-500 focus:ring-indigo-500"> |
| <span class="ml-2 text-sm text-gray-700">Extract functions/methods</span> |
| </label> |
| <label class="inline-flex items-center"> |
| <input type="checkbox" v-model="extractClasses" class="rounded border-gray-300 text-indigo-600 shadow-sm focus:border-indigo-500 focus:ring-indigo-500"> |
| <span class="ml-2 text-sm text-gray-700">Extract classes</span> |
| </label> |
| <label class="inline-flex items-center"> |
| <input type="checkbox" v-model="includeComments" class="rounded border-gray-300 text-indigo-600 shadow-sm focus:border-indigo-500 focus:ring-indigo-500"> |
| <span class="ml-2 text-sm text-gray-700">Include comments/docstrings</span> |
| </label> |
| <label class="inline-flex items-center"> |
| <input type="checkbox" v-model="cleanCode" class="rounded border-gray-300 text-indigo-600 shadow-sm focus:border-indigo-500 focus:ring-indigo-500"> |
| <span class="ml-2 text-sm text-gray-700">Clean code (remove noise)</span> |
| </label> |
| </div> |
| </div> |
|
|
| |
| <button |
| @click="processFiles" |
| :disabled="selectedFiles.length === 0 || processing" |
| class="w-full py-3 px-4 bg-indigo-600 hover:bg-indigo-700 text-white font-medium rounded-md transition flex items-center justify-center" |
| :class="{ 'opacity-50 cursor-not-allowed': selectedFiles.length === 0 || processing }" |
| > |
| <span v-if="!processing"> |
| <i class="fas fa-cogs mr-2"></i>Generate Dataset |
| </span> |
| <span v-else> |
| <i class="fas fa-spinner fa-spin mr-2"></i>Processing... |
| </span> |
| </button> |
| </div> |
| </div> |
|
|
| |
| <div class="bg-white rounded-xl shadow-md overflow-hidden"> |
| <div class="p-6"> |
| <h2 class="text-2xl font-semibold text-gray-800 mb-4"> |
| <i class="fas fa-eye mr-2 text-indigo-600"></i>Dataset Preview |
| </h2> |
| |
| |
| <div v-if="processing" class="mb-6"> |
| <div class="flex justify-between text-sm text-gray-600 mb-1"> |
| <span>Processing files...</span> |
| <span>{{ processedFiles }} / {{ selectedFiles.length }}</span> |
| </div> |
| <div class="w-full bg-gray-200 rounded-full h-2.5"> |
| <div |
| class="progress-bar bg-indigo-600 h-2.5 rounded-full" |
| :style="{ width: (processedFiles / selectedFiles.length) * 100 + '%' }" |
| ></div> |
| </div> |
| </div> |
|
|
| |
| <div v-if="previewData" class="space-y-4"> |
| <div class="flex justify-between items-center"> |
| <h3 class="text-lg font-medium text-gray-700">Generated Samples</h3> |
| <span class="text-xs bg-indigo-100 text-indigo-800 px-2 py-1 rounded-full"> |
| {{ previewData.length }} items |
| </span> |
| </div> |
| |
| |
| <div class="relative"> |
| <select |
| v-model="selectedSampleIndex" |
| class="w-full rounded-md border-gray-300 shadow-sm focus:border-indigo-500 focus:ring-indigo-500" |
| > |
| <option v-for="(item, index) in previewData" :value="index"> |
| Sample {{ index + 1 }} - {{ item.instruction.substring(0, 30) }}... |
| </option> |
| </select> |
| </div> |
| |
| |
| <div v-if="selectedSampleIndex !== null" class="space-y-3"> |
| <div> |
| <label class="block text-sm font-medium text-gray-700 mb-1">Instruction</label> |
| <div class="bg-gray-50 p-3 rounded-md text-sm"> |
| {{ previewData[selectedSampleIndex].instruction }} |
| </div> |
| </div> |
| |
| <div> |
| <label class="block text-sm font-medium text-gray-700 mb-1">Input Code</label> |
| <div class="code-preview p-3 rounded-md text-sm overflow-x-auto"> |
| <pre>{{ previewData[selectedSampleIndex].input }}</pre> |
| </div> |
| </div> |
| |
| <div> |
| <label class="block text-sm font-medium text-gray-700 mb-1">Output</label> |
| <div class="bg-gray-50 p-3 rounded-md text-sm"> |
| {{ previewData[selectedSampleIndex].output }} |
| </div> |
| </div> |
| </div> |
| |
| |
| <button |
| v-if="previewData.length > 0" |
| @click="downloadDataset" |
| class="w-full py-2 px-4 bg-green-600 hover:bg-green-700 text-white font-medium rounded-md transition flex items-center justify-center mt-4" |
| > |
| <i class="fas fa-download mr-2"></i>Download Dataset |
| </button> |
| </div> |
| |
| |
| <div v-else class="text-center py-10"> |
| <i class="fas fa-code text-4xl text-gray-300 mb-3"></i> |
| <p class="text-gray-500">Your processed dataset will appear here</p> |
| </div> |
| </div> |
| </div> |
| </div> |
|
|
| |
| <div class="mt-16"> |
| <h2 class="text-2xl font-bold text-center text-gray-800 mb-8">Key Features</h2> |
| <div class="grid grid-cols-1 md:grid-cols-3 gap-6"> |
| <div class="bg-white p-6 rounded-xl shadow-sm border border-gray-100"> |
| <div class="text-indigo-600 mb-3"> |
| <i class="fas fa-language text-3xl"></i> |
| </div> |
| <h3 class="text-lg font-semibold mb-2">Multi-language Support</h3> |
| <p class="text-gray-600">Automatically detects and processes code in Python, C/C++, Rust, Go, JavaScript and more.</p> |
| </div> |
| <div class="bg-white p-6 rounded-xl shadow-sm border border-gray-100"> |
| <div class="text-indigo-600 mb-3"> |
| <i class="fas fa-robot text-3xl"></i> |
| </div> |
| <h3 class="text-lg font-semibold mb-2">Smart Prompt Engineering</h3> |
| <p class="text-gray-600">Automatically generates meaningful instruction-output pairs from your source code.</p> |
| </div> |
| <div class="bg-white p-6 rounded-xl shadow-sm border border-gray-100"> |
| <div class="text-indigo-600 mb-3"> |
| <i class="fas fa-database text-3xl"></i> |
| </div> |
| <h3 class="text-lg font-semibold mb-2">Falcon 40B Optimized</h3> |
| <p class="text-gray-600">Tokenization and formatting specifically optimized for Falcon 40B model fine-tuning.</p> |
| </div> |
| </div> |
| </div> |
|
|
| |
| <footer class="mt-16 text-center text-gray-500 text-sm"> |
| <p>Code Dataset Generator - Transform your source code into LLM training data</p> |
| <p class="mt-1">© 2023 AI Engineering Team. All rights reserved.</p> |
| </footer> |
| </div> |
|
|
| <script> |
| const app = { |
| data() { |
| return { |
| dragOver: false, |
| selectedFiles: [], |
| outputFormat: 'jsonl', |
| languageDetection: 'auto', |
| extractFunctions: true, |
| extractClasses: true, |
| includeComments: true, |
| cleanCode: false, |
| processing: false, |
| processedFiles: 0, |
| previewData: null, |
| selectedSampleIndex: 0 |
| } |
| }, |
| methods: { |
| handleDrop(e) { |
| this.dragOver = false; |
| const files = Array.from(e.dataTransfer.files); |
| this.addFiles(files); |
| }, |
| handleFileSelect(e) { |
| const files = Array.from(e.target.files); |
| this.addFiles(files); |
| e.target.value = ''; |
| }, |
| addFiles(files) { |
| const validExtensions = ['.py', '.cpp', '.c', '.go', '.rs', '.js', '.java', '.php', '.rb', '.ts']; |
| const filteredFiles = files.filter(file => { |
| const ext = '.' + file.name.split('.').pop().toLowerCase(); |
| return validExtensions.includes(ext); |
| }); |
| |
| if (filteredFiles.length < files.length) { |
| alert('Some files were ignored. Only source code files are supported.'); |
| } |
| |
| this.selectedFiles = [...this.selectedFiles, ...filteredFiles]; |
| }, |
| removeFile(index) { |
| this.selectedFiles.splice(index, 1); |
| }, |
| getFileIcon(filename) { |
| const ext = filename.split('.').pop().toLowerCase(); |
| const icons = { |
| py: 'fab fa-python', |
| cpp: 'fas fa-file-code', |
| c: 'fas fa-file-code', |
| h: 'fas fa-file-code', |
| go: 'fab fa-golang', |
| rs: 'fas fa-rust', |
| js: 'fab fa-js-square', |
| java: 'fab fa-java', |
| php: 'fab fa-php', |
| rb: 'fas fa-gem', |
| ts: 'fas fa-file-code' |
| }; |
| return icons[ext] || 'fas fa-file-code'; |
| }, |
| formatFileSize(bytes) { |
| if (bytes === 0) return '0 Bytes'; |
| const k = 1024; |
| const sizes = ['Bytes', 'KB', 'MB', 'GB']; |
| const i = Math.floor(Math.log(bytes) / Math.log(k)); |
| return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; |
| }, |
| processFiles() { |
| if (this.selectedFiles.length === 0) return; |
| |
| this.processing = true; |
| this.processedFiles = 0; |
| this.previewData = null; |
| |
| |
| const totalFiles = this.selectedFiles.length; |
| const interval = setInterval(() => { |
| this.processedFiles += 1; |
| |
| if (this.processedFiles >= totalFiles) { |
| clearInterval(interval); |
| this.processing = false; |
| this.generatePreviewData(); |
| } |
| }, 800); |
| }, |
| generatePreviewData() { |
| |
| this.previewData = [ |
| { |
| "instruction": "Explain the purpose of this Python function.", |
| "input": "def get_user_by_id(id):\n return db.query(User).filter(User.id == id).first()", |
| "output": "This function retrieves a user from the database by their ID." |
| }, |
| { |
| "instruction": "What does this C++ function calculate?", |
| "input": "int factorial(int n) {\n if (n <= 1) return 1;\n return n * factorial(n - 1);\n}", |
| "output": "This function calculates the factorial of a given integer n using recursion." |
| }, |
| { |
| "instruction": "Convert this JavaScript function to TypeScript with proper typing.", |
| "input": "function greet(name) {\n return `Hello, ${name}!`;\n}", |
| "output": "function greet(name: string): string {\n return `Hello, ${name}!`;\n}" |
| }, |
| { |
| "instruction": "Optimize this Go function for better performance.", |
| "input": "func sum(numbers []int) int {\n total := 0\n for _, num := range numbers {\n total += num\n }\n return total\n}", |
| "output": "The function is already quite optimized. For very large slices, you might consider parallel processing using goroutines." |
| } |
| ]; |
| this.selectedSampleIndex = 0; |
| }, |
| downloadDataset() { |
| if (!this.previewData) return; |
| |
| |
| const dataStr = JSON.stringify(this.previewData, null, 2); |
| const dataBlob = new Blob([dataStr], { type: 'application/json' }); |
| const url = URL.createObjectURL(dataBlob); |
| |
| const link = document.createElement('a'); |
| link.href = url; |
| link.download = 'code_dataset.json'; |
| document.body.appendChild(link); |
| link.click(); |
| document.body.removeChild(link); |
| URL.revokeObjectURL(url); |
| } |
| } |
| }; |
| |
| Vue.createApp(app).mount('#app'); |
| </script> |
| <script src="https://unpkg.com/vue@3/dist/vue.global.js"></script> |
| <p style="border-radius: 8px; text-align: center; font-size: 12px; color: #fff; margin-top: 16px;position: fixed; left: 8px; bottom: 8px; z-index: 10; background: rgba(0, 0, 0, 0.8); padding: 4px 8px;">Made with <img src="https://enzostvs-deepsite.hf.space/logo.svg" alt="DeepSite Logo" style="width: 16px; height: 16px; vertical-align: middle;display:inline-block;margin-right:3px;filter:brightness(0) invert(1);"><a href="https://enzostvs-deepsite.hf.space" style="color: #fff;text-decoration: underline;" target="_blank" >DeepSite</a> - <a href="https://enzostvs-deepsite.hf.space?remix=Barbuuuuuuuu/code-dataset-generator" style="color: #fff;text-decoration: underline;" target="_blank" >🧬 Remix</a></p></body> |
| </html> |
