Rename LSTM_datset_converter.py to app.py

LSTM_datset_converter.py

@@ -1,103 +0,0 @@
-import json
-import os
-from typing import List, Dict, Any
-
-
-def load_and_align_unified_data(input_json_path: str, output_json_path: str) -> str:
-    """
-    Loads the Label Studio JSON (with pre-extracted words and bboxes) and
-    aligns the character-offset labels to create a token-level (Token, Label, Bbox)
-    training file (The Unified JSON).
-    """
-
-    if not os.path.exists(input_json_path):
-        raise FileNotFoundError(f"Input JSON file not found at: {input_json_path}")
-
-    with open(input_json_path, "r", encoding="utf-8") as f:
-        data = json.load(f)
-
-    # We assume the Label Studio output is a list where each item is a document/page
-    processed_documents = []
-
-    for item in data:
-        # 1. Get the pre-extracted tokens and bboxes from the 'data' field
-        words = item["data"].get("original_words", [])
-        bboxes = item["data"].get("original_bboxes", [])
-        full_text = item["data"].get("text", "")
-
-        if not words or not bboxes or not item.get("annotations"):
-            print(f"Skipping item {item.get('id', 'N/A')}: Missing words, bboxes, or annotations.")
-            continue
-
-        # Initialize labels for every token to 'O' (Outside)
-        labels = ["O"] * len(words)
-
-        # 2. Get the character-offset annotations from the 'annotations' field
-        annotations = item["annotations"][0].get("result", [])
-
-        # 3. Perform Alignment: Match the labeled text to the token list
-        for res in annotations:
-            if "value" in res and "labels" in res["value"]:
-                text_snippet = res["value"]["text"]
-                tag = res["value"]["labels"][0].upper()  # e.g., 'QUESTION'
-
-                # Tokenize the labeled snippet using simple space split
-                # (MUST match the original tokenization used to create 'original_words' if possible)
-                text_tokens = text_snippet.split()
-
-                # Find the starting index (i) where the sequence of tokens matches the snippet
-                for i in range(len(words) - len(text_tokens) + 1):
-                    # Check if the sequence of original words matches the sequence of labeled words
-                    if words[i:i + len(text_tokens)] == text_tokens:
-                        # Apply B-I-O scheme
-                        labels[i] = f"B-{tag}"
-                        for j in range(1, len(text_tokens)):
-                            # Check bounds just in case
-                            if i + j < len(labels):
-                                labels[i + j] = f"I-{tag}"
-                        break  # Found the match, move to the next annotation
-
-        # 4. Construct the final token-level output structure
-        document_tokens = []
-        for word, label, bbox in zip(words, labels, bboxes):
-            document_tokens.append({
-                "token": word,
-                "label": label,
-                "bbox": bbox
-            })
-
-        processed_documents.append(document_tokens)
-
-    # Flatten the list of documents into a single sequence if your training script expects it
-    # Note: The 'load_unified_data' function in the training script expects a flat list
-    flat_output = [token for doc in processed_documents for token in doc]
-
-    # Save the final Unified JSON
-    with open(output_json_path, "w", encoding="utf-8") as f:
-        json.dump(flat_output, f, indent=2, ensure_ascii=False)
-
-    print(f"✅ Alignment successful. Unified training data saved to: {output_json_path}")
-    print(f"Total aligned tokens: {len(flat_output)}")
-    return output_json_path
-
-
-# ==============================================================================
-
-if __name__ == '__main__':
-    # --- Configuration ---
-    # ⚠️ 1. Set the path to your uploaded Label Studio output JSON file
-    INPUT_FILE = "project-6-at-2026-01-21-07-10-460e552c.json"
-
-    # 2. Set the path for the output file (This is your Unified JSON Path)
-    OUTPUT_FILE = "unified_training_data_bluuhhhhh.json"
-
-    os.makedirs("output_data", exist_ok=True)
-
-    try:
-        # Run the alignment
-        unified_path = load_and_align_unified_data(INPUT_FILE, OUTPUT_FILE)
-        print("\nReady for Training! Use this path in your fixed training script:")
-        print(f"UNIFIED_DATA_PATH = \"{unified_path}\"")
-
-    except Exception as e:
-        print(f"\n❌ An error occurred during alignment: {e}")

app.py

@@ -0,0 +1,426 @@
+
+import gradio as gr
+import subprocess
+import os
+import sys
+from datetime import datetime
+import shutil
+
+
+TRAINING_SCRIPT = "train.py"
+
+MODEL_OUTPUT_DIR = "checkpoints"
+MODEL_FILE_NAME = "layoutlmv3_crf_passage.pth"
+MODEL_FILE_PATH = os.path.join(MODEL_OUTPUT_DIR, MODEL_FILE_NAME)
+
+
+# ----------------------------------------------------------------
+
+def retrieve_model():
+    """
+    Checks for the final model file and prepares it for download.
+    Useful for when the training job finishes server-side but the
+    client connection has timed out.
+    """
+    MODEL_OUTPUT_DIR = "checkpoints"
+    MODEL_FILE_NAME = "layoutlmv3_crf_passage.pth"
+    MODEL_FILE_PATH = os.path.join(MODEL_OUTPUT_DIR, MODEL_FILE_NAME)
+
+    if os.path.exists(MODEL_FILE_PATH):
+        file_size = os.path.getsize(MODEL_FILE_PATH) / (1024 * 1024)  # Size in MB
+
+        # CRITICAL: Copy to a simple location that Gradio can reliably serve
+        import tempfile
+        temp_dir = tempfile.gettempdir()
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        temp_model_path = os.path.join(temp_dir, f"layoutlmv3_trained_{timestamp}_recovered.pth")
+
+        try:
+            shutil.copy2(MODEL_FILE_PATH, temp_model_path)
+            download_path = temp_model_path
+
+            log_output = (
+                f"--- Model Status Check: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
+                f"🎉 SUCCESS! A trained model was found and recovered. Boobs! AASTIK MERA NAAM\n"
+                f"📦 Model file: {MODEL_FILE_PATH}\n"
+                f"📊 Model size: {file_size:.2f} MB\n"
+                f"🔗 Download path prepared: {download_path}\n\n"
+                f"⬇️ Click the '📥 Download Model' button below to save your model."
+            )
+            return log_output, download_path, gr.Button(visible=True)
+
+        except Exception as e:
+            log_output = (
+                f"--- Model Status Check FAILED ---\n"
+                f"⚠️ Trained model found, but could not prepare for download: {e}\n"
+                f"📁 Original Path: {MODEL_FILE_PATH}. Try again or check Space logs."
+            )
+            return log_output, None, gr.Button(visible=False)
+
+    else:
+        log_output = (
+            f"--- Model Status Check: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
+            f"❌ Model file not found at {MODEL_FILE_PATH}.\n"
+            f"Training may still be running or it failed. Check back later."
+        )
+        return log_output, None, gr.Button(visible=False)
+
+
+
+
+def clear_memory(dataset_file: gr.File):
+    """
+    Deletes the model output directory and the uploaded dataset file.
+    """
+    MODEL_OUTPUT_DIR = "checkpoints"
+
+    log_output = f"--- Memory Clear Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
+
+    # 1. Clear Model Checkpoints Directory
+    if os.path.exists(MODEL_OUTPUT_DIR):
+        try:
+            shutil.rmtree(MODEL_OUTPUT_DIR)
+            log_output += f"✅ Successfully deleted model directory: {MODEL_OUTPUT_DIR}\n"
+        except Exception as e:
+            log_output += f"❌ ERROR deleting model directory {MODEL_OUTPUT_DIR}: {e}\n"
+    else:
+        log_output += f"ℹ️ Model directory not found: {MODEL_OUTPUT_DIR} (Nothing to delete)\n"
+
+    # 2. Clear Uploaded Dataset File (Temporary file cleanup)
+    if dataset_file is not None:
+        input_path = dataset_file.name if hasattr(dataset_file, 'name') else str(dataset_file)
+        if os.path.exists(input_path):
+            try:
+                os.remove(input_path)
+                log_output += f"✅ Successfully deleted uploaded dataset file: {input_path}\n"
+            except Exception as e:
+                log_output += f"❌ ERROR deleting dataset file {input_path}: {e}\n"
+        else:
+            log_output += f"ℹ️ Uploaded dataset file not found at {input_path}.\n"
+    else:
+        log_output += f"ℹ️ No dataset file currently tracked for deletion.\n"
+
+    # 3. Final message and state reset
+    log_output += f"--- Memory Clear Complete: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
+    log_output += "✨ Files and checkpoints have been removed. You can now start a fresh training run."
+
+    # Reset log_output, model_path_state, download_btn visibility, and model_download component
+    return log_output, None, gr.Button(visible=False), None
+
+
+
+def train_model(dataset_file: gr.File, batch_size: int, epochs: int, lr: float, max_len: int, progress=gr.Progress()):
+    """
+    Handles the Gradio submission and executes the training script using subprocess.
+    Yields logs in real-time for user feedback.
+    """
+
+    # 1. Setup: Create output directory if it doesn't exist
+    os.makedirs(MODEL_OUTPUT_DIR, exist_ok=True)
+
+    # 2. File Handling: Use the temporary path of the uploaded file
+    if dataset_file is None:
+        yield "❌ ERROR: Please upload a file.", None, gr.Button(visible=False)
+        return
+
+    # CRITICAL FIX: dataset_file is a gradio.File object, use .name to get the path
+    # This is a temporary file path like /tmp/gradio/.../filename.json
+    input_path = dataset_file.name if hasattr(dataset_file, 'name') else str(dataset_file)
+
+    # Verify the file actually exists before proceeding
+    if not os.path.exists(input_path):
+        error_msg = f"❌ ERROR: Uploaded file not found at {input_path}. Please try uploading again."
+        yield error_msg, None, gr.Button(visible=False)
+        return
+
+    if not input_path.lower().endswith(".json"):
+        yield "❌ ERROR: Please upload a valid Label Studio JSON file (.json).", None, gr.Button(visible=False)
+        return
+
+    progress(0.1, desc="Starting LayoutLMv3 Training...")
+
+    log_output = f"--- Training Started: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')} ---\n"
+
+    # 3. Construct the subprocess command
+    command = [
+        sys.executable,
+        TRAINING_SCRIPT,
+        "--mode", "train",
+        "--input", input_path,
+        "--batch_size", str(batch_size),
+        "--epochs", str(epochs),
+        "--lr", str(lr),
+        "--max_len", str(max_len)
+    ]
+
+    log_output += f"Executing command: {' '.join(command)}\n\n"
+    yield log_output, None, gr.Button(visible=False)  # Initial yield
+
+    try:
+        # 4. Run the training script and capture output
+        process = subprocess.Popen(
+            command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            bufsize=1
+        )
+
+        # Stream logs in real-time
+        for line in iter(process.stdout.readline, ""):
+            log_output += line
+            # Print to console as well for debugging
+            print(line, end='')
+            # Yield updated logs in real-time
+            yield log_output, None, gr.Button(visible=False)
+
+        process.stdout.close()
+        return_code = process.wait()
+
+        # 5. Check for successful completion
+        if return_code == 0:
+            log_output += "\n" + "=" * 60 + "\n"
+            log_output += "✅ TRAINING COMPLETE! Model saved successfully.\n"
+            log_output += "=" * 60 + "\n"
+            print("\n✅ TRAINING COMPLETE! Model saved.")
+
+            # 6. Verify model file exists
+            if os.path.exists(MODEL_FILE_PATH):
+                file_size = os.path.getsize(MODEL_FILE_PATH) / (1024 * 1024)  # Size in MB
+                log_output += f"\n📦 Model file found: {MODEL_FILE_PATH}"
+                log_output += f"\n📊 Model size: {file_size:.2f} MB"
+
+                print(f"\n✅ Model exists at: {MODEL_FILE_PATH} ({file_size:.2f} MB)")
+
+                # CRITICAL: Copy to a simple location that Gradio can reliably serve
+                # Use the same temp directory pattern as the uploaded JSON file
+                import tempfile
+                temp_dir = tempfile.gettempdir()
+                timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+                # Create filename in temp directory
+                temp_model_path = os.path.join(temp_dir, f"layoutlmv3_trained_{timestamp}.pth")
+
+                try:
+                    # Copy the model to temp directory
+                    shutil.copy2(MODEL_FILE_PATH, temp_model_path)
+                    log_output += f"\n📋 Model copied to temporary download location"
+                    log_output += f"\n🔗 Download path: {temp_model_path}"
+                    print(f"✅ Model copied to temp location: {temp_model_path}")
+
+                    # Verify the copy exists
+                    if os.path.exists(temp_model_path):
+                        log_output += f"\n✅ Download file verified and ready!"
+                        download_path = temp_model_path
+                    else:
+                        log_output += f"\n⚠️ Warning: Temp copy verification failed, using original path"
+                        download_path = MODEL_FILE_PATH
+
+                except Exception as e:
+                    log_output += f"\n⚠️ Could not create temp copy: {e}"
+                    log_output += f"\n📁 Using original path: {MODEL_FILE_PATH}"
+                    print(f"⚠️ Copy failed: {e}, using original path")
+                    download_path = MODEL_FILE_PATH
+
+                # Final success message
+                log_output += f"\n\n{'=' * 60}"
+                log_output += f"\n🎉 SUCCESS! Your model is ready for download."
+                log_output += f"\n{'=' * 60}"
+                log_output += f"\n\n⬇️ Click the '📥 Download Model' button below to save your model."
+                log_output += f"\n⚠️ CRITICAL: Download NOW! File will be deleted when:"
+                log_output += f"\n   - This tab is closed"
+                log_output += f"\n   - Space restarts or goes idle"
+                log_output += f"\n   - System clears temp files"
+                log_output += f"\n\n📥 The file will download as a .pth file to your computer's Downloads folder."
+                log_output += f"\n\n{'=' * 60}\n"
+
+                # Return final logs and make download button visible
+                # IMPORTANT: Return the path that Gradio can access
+                yield log_output, download_path, gr.Button(visible=True)
+                return
+            else:
+                log_output += f"\n⚠️ WARNING: Training completed, but model file not found at expected path ({MODEL_FILE_PATH})."
+                log_output += f"\n🔍 Checking directory contents..."
+
+                # List files in checkpoints directory for debugging
+                if os.path.exists(MODEL_OUTPUT_DIR):
+                    files = os.listdir(MODEL_OUTPUT_DIR)
+                    log_output += f"\n📁 Files in {MODEL_OUTPUT_DIR}: {files}"
+                else:
+                    log_output += f"\n❌ Directory {MODEL_OUTPUT_DIR} does not exist!"
+
+                yield log_output, None, gr.Button(visible=False)
+                return
+        else:
+            log_output += f"\n\n{'=' * 60}\n"
+            log_output += f"❌ TRAINING FAILED with return code {return_code}\n"
+            log_output += f"{'=' * 60}\n"
+            log_output += f"\nPlease check the logs above for error details.\n"
+            yield log_output, None, gr.Button(visible=False)
+            return
+
+    except FileNotFoundError:
+        error_msg = f"❌ ERROR: The training script '{TRAINING_SCRIPT}' was not found. Ensure it is in the root directory of your Space."
+        print(error_msg)
+        yield log_output + "\n" + error_msg, None, gr.Button(visible=False)
+        return
+    except Exception as e:
+        error_msg = f"❌ An unexpected error occurred: {e}"
+        print(error_msg)
+        import traceback
+        print(traceback.format_exc())
+        yield log_output + "\n" + error_msg, None, gr.Button(visible=False)
+        return
+
+
+# --- Gradio Interface Setup (using Blocks for a nicer layout) ---
+with gr.Blocks(title="LayoutLMv3 Fine-Tuning App by Aastik", theme=gr.themes.Soft()) as demo:
+    gr.Markdown("# 🚀 LayoutLMv3 Fine-Tuning on Hugging Face Spaces")
+    gr.Markdown(
+        """
+        Upload your Label Studio JSON file, set your hyperparameters, and click **Train Model** to fine-tune the LayoutLMv3 model.
+
+        **⚠️ IMPORTANT - Free Tier Users:**
+        - **Download your model IMMEDIATELY** after training completes! 
+        - The model file is **temporary** and will be deleted when the Space restarts.
+        - A download button will appear below once training is complete.
+        - **Real-time logs** will stream during training so you can monitor progress.
+
+        **⏱️ Timeout Note:** Training may timeout on free tier. Consider reducing epochs or batch size for faster training.
+        """
+    )
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown("### 📁 Dataset Upload")
+            file_input = gr.File(
+                label="Upload Label Studio JSON Dataset",
+                file_types=[".json"]
+            )
+
+            gr.Markdown("---")
+            gr.Markdown("### ⚙️ Training Parameters")
+
+            batch_size_input = gr.Slider(
+                minimum=1, maximum=16, step=1, value=4,
+                label="Batch Size",
+                info="Smaller = less memory, slower training"
+            )
+            epochs_input = gr.Slider(
+                minimum=1, maximum=10, step=1, value=3,
+                label="Epochs",
+                info="Fewer epochs = faster training (recommended: 3-5)"
+            )
+            lr_input = gr.Number(
+                value=5e-5, label="Learning Rate",
+                info="Default: 5e-5"
+            )
+            max_len_input = gr.Slider(
+                minimum=128, maximum=512, step=128, value=512,
+                label="Max Sequence Length",
+                info="Shorter = faster training, less memory"
+            )
+
+            train_button = gr.Button("🔥 Start Training", variant="primary", size="lg")
+            check_button = gr.Button("🔍 Check Model Status/Download", variant="secondary", size="lg")
+            clear_button = gr.Button("🧹 Clear Model/Dataset Files", variant="stop", size="lg")
+
+        with gr.Column(scale=2):
+            gr.Markdown("### 📊 Training Progress (Real-Time Logs)")
+
+            log_output = gr.Textbox(
+                label="Training Logs - Updates in Real-Time",
+                lines=25,
+                max_lines=30,
+                autoscroll=True,
+                show_copy_button=True,
+                placeholder="Click 'Start Training' to begin...\n\nLogs will stream here in real-time as training progresses."
+            )
+
+            gr.Markdown("### ⬇️ Download Trained Model")
+
+            # Hidden state to store the file path
+            model_path_state = gr.State(value=None)
+
+            # Download button (initially hidden)
+            download_btn = gr.Button(
+                "📥 Download Model (.pth file)",
+                variant="primary",
+                size="lg",
+                visible=False
+            )
+
+            check_button.click(
+                fn=retrieve_model,  # A new function we'll define
+                inputs=[],
+                outputs=[log_output, model_path_state, download_btn]
+            )
+
+
+
+            # File output for download
+            model_download = gr.File(
+                label="Your trained model will appear here after clicking Download",
+                interactive=False,
+                visible=True
+            )
+
+            clear_button.click(
+                fn=clear_memory,
+                inputs=[file_input],  # Pass the uploaded file object to delete the temp file
+                outputs=[log_output, model_path_state, download_btn, model_download]
+            )
+
+            gr.Markdown(
+                """
+                **📥 Download Instructions:**
+                1. Wait for training to complete - watch the real-time logs above
+                2. Look for **"✅ TRAINING COMPLETE!"** message
+                3. Click the **"📥 Download Model"** button that appears above
+                4. Save the `.pth` file to your local machine
+                5. **Do this immediately** - file is temporary and will be deleted on Space restart!
+
+                **🔧 Troubleshooting:**
+                - If download button doesn't appear, check the logs for errors
+                - Try reducing epochs or batch size if timeout occurs
+                - Ensure your JSON file is properly formatted
+                - Logs update in real-time - you can monitor training progress
+                """
+            )
+
+    # Define the training action - now with real-time log streaming via yield
+    train_button.click(
+        fn=train_model,
+        inputs=[file_input, batch_size_input, epochs_input, lr_input, max_len_input],
+        outputs=[log_output, model_path_state, download_btn],
+        api_name="train"
+    )
+
+    # Define the download action
+    download_btn.click(
+        fn=lambda path: path,
+        inputs=[model_path_state],
+        outputs=[model_download]
+    )
+
+    gr.Markdown(
+        """
+        ---
+        ### 📖 About
+        This Space fine-tunes LayoutLMv3 with CRF for document understanding tasks including:
+        - Questions, Options, Answers
+        - Section Headings
+        - Passages
+
+        **Model Details:** LayoutLMv3-base + CRF layer for sequence labeling
+
+        **Features:**
+        - ✅ Real-time log streaming during training
+        - ✅ Progress monitoring with epoch/batch updates
+        - ✅ Immediate model download after completion
+        - ✅ Automatic file preparation for download
+        """
+    )
+
+if __name__ == "__main__":
+    demo.launch()