init

.gitignore

@@ -0,0 +1,2 @@
+#.env
+.env

Dockerfile

@@ -0,0 +1,16 @@
+# Read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
+# you will also find guides on how best to write your Dockerfile
+
+FROM python:3.9
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,93 @@
+from fastapi import FastAPI, File, UploadFile, Form, BackgroundTasks, HTTPException
+from fastapi.responses import HTMLResponse, JSONResponse
+from pathlib import Path
+from typing import List
+from uploader import HuggingFaceUploader
+import shutil
+import uuid
+import os
+
+app = FastAPI()
+
+# Initialize HuggingFaceUploader with your token and repo_id
+hf_token = os.getenv("HF_TOKEN")
+repo_id = os.getenv("REPO_ID")
+uploader = HuggingFaceUploader(hf_token, repo_id)
+
+# Directories for temporary uploads and status tracking
+UPLOAD_FOLDER = Path("temp_uploads")
+STATUS_FOLDER = Path("status")
+UPLOAD_FOLDER.mkdir(exist_ok=True)
+STATUS_FOLDER.mkdir(exist_ok=True)
+
+def save_progress(status_file, filename, status):
+    """Helper function to update progress to a status file."""
+    with status_file.open("a", encoding="utf-8") as f:
+        f.write(f"{filename}: {status}\n")
+
+def upload_file_task(temp_folder_path: Path, destination_folder: str, status_file: Path):
+    """Background task to upload files and log progress."""
+    try:
+        total_files = len(list(temp_folder_path.rglob('*')))
+        uploaded_files = 0
+
+        for local_path in temp_folder_path.rglob('*'):
+            if local_path.is_file():
+                relative_path = local_path.relative_to(temp_folder_path)
+                path_in_repo = str(Path(destination_folder) / relative_path).replace("\\", "/")
+                
+                uploader.api.upload_file(
+                    path_or_fileobj=str(local_path),
+                    path_in_repo=path_in_repo,
+                    repo_id=uploader.repo_id,
+                    repo_type="model"
+                )
+
+                uploaded_files += 1
+                save_progress(status_file, str(local_path.name), "Uploaded")
+                # Log current progress
+                save_progress(status_file, "Progress", f"{uploaded_files}/{total_files} files uploaded.")
+
+    except Exception as e:
+        save_progress(status_file, "error", str(e))
+    finally:
+        # Clean up files
+        for file in temp_folder_path.glob('*'):
+            file.unlink()
+        temp_folder_path.rmdir()
+
+@app.get("/", response_class=HTMLResponse)
+async def main():
+    with open("index.html") as f:
+        return HTMLResponse(content=f.read())
+
+@app.post("/upload")
+async def upload_files(
+    background_tasks: BackgroundTasks,
+    destination_folder: str = Form(...),
+    files: List[UploadFile] = File(...)
+):
+    temp_folder_path = UPLOAD_FOLDER / uuid.uuid4().hex
+    temp_folder_path.mkdir(parents=True, exist_ok=True)
+
+    # Save uploaded files temporarily
+    for file in files:
+        file_path = temp_folder_path / file.filename
+        with file_path.open("wb") as buffer:
+            shutil.copyfileobj(file.file, buffer)
+    
+    # Create a status file for tracking progress
+    status_file = STATUS_FOLDER / f"{temp_folder_path.name}.txt"
+    background_tasks.add_task(upload_file_task, temp_folder_path, destination_folder, status_file)
+    return JSONResponse(content={"status_id": temp_folder_path.name})
+
+@app.get("/progress/{status_id}")
+async def get_progress(status_id: str):
+    """Get the progress of the upload from the status file."""
+    status_file = STATUS_FOLDER / f"{status_id}.txt"
+    if not status_file.exists():
+        raise HTTPException(status_code=404, detail="Status not found")
+    
+    with status_file.open("r", encoding="utf-8") as f:
+        progress = f.readlines()
+    return JSONResponse(content={"progress": progress})

index.html

@@ -0,0 +1,58 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>File Upload with Progress</title>
+</head>
+<body>
+    <h2>Upload Files to Hugging Face Repository</h2>
+    <form id="uploadForm" enctype="multipart/form-data">
+        <label for="destination_folder">Destination Folder:</label>
+        <input type="text" id="destination_folder" name="destination_folder" required>
+        
+        <label for="files">Select Files:</label>
+        <input type="file" id="files" name="files" multiple required>
+
+        <button type="submit">Upload</button>
+    </form>
+    
+    <div id="progress">
+        <h3>Upload Progress:</h3>
+        <pre id="progressText">No upload in progress...</pre>
+    </div>
+
+    <script>
+        document.getElementById("uploadForm").onsubmit = async function(event) {
+            event.preventDefault();
+            const formData = new FormData(this);
+
+            try {
+                const response = await fetch("/upload", {
+                    method: "POST",
+                    body: formData
+                });
+                const result = await response.json();
+                const statusId = result.status_id;
+                
+                // Start polling for progress
+                const progressText = document.getElementById("progressText");
+                progressText.textContent = "Starting upload...";
+                
+                const intervalId = setInterval(async () => {
+                    const progressResponse = await fetch(`/progress/${statusId}`);
+                    if (progressResponse.ok) {
+                        const progressResult = await progressResponse.json();
+                        progressText.textContent = progressResult.progress.join("");
+                    } else {
+                        clearInterval(intervalId);
+                        progressText.textContent += "\nUpload completed or status not found.";
+                    }
+                }, 2000);
+            } catch (error) {
+                document.getElementById("progressText").textContent = "Upload failed.";
+            }
+        };
+    </script>
+</body>
+</html>

requirements.txt

@@ -0,0 +1,4 @@
+fastapi
+uvicorn[standard]
+requests
+aiofiles

uploader.py

@@ -0,0 +1,34 @@
+from huggingface_hub import HfApi
+from pathlib import Path
+
+class HuggingFaceUploader:
+    def __init__(self, hf_token: str, repo_id: str):
+        self.api = HfApi(token=hf_token)
+        self.repo_id = repo_id
+
+    def upload_folder_with_structure(self, folder_path: str, destination_folder: str):
+        folder_path = Path(folder_path)
+        destination_folder = Path(destination_folder)
+
+        for local_path in folder_path.rglob('*'):
+            if local_path.is_file():
+                # Define the path in the repository
+                relative_path = local_path.relative_to(folder_path)
+                path_in_repo = str(destination_folder / relative_path).replace("\\", "/")  # Ensuring forward slashes
+
+                # Upload each file to the correct path in the repository
+                self.api.upload_file(
+                    path_or_fileobj=str(local_path),
+                    path_in_repo=path_in_repo,
+                    repo_id=self.repo_id,
+                    repo_type="model"
+                )
+                print(f"Uploaded '{local_path}' to '{path_in_repo}' in the Hugging Face repository '{self.repo_id}'.")
+
+if __name__ == "__main__":
+    hf_token = ""
+    repo_id = ""
+    folder_path = "content"  # Set the local folder path
+    destination_folder = "hindi"  # Adjust the destination folder structure as needed
+    uploader = HuggingFaceUploader(hf_token, repo_id)
+    uploader.upload_folder_with_structure(folder_path, destination_folder)