Upload folder using huggingface_hub

main.py

@@ -5,10 +5,11 @@ import json
 from datetime import datetime
 from typing import List, Optional, Dict, Any
 
-from fastapi import FastAPI, HTTPException, Request
+from fastapi import FastAPI, HTTPException, Request, Body
 from fastapi.responses import HTMLResponse
 from fastapi.middleware.cors import CORSMiddleware
 from huggingface_hub import HfApi, hf_hub_download, login
+from pydantic import BaseModel
 
 # Setup logging
 logging.basicConfig(level=logging.INFO)
@@ -17,21 +18,30 @@ logger = logging.getLogger(__name__)
 # Configuration
 TOKEN = os.getenv("HF_TOKEN")
 REPO_ID = "gebsunamy/MemoryAndSkills_Data" 
+OFFLINE_MODE = TOKEN is None
 
-if TOKEN:
-    login(token=TOKEN)
-
-api = HfApi()
-
-# Ensure the repository exists
-try:
-    api.create_repo(repo_id=REPO_ID, repo_type="dataset", private=True)
-    logger.info(f"Created private dataset repository: {REPO_ID}")
-except Exception:
-    pass
+if not OFFLINE_MODE:
+    try:
+        login(token=TOKEN)
+        api = HfApi()
+        # Ensure the repository exists
+        try:
+            api.create_repo(repo_id=REPO_ID, repo_type="dataset", private=True)
+            logger.info(f"Connected to HF: {REPO_ID}")
+        except Exception:
+            pass 
+    except Exception as e:
+        logger.error(f"HF Login failed: {e}. Switching to OFFLINE MODE.")
+        OFFLINE_MODE = True
 
-# --- DATA HELPERS (JSON) ---
+# --- DATA HELPERS (JSON + LOCAL FALLBACK) ---
 def load_json_data(file_name: str) -> List[Dict]:
+    if OFFLINE_MODE:
+        if os.path.exists(file_name):
+            with open(file_name, "r", encoding="utf-8") as f:
+                return json.load(f)
+        return []
+    
     try:
         path = hf_hub_download(
             repo_id=REPO_ID,
@@ -42,15 +52,22 @@ def load_json_data(file_name: str) -> List[Dict]:
         with open(path, "r", encoding="utf-8") as f:
             return json.load(f)
     except Exception as e:
-        logger.warning(f"File {file_name} not found or error: {e}. Returning empty list.")
+        logger.warning(f"File {file_name} not found on HF: {e}. Returning empty list.")
         return []
 
 def save_json_data(data: List[Dict], file_name: str):
+    if OFFLINE_MODE:
+        with open(file_name, "w", encoding="utf-8") as f:
+            json.dump(data, f, ensure_ascii=False, indent=2)
+        logger.info(f"Saved {file_name} locally.")
+        return
+
     try:
         temp_file = f"temp_{file_name}"
         with open(temp_file, "w", encoding="utf-8") as f:
             json.dump(data, f, ensure_ascii=False, indent=2)
         
+        api = HfApi()
         api.upload_file(
             path_or_fileobj=temp_file,
             path_in_repo=file_name,
@@ -59,27 +76,31 @@ def save_json_data(data: List[Dict], file_name: str):
             token=TOKEN
         )
         os.remove(temp_file)
-        logger.info(f"Successfully uploaded {file_name} to HF Hub.")
+        logger.info(f"Uploaded {file_name} to HF Hub.")
     except Exception as e:
         logger.error(f"Failed to save {file_name}: {e}")
 
+# --- SCHEMAS ---
+class MemoryCreate(BaseModel):
+    text: str
+    metadata: Optional[Dict] = {}
+
+class SkillCreate(BaseModel):
+    skill_name: str
+    skill_content: str
+
 # --- SUB-APPS ---
 
 # 1. MEMORY APP
-memory_app = FastAPI(title="Memory API", version="2.0.0")
+memory_app = FastAPI(title="Memory API", version="3.0.0")
 
-@memory_app.get("/create")
-async def create_memory(text: str, metadata: str = "{}"):
+@memory_app.post("/create")
+async def create_memory(item: MemoryCreate):
     data = load_json_data("memories.json")
-    try:
-        meta_dict = json.loads(metadata)
-    except:
-        meta_dict = {"raw": metadata}
-    
     new_item = {
         "id": str(uuid.uuid4()),
-        "text": text,
-        "metadata": meta_dict,
+        "text": item.text,
+        "metadata": item.metadata,
         "timestamp": datetime.utcnow().isoformat()
     }
     data.append(new_item)
@@ -90,14 +111,6 @@ async def create_memory(text: str, metadata: str = "{}"):
 async def get_all_memories():
     return load_json_data("memories.json")
 
-@memory_app.get("/get_once")
-async def get_memory(id: str):
-    data = load_json_data("memories.json")
-    item = next((m for m in data if m["id"] == id), None)
-    if not item:
-        raise HTTPException(status_code=404, detail="Memory not found")
-    return item
-
 @memory_app.get("/search")
 async def search_memories(query: str):
     data = load_json_data("memories.json")
@@ -112,15 +125,15 @@ async def delete_memory(id: str):
 
 
 # 2. SKILLS APP
-skills_app = FastAPI(title="Skills API", version="2.0.0")
+skills_app = FastAPI(title="Skills API", version="3.0.0")
 
-@skills_app.get("/create")
-async def create_skill(skill_name: str, skill_content: str):
+@skills_app.post("/create")
+async def create_skill(item: SkillCreate):
     data = load_json_data("skills.json")
     new_item = {
         "skill_id": str(uuid.uuid4()),
-        "skill_name": skill_name,
-        "skill_content": skill_content,
+        "skill_name": item.skill_name,
+        "skill_content": item.skill_content,
         "timestamp": datetime.utcnow().isoformat()
     }
     data.append(new_item)
@@ -162,51 +175,39 @@ app.mount("/memory", memory_app)
 app.mount("/skills", skills_app)
 
 @app.get("/", response_class=HTMLResponse)
-async def home(request: Request):
+async def home():
+    mode_text = "OFFLINE (Local Files)" if OFFLINE_MODE else "ONLINE (Hugging Face)"
     return f"""
     <html>
         <head>
-            <title>Memory & Skills Hub</title>
+            <title>Memory Hub</title>
             <style>
                 body {{ font-family: -apple-system, sans-serif; max-width: 900px; margin: 40px auto; background: #f8f9fa; color: #212529; padding: 20px; }}
                 .container {{ background: white; padding: 30px; border-radius: 12px; box-shadow: 0 4px 20px rgba(0,0,0,0.08); }}
-                h1 {{ color: #0d6efd; border-bottom: 2px solid #0d6efd; padding-bottom: 10px; }}
+                .badge {{ background: #ffc107; padding: 5px 10px; border-radius: 5px; font-size: 0.8em; font-weight: bold; }}
+                h1 {{ color: #0d6efd; }}
                 .section {{ margin-top: 30px; padding: 20px; border: 1px solid #dee2e6; border-radius: 8px; }}
-                .btn {{ display: inline-block; padding: 12px 24px; background: #0d6efd; color: white; text-decoration: none; border-radius: 6px; font-weight: bold; margin: 5px; transition: opacity 0.2s; }}
-                .btn-green {{ background: #198754; }}
-                .btn:hover {{ opacity: 0.85; }}
-                code {{ background: #e9ecef; padding: 3px 6px; border-radius: 4px; font-size: 0.9em; }}
-                .links {{ margin-top: 10px; }}
-                .links a {{ color: #6c757d; font-size: 0.85em; margin-right: 15px; }}
+                .btn {{ display: inline-block; padding: 12px 24px; background: #0d6efd; color: white; text-decoration: none; border-radius: 6px; font-weight: bold; margin: 5px; }}
             </style>
         </head>
         <body>
             <div class="container">
-                <h1>🧠 Memory & Skills Hub (JSON Engine)</h1>
-                <p>Universal <code>GET</code>-only storage for AI. Data is persisted as JSON on Hugging Face.</p>
+                <h1>🧠 Memory & Skills Hub <span class="badge">{mode_text}</span></h1>
+                <p>Universal storage. Creation uses <code>POST</code> (to handle large data), Reading uses <code>GET</code>.</p>
                 
                 <div class="section">
                     <h3>📂 Memories</h3>
-                    <p>Manage facts, history, and observations.</p>
-                    <a class="btn" href="/memory/docs">Swagger Docs</a>
-                    <div class="links">
-                        <a href="/memory/openapi.json">openapi.json</a>
-                    </div>
+                    <a class="btn" href="/memory/docs">Docs</a>
                 </div>
 
                 <div class="section">
                     <h3>🛠 Skills</h3>
-                    <p>Store specialized knowledge and capabilities.</p>
-                    <a class="btn" href="/skills/docs">Swagger Docs</a>
-                    <div class="links">
-                        <a href="/skills/openapi.json">openapi.json</a>
-                    </div>
+                    <a class="btn" href="/skills/docs">Docs</a>
                 </div>
 
                 <div class="section">
-                    <h3>⚙️ Control Panel</h3>
-                    <p>Add or remove data via a web interface.</p>
-                    <a class="btn btn-green" href="/admin">Open Admin Dashboard</a>
+                    <h3>⚙️ Admin</h3>
+                    <a class="btn" style="background:#198754" href="/admin">Open Admin Dashboard</a>
                 </div>
             </div>
         </body>
@@ -214,20 +215,17 @@ async def home(request: Request):
     """
 
 @app.get("/admin", response_class=HTMLResponse)
-async def admin_dashboard(request: Request):
+async def admin_dashboard():
     return f"""
     <html>
         <head>
             <title>Admin Dashboard</title>
             <style>
-                body {{ font-family: sans-serif; padding: 20px; background: #f4f4f9; color: #333; }}
+                body {{ font-family: sans-serif; padding: 20px; background: #f4f4f9; }}
                 .card {{ background: white; padding: 20px; margin-bottom: 25px; border-radius: 10px; box-shadow: 0 2px 10px rgba(0,0,0,0.05); }}
                 input, textarea {{ width: 100%; margin: 10px 0; padding: 12px; border: 1px solid #ddd; border-radius: 5px; box-sizing: border-box; }}
-                button {{ background: #0d6efd; color: white; border: none; padding: 12px 20px; cursor: pointer; border-radius: 5px; font-weight: bold; }}
-                button.delete {{ background: #dc3545; padding: 5px 10px; font-size: 0.8em; }}
-                .item-row {{ border-bottom: 1px solid #eee; padding: 10px 0; display: flex; justify-content: space-between; align-items: center; }}
-                .item-info {{ flex-grow: 1; }}
-                h2 {{ margin-top: 0; color: #0d6efd; }}
+                button {{ background: #0d6efd; color: white; border: none; padding: 12px 20px; cursor: pointer; border-radius: 5px; }}
+                .item-row {{ border-bottom: 1px solid #eee; padding: 10px 0; display: flex; justify-content: space-between; }}
             </style>
         </head>
         <body>
@@ -236,77 +234,63 @@ async def admin_dashboard(request: Request):
 
             <div class="card">
                 <h2>➕ New Memory</h2>
-                <input id="m_text" placeholder="What should I remember?">
-                <button onclick="addMemory()">Save Memory</button>
+                <input id="m_text" placeholder="Text...">
+                <button onclick="addMemory()">Save (POST)</button>
             </div>
 
             <div class="card">
                 <h2>➕ New Skill</h2>
-                <input id="s_name" placeholder="Skill Name (e.g. Python Programming)">
-                <textarea id="s_content" placeholder="Skill Content / Details..." rows="3"></textarea>
-                <button onclick="addSkill()">Save Skill</button>
+                <input id="s_name" placeholder="Skill Name">
+                <textarea id="s_content" placeholder="Skill Content (Unlimited Size)..." rows="5"></textarea>
+                <button onclick="addSkill()">Save (POST)</button>
             </div>
 
             <div class="card">
                 <h2>📦 Managed Data</h2>
                 <button onclick="loadData()">🔄 Refresh</button>
-                <div id="status"></div>
-                <h3>Memories</h3>
-                <div id="memories">Loading...</div>
-                <h3>Skills</h3>
-                <div id="skills">Loading...</div>
+                <div id="memories"></div>
+                <div id="skills"></div>
             </div>
 
             <script>
                 async function addMemory() {{
                     const text = document.getElementById('m_text').value;
-                    if(!text) return;
-                    await fetch(`/memory/create?text=${{encodeURIComponent(text)}}`);
+                    await fetch('/memory/create', {{
+                        method: 'POST',
+                        headers: {{ 'Content-Type': 'application/json' }},
+                        body: JSON.stringify({{ text, metadata: {{}} }})
+                    }});
                     document.getElementById('m_text').value = '';
                     loadData();
                 }}
                 async function addSkill() {{
-                    const name = document.getElementById('s_name').value;
-                    const content = document.getElementById('s_content').value;
-                    if(!name || !content) return;
-                    await fetch(`/skills/create?skill_name=${{encodeURIComponent(name)}}&skill_content=${{encodeURIComponent(content)}}`);
+                    const skill_name = document.getElementById('s_name').value;
+                    const skill_content = document.getElementById('s_content').value;
+                    await fetch('/skills/create', {{
+                        method: 'POST',
+                        headers: {{ 'Content-Type': 'application/json' }},
+                        body: JSON.stringify({{ skill_name, skill_content }})
+                    }});
                     document.getElementById('s_name').value = '';
                     document.getElementById('s_content').value = '';
                     loadData();
                 }}
-                async function deleteMemory(id) {{
-                    if(!confirm('Delete this memory?')) return;
-                    await fetch(`/memory/delete?id=${{id}}`);
-                    loadData();
-                }}
-                async function deleteSkill(id) {{
-                    if(!confirm('Delete this skill?')) return;
-                    await fetch(`/skills/delete?skill_id=${{id}}`);
+                async function deleteItem(type, id_param, id) {{
+                    await fetch(`/${{type}}/delete?${{id_param}}=${{id}}`);
                     loadData();
                 }}
                 async function loadData() {{
-                    try {{
-                        const mRes = await fetch('/memory/get_all');
-                        const mData = await mRes.json();
-                        document.getElementById('memories').innerHTML = mData.map(m => `
-                            <div class="item-row">
-                                <div class="item-info">${{m.text}}</div>
-                                <button class="delete" onclick="deleteMemory('${{m.id}}')">Delete</button>
-                            </div>
-                        `).join('') || 'No memories found.';
-
-                        const sRes = await fetch('/skills/get_all');
-                        const sData = await sRes.json();
-                        document.getElementById('skills').innerHTML = sData.map(s => `
-                            <div class="item-row">
-                                <div class="item-info"><b>${{s.skill_name}}</b>: ${{s.skill_content}}</div>
-                                <button class="delete" onclick="deleteSkill('${{s.skill_id}}')">Delete</button>
-                            </div>
-                        `).join('') || 'No skills found.';
-                    }} catch (err) {{
-                        console.error('Error loading data:', err);
-                        document.getElementById('status').innerText = 'Error loading data.';
-                    }}
+                    const mRes = await fetch('/memory/get_all');
+                    const mData = await mRes.json();
+                    document.getElementById('memories').innerHTML = '<h3>Memories</h3>' + mData.map(m => `
+                        <div class="item-row"><span>${{m.text}}</span> <button onclick="deleteItem('memory', 'id', '${{m.id}}')">Del</button></div>
+                    `).join('');
+
+                    const sRes = await fetch('/skills/get_all');
+                    const sData = await sRes.json();
+                    document.getElementById('skills').innerHTML = '<h3>Skills</h3>' + sData.map(s => `
+                        <div class="item-row"><span><b>${{s.skill_name}}</b></span> <button onclick="deleteItem('skills', 'skill_id', '${{s.skill_id}}')">Del</button></div>
+                    `).join('');
                 }}
                 window.onload = loadData;
             </script>

test_offline.py

@@ -0,0 +1,70 @@
+import unittest
+import os
+import json
+import time
+from multiprocessing import Process
+import uvicorn
+import httpx
+from main import app
+
+PORT = 8001
+
+def run_server():
+    uvicorn.run(app, host="127.0.0.1", port=PORT, log_level="error")
+
+class TestOfflineHub(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        if "HF_TOKEN" in os.environ:
+            del os.environ["HF_TOKEN"]
+        
+        cls.server_process = Process(target=run_server)
+        cls.server_process.start()
+        
+        # Wait for server to be ready
+        for _ in range(10):
+            try:
+                with httpx.Client() as client:
+                    client.get(f"http://127.0.0.1:{PORT}/")
+                break
+            except:
+                time.sleep(1)
+        else:
+            cls.server_process.terminate()
+            raise RuntimeError("Server failed to start")
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.server_process.terminate()
+        for f in ["memories.json", "skills.json"]:
+            if os.path.exists(f):
+                os.remove(f)
+
+    def test_memory_lifecycle(self):
+        url = f"http://127.0.0.1:{PORT}"
+        res = httpx.post(f"{url}/memory/create", json={"text": "Test Fact", "metadata": {}})
+        self.assertEqual(res.status_code, 200)
+        item_id = res.json()["id"]
+
+        res = httpx.get(f"{url}/memory/get_all")
+        self.assertEqual(len(res.json()), 1)
+
+        res = httpx.get(f"{url}/memory/search?query=Fact")
+        self.assertEqual(len(res.json()), 1)
+
+        res = httpx.get(f"{url}/memory/delete?id={item_id}")
+        self.assertEqual(res.json()["status"], "deleted")
+
+    def test_skills_lifecycle(self):
+        url = f"http://127.0.0.1:{PORT}"
+        large_content = "A" * 10000
+        res = httpx.post(f"{url}/skills/create", json={"skill_name": "Python", "skill_content": large_content})
+        self.assertEqual(res.status_code, 200)
+        skill_id = res.json()["skill_id"]
+
+        res = httpx.get(f"{url}/skills/get_once?skill_id={skill_id}")
+        self.assertEqual(res.json()["skill_name"], "Python")
+        self.assertEqual(len(res.json()["skill_content"]), 10000)
+
+if __name__ == "__main__":
+    unittest.main()