Upload 7 files

.gitattributes

@@ -0,0 +1,2 @@
+embeddings_glossary.json filter=lfs diff=lfs merge=lfs -text
+embeddings_quality.json filter=lfs diff=lfs merge=lfs -text

README (1).md

@@ -0,0 +1,13 @@
+---
+title: DustLookGradio
+emoji: 🌖
+colorFrom: green
+colorTo: red
+sdk: gradio
+sdk_version: 5.49.1
+app_file: app.py
+pinned: false
+license: unknown
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,181 @@
+import json, html, numpy as np, torch, gradio as gr
+from sentence_transformers import SentenceTransformer
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
+from threading import Thread
+
+
+class Config:
+    EMBEDDINGS_FILE = "embeddings_quality.json"
+    MODEL_ID = "HuggingFaceTB/SmolLM2-135M-Instruct"
+    TOP_K = 5
+    SIM_THRESHOLD = 0.36
+    MAX_NEW_TOKENS = 512
+    DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
+
+
+cfg = Config()
+
+
+def safe_strip(x: str) -> str:
+    return x.replace("\n", " ").replace("\r", " ").strip() if isinstance(x, str) else ""
+
+
+def load_entries(path):
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    entries = []
+    for v in data.values():
+        m = v.get("metadata", {})
+        title = m.get("title") or v.get("title") or ""
+        definition = m.get("definition") or v.get("definition") or m.get("content") or ""
+        source = m.get("source") or v.get("source") or ""
+        emb = np.array(v.get("embedding", []), dtype=np.float32)
+        if emb.size == 0:
+            continue
+        emb = emb / np.linalg.norm(emb)
+        entries.append({
+            "title": safe_strip(title),
+            "definition": safe_strip(definition),
+            "source": safe_strip(source),
+            "embedding": emb
+        })
+    vectors = np.stack([e["embedding"] for e in entries])
+    return entries, vectors
+
+
+def init_models():
+    """Initialize all models and load data"""
+    # 1. Load embedding model for semantic search
+    embed_model = SentenceTransformer("all-MiniLM-L6-v2", device=cfg.DEVICE)
+
+    # 2. Load tokenizer and model for text generation
+    tokenizer = AutoTokenizer.from_pretrained(cfg.MODEL_ID)
+
+    # Add pad token if missing
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    # Load model without quantization to avoid bitsandbytes issues
+    model = AutoModelForCausalLM.from_pretrained(
+        cfg.MODEL_ID,
+        device_map="auto",
+        torch_dtype=torch.float16 if cfg.DEVICE == "cuda" else torch.float32
+    )
+
+    # 3. Load entries and vectors
+    entries, vectors = load_entries(cfg.EMBEDDINGS_FILE)
+
+    return embed_model, tokenizer, model, entries, vectors
+
+
+embed_model, tokenizer, model, entries, vectors = init_models()
+
+
+def search_chunks(query, top_k=cfg.TOP_K, batch_size=512):
+    """Memory-efficient cosine similarity search."""
+    import heapq
+
+    # Encode query as normalized float32 vector
+    qv = embed_model.encode([query], normalize_embeddings=True,
+                            convert_to_numpy=True).astype("float32")[0]
+
+    # Use a small max-heap to store the best results
+    heap = []  # stores (-similarity, index)
+
+    n = len(entries)
+    for start in range(0, n, batch_size):
+        end = min(start + batch_size, n)
+        # Instead of dotting all vectors, dot only a slice
+        sims = np.dot(vectors[start:end], qv)
+        for j, s in enumerate(sims):
+            if s < cfg.SIM_THRESHOLD:
+                continue
+            heapq.heappush(heap, (-s, start + j))
+            if len(heap) > top_k:
+                heapq.heappop(heap)  # maintain top_k only
+
+    # Convert heap to sorted list (descending order)
+    results = [(-s, entries[i]) for s, i in sorted(heap)]
+    return [(e, float(s)) for s, e in results]
+
+
+def build_context(entries, tokenizer, max_tokens=1500):
+    ctx, t = [], 0
+    for e in entries:
+        txt = f"{e['title']}: {e['definition']}\n"
+        tok = tokenizer.encode(txt, add_special_tokens=False)
+        if t + len(tok) > max_tokens:
+            break
+        ctx.append(txt)
+        t += len(tok)
+    return "\n".join(ctx)
+
+
+def generate_answer(question, context_entries):
+    ctx_text = build_context(context_entries, tokenizer)
+    prompt = f"""<|im_start|>system
+You are an expert on fighting game terminology. Use only the CONTEXT below to answer the QUESTION clearly using english language and proper structure.
+<|im_end|>
+<|im_start|>user
+CONTEXT:
+{ctx_text}
+
+QUESTION: {question}
+<|im_end|>
+<|im_start|>assistant
+"""
+    inputs = tokenizer(prompt, return_tensors="pt").to(cfg.DEVICE)
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+    kwargs = dict(
+        **inputs,
+        max_new_tokens=cfg.MAX_NEW_TOKENS,
+        temperature=0.5,
+        top_p=0.9,
+        do_sample=True,
+        pad_token_id=tokenizer.eos_token_id,
+        eos_token_id=tokenizer.eos_token_id,
+        streamer=streamer
+    )
+    Thread(target=model.generate, kwargs=kwargs).start()
+    partial = ""
+    for token in streamer:
+        partial += token
+        yield partial
+
+
+def qa_pipeline(question):
+    """Unified pipeline that streams search results first, then LLM answer."""
+    results = search_chunks(question)
+
+    # Build top results HTML immediately
+    if not results:
+        top_html = "<p>No relevant entries found.</p>"
+        yield top_html, "Your question is out of scope."
+        return
+
+    html_out = "<h4>Top Relevant Entries:</h4>"
+    for i, (e, s) in enumerate(results, 1):
+        html_out += f"<details open><summary><b>[{i}] (score: {s:.3f}) {html.escape(e['title'])}</b></summary><p>{html.escape(e['definition'][:500])}</p><p><i>{html.escape(e['source'])}</i></p></details>"
+
+    # Yield search results immediately
+    yield html_out, ""
+
+    # Then stream the LLM response
+    entries_only = [r[0] for r in results]
+    partial = ""
+    for token in generate_answer(question, entries_only):
+        partial = token
+        yield html_out, partial
+
+
+with gr.Blocks(title="Fighting Game Glossary QA") as demo:
+    gr.Markdown("## 🎮 Fighting Game Glossary QA\nAsk about any fighting game term.")
+    q = gr.Textbox(label="Ask a question:", placeholder="e.g., What is a Roman Cancel?")
+    top = gr.HTML(label="Top Matches")
+    out = gr.Textbox(label="LLM Answer", lines=15, interactive=False, show_copy_button=True)
+    btn = gr.Button("Search & Answer")
+
+    # Use a single click event that streams both outputs
+    btn.click(fn=qa_pipeline, inputs=q, outputs=[top, out], queue=True)
+
+demo.queue().launch()

embeddings_glossary.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f9522ed71ee174349ba2ef3789f0db9a85f832f630b5c2db64be8db569425dcb
+size 11349754

embeddings_quality.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acdebed75d66a96d60a877262026e832faa3eae141a351d824b4226a03fc6afe
+size 330053950

gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+embeddings_all.json filter=lfs diff=lfs merge=lfs -text

init_models.py

@@ -0,0 +1,69 @@
+import torch
+from sentence_transformers import SentenceTransformer
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
+
+
+from projeto.app import cfg, load_entries
+
+
+def debug_json_structure(path):
+    """Debug para ver a estrutura real do JSON"""
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+
+    print("🔍 DEBUG DA ESTRUTURA DO JSON:")
+
+    # Ver as primeiras 3 chaves para ver a estrutura completa
+    for i, (key, value) in enumerate(list(data.items())[:3]):
+        print(f"\n--- Chave {i + 1}: '{key}' ---")
+        print(f"Tipo do valor: {type(value)}")
+        if isinstance(value, dict):
+            print(f"Campos: {list(value.keys())}")
+            for k, v in value.items():
+                if k == "embedding":
+                    print(f"  {k}: [lista com {len(v) if isinstance(v, list) else '?'} elementos]")
+                else:
+                    print(f"  {k}: {str(v)[:100]}{'...' if len(str(v)) > 100 else ''}")
+        print("---")
+
+    # Procurar especificamente por "Faust" para ver sua estrutura
+    print("\n🔍 PROCURANDO POR 'Faust' NO JSON:")
+    faust_found = False
+    for key, value in data.items():
+        if "Faust" in key or (
+                isinstance(value, dict) and "Faust" in str(value.get('title', '')) + str(value.get('term', ''))):
+            print(f"Encontrado Faust na chave: '{key}'")
+            print(f"Estrutura: {value}")
+            faust_found = True
+            break
+
+    if not faust_found:
+        print("Faust não encontrado nas primeiras verificações")
+
+def init_models():
+    """Initialize all models and load data"""
+    # 1. Load embedding model for semantic search
+    embed_model = SentenceTransformer("all-MiniLM-L6-v2", device=cfg.DEVICE)
+
+    # 2. Load tokenizer and model for text generation
+    tokenizer = AutoTokenizer.from_pretrained(cfg.MODEL_ID)
+
+    # Add pad token if missing
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+
+    # Load model without quantization to avoid bitsandbytes issues
+    model = AutoModelForCausalLM.from_pretrained(
+        cfg.MODEL_ID,
+        device_map="auto",
+        torch_dtype=torch.float16 if cfg.DEVICE == "cuda" else torch.float32
+    )
+
+    # 3. Load entries and vectors
+    print("Carregando embeddings...")
+    entries, vectors = load_entries(cfg.EMBEDDINGS_FILE)
+
+    if len(entries) == 0:
+        raise Exception("Nenhuma entrada foi carregada! Verifique o arquivo de embeddings.")
+
+    return embed_model, tokenizer, model, entries, vectors

requirements.txt

@@ -0,0 +1,7 @@
+bitsandbytes
+torch>=2.0.0
+transformers>=4.30.0
+sentence-transformers>=2.2.0
+gradio>=4.0.0
+accelerate>=0.20.0
+numpy>=1.21.0