v1

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.11-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+RUN chmod +x start.sh
+
+EXPOSE 7860
+
+CMD ["./start.sh"]

backend/embedder.py

@@ -0,0 +1,23 @@
+import os
+from openai import OpenAI
+
+_client: OpenAI | None = None
+EMBED_MODEL = "text-embedding-3-small"
+BATCH_SIZE = 50
+
+
+def _get_client() -> OpenAI:
+    global _client
+    if _client is None:
+        _client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+    return _client
+
+
+def embed_texts(texts: list[str]) -> list[list[float]]:
+    client = _get_client()
+    vectors = []
+    for i in range(0, len(texts), BATCH_SIZE):
+        batch = texts[i : i + BATCH_SIZE]
+        response = client.embeddings.create(model=EMBED_MODEL, input=batch)
+        vectors.extend([item.embedding for item in response.data])
+    return vectors

backend/jobs.py

@@ -0,0 +1,36 @@
+import uuid
+from dataclasses import dataclass, field
+from typing import Optional
+
+_jobs: dict[str, "Job"] = {}
+
+
+@dataclass
+class Job:
+    id: str
+    status: str = "pending"   # pending | running | done | error
+    progress: int = 0          # 0-100
+    message: str = ""
+    error: Optional[str] = None
+    total_rows: int = 0
+    processed_rows: int = 0
+
+
+def create_job() -> Job:
+    job = Job(id=str(uuid.uuid4()))
+    _jobs[job.id] = job
+    return job
+
+
+def get_job(job_id: str) -> Optional[Job]:
+    return _jobs.get(job_id)
+
+
+def update_job(job_id: str, **kwargs) -> None:
+    job = _jobs.get(job_id)
+    if not job:
+        return
+    for k, v in kwargs.items():
+        setattr(job, k, v)
+    if job.total_rows > 0:
+        job.progress = int((job.processed_rows / job.total_rows) * 100)

backend/main.py

@@ -0,0 +1,119 @@
+import os
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, UploadFile, File, HTTPException, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+from dotenv import load_dotenv
+
+from backend.jobs import create_job, get_job, update_job
+from backend.parser import parse_file, row_to_text, SUPPORTED_EXTENSIONS
+from backend.embedder import embed_texts, BATCH_SIZE
+from backend.vectordb import ensure_collection, upsert_points, search, list_source_files
+
+load_dotenv()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    ensure_collection()
+    yield
+
+
+app = FastAPI(title="SemanticSearch API", lifespan=lifespan)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+# ---------------------------------------------------------------------------
+# Background task
+# ---------------------------------------------------------------------------
+
+def _ingest(job_id: str, filename: str, content: bytes) -> None:
+    try:
+        update_job(job_id, status="running", message="Parsing file...")
+        rows = parse_file(filename, content)
+        total = len(rows)
+        update_job(job_id, total_rows=total, message=f"Parsed {total} rows. Embedding...")
+
+        texts = [row_to_text(r) for r in rows]
+
+        all_vectors: list[list[float]] = []
+        for i in range(0, total, BATCH_SIZE):
+            batch_texts = texts[i : i + BATCH_SIZE]
+            batch_vectors = embed_texts(batch_texts)
+            all_vectors.extend(batch_vectors)
+            update_job(
+                job_id,
+                processed_rows=min(i + BATCH_SIZE, total),
+                message=f"Embedded {min(i + BATCH_SIZE, total)}/{total} rows...",
+            )
+
+        update_job(job_id, message="Storing vectors...")
+        upsert_points(all_vectors, rows, source_file=filename)
+        update_job(job_id, status="done", progress=100, message="Done!")
+
+    except Exception as exc:
+        update_job(job_id, status="error", message="Failed.", error=str(exc))
+
+
+# ---------------------------------------------------------------------------
+# Routes
+# ---------------------------------------------------------------------------
+
+@app.post("/upload")
+async def upload_file(background_tasks: BackgroundTasks, file: UploadFile = File(...)):
+    ext = "." + file.filename.rsplit(".", 1)[-1].lower()
+    if ext not in SUPPORTED_EXTENSIONS:
+        raise HTTPException(
+            status_code=400,
+            detail=f"Unsupported file type. Allowed: {', '.join(SUPPORTED_EXTENSIONS)}",
+        )
+    content = await file.read()
+    job = create_job()
+    background_tasks.add_task(_ingest, job.id, file.filename, content)
+    return {"job_id": job.id}
+
+
+@app.get("/status/{job_id}")
+def job_status(job_id: str):
+    job = get_job(job_id)
+    if not job:
+        raise HTTPException(status_code=404, detail="Job not found")
+    return {
+        "job_id": job.id,
+        "status": job.status,
+        "progress": job.progress,
+        "message": job.message,
+        "error": job.error,
+        "total_rows": job.total_rows,
+        "processed_rows": job.processed_rows,
+    }
+
+
+class SearchRequest(BaseModel):
+    query: str
+    top_k: int = 10
+
+
+@app.post("/search")
+def semantic_search(req: SearchRequest):
+    if not req.query.strip():
+        raise HTTPException(status_code=400, detail="Query cannot be empty")
+    vectors = embed_texts([req.query])
+    results = search(vectors[0], top_k=req.top_k)
+    return {"results": results}
+
+
+@app.get("/collections")
+def get_collections():
+    return {"files": list_source_files()}
+
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}

backend/parser.py

@@ -0,0 +1,30 @@
+import io
+import json
+import pandas as pd
+
+
+SUPPORTED_EXTENSIONS = {".csv", ".xlsx", ".xls", ".json"}
+
+
+def parse_file(filename: str, content: bytes) -> list[dict]:
+    ext = _ext(filename)
+    if ext == ".csv":
+        df = pd.read_csv(io.BytesIO(content))
+    elif ext in {".xlsx", ".xls"}:
+        df = pd.read_excel(io.BytesIO(content))
+    elif ext == ".json":
+        data = json.loads(content)
+        df = pd.DataFrame(data if isinstance(data, list) else [data])
+    else:
+        raise ValueError(f"Unsupported file type: {ext}")
+
+    df = df.dropna(how="all").fillna("")
+    return df.to_dict(orient="records")
+
+
+def row_to_text(row: dict) -> str:
+    return " | ".join(f"{k}: {v}" for k, v in row.items() if str(v).strip())
+
+
+def _ext(filename: str) -> str:
+    return "." + filename.rsplit(".", 1)[-1].lower()

backend/vectordb.py

@@ -0,0 +1,90 @@
+import os
+import uuid
+from qdrant_client import QdrantClient
+from qdrant_client.models import (
+    Distance,
+    VectorParams,
+    PointStruct,
+    Filter,
+    FieldCondition,
+    MatchValue,
+)
+
+VECTOR_SIZE = 1536  # text-embedding-3-small dimension
+
+_client: QdrantClient | None = None
+
+
+def _get_client() -> QdrantClient:
+    global _client
+    if _client is None:
+        _client = QdrantClient(
+            url=os.environ["QDRANT_URL"],
+            api_key=os.environ["QDRANT_API_KEY"],
+        )
+    return _client
+
+
+def _collection() -> str:
+    return os.environ.get("QDRANT_COLLECTION", "documents")
+
+
+def ensure_collection() -> None:
+    client = _get_client()
+    col = _collection()
+    existing = [c.name for c in client.get_collections().collections]
+    if col not in existing:
+        client.create_collection(
+            collection_name=col,
+            vectors_config=VectorParams(size=VECTOR_SIZE, distance=Distance.COSINE),
+        )
+
+
+def upsert_points(
+    vectors: list[list[float]],
+    payloads: list[dict],
+    source_file: str,
+) -> None:
+    client = _get_client()
+    points = [
+        PointStruct(
+            id=str(uuid.uuid4()),
+            vector=vec,
+            payload={**payload, "source_file": source_file},
+        )
+        for vec, payload in zip(vectors, payloads)
+    ]
+    client.upsert(collection_name=_collection(), points=points)
+
+
+def search(query_vector: list[float], top_k: int = 10) -> list[dict]:
+    client = _get_client()
+    results = client.search(
+        collection_name=_collection(),
+        query_vector=query_vector,
+        limit=top_k,
+        with_payload=True,
+    )
+    return [
+        {"score": round(hit.score, 4), **hit.payload}
+        for hit in results
+    ]
+
+
+def list_source_files() -> list[str]:
+    client = _get_client()
+    seen: set[str] = set()
+    offset = None
+    while True:
+        records, offset = client.scroll(
+            collection_name=_collection(),
+            with_payload=["source_file"],
+            limit=256,
+            offset=offset,
+        )
+        for r in records:
+            if r.payload and "source_file" in r.payload:
+                seen.add(r.payload["source_file"])
+        if offset is None:
+            break
+    return sorted(seen)

frontend/app.py

@@ -0,0 +1,124 @@
+import time
+import httpx
+import streamlit as st
+
+API_BASE = "http://localhost:8000"
+
+st.set_page_config(page_title="Semantic Search", page_icon="🔍", layout="wide")
+st.title("🔍 Semantic Search")
+
+tab_upload, tab_search, tab_files = st.tabs(["Upload", "Search", "Indexed Files"])
+
+
+# ---------------------------------------------------------------------------
+# Upload tab
+# ---------------------------------------------------------------------------
+with tab_upload:
+    st.subheader("Upload a file to index")
+    st.caption("Supported formats: CSV, XLSX, XLS, JSON — one file at a time")
+
+    uploaded = st.file_uploader(
+        "Choose a file",
+        type=["csv", "xlsx", "xls", "json"],
+        label_visibility="collapsed",
+    )
+
+    if uploaded and st.button("Upload & Embed", type="primary"):
+        with st.spinner("Sending file to API..."):
+            try:
+                resp = httpx.post(
+                    f"{API_BASE}/upload",
+                    files={"file": (uploaded.name, uploaded.getvalue(), "application/octet-stream")},
+                    timeout=30,
+                )
+                resp.raise_for_status()
+                job_id = resp.json()["job_id"]
+            except Exception as e:
+                st.error(f"Upload failed: {e}")
+                st.stop()
+
+        st.info(f"Job started: `{job_id}`")
+        progress_bar = st.progress(0)
+        status_text = st.empty()
+
+        while True:
+            try:
+                status_resp = httpx.get(f"{API_BASE}/status/{job_id}", timeout=10)
+                data = status_resp.json()
+            except Exception as e:
+                st.error(f"Could not poll status: {e}")
+                break
+
+            pct = data.get("progress", 0)
+            msg = data.get("message", "")
+            state = data.get("status", "running")
+
+            progress_bar.progress(pct)
+            status_text.text(msg)
+
+            if state == "done":
+                st.success(f"Indexed {data['total_rows']} rows from **{uploaded.name}**")
+                break
+            elif state == "error":
+                st.error(f"Error: {data.get('error', 'Unknown error')}")
+                break
+
+            time.sleep(1)
+
+
+# ---------------------------------------------------------------------------
+# Search tab
+# ---------------------------------------------------------------------------
+with tab_search:
+    st.subheader("Search across indexed documents")
+
+    query = st.text_input("Enter your search query", placeholder="e.g. high revenue customers in Q3")
+    top_k = st.slider("Results to return", min_value=1, max_value=25, value=10)
+
+    if st.button("Search", type="primary") and query.strip():
+        with st.spinner("Searching..."):
+            try:
+                resp = httpx.post(
+                    f"{API_BASE}/search",
+                    json={"query": query, "top_k": top_k},
+                    timeout=30,
+                )
+                resp.raise_for_status()
+                results = resp.json()["results"]
+            except Exception as e:
+                st.error(f"Search failed: {e}")
+                st.stop()
+
+        if not results:
+            st.warning("No results found.")
+        else:
+            st.success(f"{len(results)} results")
+            for i, r in enumerate(results, 1):
+                score = r.pop("score", None)
+                source = r.pop("source_file", "unknown")
+                with st.expander(f"#{i} — score: {score}  |  source: {source}"):
+                    st.json(r)
+
+
+# ---------------------------------------------------------------------------
+# Files tab
+# ---------------------------------------------------------------------------
+with tab_files:
+    st.subheader("Indexed source files")
+
+    if st.button("Refresh", type="secondary"):
+        st.rerun()
+
+    try:
+        resp = httpx.get(f"{API_BASE}/collections", timeout=10)
+        resp.raise_for_status()
+        files = resp.json()["files"]
+    except Exception as e:
+        st.error(f"Could not fetch files: {e}")
+        files = []
+
+    if not files:
+        st.info("No files indexed yet. Upload one in the Upload tab.")
+    else:
+        for f in files:
+            st.markdown(f"- `{f}`")

requirements.txt

@@ -0,0 +1,10 @@
+fastapi==0.115.0
+uvicorn[standard]==0.30.6
+python-multipart==0.0.12
+streamlit==1.38.0
+openai==1.50.0
+qdrant-client==1.11.3
+pandas==2.2.3
+openpyxl==3.1.5
+python-dotenv==1.0.1
+httpx==0.27.2

start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+uvicorn backend.main:app --host 0.0.0.0 --port 8000 &
+
+streamlit run frontend/app.py \
+  --server.port 7860 \
+  --server.address 0.0.0.0 \
+  --server.headless true \
+  --server.fileWatcherType none