Upload 13 files

hf-vector-match-api/.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

hf-vector-match-api/.gitignore

@@ -0,0 +1,5 @@
+__pycache__/
+*.pyc
+.env
+.env.local
+data/uploads/

hf-vector-match-api/Dockerfile

@@ -0,0 +1,16 @@
+FROM python:3.10-slim
+
+RUN useradd -m -u 1000 user
+USER user
+ENV PATH="/home/user/.local/bin:$PATH"
+
+WORKDIR /app
+
+COPY --chown=user requirements.txt .
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY --chown=user . /app
+
+RUN mkdir -p data/uploads
+
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "7860"]

hf-vector-match-api/README.md

@@ -0,0 +1,10 @@
+---
+title: Vector Match Api
+emoji: 😻
+colorFrom: pink
+colorTo: purple
+sdk: docker
+pinned: false
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

hf-vector-match-api/database.py

@@ -0,0 +1,172 @@
+import os
+from pathlib import Path
+from urllib.parse import quote_plus
+
+from dotenv import load_dotenv
+from sqlalchemy import create_engine
+
+_env_dir = Path(__file__).resolve().parent
+load_dotenv(_env_dir / ".env.local", override=False)   # 本地开发优先
+load_dotenv(_env_dir / ".env", override=False)          # 兜底（Docker/线上）
+from sqlalchemy.orm import sessionmaker, declarative_base
+
+# 可选：整串 URL（优先级最高），例如 postgresql+psycopg2://user:pass@host:5432/dbname
+_database_url = os.environ.get("DATABASE_URL", "").strip()
+
+if _database_url:
+    SQLALCHEMY_DATABASE_URL = _database_url
+else:
+    PG_HOST = os.environ.get("PG_HOST", "localhost")
+    PG_PORT = os.environ.get("PG_PORT", "5432")
+    PG_USER = os.environ.get("PG_USER", "postgres")
+    PG_PASSWORD = os.environ.get("PG_PASSWORD", "postgres")
+    PG_DB = os.environ.get("PG_DB", "vector_match")
+    _pw = quote_plus(PG_PASSWORD)
+    SQLALCHEMY_DATABASE_URL = (
+        f"postgresql+psycopg2://{PG_USER}:{_pw}@{PG_HOST}:{PG_PORT}/{PG_DB}"
+    )
+
+PG_SCHEMA = os.environ.get("PG_SCHEMA", "vector_match")
+
+engine = create_engine(
+    SQLALCHEMY_DATABASE_URL,
+    pool_pre_ping=True,
+    pool_size=20,
+    pool_recycle=180,
+    pool_timeout=60,
+    max_overflow=10,
+)
+
+# 每次连接自动切换到 vector_match schema
+from sqlalchemy import event
+
+@event.listens_for(engine, "connect")
+def _set_search_path(dbapi_conn, connection_record):
+    cursor = dbapi_conn.cursor()
+    cursor.execute(f"SET search_path TO {PG_SCHEMA}, public")
+    cursor.close()
+
+SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+Base = declarative_base()
+
+DATA_DIR = os.path.join(os.path.dirname(__file__), "data")
+os.makedirs(DATA_DIR, exist_ok=True)
+
+_SCHEMA_TABLES = (
+    "vector_match_task",
+    "vector_dataset",
+    "vector_data_row",
+    "vector_embedding",
+    "match_result",
+)
+
+
+def get_db():
+    db = SessionLocal()
+    try:
+        yield db
+    except Exception:
+        db.rollback()
+        raise
+    finally:
+        db.close()
+
+
+def _table_sql_name(table: str) -> str:
+    if engine.dialect.name == "postgresql":
+        return f'"{PG_SCHEMA}"."{table}"'
+    return table
+
+
+def _column_names_conn(conn, table: str) -> set:
+    """
+    与当前连接共用同一事务，避免在持有 ALTER 锁的事务内再用 inspect(engine) 开新连接查目录，
+    否则 PostgreSQL 上会自锁（会话 A 持锁等 B 查元数据，B 等 A 释放锁）。
+    """
+    from sqlalchemy import inspect
+
+    insp = inspect(conn)
+    schema = PG_SCHEMA if engine.dialect.name == "postgresql" else None
+    return {c["name"] for c in insp.get_columns(table, schema=schema)}
+
+
+def _ensure_is_archived_column():
+    """旧库无 is_archived 时补列。"""
+    from sqlalchemy import inspect, text
+
+    insp = inspect(engine)
+    schema = PG_SCHEMA if engine.dialect.name == "postgresql" else None
+    try:
+        cols = insp.get_columns("vector_match_task", schema=schema)
+    except Exception:
+        return
+    if any(c["name"] == "is_archived" for c in cols):
+        return
+    ft = _table_sql_name("vector_match_task")
+    ddl = (
+        f"ALTER TABLE {ft} ADD COLUMN is_archived INTEGER NOT NULL DEFAULT 0"
+        if engine.dialect.name == "postgresql"
+        else "ALTER TABLE vector_match_task ADD COLUMN is_archived INTEGER NOT NULL DEFAULT 0"
+    )
+    with engine.begin() as conn:
+        conn.execute(text(ddl))
+
+
+def _ensure_time_is_delete_columns():
+    """
+    统一：created_at→created_time；任务表 updated_at→updated_time；
+    各表补 is_delete；is_deleted→is_delete；遗留 deleted_at 迁移后删除。
+    """
+    from sqlalchemy import text
+
+    ft_task = _table_sql_name("vector_match_task")
+
+    with engine.begin() as conn:
+        cols = _column_names_conn(conn, "vector_match_task")
+        if "created_at" in cols and "created_time" not in cols:
+            conn.execute(text(f"ALTER TABLE {ft_task} RENAME COLUMN created_at TO created_time"))
+        if "updated_at" in cols and "updated_time" not in cols:
+            conn.execute(text(f"ALTER TABLE {ft_task} RENAME COLUMN updated_at TO updated_time"))
+        if "is_deleted" in cols and "is_delete" not in cols:
+            conn.execute(text(f"ALTER TABLE {ft_task} RENAME COLUMN is_deleted TO is_delete"))
+        cols = _column_names_conn(conn, "vector_match_task")
+        if "deleted_at" in cols:
+            if "is_delete" not in cols:
+                conn.execute(
+                    text(f"ALTER TABLE {ft_task} ADD COLUMN is_delete INTEGER NOT NULL DEFAULT 0")
+                )
+            conn.execute(text(f"UPDATE {ft_task} SET is_delete = 1 WHERE deleted_at IS NOT NULL"))
+            conn.execute(text(f"ALTER TABLE {ft_task} DROP COLUMN deleted_at"))
+
+    for table in _SCHEMA_TABLES:
+        ft = _table_sql_name(table)
+        with engine.begin() as conn:
+            cols = _column_names_conn(conn, table)
+            if "created_at" in cols and "created_time" not in cols:
+                conn.execute(text(f"ALTER TABLE {ft} RENAME COLUMN created_at TO created_time"))
+            cols = _column_names_conn(conn, table)
+            if "is_delete" not in cols:
+                conn.execute(
+                    text(f"ALTER TABLE {ft} ADD COLUMN is_delete INTEGER NOT NULL DEFAULT 0")
+                )
+            cols = _column_names_conn(conn, table)
+            if "updated_time" not in cols:
+                conn.execute(text(f"ALTER TABLE {ft} ADD COLUMN updated_time TIMESTAMP NULL"))
+                if "created_time" in cols:
+                    conn.execute(
+                        text(
+                            f"UPDATE {ft} SET updated_time = created_time "
+                            f"WHERE updated_time IS NULL"
+                        )
+                    )
+
+
+def init_db():
+    from models import Base as ModelBase  # noqa: F401
+    from sqlalchemy import text
+    with engine.connect() as conn:
+        conn.execute(text(f"CREATE SCHEMA IF NOT EXISTS {PG_SCHEMA}"))
+        conn.commit()
+    ModelBase.metadata.create_all(bind=engine)
+    _ensure_is_archived_column()
+    _ensure_time_is_delete_columns()

hf-vector-match-api/main.py

@@ -0,0 +1,599 @@
+import os
+import asyncio
+import json
+import time
+import datetime
+import httpx
+from typing import List, Optional
+
+from fastapi import FastAPI, UploadFile, File, Form, Depends, Query, HTTPException, BackgroundTasks
+from fastapi.middleware.cors import CORSMiddleware
+from sqlalchemy.orm import Session
+
+from database import get_db, init_db, SessionLocal
+from models import (
+    VectorMatchTask, VectorDataset, VectorDataRow,
+    VectorEmbedding, MatchResult,
+)
+from schemas import (
+    TaskCreate, TaskDetail, TaskProgress, TaskListItem,
+    MatchResultItem, MatchResultPage, SourceWithCandidates, CandidateDetail,
+    UploadResponse, SettingItem, SettingsResponse, DatasetInfo,
+)
+from services.excel_service import save_upload_file, get_sheet_info, parse_excel_rows
+from services.match_service import run_match_task
+
+
+app = FastAPI(title="VectorMatch API", version="1.0.0")
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+import logging, traceback
+from starlette.requests import Request
+from starlette.responses import JSONResponse
+
+logger = logging.getLogger("uvicorn.error")
+
+@app.exception_handler(Exception)
+async def global_exception_handler(request: Request, exc: Exception):
+    logger.error(f"Unhandled error on {request.method} {request.url}:\n{traceback.format_exc()}")
+    return JSONResponse(status_code=500, content={"detail": str(exc)})
+
+
+# ─── 健康状态缓存 ─────────────────────────────────────────────────────────
+_health_cache = {
+    "result": {"embedding_ok": False, "reranker_ok": False, "embedding_model": "",
+               "reranker_model": "", "reranker_enabled": False, "has_api_key": False},
+    "updated_at": 0,
+}
+_HEALTH_TTL = 30  # 缓存有效期（秒）
+
+
+async def _do_health_check():
+    """执行真正的 API 探活，更新缓存"""
+    import services.embedding_service as es
+    api_key = es.SILICONFLOW_API_KEY
+    result = {
+        "embedding_ok": False,
+        "reranker_ok": False,
+        "embedding_model": es.EMBEDDING_MODEL,
+        "reranker_model": es.RERANKER_MODEL,
+        "reranker_enabled": es.RERANKER_ENABLED,
+        "has_api_key": bool(api_key),
+    }
+    if api_key:
+        try:
+            async with httpx.AsyncClient(timeout=5.0, proxies={}) as client:
+                try:
+                    emb_resp = await client.post(
+                        "https://api.siliconflow.cn/v1/embeddings",
+                        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+                        json={"model": es.EMBEDDING_MODEL, "input": ["ping"]},
+                    )
+                    result["embedding_ok"] = emb_resp.status_code == 200
+                except Exception:
+                    pass
+                if es.RERANKER_ENABLED:
+                    try:
+                        rerank_resp = await client.post(
+                            "https://api.siliconflow.cn/v1/rerank",
+                            headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
+                            json={"model": es.RERANKER_MODEL, "query": "ping", "documents": ["pong"], "top_n": 1},
+                        )
+                        result["reranker_ok"] = rerank_resp.status_code == 200
+                    except Exception:
+                        pass
+        except Exception:
+            pass
+    _health_cache["result"] = result
+    _health_cache["updated_at"] = time.time()
+    return result
+
+
+async def _health_polling_loop():
+    """后台定时探活循环"""
+    while True:
+        try:
+            await _do_health_check()
+        except Exception:
+            pass
+        await asyncio.sleep(_HEALTH_TTL)
+
+
+@app.on_event("startup")
+async def startup():
+    init_db()
+    # 启动后台健康检查循环
+    asyncio.create_task(_health_polling_loop())
+
+
+# ─── Upload Excel ───────────────────────────────────────────────────────────
+@app.post("/api/upload", response_model=UploadResponse)
+async def upload_excel(
+    file: UploadFile = File(...),
+    dataset_role: str = Form("source"),
+    db: Session = Depends(get_db),
+):
+    content = await file.read()
+    filepath = save_upload_file(content, file.filename)
+    info = get_sheet_info(filepath)
+
+    dataset = VectorDataset(
+        name=file.filename,
+        file_name=file.filename,
+        dataset_role=dataset_role,
+        data_scope="task",
+    )
+    db.add(dataset)
+    db.commit()
+    db.refresh(dataset)
+
+    return UploadResponse(
+        dataset_id=dataset.id,
+        file_name=file.filename,
+        sheet_names=info["sheet_names"],
+        columns=info["columns"],
+        all_columns=info.get("all_columns", info["columns"]),
+    )
+
+
+# ─── Configure dataset (sheet, fields) ─────────────────────────────────────
+@app.post("/api/dataset/{dataset_id}/configure")
+def configure_dataset(
+    dataset_id: int,
+    sheet_name: str = Form(...),
+    vector_fields: str = Form(...),
+    db: Session = Depends(get_db),
+):
+    dataset = db.query(VectorDataset).get(dataset_id)
+    if not dataset:
+        raise HTTPException(404, "Dataset not found")
+
+    dataset.sheet_name = sheet_name
+    dataset.vector_fields = vector_fields
+    db.commit()
+
+    fields = json.loads(vector_fields)
+    import os
+    filepath = os.path.join(
+        os.path.dirname(__file__), "data", "uploads", dataset.file_name
+    )
+    rows = parse_excel_rows(filepath, sheet_name, fields)
+
+    for row_data in rows:
+        dr = VectorDataRow(
+            dataset_id=dataset.id,
+            dataset_role=dataset.dataset_role,
+            data_scope=dataset.data_scope,
+            row_number=row_data["row_number"],
+            raw_text=row_data["raw_text"],
+            text_hash=row_data["text_hash"],
+            field_values=row_data["field_values"],
+        )
+        db.add(dr)
+    dataset.row_count = len(rows)
+    db.commit()
+
+    return {"status": "ok", "row_count": len(rows)}
+
+
+# ─── Get dataset info ──────────────────────────────────────────────────────
+@app.get("/api/dataset/{dataset_id}", response_model=DatasetInfo)
+def get_dataset(dataset_id: int, db: Session = Depends(get_db)):
+    dataset = db.query(VectorDataset).get(dataset_id)
+    if not dataset:
+        raise HTTPException(404, "Dataset not found")
+    return dataset
+
+
+# ─── Create & start task ───────────────────────────────────────────────────
+@app.post("/api/task", response_model=TaskDetail)
+async def create_task(
+    background_tasks: BackgroundTasks,
+    source_dataset_id: int = Form(...),
+    target_dataset_id: int = Form(...),
+    match_mode: str = Form("two_file"),
+    top_k: int = Form(10),
+    rerank_top_k: int = Form(3),
+    min_threshold: float = Form(0.70),
+    candidate_scope: str = Form("current_task_target"),
+    db: Session = Depends(get_db),
+):
+    now = datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=8)))
+    task_code = now.strftime("%Y%m%d%H%M%S") + f"{now.microsecond // 1000:03d}"
+
+    src = db.query(VectorDataset).get(source_dataset_id)
+    tgt = db.query(VectorDataset).get(target_dataset_id)
+    if not src or not tgt:
+        raise HTTPException(400, "Source or target dataset not found")
+
+    task = VectorMatchTask(
+        task_code=task_code,
+        match_mode=match_mode,
+        candidate_scope=candidate_scope,
+        source_dataset_id=source_dataset_id,
+        target_dataset_id=target_dataset_id,
+        top_k=top_k,
+        rerank_top_k=rerank_top_k,
+        min_threshold=min_threshold,
+        status="pending",
+    )
+    db.add(task)
+    db.commit()
+    db.refresh(task)
+
+    src.task_id = task.id
+    tgt.task_id = task.id
+    db.query(VectorDataRow).filter(VectorDataRow.dataset_id == src.id).update({"task_id": task.id})
+    db.query(VectorDataRow).filter(VectorDataRow.dataset_id == tgt.id).update({"task_id": task.id})
+    db.commit()
+
+    background_tasks.add_task(_run_task_in_background, task.id)
+
+    db.refresh(task)
+    return task
+
+
+def _run_task_in_background(task_id: int):
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    loop.run_until_complete(run_match_task(task_id, SessionLocal))
+    loop.close()
+
+
+def _get_alive_task(db: Session, task_id: int) -> Optional[VectorMatchTask]:
+    """未软删除的任务（is_delete=0）。"""
+    task = db.query(VectorMatchTask).get(task_id)
+    if not task or (task.is_delete or 0) == 1:
+        return None
+    return task
+
+
+# ─── Task progress ─────────────────────────────────────────────────────────
+@app.get("/api/task/{task_id}/progress", response_model=TaskProgress)
+def get_task_progress(task_id: int, db: Session = Depends(get_db)):
+    task = _get_alive_task(db, task_id)
+    if not task:
+        raise HTTPException(404, "Task not found")
+    return task
+
+
+# ─── Task detail ───────────────────────────────────────────────────────────
+@app.get("/api/task/{task_id}", response_model=TaskDetail)
+def get_task_detail(task_id: int, db: Session = Depends(get_db)):
+    task = _get_alive_task(db, task_id)
+    if not task:
+        raise HTTPException(404, "Task not found")
+    return task
+
+
+# ─── Task list ─────────────────────────────────────────────────────────────
+@app.get("/api/tasks", response_model=List[TaskListItem])
+def list_tasks(
+    scope: str = Query("active", description="active=未归档, archived=仅归档, deleted=回收站"),
+    db: Session = Depends(get_db),
+):
+    if scope not in ("active", "archived", "deleted"):
+        raise HTTPException(400, "scope 须为 active、archived 或 deleted")
+    q = db.query(VectorMatchTask)
+    if scope == "deleted":
+        q = q.filter(VectorMatchTask.is_delete == 1)
+    else:
+        q = q.filter(VectorMatchTask.is_delete == 0)
+        if scope == "archived":
+            q = q.filter(VectorMatchTask.is_archived == 1)
+        else:
+            q = q.filter(VectorMatchTask.is_archived == 0)
+    tasks = q.order_by(VectorMatchTask.created_time.desc()).all()
+    result = []
+    for t in tasks:
+        src_name = t.source_dataset.name if t.source_dataset else None
+        tgt_name = t.target_dataset.name if t.target_dataset else None
+        result.append(TaskListItem(
+            id=t.id,
+            task_code=t.task_code,
+            match_mode=t.match_mode,
+            candidate_scope=t.candidate_scope,
+            source_dataset_name=src_name,
+            target_dataset_name=tgt_name,
+            status=t.status,
+            is_archived=t.is_archived or 0,
+            is_delete=t.is_delete or 0,
+            created_time=t.created_time,
+        ))
+    return result
+
+
+@app.post("/api/task/{task_id}/archive")
+def archive_task(task_id: int, db: Session = Depends(get_db)):
+    task = _get_alive_task(db, task_id)
+    if not task:
+        raise HTTPException(404, "Task not found")
+    task.is_archived = 1
+    db.commit()
+    return {"status": "ok"}
+
+
+@app.post("/api/task/{task_id}/unarchive")
+def unarchive_task(task_id: int, db: Session = Depends(get_db)):
+    task = _get_alive_task(db, task_id)
+    if not task:
+        raise HTTPException(404, "Task not found")
+    task.is_archived = 0
+    db.commit()
+    return {"status": "ok"}
+
+
+@app.delete("/api/task/{task_id}")
+def delete_task(task_id: int, db: Session = Depends(get_db)):
+    """软删除：is_delete=1，数据仍保留在库中。"""
+    task = _get_alive_task(db, task_id)
+    if not task:
+        raise HTTPException(404, "Task not found")
+    task.is_delete = 1
+    db.commit()
+    return {"status": "ok"}
+
+
+@app.post("/api/task/{task_id}/restore")
+def restore_task(task_id: int, db: Session = Depends(get_db)):
+    """从回收站恢复。"""
+    task = db.query(VectorMatchTask).get(task_id)
+    if not task or (task.is_delete or 0) != 1:
+        raise HTTPException(404, "Task not found or not deleted")
+    task.is_delete = 0
+    db.commit()
+    return {"status": "ok"}
+
+
+# ─── Match results ─────────────────────────────────────────────────────────
+@app.get("/api/task/{task_id}/results", response_model=MatchResultPage)
+def get_task_results(
+    task_id: int,
+    page: int = Query(1, ge=1),
+    page_size: int = Query(20, ge=1, le=100),
+    search: Optional[str] = None,
+    level: Optional[str] = None,
+    sort: str = "score_desc",
+    db: Session = Depends(get_db),
+):
+    if not _get_alive_task(db, task_id):
+        raise HTTPException(404, "Task not found")
+    query = (
+        db.query(MatchResult)
+        .filter(MatchResult.task_id == task_id, MatchResult.rank == 1)
+    )
+    if level and level != "all":
+        query = query.filter(MatchResult.match_level == level)
+
+    if sort == "score_desc":
+        query = query.order_by(MatchResult.similarity_score.desc())
+    elif sort == "score_asc":
+        query = query.order_by(MatchResult.similarity_score.asc())
+    else:
+        query = query.order_by(MatchResult.source_row_id)
+
+    total = query.count()
+    results = query.offset((page - 1) * page_size).limit(page_size).all()
+
+    items = []
+    for r in results:
+        src_row = db.query(VectorDataRow).get(r.source_row_id)
+        tgt_row = db.query(VectorDataRow).get(r.target_row_id)
+        if search:
+            if search.lower() not in (src_row.raw_text or "").lower() and \
+               search.lower() not in (tgt_row.raw_text or "").lower():
+                continue
+        items.append(MatchResultItem(
+            id=r.id,
+            source_row_id=r.source_row_id,
+            source_row_number=src_row.row_number if src_row else 0,
+            source_text=src_row.raw_text if src_row else "",
+            target_text=tgt_row.raw_text if tgt_row else "",
+            similarity_score=r.similarity_score,
+            rerank_score=r.rerank_score,
+            match_level=r.match_level or "",
+            candidate_scope=r.candidate_scope,
+            is_confirmed=r.is_confirmed,
+        ))
+
+    return MatchResultPage(items=items, total=total, page=page, page_size=page_size)
+
+
+# ─── Candidate details for a source row ────────────────────────────────────
+@app.get("/api/task/{task_id}/candidates/{source_row_id}", response_model=SourceWithCandidates)
+def get_candidates(task_id: int, source_row_id: int, db: Session = Depends(get_db)):
+    if not _get_alive_task(db, task_id):
+        raise HTTPException(404, "Task not found")
+    src_row = db.query(VectorDataRow).get(source_row_id)
+    if not src_row:
+        raise HTTPException(404, "Source row not found")
+
+    results = (
+        db.query(MatchResult)
+        .filter(MatchResult.task_id == task_id, MatchResult.source_row_id == source_row_id)
+        .order_by(MatchResult.rank)
+        .all()
+    )
+
+    candidates = []
+    for r in results:
+        tgt_row = db.query(VectorDataRow).get(r.target_row_id)
+        candidates.append(CandidateDetail(
+            rank=r.rank,
+            rerank_rank=r.rerank_rank,
+            target_row_id=r.target_row_id,
+            target_text=tgt_row.raw_text if tgt_row else "",
+            similarity_score=r.similarity_score,
+            rerank_score=r.rerank_score,
+            match_level=r.match_level or "",
+            dataset_role="target",
+            candidate_scope=r.candidate_scope,
+            data_row_id=tgt_row.id if tgt_row else 0,
+            is_confirmed=r.is_confirmed,
+        ))
+
+    return SourceWithCandidates(
+        source_row_id=src_row.id,
+        source_text=src_row.raw_text,
+        source_row_number=src_row.row_number,
+        dataset_role=src_row.dataset_role,
+        data_row_id=src_row.id,
+        candidates=candidates,
+    )
+
+
+# ─── Confirm match ─────────────────────────────────────────────────────────
+@app.post("/api/result/{result_id}/confirm")
+def confirm_match(result_id: int, db: Session = Depends(get_db)):
+    result = db.query(MatchResult).get(result_id)
+    if not result:
+        raise HTTPException(404, "Result not found")
+    result.is_confirmed = 1
+    db.commit()
+    return {"status": "ok"}
+
+
+@app.post("/api/result/{result_id}/ignore")
+def ignore_match(result_id: int, db: Session = Depends(get_db)):
+    result = db.query(MatchResult).get(result_id)
+    if not result:
+        raise HTTPException(404, "Result not found")
+    result.is_confirmed = -1
+    db.commit()
+    return {"status": "ok"}
+
+
+# ─── Settings (read/write .env) ────────────────────────────────────────────
+_backend_dir = os.path.dirname(os.path.abspath(__file__))
+_env_local = os.path.join(_backend_dir, ".env.local")
+ENV_PATH = _env_local if os.path.exists(_env_local) else os.path.join(_backend_dir, ".env")
+
+def _read_env() -> dict:
+    result = {}
+    if os.path.exists(ENV_PATH):
+        with open(ENV_PATH, "r", encoding="utf-8") as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#") and "=" in line:
+                    k, v = line.split("=", 1)
+                    result[k.strip()] = v.strip()
+    return result
+
+def _write_env(settings: dict):
+    with open(ENV_PATH, "w", encoding="utf-8") as f:
+        for k, v in settings.items():
+            f.write(f"{k}={v}\n")
+
+@app.get("/api/settings", response_model=SettingsResponse)
+def get_settings():
+    return SettingsResponse(settings=_read_env())
+
+@app.post("/api/settings")
+async def update_settings(items: List[SettingItem]):
+    current = _read_env()
+    for item in items:
+        current[item.key] = item.value
+    _write_env(current)
+    # 保存后自动重载环境变量，无需手动重启
+    from dotenv import load_dotenv
+    load_dotenv(ENV_PATH, override=True)
+    # 同步更新 embedding_service 模块中的配置常量
+    import services.embedding_service as es
+    es.SILICONFLOW_API_KEY = os.environ.get("SILICONFLOW_API_KEY", "")
+    es.EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-m3")
+    es.EMBEDDING_DIM = int(os.environ.get("EMBEDDING_DIM", "1024"))
+    es.RERANKER_MODEL = os.environ.get("RERANKER_MODEL", "Qwen/Qwen3-VL-Reranker-8B")
+    es.RERANKER_ENABLED = os.environ.get("RERANKER_ENABLED", "true").lower() == "true"
+    # 立即刷新健康缓存，前端下次请求即可拿到最新状态
+    await _do_health_check()
+    return {"status": "ok", "message": "已保存��配置已实时生效"}
+
+
+# ─── 健康检查（返回后端缓存，秒级响应）────────────────────────────────────────
+@app.get("/api/health")
+async def health_check(force: bool = False):
+    """返回缓存的健康状态，force=true 时立即刷新"""
+    if force or time.time() - _health_cache["updated_at"] > _HEALTH_TTL:
+        await _do_health_check()
+    return _health_cache["result"]
+
+
+# ─── Export results ────────────────────────────────────────────────────────
+@app.get("/api/task/{task_id}/export")
+def export_results(task_id: int, db: Session = Depends(get_db)):
+    import io
+    import openpyxl
+    from openpyxl.styles import Font, Alignment, PatternFill
+    from fastapi.responses import StreamingResponse
+
+    task = _get_alive_task(db, task_id)
+    if not task:
+        raise HTTPException(404, "Task not found")
+
+    results = (
+        db.query(MatchResult)
+        .filter(MatchResult.task_id == task_id)
+        .order_by(MatchResult.source_row_id, MatchResult.rank)
+        .all()
+    )
+
+    from openpyxl.styles import Font, PatternFill, Alignment
+
+    wb = openpyxl.Workbook()
+    ws = wb.active
+    ws.title = "匹配结果"
+
+    headers = ["源行号", "源数据内容", "候选排名", "目标候选内容", "相似度(%)", "精排分", "匹配等级", "候选来源"]
+    ws.append(headers)
+
+    # Header styling
+    header_font = Font(bold=True, color="FFFFFF")
+    header_fill = PatternFill(start_color="1F4E79", end_color="1F4E79", fill_type="solid")
+    for cell in ws[1]:
+        cell.font = header_font
+        cell.fill = header_fill
+        cell.alignment = Alignment(horizontal="center", vertical="center")
+
+    level_map = {"high": "高度匹配", "possible": "可能匹配", "low_confidence": "低置信", "no_match": "不匹配"}
+    scope_map = {"current_task_target": "目标候选集", "history": "历史数据", "standard": "标准库"}
+
+    for r in results:
+        src = db.query(VectorDataRow).get(r.source_row_id)
+        tgt = db.query(VectorDataRow).get(r.target_row_id)
+        ws.append([
+            src.row_number if src else "",
+            src.raw_text if src else "",
+            r.rank,
+            tgt.raw_text if tgt else "",
+            round(r.similarity_score * 100, 2),
+            round(r.rerank_score, 4) if r.rerank_score is not None else "",
+            level_map.get(r.match_level, r.match_level),
+            scope_map.get(r.candidate_scope, r.candidate_scope or ""),
+        ])
+
+    # Column widths
+    col_widths = [8, 40, 10, 40, 12, 12, 12, 14]
+    for i, w in enumerate(col_widths, 1):
+        ws.column_dimensions[chr(64 + i)].width = w
+
+    output = io.BytesIO()
+    wb.save(output)
+    output.seek(0)
+
+    return StreamingResponse(
+        output,
+        media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        headers={"Content-Disposition": f"attachment; filename=match_result_{task.task_code}.xlsx"},
+    )
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run("main:app", host="0.0.0.0", port=8000, reload=True)

hf-vector-match-api/models.py

@@ -0,0 +1,128 @@
+import datetime
+from sqlalchemy import Column, Integer, String, Float, DateTime, Text, LargeBinary, ForeignKey
+from sqlalchemy.orm import relationship
+from database import Base
+
+_TZ_BEIJING = datetime.timezone(datetime.timedelta(hours=8))
+
+def _now_beijing():
+    return datetime.datetime.now(_TZ_BEIJING).replace(tzinfo=None)
+
+
+class VectorMatchTask(Base):
+    __tablename__ = "vector_match_task"
+    __table_args__ = {"comment": "向量匹配任务表"}
+
+    id = Column(Integer, primary_key=True, autoincrement=True, comment="主键ID")
+    task_code = Column(String(30), unique=True, nullable=False, index=True, comment="任务编号，格式：YYYYMMDDHHMMSSmmm")
+    match_mode = Column(String(50), nullable=False, default="two_file", comment="匹配模式：two_file/history/standard")
+    candidate_scope = Column(String(50), nullable=False, default="current_task_target", comment="候选范围：current_task_target/history/standard")
+    source_dataset_id = Column(Integer, ForeignKey("vector_dataset.id"), nullable=True, comment="源数据集ID")
+    target_dataset_id = Column(Integer, ForeignKey("vector_dataset.id"), nullable=True, comment="目标候选集ID")
+    top_k = Column(Integer, default=10, comment="每条源数据保留的Top-K候选数")
+    rerank_top_k = Column(Integer, default=3, comment="Reranker重排序后保留的Top-K数")
+    min_threshold = Column(Float, default=0.70, comment="最低相似度阈值")
+    status = Column(String(20), default="pending", comment="任务状态：pending/running/completed/failed")
+    source_row_count = Column(Integer, default=0, comment="源数据行数")
+    target_row_count = Column(Integer, default=0, comment="目标候选行数")
+    high_match_count = Column(Integer, default=0, comment="高度匹配数量(score>=0.90)")
+    low_confidence_count = Column(Integer, default=0, comment="低置信数量(score<0.70)")
+    reused_vectors = Column(Integer, default=0, comment="通过text_hash复用的向量数")
+    new_vectors = Column(Integer, default=0, comment="新生成的向量数")
+    progress_parse_source = Column(Integer, default=0, comment="解析源数据集进度(0-100)")
+    progress_parse_target = Column(Integer, default=0, comment="解析目标候选集进度(0-100)")
+    progress_vectorize = Column(Integer, default=0, comment="向量化进度(0-100)")
+    progress_load_candidates = Column(Integer, default=0, comment="加载候选范围进度(0-100)")
+    progress_similarity = Column(Integer, default=0, comment="相似度计算进度(0-100)")
+    progress_rerank = Column(Integer, default=0, comment="Reranker重排序进度(0-100)")
+    progress_save_results = Column(Integer, default=0, comment="保存结果进度(0-100)")
+    created_time = Column(DateTime, default=_now_beijing, comment="创建时间")
+    updated_time = Column(DateTime, default=_now_beijing, onupdate=_now_beijing, comment="更新时间")
+    is_archived = Column(Integer, default=0, comment="是否归档：0=未归档,1=已归档")
+    is_delete = Column(Integer, default=0, comment="是否删除：0=未删除,1=已删除")
+
+    source_dataset = relationship("VectorDataset", foreign_keys=[source_dataset_id])
+    target_dataset = relationship("VectorDataset", foreign_keys=[target_dataset_id])
+    results = relationship("MatchResult", back_populates="task")
+
+
+class VectorDataset(Base):
+    __tablename__ = "vector_dataset"
+    __table_args__ = {"comment": "向量数据集表（上传或逻辑数据集）"}
+
+    id = Column(Integer, primary_key=True, autoincrement=True, comment="主键ID")
+    task_id = Column(Integer, ForeignKey("vector_match_task.id"), nullable=True, comment="所属任务ID")
+    name = Column(String(255), nullable=False, comment="数据集名称")
+    file_name = Column(String(255), nullable=True, comment="上传文件名")
+    sheet_name = Column(String(100), nullable=True, comment="Excel工作表名")
+    dataset_role = Column(String(20), nullable=False, comment="数据集角色：source(源)/target(目标候选)")
+    data_scope = Column(String(20), default="task", comment="数据范围：task/history/standard")
+    vector_fields = Column(Text, nullable=True, comment="参与向量化的字段列表(JSON)")
+    row_count = Column(Integer, default=0, comment="数据行数")
+    is_delete = Column(Integer, default=0, nullable=False, index=True, comment="软删除标记：0=有效，1=已删除")
+    created_time = Column(DateTime, default=_now_beijing, comment="创建时间")
+    updated_time = Column(DateTime, default=_now_beijing, onupdate=_now_beijing, comment="更新时间")
+
+    rows = relationship("VectorDataRow", back_populates="dataset")
+
+
+class VectorDataRow(Base):
+    __tablename__ = "vector_data_row"
+    __table_args__ = {"comment": "向量数据行表（单行物料/申报项等）"}
+
+    id = Column(Integer, primary_key=True, autoincrement=True, comment="主键ID")
+    dataset_id = Column(Integer, ForeignKey("vector_dataset.id"), nullable=False, index=True, comment="所属数据集ID")
+    task_id = Column(Integer, nullable=True, index=True, comment="所属任务ID")
+    dataset_role = Column(String(20), nullable=False, index=True, comment="数据集角色：source/target")
+    data_scope = Column(String(20), default="task", index=True, comment="数据范围：task/history/standard")
+    row_number = Column(Integer, nullable=False, comment="Excel中的行号")
+    raw_text = Column(Text, nullable=False, comment="拼接后的原始文本")
+    text_hash = Column(String(64), nullable=True, index=True, comment="文本SHA256哈希，用于向量复用")
+    field_values = Column(Text, nullable=True, comment="各字段值(JSON)")
+    is_delete = Column(Integer, default=0, nullable=False, index=True, comment="软删除标记：0=有效，1=已删除")
+    created_time = Column(DateTime, default=_now_beijing, comment="创建时间")
+    updated_time = Column(DateTime, default=_now_beijing, onupdate=_now_beijing, comment="更新时间")
+
+    dataset = relationship("VectorDataset", back_populates="rows")
+    embedding = relationship("VectorEmbedding", back_populates="data_row", uselist=False)
+
+
+class VectorEmbedding(Base):
+    __tablename__ = "vector_embedding"
+    __table_args__ = {"comment": "向量嵌入表（与数据行一对一）"}
+
+    id = Column(Integer, primary_key=True, autoincrement=True, comment="主键ID")
+    data_row_id = Column(Integer, ForeignKey("vector_data_row.id"), unique=True, nullable=False, index=True, comment="关联 vector_data_row.id")
+    text_hash = Column(String(64), nullable=False, index=True, comment="与行一致的文本哈希")
+    embedding = Column(LargeBinary(length=65536), nullable=False, comment="float32 数组二进制存储")
+    model_name = Column(String(100), nullable=True, comment="生成向量所用模型名")
+    dimension = Column(Integer, nullable=True, comment="向量维度")
+    is_delete = Column(Integer, default=0, nullable=False, index=True, comment="软删除标记：0=有效，1=已删除")
+    created_time = Column(DateTime, default=_now_beijing, comment="创建时间")
+    updated_time = Column(DateTime, default=_now_beijing, onupdate=_now_beijing, comment="更新时间")
+
+    data_row = relationship("VectorDataRow", back_populates="embedding")
+
+
+class MatchResult(Base):
+    __tablename__ = "match_result"
+    __table_args__ = {"comment": "匹配结果表（源行与候选的关联及得分）"}
+
+    id = Column(Integer, primary_key=True, autoincrement=True, comment="主键ID")
+    task_id = Column(Integer, ForeignKey("vector_match_task.id"), nullable=False, index=True, comment="所属任务ID")
+    source_row_id = Column(Integer, ForeignKey("vector_data_row.id"), nullable=False, comment="源数据行ID")
+    target_row_id = Column(Integer, ForeignKey("vector_data_row.id"), nullable=False, comment="目标候选行ID")
+    similarity_score = Column(Float, nullable=False, comment="余弦相似度分数(0-1)")
+    rerank_score = Column(Float, nullable=True, comment="Reranker精排分数，越高越相关")
+    rank = Column(Integer, nullable=False, comment="排名(1=最相似)")
+    rerank_rank = Column(Integer, nullable=True, comment="Reranker重排后的排名")
+    candidate_scope = Column(String(50), nullable=True, comment="候选来源范围")
+    match_level = Column(String(20), nullable=True, comment="匹配等级：high/possible/low_confidence/no_match")
+    is_confirmed = Column(Integer, default=0, comment="是否已人工确认：0=未确认,1=已确认,-1=已忽略")
+    is_delete = Column(Integer, default=0, nullable=False, index=True, comment="软删除标记：0=有效，1=已删除")
+    created_time = Column(DateTime, default=_now_beijing, comment="创建时间")
+    updated_time = Column(DateTime, default=_now_beijing, onupdate=_now_beijing, comment="更新时间")
+
+    task = relationship("VectorMatchTask", back_populates="results")
+    source_row = relationship("VectorDataRow", foreign_keys=[source_row_id])
+    target_row = relationship("VectorDataRow", foreign_keys=[target_row_id])

hf-vector-match-api/requirements.txt

@@ -0,0 +1,13 @@
+fastapi==0.115.0
+uvicorn==0.30.6
+sqlalchemy==2.0.35
+python-multipart==0.0.12
+openpyxl==3.1.5
+numpy==1.24.4
+pandas==2.0.3
+httpx==0.26.0
+pydantic==2.9.2
+psycopg2-binary==2.9.9
+pymysql==1.1.1
+cryptography==43.0.1
+python-dotenv==1.0.1

hf-vector-match-api/schemas.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+from pydantic import BaseModel
+from typing import Optional, List
+from datetime import datetime
+
+
+class DatasetInfo(BaseModel):
+    id: int
+    name: str
+    file_name: Optional[str] = None
+    sheet_name: Optional[str] = None
+    dataset_role: str
+    data_scope: str
+    vector_fields: Optional[str] = None
+    row_count: int = 0
+
+    class Config:
+        from_attributes = True
+
+
+class TaskCreate(BaseModel):
+    match_mode: str = "two_file"
+    top_k: int = 3
+    min_threshold: float = 0.70
+    candidate_scope: str = "current_task_target"
+
+
+class TaskProgress(BaseModel):
+    id: int
+    task_code: str
+    status: str
+    source_row_count: int
+    target_row_count: int
+    reused_vectors: int
+    new_vectors: int
+    progress_parse_source: int
+    progress_parse_target: int
+    progress_vectorize: int
+    progress_load_candidates: int
+    progress_similarity: int
+    progress_rerank: int = 0
+    progress_save_results: int
+
+    class Config:
+        from_attributes = True
+
+
+class TaskDetail(BaseModel):
+    id: int
+    task_code: str
+    match_mode: str
+    candidate_scope: str
+    top_k: int
+    min_threshold: float
+    status: str
+    source_row_count: int
+    target_row_count: int
+    high_match_count: int
+    low_confidence_count: int
+    reused_vectors: int
+    new_vectors: int
+    source_dataset: Optional[DatasetInfo] = None
+    target_dataset: Optional[DatasetInfo] = None
+    created_time: Optional[datetime] = None
+    updated_time: Optional[datetime] = None
+
+    class Config:
+        from_attributes = True
+
+
+class TaskListItem(BaseModel):
+    id: int
+    task_code: str
+    match_mode: str
+    candidate_scope: str
+    source_dataset_name: Optional[str] = None
+    target_dataset_name: Optional[str] = None
+    status: str
+    is_archived: int = 0
+    is_delete: int = 0
+    created_time: Optional[datetime] = None
+
+
+class MatchResultItem(BaseModel):
+    id: int
+    source_row_id: int
+    source_row_number: int
+    source_text: str
+    target_text: str
+    similarity_score: float
+    rerank_score: Optional[float] = None
+    match_level: str
+    candidate_scope: Optional[str] = None
+    is_confirmed: int = 0
+
+
+class MatchResultPage(BaseModel):
+    items: List[MatchResultItem]
+    total: int
+    page: int
+    page_size: int
+
+
+class CandidateDetail(BaseModel):
+    rank: int
+    rerank_rank: Optional[int] = None
+    target_row_id: int
+    target_text: str
+    similarity_score: float
+    rerank_score: Optional[float] = None
+    match_level: str
+    dataset_role: str
+    candidate_scope: Optional[str] = None
+    data_row_id: int
+    is_confirmed: int = 0
+
+
+class SourceWithCandidates(BaseModel):
+    source_row_id: int
+    source_text: str
+    source_row_number: int
+    dataset_role: str
+    data_row_id: int
+    candidates: List[CandidateDetail]
+
+
+class SheetInfo(BaseModel):
+    sheet_names: List[str]
+    columns: dict
+
+
+class UploadResponse(BaseModel):
+    dataset_id: int
+    file_name: str
+    sheet_names: List[str]
+    columns: dict
+    all_columns: dict = {}
+
+
+class SettingItem(BaseModel):
+    key: str
+    value: str
+
+
+class SettingsResponse(BaseModel):
+    settings: dict

hf-vector-match-api/services/embedding_service.py

@@ -0,0 +1,231 @@
+import os
+import json
+import hashlib
+import numpy as np
+from typing import List, Optional, Dict
+import httpx
+from dotenv import load_dotenv
+
+load_dotenv(os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".env"))
+
+EMBEDDING_API_URL = os.environ.get("EMBEDDING_API_URL", "https://api.siliconflow.cn/v1/embeddings")
+# EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "Qwen/Qwen3-VL-Embedding-8B")
+# EMBEDDING_DIM = int(os.environ.get("EMBEDDING_DIM", "4096"))
+EMBEDDING_MODEL = os.environ.get("EMBEDDING_MODEL", "BAAI/bge-m3")
+EMBEDDING_DIM = int(os.environ.get("EMBEDDING_DIM", "1024"))
+EMBEDDING_PROVIDER = os.environ.get("EMBEDDING_PROVIDER", "siliconflow")
+SILICONFLOW_API_KEY = os.environ.get("SILICONFLOW_API_KEY", "")
+
+RERANKER_MODEL = os.environ.get("RERANKER_MODEL", "Qwen/Qwen3-VL-Reranker-8B")
+RERANKER_API_URL = os.environ.get("RERANKER_API_URL", "https://api.siliconflow.cn/v1/rerank")
+RERANKER_ENABLED = os.environ.get("RERANKER_ENABLED", "true").lower() == "true"
+
+# Bypass system proxy for SiliconFlow API calls
+if "NO_PROXY" not in os.environ:
+    os.environ["NO_PROXY"] = "api.siliconflow.cn"
+elif "siliconflow" not in os.environ.get("NO_PROXY", ""):
+    os.environ["NO_PROXY"] = os.environ["NO_PROXY"] + ",api.siliconflow.cn"
+
+
+def _build_simple_embedding(text: str, dim: int = 768) -> np.ndarray:
+    """Fallback: deterministic pseudo-embedding based on character hashing.
+    Only for testing when no real embedding API is available."""
+    h = hashlib.sha512(text.encode("utf-8")).digest()
+    seed = int.from_bytes(h[:4], "big")
+    rng = np.random.RandomState(seed)
+    vec = rng.randn(dim).astype(np.float32)
+    norm = np.linalg.norm(vec)
+    if norm > 0:
+        vec = vec / norm
+    return vec
+
+
+async def get_embeddings_batch(texts: List[str], model: Optional[str] = None) -> List[np.ndarray]:
+    """Generate embeddings for a batch of texts."""
+    model = model or EMBEDDING_MODEL
+    provider = EMBEDDING_PROVIDER.lower()
+
+    if provider == "siliconflow":
+        return await _siliconflow_embeddings(texts, model)
+    elif provider == "ollama":
+        return await _ollama_embeddings(texts, model)
+    elif provider == "openai":
+        return await _openai_embeddings(texts, model)
+    else:
+        return [_build_simple_embedding(t, EMBEDDING_DIM) for t in texts]
+
+
+async def _siliconflow_embeddings(texts: List[str], model: str) -> List[np.ndarray]:
+    """Call SiliconFlow (硅基流动) embedding API.
+    API docs: https://docs.siliconflow.cn/api-reference/embeddings
+    Compatible with OpenAI format, supports batch input."""
+    api_url = EMBEDDING_API_URL or "https://api.siliconflow.cn/v1/embeddings"
+    api_key = SILICONFLOW_API_KEY
+    if not api_key:
+        print("[WARN] SILICONFLOW_API_KEY not set, falling back to pseudo embeddings")
+        return [_build_simple_embedding(t, EMBEDDING_DIM) for t in texts]
+
+    results = []
+    try:
+        async with httpx.AsyncClient(timeout=120.0, proxies={}) as client:
+            # SiliconFlow supports batch, but limit to 64 per request
+            for i in range(0, len(texts), 64):
+                batch = texts[i : i + 64]
+                resp = await client.post(
+                    api_url,
+                    headers={
+                        "Authorization": f"Bearer {api_key}",
+                        "Content-Type": "application/json",
+                    },
+                    json={"model": model, "input": batch, "encoding_format": "float"},
+                )
+                if resp.status_code == 200:
+                    data = resp.json()
+                    for item in sorted(data["data"], key=lambda x: x["index"]):
+                        vec = np.array(item["embedding"], dtype=np.float32)
+                        results.append(vec)
+                else:
+                    print(f"[ERROR] SiliconFlow API returned {resp.status_code}: {resp.text[:200]}")
+                    results.extend([_build_simple_embedding(t, EMBEDDING_DIM) for t in batch])
+    except Exception as e:
+        print(f"[ERROR] SiliconFlow API call failed: {e}")
+        results = [_build_simple_embedding(t, EMBEDDING_DIM) for t in texts]
+    return results
+
+
+async def _ollama_embeddings(texts: List[str], model: str) -> List[np.ndarray]:
+    """Call Ollama embedding API."""
+    results = []
+    try:
+        async with httpx.AsyncClient(timeout=120.0, proxies={}) as client:
+            for text in texts:
+                resp = await client.post(
+                    EMBEDDING_API_URL,
+                    json={"model": model, "input": text}
+                )
+                if resp.status_code == 200:
+                    data = resp.json()
+                    if "embeddings" in data:
+                        vec = np.array(data["embeddings"][0], dtype=np.float32)
+                    elif "embedding" in data:
+                        vec = np.array(data["embedding"], dtype=np.float32)
+                    else:
+                        vec = _build_simple_embedding(text, EMBEDDING_DIM)
+                    results.append(vec)
+                else:
+                    results.append(_build_simple_embedding(text, EMBEDDING_DIM))
+    except Exception:
+        results = [_build_simple_embedding(t, EMBEDDING_DIM) for t in texts]
+    return results
+
+
+async def _openai_embeddings(texts: List[str], model: str) -> List[np.ndarray]:
+    """Call OpenAI-compatible embedding API (e.g., vLLM)."""
+    api_url = os.environ.get("OPENAI_API_BASE", "http://localhost:8000") + "/v1/embeddings"
+    api_key = os.environ.get("OPENAI_API_KEY", "no-key")
+    results = []
+    try:
+        async with httpx.AsyncClient(timeout=120.0, proxies={}) as client:
+            resp = await client.post(
+                api_url,
+                headers={"Authorization": f"Bearer {api_key}"},
+                json={"model": model, "input": texts}
+            )
+            if resp.status_code == 200:
+                data = resp.json()
+                for item in data["data"]:
+                    vec = np.array(item["embedding"], dtype=np.float32)
+                    results.append(vec)
+            else:
+                results = [_build_simple_embedding(t, EMBEDDING_DIM) for t in texts]
+    except Exception:
+        results = [_build_simple_embedding(t, EMBEDDING_DIM) for t in texts]
+    return results
+
+
+async def rerank_candidates(
+    query: str,
+    documents: List[str],
+    top_n: Optional[int] = None,
+    model: Optional[str] = None,
+) -> List[Dict]:
+    """Call SiliconFlow Reranker API (Qwen/Qwen3-VL-Reranker-8B).
+    Returns list of {"index": int, "relevance_score": float} sorted by score desc."""
+    model = model or RERANKER_MODEL
+    api_key = SILICONFLOW_API_KEY
+
+    if not api_key or not RERANKER_ENABLED:
+        return [{"index": i, "relevance_score": 0.0} for i in range(len(documents))]
+
+    if not documents:
+        return []
+
+    top_n = top_n or len(documents)
+
+    try:
+        async with httpx.AsyncClient(timeout=120.0, proxies={}) as client:
+            resp = await client.post(
+                RERANKER_API_URL,
+                headers={
+                    "Authorization": f"Bearer {api_key}",
+                    "Content-Type": "application/json",
+                },
+                json={
+                    "model": model,
+                    "query": query,
+                    "documents": documents,
+                    "top_n": top_n,
+                    "return_documents": False,
+                },
+            )
+            if resp.status_code == 200:
+                data = resp.json()
+                results = data.get("results", [])
+                return sorted(results, key=lambda x: x["relevance_score"], reverse=True)
+            else:
+                print(f"[ERROR] Reranker API returned {resp.status_code}: {resp.text[:200]}")
+                return [{"index": i, "relevance_score": 0.0} for i in range(len(documents))]
+    except Exception as e:
+        print(f"[ERROR] Reranker API call failed: {e}")
+        return [{"index": i, "relevance_score": 0.0} for i in range(len(documents))]
+
+
+def cosine_similarity(a: np.ndarray, b: np.ndarray) -> float:
+    """Compute cosine similarity between two vectors."""
+    norm_a = np.linalg.norm(a)
+    norm_b = np.linalg.norm(b)
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+    return float(np.dot(a, b) / (norm_a * norm_b))
+
+
+def batch_cosine_similarity(source_vecs: np.ndarray, target_vecs: np.ndarray) -> np.ndarray:
+    """Compute pairwise cosine similarity matrix.
+    source_vecs: (M, D), target_vecs: (N, D)
+    Returns: (M, N) similarity matrix"""
+    source_norms = np.linalg.norm(source_vecs, axis=1, keepdims=True)
+    target_norms = np.linalg.norm(target_vecs, axis=1, keepdims=True)
+    source_norms = np.where(source_norms == 0, 1, source_norms)
+    target_norms = np.where(target_norms == 0, 1, target_norms)
+    source_normed = source_vecs / source_norms
+    target_normed = target_vecs / target_norms
+    return source_normed @ target_normed.T
+
+
+def embedding_to_bytes(vec: np.ndarray) -> bytes:
+    return vec.astype(np.float32).tobytes()
+
+
+def bytes_to_embedding(data: bytes) -> np.ndarray:
+    return np.frombuffer(data, dtype=np.float32)
+
+
+def get_match_level(score: float) -> str:
+    if score >= 0.90:
+        return "high"
+    elif score >= 0.80:
+        return "possible"
+    elif score >= 0.70:
+        return "low_confidence"
+    else:
+        return "no_match"

hf-vector-match-api/services/excel_service.py

@@ -0,0 +1,73 @@
+import os
+import hashlib
+import json
+import openpyxl
+from typing import List, Dict, Tuple
+
+
+UPLOAD_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "data", "uploads")
+os.makedirs(UPLOAD_DIR, exist_ok=True)
+
+
+def save_upload_file(file_bytes: bytes, filename: str) -> str:
+    filepath = os.path.join(UPLOAD_DIR, filename)
+    with open(filepath, "wb") as f:
+        f.write(file_bytes)
+    return filepath
+
+
+EXCLUDED_FIELDS = {"序号", "行号", "编号", "id", "ID", "Id", "no", "No", "NO", "行", "#"}
+
+def get_sheet_info(filepath: str) -> Dict:
+    wb = openpyxl.load_workbook(filepath, read_only=True)
+    result = {"sheet_names": wb.sheetnames, "columns": {}, "all_columns": {}}
+    for sheet_name in wb.sheetnames:
+        ws = wb[sheet_name]
+        headers = []
+        for row in ws.iter_rows(min_row=1, max_row=1, values_only=True):
+            headers = [str(c) if c else f"列{i+1}" for i, c in enumerate(row)]
+        result["all_columns"][sheet_name] = headers
+        result["columns"][sheet_name] = [
+            h for h in headers if h.strip() not in EXCLUDED_FIELDS
+        ]
+    wb.close()
+    return result
+
+
+def parse_excel_rows(
+    filepath: str,
+    sheet_name: str,
+    vector_fields: List[str],
+) -> List[Dict]:
+    wb = openpyxl.load_workbook(filepath, read_only=True)
+    ws = wb[sheet_name]
+    rows_data = []
+    headers = []
+    for row_idx, row in enumerate(ws.iter_rows(values_only=True)):
+        if row_idx == 0:
+            headers = [str(c) if c else f"列{i+1}" for i, c in enumerate(row)]
+            continue
+        row_dict = {}
+        for i, val in enumerate(row):
+            if i < len(headers):
+                row_dict[headers[i]] = str(val) if val is not None else ""
+
+        text_parts = []
+        for field in vector_fields:
+            if field in row_dict and row_dict[field]:
+                text_parts.append(row_dict[field])
+        raw_text = " ".join(text_parts)
+
+        if not raw_text.strip():
+            continue
+
+        text_hash = hashlib.sha256(raw_text.encode("utf-8")).hexdigest()
+
+        rows_data.append({
+            "row_number": row_idx + 1,
+            "raw_text": raw_text,
+            "text_hash": text_hash,
+            "field_values": json.dumps(row_dict, ensure_ascii=False),
+        })
+    wb.close()
+    return rows_data

hf-vector-match-api/services/match_service.py

@@ -0,0 +1,260 @@
+import asyncio
+import numpy as np
+from typing import List, Dict, Optional
+from sqlalchemy.orm import Session
+
+from models import (
+    VectorMatchTask, VectorDataset, VectorDataRow,
+    VectorEmbedding, MatchResult
+)
+from services.embedding_service import (
+    get_embeddings_batch, batch_cosine_similarity,
+    embedding_to_bytes, bytes_to_embedding, get_match_level,
+    rerank_candidates, RERANKER_ENABLED
+)
+
+
+BATCH_SIZE = 32
+
+
+def _safe_commit(db):
+    """提交事务，连接断开时自动回滚并重试"""
+    try:
+        db.commit()
+    except Exception:
+        db.rollback()
+        try:
+            db.commit()
+        except Exception:
+            db.rollback()
+
+
+async def run_match_task(task_id: int, db_factory):
+    """Main matching pipeline: parse → vectorize → match → save results."""
+    db: Session = db_factory()
+    try:
+        task = db.query(VectorMatchTask).get(task_id)
+        if not task:
+            return
+        task.status = "running"
+        _safe_commit(db)
+
+        # Step 1: Parse source
+        task.progress_parse_source = 100
+        _safe_commit(db)
+
+        # Step 2: Parse target
+        task.progress_parse_target = 100
+        _safe_commit(db)
+
+        # Step 3: Vectorize
+        source_rows = (
+            db.query(VectorDataRow)
+            .filter(VectorDataRow.dataset_id == task.source_dataset_id)
+            .all()
+        )
+        target_rows = (
+            db.query(VectorDataRow)
+            .filter(VectorDataRow.dataset_id == task.target_dataset_id)
+            .all()
+        )
+
+        task.source_row_count = len(source_rows)
+        task.target_row_count = len(target_rows)
+        _safe_commit(db)
+
+        all_rows = source_rows + target_rows
+        reused = 0
+        new_count = 0
+
+        for i in range(0, len(all_rows), BATCH_SIZE):
+            batch = all_rows[i : i + BATCH_SIZE]
+            texts_to_embed = []
+            rows_to_embed = []
+
+            for row in batch:
+                existing = (
+                    db.query(VectorEmbedding)
+                    .filter(VectorEmbedding.text_hash == row.text_hash)
+                    .first()
+                )
+                if existing and existing.data_row_id != row.id:
+                    new_emb = VectorEmbedding(
+                        data_row_id=row.id,
+                        text_hash=row.text_hash,
+                        embedding=existing.embedding,
+                        model_name=existing.model_name,
+                        dimension=existing.dimension,
+                    )
+                    db.add(new_emb)
+                    reused += 1
+                elif existing:
+                    reused += 1
+                else:
+                    texts_to_embed.append(row.raw_text)
+                    rows_to_embed.append(row)
+
+            if texts_to_embed:
+                embeddings = await get_embeddings_batch(texts_to_embed)
+                for row, vec in zip(rows_to_embed, embeddings):
+                    emb = VectorEmbedding(
+                        data_row_id=row.id,
+                        text_hash=row.text_hash,
+                        embedding=embedding_to_bytes(vec),
+                        model_name="default",
+                        dimension=len(vec),
+                    )
+                    db.add(emb)
+                    new_count += 1
+
+            progress = min(100, int((i + len(batch)) / max(len(all_rows), 1) * 100))
+            task.progress_vectorize = progress
+            task.reused_vectors = reused
+            task.new_vectors = new_count
+            _safe_commit(db)
+
+        task.progress_vectorize = 100
+        _safe_commit(db)
+
+        # Step 4: Load candidate range
+        task.progress_load_candidates = 100
+        _safe_commit(db)
+
+        # Step 5: Similarity calculation
+        source_embeddings = []
+        source_row_ids = []
+        for row in source_rows:
+            emb = db.query(VectorEmbedding).filter(VectorEmbedding.data_row_id == row.id).first()
+            if emb:
+                source_embeddings.append(bytes_to_embedding(emb.embedding))
+                source_row_ids.append(row.id)
+
+        target_embeddings = []
+        target_row_ids = []
+        for row in target_rows:
+            emb = db.query(VectorEmbedding).filter(VectorEmbedding.data_row_id == row.id).first()
+            if emb:
+                target_embeddings.append(bytes_to_embedding(emb.embedding))
+                target_row_ids.append(row.id)
+
+        if not source_embeddings or not target_embeddings:
+            task.status = "completed"
+            task.progress_similarity = 100
+            task.progress_save_results = 100
+            _safe_commit(db)
+            return
+
+        source_matrix = np.stack(source_embeddings)
+        target_matrix = np.stack(target_embeddings)
+
+        sim_matrix = batch_cosine_similarity(source_matrix, target_matrix)
+        task.progress_similarity = 100
+        _safe_commit(db)
+
+        # Step 6: Collect Top-K candidates per source row
+        # top_k 为初始候选数，rerank_top_k 为重排序后保留数
+        initial_k = task.top_k
+        initial_k = min(initial_k, len(target_row_ids))
+
+        # Build raw_text lookup for reranker
+        source_text_map = {}
+        target_text_map = {}
+        if RERANKER_ENABLED:
+            for row in source_rows:
+                source_text_map[row.id] = row.raw_text
+            for row in target_rows:
+                target_text_map[row.id] = row.raw_text
+
+        high_count = 0
+        low_count = 0
+        total_source = len(source_row_ids)
+
+        for idx, src_id in enumerate(source_row_ids):
+            scores = sim_matrix[idx]
+            top_indices = np.argsort(scores)[::-1][:initial_k]
+
+            candidates = []
+            for tgt_idx in top_indices:
+                candidates.append({
+                    "tgt_idx": tgt_idx,
+                    "tgt_row_id": target_row_ids[tgt_idx],
+                    "sim_score": float(scores[tgt_idx]),
+                    "rerank_score": None,
+                })
+
+            # Step 6.5: Rerank candidates
+            if RERANKER_ENABLED and candidates:
+                query_text = source_text_map.get(src_id, "")
+                doc_texts = [target_text_map.get(c["tgt_row_id"], "") for c in candidates]
+                try:
+                    rerank_top_k = task.rerank_top_k or task.top_k
+                    rerank_results = await rerank_candidates(
+                        query=query_text,
+                        documents=doc_texts,
+                        top_n=rerank_top_k,
+                    )
+                    # Map rerank scores back to candidates
+                    for rr in rerank_results:
+                        orig_idx = rr["index"]
+                        if orig_idx < len(candidates):
+                            candidates[orig_idx]["rerank_score"] = rr["relevance_score"]
+
+                    # Sort by rerank_score (desc), keep rerank_top_k
+                    candidates.sort(
+                        key=lambda c: c["rerank_score"] if c["rerank_score"] is not None else -1,
+                        reverse=True,
+                    )
+                    candidates = candidates[:rerank_top_k]
+                except Exception as e:
+                    print(f"[WARN] Rerank failed for source {src_id}: {e}")
+                    candidates = candidates[:task.top_k]
+
+                progress = min(100, int((idx + 1) / total_source * 100))
+                task.progress_rerank = progress
+                if idx % 20 == 0:
+                    _safe_commit(db)
+            else:
+                candidates = candidates[:task.top_k]
+
+            # Save results
+            for rank, c in enumerate(candidates):
+                level = get_match_level(c["sim_score"])
+                result = MatchResult(
+                    task_id=task.id,
+                    source_row_id=src_id,
+                    target_row_id=c["tgt_row_id"],
+                    similarity_score=c["sim_score"],
+                    rerank_score=c["rerank_score"],
+                    rank=rank + 1,
+                    rerank_rank=rank + 1 if c["rerank_score"] is not None else None,
+                    candidate_scope=task.candidate_scope,
+                    match_level=level,
+                )
+                db.add(result)
+
+                if rank == 0:
+                    if c["sim_score"] >= 0.90:
+                        high_count += 1
+                    elif c["sim_score"] < 0.70:
+                        low_count += 1
+
+            progress = min(100, int((idx + 1) / total_source * 100))
+            task.progress_save_results = progress
+            if idx % 50 == 0:
+                _safe_commit(db)
+
+        task.high_match_count = high_count
+        task.low_confidence_count = low_count
+        task.progress_rerank = 100
+        task.progress_save_results = 100
+        task.status = "completed"
+        _safe_commit(db)
+
+    except Exception as e:
+        task = db.query(VectorMatchTask).get(task_id)
+        if task:
+            task.status = "failed"
+            _safe_commit(db)
+        raise e
+    finally:
+        db.close()