Spaces:
Paused
Paused
Merge pull request #1 from will-rice/feat/papers-mcp
Browse filespapers-mcp: MCP servers over the lipsync and TTS paper corpora
- .dockerignore +8 -0
- .gitignore +6 -0
- .pre-commit-config.yaml +14 -0
- Dockerfile +21 -0
- README.md +39 -0
- pyproject.toml +40 -0
- src/papers_mcp/__init__.py +0 -0
- src/papers_mcp/corpus.py +95 -0
- src/papers_mcp/search.py +46 -0
- src/papers_mcp/server.py +181 -0
- tests/__init__.py +0 -0
- tests/conftest.py +24 -0
- tests/test_corpus.py +31 -0
- tests/test_search.py +29 -0
- tests/test_server.py +99 -0
- uv.lock +0 -0
.dockerignore
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.git
|
| 2 |
+
.venv
|
| 3 |
+
data/
|
| 4 |
+
tests/.cache
|
| 5 |
+
__pycache__
|
| 6 |
+
.pytest_cache
|
| 7 |
+
docs/
|
| 8 |
+
.superpowers/
|
.gitignore
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
__pycache__/
|
| 2 |
+
*.egg-info/
|
| 3 |
+
.venv/
|
| 4 |
+
.pytest_cache/
|
| 5 |
+
data/
|
| 6 |
+
tests/.cache/
|
.pre-commit-config.yaml
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
repos:
|
| 2 |
+
- repo: https://github.com/pre-commit/pre-commit-hooks
|
| 3 |
+
rev: v5.0.0
|
| 4 |
+
hooks:
|
| 5 |
+
- id: trailing-whitespace
|
| 6 |
+
- id: end-of-file-fixer
|
| 7 |
+
- id: check-yaml
|
| 8 |
+
- id: check-added-large-files
|
| 9 |
+
- repo: https://github.com/astral-sh/ruff-pre-commit
|
| 10 |
+
rev: v0.12.0
|
| 11 |
+
hooks:
|
| 12 |
+
- id: ruff
|
| 13 |
+
args: [--fix]
|
| 14 |
+
- id: ruff-format
|
Dockerfile
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM python:3.13-slim
|
| 2 |
+
|
| 3 |
+
RUN apt-get update && apt-get install -y --no-install-recommends git \
|
| 4 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 5 |
+
COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
|
| 6 |
+
|
| 7 |
+
RUN useradd -m -u 1000 user
|
| 8 |
+
USER user
|
| 9 |
+
ENV HOME=/home/user \
|
| 10 |
+
HF_HOME=/home/user/.cache/huggingface \
|
| 11 |
+
UV_PROJECT_ENVIRONMENT=/home/user/.venv
|
| 12 |
+
WORKDIR /app
|
| 13 |
+
|
| 14 |
+
COPY --chown=user pyproject.toml uv.lock ./
|
| 15 |
+
RUN uv sync --frozen --no-dev --no-install-project
|
| 16 |
+
COPY --chown=user . .
|
| 17 |
+
RUN uv sync --frozen --no-dev
|
| 18 |
+
|
| 19 |
+
EXPOSE 7860
|
| 20 |
+
CMD ["uv", "run", "--no-sync", "uvicorn", "--factory", "papers_mcp.server:create_app", \
|
| 21 |
+
"--host", "0.0.0.0", "--port", "7860"]
|
README.md
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
title: papers-mcp
|
| 3 |
+
emoji: 📚
|
| 4 |
+
colorFrom: blue
|
| 5 |
+
colorTo: purple
|
| 6 |
+
sdk: docker
|
| 7 |
+
app_port: 7860
|
| 8 |
+
pinned: false
|
| 9 |
+
---
|
| 10 |
+
|
| 11 |
+
# papers-mcp
|
| 12 |
+
|
| 13 |
+
MCP servers over the [lipsync-papers](https://github.com/will-rice/lipsync-papers)
|
| 14 |
+
and [tts-papers](https://github.com/will-rice/tts-papers) research corpora.
|
| 15 |
+
Public, read-only, streamable HTTP.
|
| 16 |
+
|
| 17 |
+
| Endpoint | Corpus |
|
| 18 |
+
|---|---|
|
| 19 |
+
| `https://wrice-papers-mcp.hf.space/lipsync/mcp` | lipsync-papers |
|
| 20 |
+
| `https://wrice-papers-mcp.hf.space/tts/mcp` | tts-papers |
|
| 21 |
+
|
| 22 |
+
Tools per server: `search_papers` (hybrid BM25 + embedding search),
|
| 23 |
+
`get_paper` (full markdown), `get_citations` (in-corpus citation graph),
|
| 24 |
+
`list_recent`. Corpora re-sync from GitHub every 6 hours.
|
| 25 |
+
|
| 26 |
+
## Connect
|
| 27 |
+
|
| 28 |
+
```bash
|
| 29 |
+
claude mcp add --transport http lipsync-papers https://wrice-papers-mcp.hf.space/lipsync/mcp
|
| 30 |
+
claude mcp add --transport http tts-papers https://wrice-papers-mcp.hf.space/tts/mcp
|
| 31 |
+
```
|
| 32 |
+
|
| 33 |
+
## Develop
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
uv sync
|
| 37 |
+
uv run pytest
|
| 38 |
+
uv run uvicorn --factory papers_mcp.server:create_app --port 7860
|
| 39 |
+
```
|
pyproject.toml
ADDED
|
@@ -0,0 +1,40 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[project]
|
| 2 |
+
name = "papers-mcp"
|
| 3 |
+
version = "0.1.0"
|
| 4 |
+
description = "MCP servers over the lipsync-papers and tts-papers research corpora"
|
| 5 |
+
requires-python = ">=3.11"
|
| 6 |
+
dependencies = [
|
| 7 |
+
"mcp>=1.27,<2",
|
| 8 |
+
"rank-bm25>=0.2.2",
|
| 9 |
+
"sentence-transformers>=3.0",
|
| 10 |
+
"starlette",
|
| 11 |
+
"uvicorn>=0.30",
|
| 12 |
+
]
|
| 13 |
+
|
| 14 |
+
[dependency-groups]
|
| 15 |
+
dev = [
|
| 16 |
+
"httpx>=0.27",
|
| 17 |
+
"pre-commit>=4.0",
|
| 18 |
+
"pytest>=8.0",
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
[tool.uv.sources]
|
| 22 |
+
torch = [{ index = "pytorch-cpu", marker = "sys_platform == 'linux'" }]
|
| 23 |
+
|
| 24 |
+
[[tool.uv.index]]
|
| 25 |
+
name = "pytorch-cpu"
|
| 26 |
+
url = "https://download.pytorch.org/whl/cpu"
|
| 27 |
+
explicit = true
|
| 28 |
+
|
| 29 |
+
[build-system]
|
| 30 |
+
requires = ["hatchling"]
|
| 31 |
+
build-backend = "hatchling.build"
|
| 32 |
+
|
| 33 |
+
[tool.hatch.build.targets.wheel]
|
| 34 |
+
packages = ["src/papers_mcp"]
|
| 35 |
+
|
| 36 |
+
[tool.ruff]
|
| 37 |
+
line-length = 100
|
| 38 |
+
|
| 39 |
+
[tool.pytest.ini_options]
|
| 40 |
+
testpaths = ["tests"]
|
src/papers_mcp/__init__.py
ADDED
|
File without changes
|
src/papers_mcp/corpus.py
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Load and sync a research-papers corpus (papers.csv + markdown) from GitHub."""
|
| 2 |
+
|
| 3 |
+
import csv
|
| 4 |
+
import logging
|
| 5 |
+
import re
|
| 6 |
+
import subprocess
|
| 7 |
+
from dataclasses import dataclass, field
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
# In-corpus citation links: `](../<year>/<id>.md)` or same-directory `](<id>.md)`.
|
| 11 |
+
CITATION_LINK_RE = re.compile(r"\]\((?:\.\./\d{4}/)?([^/()\s]+)\.md\)")
|
| 12 |
+
|
| 13 |
+
|
| 14 |
+
@dataclass
|
| 15 |
+
class Paper:
|
| 16 |
+
"""One paper's metadata, markdown location, and in-corpus citation edges."""
|
| 17 |
+
|
| 18 |
+
paper_id: str
|
| 19 |
+
title: str
|
| 20 |
+
authors: str
|
| 21 |
+
submitted: str
|
| 22 |
+
url: str
|
| 23 |
+
abstract: str
|
| 24 |
+
md_path: Path | None = None
|
| 25 |
+
markdown: str = ""
|
| 26 |
+
cites: list[str] = field(default_factory=list)
|
| 27 |
+
cited_by: list[str] = field(default_factory=list)
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@dataclass
|
| 31 |
+
class Corpus:
|
| 32 |
+
"""A cloned corpus repo and its loaded papers, keyed by paper id."""
|
| 33 |
+
|
| 34 |
+
name: str
|
| 35 |
+
repo_url: str
|
| 36 |
+
clone_dir: Path
|
| 37 |
+
papers: dict[str, Paper] = field(default_factory=dict)
|
| 38 |
+
|
| 39 |
+
def sync(self) -> None:
|
| 40 |
+
"""Clone the corpus repo if absent, otherwise hard-reset to the latest origin HEAD.
|
| 41 |
+
|
| 42 |
+
A reset-to-fetched-ref mirror (rather than `pull --ff-only`) is immune to the
|
| 43 |
+
upstream repo ever force-pushing, which would otherwise fail every refresh forever.
|
| 44 |
+
"""
|
| 45 |
+
if (self.clone_dir / ".git").exists():
|
| 46 |
+
self._git(["-C", str(self.clone_dir), "fetch", "--depth", "1", "origin", "HEAD"])
|
| 47 |
+
self._git(["-C", str(self.clone_dir), "reset", "--hard", "FETCH_HEAD"])
|
| 48 |
+
else:
|
| 49 |
+
self.clone_dir.parent.mkdir(parents=True, exist_ok=True)
|
| 50 |
+
self._git(["clone", "--depth", "1", self.repo_url, str(self.clone_dir)])
|
| 51 |
+
logging.info("synced %s corpus at %s", self.name, self.clone_dir)
|
| 52 |
+
|
| 53 |
+
def _git(self, args: list[str]) -> None:
|
| 54 |
+
"""Run a git command, surfacing its stderr on failure instead of swallowing it."""
|
| 55 |
+
try:
|
| 56 |
+
subprocess.run(["git", *args], check=True, capture_output=True, text=True)
|
| 57 |
+
except subprocess.CalledProcessError as exc:
|
| 58 |
+
raise RuntimeError(f"git {args} failed for {self.name}: {exc.stderr.strip()}") from exc
|
| 59 |
+
|
| 60 |
+
def load(self) -> None:
|
| 61 |
+
"""Load papers.csv, locate corpus markdown files, and build the citation graph."""
|
| 62 |
+
papers: dict[str, Paper] = {}
|
| 63 |
+
with (self.clone_dir / "papers.csv").open(newline="", encoding="utf-8") as f:
|
| 64 |
+
for row in csv.DictReader(f):
|
| 65 |
+
papers[row["arxiv_id"]] = Paper(
|
| 66 |
+
paper_id=row["arxiv_id"],
|
| 67 |
+
title=row["title"],
|
| 68 |
+
authors=row["authors"],
|
| 69 |
+
submitted=row["submitted"],
|
| 70 |
+
url=row["url"],
|
| 71 |
+
abstract=row["abstract"],
|
| 72 |
+
)
|
| 73 |
+
|
| 74 |
+
for md_path in sorted(self.clone_dir.glob("papers/*/*.md")):
|
| 75 |
+
paper = papers.get(md_path.stem) # skips per-year README.md files
|
| 76 |
+
if paper:
|
| 77 |
+
paper.md_path = md_path
|
| 78 |
+
|
| 79 |
+
for paper in papers.values():
|
| 80 |
+
if paper.md_path is None:
|
| 81 |
+
continue
|
| 82 |
+
paper.markdown = paper.md_path.read_text(encoding="utf-8")
|
| 83 |
+
for cited_id in CITATION_LINK_RE.findall(paper.markdown):
|
| 84 |
+
if (
|
| 85 |
+
cited_id != paper.paper_id
|
| 86 |
+
and cited_id in papers
|
| 87 |
+
and cited_id not in paper.cites
|
| 88 |
+
):
|
| 89 |
+
paper.cites.append(cited_id)
|
| 90 |
+
for paper in papers.values():
|
| 91 |
+
for cited_id in paper.cites:
|
| 92 |
+
papers[cited_id].cited_by.append(paper.paper_id)
|
| 93 |
+
|
| 94 |
+
self.papers = papers
|
| 95 |
+
logging.info("loaded %d papers for %s corpus", len(papers), self.name)
|
src/papers_mcp/search.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Hybrid BM25 + embedding search over a corpus's papers."""
|
| 2 |
+
|
| 3 |
+
import re
|
| 4 |
+
|
| 5 |
+
import torch
|
| 6 |
+
from rank_bm25 import BM25Okapi
|
| 7 |
+
from sentence_transformers import SentenceTransformer
|
| 8 |
+
|
| 9 |
+
from papers_mcp.corpus import Paper
|
| 10 |
+
|
| 11 |
+
EMBEDDING_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
|
| 12 |
+
RRF_K = 60
|
| 13 |
+
|
| 14 |
+
TOKEN_RE = re.compile(r"[a-z0-9]+")
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def tokenize(text: str) -> list[str]:
|
| 18 |
+
"""Lowercase alphanumeric tokens for BM25."""
|
| 19 |
+
return TOKEN_RE.findall(text.lower())
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
class SearchIndex:
|
| 23 |
+
"""Ranks papers by reciprocal-rank fusion of BM25 and embedding-cosine ranks."""
|
| 24 |
+
|
| 25 |
+
def __init__(self, papers: list[Paper], model: SentenceTransformer) -> None:
|
| 26 |
+
self.papers = papers
|
| 27 |
+
self.model = model
|
| 28 |
+
texts = [f"{p.title} {p.abstract}" for p in papers]
|
| 29 |
+
self.bm25 = BM25Okapi([tokenize(f"{text} {p.authors}") for text, p in zip(texts, papers)])
|
| 30 |
+
self.embeddings = model.encode(texts, convert_to_tensor=True, normalize_embeddings=True)
|
| 31 |
+
|
| 32 |
+
def search(self, query: str, limit: int) -> list[Paper]:
|
| 33 |
+
"""Return the top *limit* papers for *query* by fused BM25 + cosine rank."""
|
| 34 |
+
bm25_scores = torch.tensor(self.bm25.get_scores(tokenize(query)))
|
| 35 |
+
query_embedding = self.model.encode(
|
| 36 |
+
[query], convert_to_tensor=True, normalize_embeddings=True
|
| 37 |
+
)
|
| 38 |
+
cosine_scores = (self.embeddings @ query_embedding.T).flatten()
|
| 39 |
+
|
| 40 |
+
fused = torch.zeros(len(self.papers))
|
| 41 |
+
for scores in (bm25_scores, cosine_scores):
|
| 42 |
+
ranks = scores.argsort(descending=True).argsort()
|
| 43 |
+
fused += 1.0 / (RRF_K + 1 + ranks)
|
| 44 |
+
|
| 45 |
+
top = fused.argsort(descending=True)[:limit]
|
| 46 |
+
return [self.papers[i] for i in top.tolist()]
|
src/papers_mcp/server.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""MCP servers over research paper corpora — one streamable-HTTP endpoint per corpus."""
|
| 2 |
+
|
| 3 |
+
import contextlib
|
| 4 |
+
import logging
|
| 5 |
+
import threading
|
| 6 |
+
import time
|
| 7 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 8 |
+
from datetime import date, timedelta
|
| 9 |
+
from pathlib import Path
|
| 10 |
+
|
| 11 |
+
from mcp.server.fastmcp import FastMCP
|
| 12 |
+
from mcp.server.transport_security import TransportSecuritySettings
|
| 13 |
+
from sentence_transformers import SentenceTransformer
|
| 14 |
+
from starlette.applications import Starlette
|
| 15 |
+
from starlette.requests import Request
|
| 16 |
+
from starlette.responses import PlainTextResponse
|
| 17 |
+
from starlette.routing import Mount, Route
|
| 18 |
+
|
| 19 |
+
from papers_mcp.corpus import Corpus, Paper
|
| 20 |
+
from papers_mcp.search import EMBEDDING_MODEL, SearchIndex
|
| 21 |
+
|
| 22 |
+
CORPORA = {
|
| 23 |
+
"lipsync": "https://github.com/will-rice/lipsync-papers",
|
| 24 |
+
"tts": "https://github.com/will-rice/tts-papers",
|
| 25 |
+
}
|
| 26 |
+
DATA_DIR = Path("data")
|
| 27 |
+
REFRESH_INTERVAL_SECONDS = 6 * 60 * 60
|
| 28 |
+
MAX_SEARCH_LIMIT = 50
|
| 29 |
+
|
| 30 |
+
corpora: dict[str, Corpus] = {}
|
| 31 |
+
indexes: dict[str, SearchIndex] = {}
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def create_app() -> Starlette:
|
| 35 |
+
"""Build the Starlette app mounting one MCP server per corpus (uvicorn factory)."""
|
| 36 |
+
logging.basicConfig(level=logging.INFO)
|
| 37 |
+
servers = {name: make_server(name) for name in CORPORA}
|
| 38 |
+
|
| 39 |
+
@contextlib.asynccontextmanager
|
| 40 |
+
async def lifespan(app: Starlette):
|
| 41 |
+
model = SentenceTransformer(EMBEDDING_MODEL, device="cpu")
|
| 42 |
+
with ThreadPoolExecutor() as pool:
|
| 43 |
+
list(pool.map(lambda name: build_corpus(name, model), CORPORA))
|
| 44 |
+
threading.Thread(target=refresh_loop, args=(model,), daemon=True).start()
|
| 45 |
+
async with contextlib.AsyncExitStack() as stack:
|
| 46 |
+
for server in servers.values():
|
| 47 |
+
await stack.enter_async_context(server.session_manager.run())
|
| 48 |
+
yield
|
| 49 |
+
|
| 50 |
+
async def index_page(request: Request) -> PlainTextResponse:
|
| 51 |
+
lines = ["papers-mcp — MCP servers over research paper corpora", ""]
|
| 52 |
+
lines += [f" {CORPORA[name]} → /{name}/mcp" for name in CORPORA]
|
| 53 |
+
return PlainTextResponse("\n".join(lines))
|
| 54 |
+
|
| 55 |
+
routes: list[Mount | Route] = [Route("/", index_page)]
|
| 56 |
+
routes += [Mount(f"/{name}", server.streamable_http_app()) for name, server in servers.items()]
|
| 57 |
+
return Starlette(routes=routes, lifespan=lifespan)
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
def make_server(name: str) -> FastMCP:
|
| 61 |
+
"""Create the FastMCP server (and its four tools) for one corpus."""
|
| 62 |
+
import os
|
| 63 |
+
|
| 64 |
+
enable_dns_rebinding_protection = os.getenv("MCP_ENABLE_DNS_REBINDING_PROTECTION", "0") == "1"
|
| 65 |
+
|
| 66 |
+
mcp = FastMCP(
|
| 67 |
+
name=f"{name}-papers",
|
| 68 |
+
instructions=(
|
| 69 |
+
f"Query the {name}-papers research corpus ({CORPORA[name]}): "
|
| 70 |
+
"search titles/abstracts, read full papers as markdown, follow the "
|
| 71 |
+
"in-corpus citation graph, and list recent papers."
|
| 72 |
+
),
|
| 73 |
+
stateless_http=True,
|
| 74 |
+
json_response=True,
|
| 75 |
+
# This server is mounted into a Starlette app (not run standalone via
|
| 76 |
+
# mcp.run()), so FastMCP's own Host-header DNS-rebinding heuristic --
|
| 77 |
+
# which only ever allowlists 127.0.0.1/localhost -- would 421 every
|
| 78 |
+
# request once deployed under a real hostname. Access control belongs
|
| 79 |
+
# at the reverse-proxy/deployment layer instead.
|
| 80 |
+
transport_security=TransportSecuritySettings(
|
| 81 |
+
enable_dns_rebinding_protection=enable_dns_rebinding_protection
|
| 82 |
+
),
|
| 83 |
+
)
|
| 84 |
+
|
| 85 |
+
@mcp.tool()
|
| 86 |
+
def search_papers(query: str, limit: int = 10) -> str:
|
| 87 |
+
"""Hybrid keyword + semantic search over paper titles, abstracts, and authors.
|
| 88 |
+
|
| 89 |
+
Returns the top matches with paper id, title, authors, submission date,
|
| 90 |
+
and abstract. Use the paper id with get_paper or get_citations.
|
| 91 |
+
"""
|
| 92 |
+
if not query.strip():
|
| 93 |
+
raise ValueError("query must be non-empty")
|
| 94 |
+
if not 1 <= limit <= MAX_SEARCH_LIMIT:
|
| 95 |
+
raise ValueError(f"limit must be between 1 and {MAX_SEARCH_LIMIT}")
|
| 96 |
+
results = indexes[name].search(query, limit)
|
| 97 |
+
return "\n\n".join(format_paper(paper) for paper in results)
|
| 98 |
+
|
| 99 |
+
@mcp.tool()
|
| 100 |
+
def get_paper(paper_id: str) -> str:
|
| 101 |
+
"""Return the paper's full converted markdown (methods, figures, references)."""
|
| 102 |
+
paper = lookup(corpora[name].papers, name, paper_id)
|
| 103 |
+
if paper.md_path is None:
|
| 104 |
+
raise ValueError(
|
| 105 |
+
f"{paper_id} has no converted markdown; its metadata and abstract "
|
| 106 |
+
"are available via search_papers"
|
| 107 |
+
)
|
| 108 |
+
return paper.markdown
|
| 109 |
+
|
| 110 |
+
@mcp.tool()
|
| 111 |
+
def get_citations(paper_id: str) -> str:
|
| 112 |
+
"""List in-corpus papers this paper cites, and in-corpus papers citing it."""
|
| 113 |
+
# One snapshot: the refresh loop may swap corpora[name] between reads,
|
| 114 |
+
# so resolve the paper and its cited titles from the same generation.
|
| 115 |
+
papers = corpora[name].papers
|
| 116 |
+
paper = lookup(papers, name, paper_id)
|
| 117 |
+
|
| 118 |
+
def title_list(ids: list[str]) -> str:
|
| 119 |
+
if not ids:
|
| 120 |
+
return "(none in corpus)"
|
| 121 |
+
return "\n".join(f"- {pid}: {papers[pid].title}" for pid in ids)
|
| 122 |
+
|
| 123 |
+
return (
|
| 124 |
+
f"## Cites ({len(paper.cites)})\n{title_list(paper.cites)}\n\n"
|
| 125 |
+
f"## Cited by ({len(paper.cited_by)})\n{title_list(paper.cited_by)}"
|
| 126 |
+
)
|
| 127 |
+
|
| 128 |
+
@mcp.tool()
|
| 129 |
+
def list_recent(days: int = 30) -> str:
|
| 130 |
+
"""List papers submitted in the last N days, newest first."""
|
| 131 |
+
if days < 1:
|
| 132 |
+
raise ValueError("days must be at least 1")
|
| 133 |
+
cutoff = (date.today() - timedelta(days=days)).isoformat()
|
| 134 |
+
recent = sorted(
|
| 135 |
+
(p for p in corpora[name].papers.values() if p.submitted >= cutoff),
|
| 136 |
+
key=lambda p: p.submitted,
|
| 137 |
+
reverse=True,
|
| 138 |
+
)
|
| 139 |
+
if not recent:
|
| 140 |
+
return f"No papers submitted in the last {days} days."
|
| 141 |
+
return "\n\n".join(format_paper(paper) for paper in recent)
|
| 142 |
+
|
| 143 |
+
return mcp
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def build_corpus(name: str, model: SentenceTransformer) -> None:
|
| 147 |
+
"""Sync, load, and index one corpus, then swap it into the registry."""
|
| 148 |
+
corpus = Corpus(name=name, repo_url=CORPORA[name], clone_dir=DATA_DIR / f"{name}-papers")
|
| 149 |
+
corpus.sync()
|
| 150 |
+
corpus.load()
|
| 151 |
+
index = SearchIndex(list(corpus.papers.values()), model)
|
| 152 |
+
corpora[name] = corpus
|
| 153 |
+
indexes[name] = index
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
def refresh_loop(model: SentenceTransformer) -> None:
|
| 157 |
+
"""Re-sync and re-index every corpus on an interval; keep the old index on failure."""
|
| 158 |
+
while True:
|
| 159 |
+
time.sleep(REFRESH_INTERVAL_SECONDS)
|
| 160 |
+
for name in CORPORA:
|
| 161 |
+
try:
|
| 162 |
+
build_corpus(name, model)
|
| 163 |
+
except Exception:
|
| 164 |
+
logging.exception("refresh failed for %s; serving previous index", name)
|
| 165 |
+
|
| 166 |
+
|
| 167 |
+
def lookup(papers: dict[str, Paper], name: str, paper_id: str) -> Paper:
|
| 168 |
+
"""Return the paper for *paper_id* in *papers*, raising a concise error when unknown."""
|
| 169 |
+
paper = papers.get(paper_id)
|
| 170 |
+
if paper is None:
|
| 171 |
+
raise ValueError(f"paper id {paper_id!r} not found in the {name} corpus")
|
| 172 |
+
return paper
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def format_paper(paper: Paper) -> str:
|
| 176 |
+
"""One search/listing hit as compact markdown."""
|
| 177 |
+
return (
|
| 178 |
+
f"**{paper.title}** ({paper.paper_id}, {paper.submitted})\n"
|
| 179 |
+
f"{paper.authors}\n"
|
| 180 |
+
f"{paper.abstract}"
|
| 181 |
+
)
|
tests/__init__.py
ADDED
|
File without changes
|
tests/conftest.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Shared fixtures: a real lipsync-papers clone, cached across test runs."""
|
| 2 |
+
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
import pytest
|
| 6 |
+
|
| 7 |
+
from papers_mcp.corpus import Corpus
|
| 8 |
+
|
| 9 |
+
CACHE_DIR = Path(__file__).parent / ".cache"
|
| 10 |
+
LIPSYNC_REPO = "https://github.com/will-rice/lipsync-papers"
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@pytest.fixture(scope="session")
|
| 14 |
+
def lipsync_corpus() -> Corpus:
|
| 15 |
+
corpus = Corpus(name="lipsync", repo_url=LIPSYNC_REPO, clone_dir=CACHE_DIR / "lipsync-papers")
|
| 16 |
+
corpus.sync()
|
| 17 |
+
corpus.load()
|
| 18 |
+
return corpus
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@pytest.fixture(scope="module")
|
| 22 |
+
def monkeypatch_module():
|
| 23 |
+
with pytest.MonkeyPatch.context() as mp:
|
| 24 |
+
yield mp
|
tests/test_corpus.py
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Tests for corpus syncing and loading against the real lipsync-papers repo."""
|
| 2 |
+
|
| 3 |
+
from papers_mcp.corpus import Corpus
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def test_sync_clones_then_pulls(lipsync_corpus: Corpus) -> None:
|
| 7 |
+
assert (lipsync_corpus.clone_dir / "papers.csv").exists()
|
| 8 |
+
# Second sync takes the pull path and must not raise.
|
| 9 |
+
lipsync_corpus.sync()
|
| 10 |
+
assert (lipsync_corpus.clone_dir / "papers.csv").exists()
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def test_load_populates_papers(lipsync_corpus: Corpus) -> None:
|
| 14 |
+
assert len(lipsync_corpus.papers) > 500
|
| 15 |
+
latentsync = lipsync_corpus.papers["2412.09262"]
|
| 16 |
+
assert "LatentSync" in latentsync.title
|
| 17 |
+
assert latentsync.md_path is not None and latentsync.md_path.exists()
|
| 18 |
+
assert latentsync.submitted == "2024-12-12"
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def test_readme_files_are_not_papers(lipsync_corpus: Corpus) -> None:
|
| 22 |
+
assert "README" not in lipsync_corpus.papers
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
def test_citation_graph_is_consistent(lipsync_corpus: Corpus) -> None:
|
| 26 |
+
papers = lipsync_corpus.papers
|
| 27 |
+
assert any(p.cites for p in papers.values())
|
| 28 |
+
for paper in papers.values():
|
| 29 |
+
for cited_id in paper.cites:
|
| 30 |
+
assert cited_id in papers
|
| 31 |
+
assert paper.paper_id in papers[cited_id].cited_by
|
tests/test_search.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Search tests against the real lipsync corpus."""
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
from sentence_transformers import SentenceTransformer
|
| 5 |
+
|
| 6 |
+
from papers_mcp.corpus import Corpus
|
| 7 |
+
from papers_mcp.search import EMBEDDING_MODEL, SearchIndex
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
@pytest.fixture(scope="session")
|
| 11 |
+
def lipsync_index(lipsync_corpus: Corpus) -> SearchIndex:
|
| 12 |
+
model = SentenceTransformer(EMBEDDING_MODEL, device="cpu")
|
| 13 |
+
return SearchIndex(list(lipsync_corpus.papers.values()), model)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def test_keyword_query_finds_latentsync(lipsync_index: SearchIndex) -> None:
|
| 17 |
+
results = lipsync_index.search("latent diffusion lip sync SyncNet", limit=10)
|
| 18 |
+
assert "2412.09262" in [p.paper_id for p in results]
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def test_semantic_query_returns_relevant_papers(lipsync_index: SearchIndex) -> None:
|
| 22 |
+
results = lipsync_index.search("make the mouth match new audio in a video", limit=5)
|
| 23 |
+
assert len(results) == 5
|
| 24 |
+
haystack = " ".join(f"{p.title} {p.abstract}".lower() for p in results)
|
| 25 |
+
assert "lip" in haystack
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def test_limit_is_respected(lipsync_index: SearchIndex) -> None:
|
| 29 |
+
assert len(lipsync_index.search("talking head", limit=3)) == 3
|
tests/test_server.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""End-to-end MCP wire tests over the mounted lipsync endpoint."""
|
| 2 |
+
|
| 3 |
+
import pytest
|
| 4 |
+
from starlette.testclient import TestClient
|
| 5 |
+
|
| 6 |
+
from papers_mcp import server
|
| 7 |
+
from tests.conftest import CACHE_DIR, LIPSYNC_REPO
|
| 8 |
+
|
| 9 |
+
MCP_HEADERS = {
|
| 10 |
+
"Accept": "application/json, text/event-stream",
|
| 11 |
+
"Content-Type": "application/json",
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def rpc(method: str, params: dict) -> dict:
|
| 16 |
+
return {"jsonrpc": "2.0", "id": 1, "method": method, "params": params}
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
@pytest.fixture(scope="module")
|
| 20 |
+
def client(monkeypatch_module, lipsync_corpus) -> TestClient:
|
| 21 |
+
monkeypatch_module.setattr(server, "CORPORA", {"lipsync": LIPSYNC_REPO})
|
| 22 |
+
monkeypatch_module.setattr(server, "DATA_DIR", CACHE_DIR)
|
| 23 |
+
app = server.create_app()
|
| 24 |
+
with TestClient(app) as test_client: # runs lifespan: sync + load + index
|
| 25 |
+
yield test_client
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def test_tools_are_listed(client: TestClient) -> None:
|
| 29 |
+
resp = client.post("/lipsync/mcp", json=rpc("tools/list", {}), headers=MCP_HEADERS)
|
| 30 |
+
assert resp.status_code == 200
|
| 31 |
+
tools = {t["name"] for t in resp.json()["result"]["tools"]}
|
| 32 |
+
assert tools == {"search_papers", "get_paper", "get_citations", "list_recent"}
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def test_search_papers_tool(client: TestClient) -> None:
|
| 36 |
+
resp = client.post(
|
| 37 |
+
"/lipsync/mcp",
|
| 38 |
+
json=rpc(
|
| 39 |
+
"tools/call",
|
| 40 |
+
{"name": "search_papers", "arguments": {"query": "latent diffusion lip sync SyncNet"}},
|
| 41 |
+
),
|
| 42 |
+
headers=MCP_HEADERS,
|
| 43 |
+
)
|
| 44 |
+
text = resp.json()["result"]["content"][0]["text"]
|
| 45 |
+
assert "2412.09262" in text
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def test_get_paper_tool(client: TestClient) -> None:
|
| 49 |
+
resp = client.post(
|
| 50 |
+
"/lipsync/mcp",
|
| 51 |
+
json=rpc("tools/call", {"name": "get_paper", "arguments": {"paper_id": "2412.09262"}}),
|
| 52 |
+
headers=MCP_HEADERS,
|
| 53 |
+
)
|
| 54 |
+
text = resp.json()["result"]["content"][0]["text"]
|
| 55 |
+
assert "LatentSync" in text and len(text) > 5000
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def test_unknown_paper_id_is_an_error(client: TestClient) -> None:
|
| 59 |
+
resp = client.post(
|
| 60 |
+
"/lipsync/mcp",
|
| 61 |
+
json=rpc("tools/call", {"name": "get_paper", "arguments": {"paper_id": "0000.00000"}}),
|
| 62 |
+
headers=MCP_HEADERS,
|
| 63 |
+
)
|
| 64 |
+
assert resp.json()["result"]["isError"] is True
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def test_blank_query_is_an_error(client: TestClient) -> None:
|
| 68 |
+
resp = client.post(
|
| 69 |
+
"/lipsync/mcp",
|
| 70 |
+
json=rpc("tools/call", {"name": "search_papers", "arguments": {"query": " "}}),
|
| 71 |
+
headers=MCP_HEADERS,
|
| 72 |
+
)
|
| 73 |
+
assert resp.json()["result"]["isError"] is True
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def test_get_citations_tool(client: TestClient) -> None:
|
| 77 |
+
resp = client.post(
|
| 78 |
+
"/lipsync/mcp",
|
| 79 |
+
json=rpc("tools/call", {"name": "get_citations", "arguments": {"paper_id": "2412.09262"}}),
|
| 80 |
+
headers=MCP_HEADERS,
|
| 81 |
+
)
|
| 82 |
+
text = resp.json()["result"]["content"][0]["text"]
|
| 83 |
+
assert "Cites" in text and "Cited by" in text
|
| 84 |
+
|
| 85 |
+
|
| 86 |
+
def test_list_recent_tool(client: TestClient) -> None:
|
| 87 |
+
import re
|
| 88 |
+
from datetime import date, timedelta
|
| 89 |
+
|
| 90 |
+
resp = client.post(
|
| 91 |
+
"/lipsync/mcp",
|
| 92 |
+
json=rpc("tools/call", {"name": "list_recent", "arguments": {"days": 365}}),
|
| 93 |
+
headers=MCP_HEADERS,
|
| 94 |
+
)
|
| 95 |
+
text = resp.json()["result"]["content"][0]["text"]
|
| 96 |
+
dates = re.findall(r", (\d{4}-\d{2}-\d{2})\)", text)
|
| 97 |
+
assert len(dates) > 5
|
| 98 |
+
assert dates == sorted(dates, reverse=True) # newest first
|
| 99 |
+
assert min(dates) >= (date.today() - timedelta(days=365)).isoformat()
|
uv.lock
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|