Update

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v5.0.0
+    rev: v6.0.0
     hooks:
       - id: check-executables-have-shebangs
       - id: check-json
@@ -14,13 +14,13 @@ repos:
       - id: requirements-txt-fixer
       - id: trailing-whitespace
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.8.6
+    rev: v0.14.10
     hooks:
       - id: ruff
         args: ["--fix"]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.14.1
+    rev: v1.19.1
     hooks:
       - id: mypy
         args: ["--ignore-missing-imports"]

README.md

@@ -4,7 +4,7 @@ emoji: 📊
 colorFrom: pink
 colorTo: pink
 sdk: gradio
-sdk_version: 5.39.0
+sdk_version: 6.1.0
 app_file: app.py
 pinned: true
 license: mit

app.py

@@ -3,115 +3,143 @@
 import datetime
 
 import gradio as gr
-import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
+import polars as pl
 
-from papers import PaperList, get_df
+from search import search
+from table import df_orig
 
 DESCRIPTION = "# [Daily Papers](https://huggingface.co/papers)"
 
-FOOT_NOTE = """\
-Related useful Spaces:
-- [Semantic Scholar Paper Recommender](https://huggingface.co/spaces/librarian-bots/recommend_similar_papers) by [davanstrien](https://huggingface.co/davanstrien)
-- [ArXiv CS RAG](https://huggingface.co/spaces/bishmoy/Arxiv-CS-RAG) by [bishmoy](https://huggingface.co/bishmoy)
-- [Paper Q&A](https://huggingface.co/spaces/chansung/paper_qa) by [chansung](https://huggingface.co/chansung)
-- [dailypapershackernews](https://huggingface.co/spaces/akhaliq/dailypapershackernews) by [akhaliq](https://huggingface.co/akhaliq)
-"""
-
-
-paper_list = PaperList(get_df())
-
-
-def update_paper_list() -> None:
-    global paper_list  # noqa: PLW0603
-    paper_list = PaperList(get_df())
-
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(func=update_paper_list, trigger="cron", hour="*", timezone="UTC", misfire_grace_time=60)
-scheduler.start()
-
-
-def update_df() -> gr.Dataframe:
-    return gr.Dataframe(value=paper_list.df_prettified)
-
-
-def update_num_papers(df: pd.DataFrame) -> str:
-    return f"{len(df)} / {len(paper_list.df_raw)}"
-
-
-def search(
+df_main = df_orig.select(
+    "date_md",
+    "paper_page_md",
+    "title",
+    "github_md",
+    "upvotes",
+    "num_comments",
+    "arxiv_id",
+    "date",
+)
+df_main = df_main.rename(
+    {
+        "date_md": "Date",
+        "title": "Title",
+        "paper_page_md": "Paper page",
+        "github_md": "GitHub",
+        "upvotes": "👍",
+        "num_comments": "💬",
+    }
+)
+
+COLUMN_INFO = {
+    "Date": ("markdown", "10%"),
+    "Paper page": ("markdown", "10%"),
+    "Title": ("str", "55%"),
+    "GitHub": ("markdown", "5%"),
+    "👍": ("number", "5%"),
+    "💬": ("number", "5%"),
+}
+
+
+def update_num_papers(df: pl.DataFrame) -> str:
+    return f"{len(df)} / {len(df_main)}"
+
+
+def update_df(
+    search_query: str,
+    candidate_pool_size: int,
+    num_results: int,
     start_date: datetime.datetime,
     end_date: datetime.datetime,
-    search_title: str,
-    search_abstract: str,
-    max_num_to_retrieve: int,
-) -> pd.DataFrame:
-    return paper_list.search(start_date, end_date, search_title, search_abstract, max_num_to_retrieve)
+) -> dict:
+    if num_results > candidate_pool_size:
+        raise gr.Error("Number of results must be less than or equal to candidate pool size", print_exception=False)
+
+    df = df_main.clone()
+    if start_date:
+        df = df.filter(pl.col("date") >= start_date)
+    if end_date:
+        df = df.filter(pl.col("date") <= end_date)
+
+    if search_query:
+        results = search(search_query, candidate_pool_size, num_results)
+        if not results:
+            df = df.head(0)
+        else:
+            df = pl.DataFrame(results).join(df, on="arxiv_id", how="inner")
+            df = df.sort("ce_score", descending=True).drop("ce_score")
+
+    columns = list(COLUMN_INFO.keys())
+    df = df.select(columns)
+    return gr.Dataframe(
+        value=df,
+        datatype=[COLUMN_INFO[col][0] for col in columns],
+        column_widths=[COLUMN_INFO[col][1] for col in columns],
+    )
 
 
-with gr.Blocks(css_paths="style.css") as demo:
+with gr.Blocks() as demo:
     gr.Markdown(DESCRIPTION)
-    with gr.Group():
-        search_title = gr.Textbox(label="Search title")
-        with gr.Row():
-            with gr.Column(scale=4):
-                search_abstract = gr.Textbox(
-                    label="Search abstract",
-                    info="The result may not be accurate as the abstract does not contain all the information.",
-                )
-            with gr.Column(scale=1):
-                max_num_to_retrieve = gr.Slider(
-                    label="Max number to retrieve",
-                    info="This is used only for search on abstracts.",
-                    minimum=1,
-                    maximum=len(paper_list.df_raw),
-                    step=1,
-                    value=100,
-                )
+    search_query = gr.Textbox(label="Search", submit_btn=True, show_label=False, placeholder="Search...")
+    with gr.Accordion(label="Search Options", open=True) as advanced_search_options:
         with gr.Row():
-            start_date = gr.DateTime(label="Start date", type="datetime", value="2023-05-05", include_time=False)
-            end_date = gr.DateTime(label="End date", type="datetime", include_time=False)
+            candidate_pool_size = gr.Slider(label="Candidate Pool Size", minimum=1, maximum=600, step=1, value=200)
+            num_results = gr.Slider(label="Number of Results", minimum=1, maximum=400, step=1, value=100)
+    with gr.Row():
+        start_date = gr.DateTime(
+            label="Start Date", value=df_orig.select(pl.col("date").min()).item(), type="datetime", include_time=False
+        )
+        end_date = gr.DateTime(
+            label="End Date", value=df_orig.select(pl.col("date").max()).item(), type="datetime", include_time=False
+        )
+
+    num_papers = gr.Textbox(label="Number of papers", value=update_num_papers(df_orig), interactive=False)
 
-    num_papers = gr.Textbox(label="Number of papers", value=update_num_papers(paper_list.df_raw), interactive=False)
     df = gr.Dataframe(
-        value=paper_list.df_prettified,
-        datatype=paper_list.column_datatype,
-        type="pandas",
+        value=df_main,
+        datatype=list(COLUMN_INFO.values()),
+        type="polars",
+        row_count=(0, "dynamic"),
+        show_row_numbers=True,
         interactive=False,
         max_height=1000,
         elem_id="table",
-        column_widths=["10%", "10%", "60%", "10%", "5%", "5%"],
-        wrap=True,
+        column_widths=[COLUMN_INFO[col][1] for col in COLUMN_INFO],
     )
 
-    gr.Markdown(FOOT_NOTE)
-
+    inputs = [
+        search_query,
+        candidate_pool_size,
+        num_results,
+        start_date,
+        end_date,
+    ]
     gr.on(
-        triggers=[start_date.change, end_date.change, search_title.submit, search_abstract.submit],
-        fn=search,
-        inputs=[start_date, end_date, search_title, search_abstract, max_num_to_retrieve],
+        triggers=[search_query.submit, start_date.change, end_date.change],
+        fn=update_df,
+        inputs=inputs,
         outputs=df,
-        api_name=False,
+        api_visibility="private",
     ).then(
         fn=update_num_papers,
         inputs=df,
         outputs=num_papers,
         queue=False,
-        api_name=False,
+        api_visibility="private",
     )
     demo.load(
         fn=update_df,
+        inputs=inputs,
         outputs=df,
-        queue=False,
-        api_name=False,
+        api_visibility="private",
     ).then(
         fn=update_num_papers,
         inputs=df,
         outputs=num_papers,
         queue=False,
-        api_name=False,
+        api_visibility="private",
     )
 
+
 if __name__ == "__main__":
-    demo.queue(api_open=False).launch(show_api=False)
+    demo.launch(css_paths="style.css", footer_links=["gradio", "settings"])

papers.py

@@ -1,153 +0,0 @@
-import datetime
-import operator
-
-import datasets
-import pandas as pd
-import tqdm.auto
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import HfApi
-from ragatouille import RAGPretrainedModel
-
-api = HfApi()
-
-INDEX_REPO_ID = "hysts-bot-data/daily-papers-abstract-index"
-INDEX_DIR_PATH = ".ragatouille/colbert/indexes/daily-papers-abstract-index/"
-api.snapshot_download(
-    repo_id=INDEX_REPO_ID,
-    repo_type="dataset",
-    local_dir=INDEX_DIR_PATH,
-)
-abstract_retriever = RAGPretrainedModel.from_index(INDEX_DIR_PATH)
-# Run once to initialize the retriever
-abstract_retriever.search("LLM")
-
-
-def update_abstract_index() -> None:
-    global abstract_retriever  # noqa: PLW0603
-
-    api.snapshot_download(
-        repo_id=INDEX_REPO_ID,
-        repo_type="dataset",
-        local_dir=INDEX_DIR_PATH,
-    )
-    abstract_retriever = RAGPretrainedModel.from_index(INDEX_DIR_PATH)
-    abstract_retriever.search("LLM")
-
-
-scheduler = BackgroundScheduler()
-scheduler.add_job(func=update_abstract_index, trigger="cron", hour="*", timezone="UTC", misfire_grace_time=3 * 60)
-scheduler.start()
-
-
-def get_df() -> pd.DataFrame:
-    df = (
-        datasets.load_dataset("hysts-bot-data/daily-papers", split="train")
-        .to_pandas()
-        .merge(
-            datasets.load_dataset("hysts-bot-data/daily-papers-stats", split="train").to_pandas(),
-            on="arxiv_id",
-        )
-    )
-    df = df[::-1].reset_index(drop=True)
-    df["date"] = df["date"].dt.strftime("%Y-%m-%d")
-    df = df.drop(columns=["authors", "abstract"])
-
-    paper_info = []
-    for _, row in tqdm.auto.tqdm(df.iterrows(), total=len(df)):
-        info = row.copy()
-        info["paper_page"] = f"https://huggingface.co/papers/{row.arxiv_id}"
-        paper_info.append(info)
-    return pd.DataFrame(paper_info)
-
-
-class Prettifier:
-    @staticmethod
-    def get_github_link(link: str) -> str:
-        if not link:
-            return ""
-        return Prettifier.create_link("github", link)
-
-    @staticmethod
-    def create_link(text: str, url: str) -> str:
-        return f'<a href="{url}" target="_blank">{text}</a>'
-
-    @staticmethod
-    def to_div(text: str | None, category_name: str) -> str:
-        if text is None:
-            text = ""
-        class_name = f"{category_name}-{text.lower()}"
-        return f'<div class="{class_name}">{text}</div>'
-
-    def __call__(self, df: pd.DataFrame) -> pd.DataFrame:
-        new_rows = []
-        for _, row in df.iterrows():
-            new_row = {
-                "date": Prettifier.create_link(row.date, f"https://huggingface.co/papers?date={row.date}"),
-                "paper_page": Prettifier.create_link(row.arxiv_id, row.paper_page),
-                "title": row["title"],
-                "github": self.get_github_link(row.github),
-                "👍": row["upvotes"],
-                "💬": row["num_comments"],
-            }
-            new_rows.append(new_row)
-        return pd.DataFrame(new_rows)
-
-
-class PaperList:
-    COLUMN_INFO = (
-        ("date", "markdown"),
-        ("paper_page", "markdown"),
-        ("title", "str"),
-        ("github", "markdown"),
-        ("👍", "number"),
-        ("💬", "number"),
-    )
-
-    def __init__(self, df: pd.DataFrame) -> None:
-        self.df_raw = df
-        self._prettifier = Prettifier()
-        self.df_prettified = self._prettifier(df).loc[:, self.column_names]
-
-    @property
-    def column_names(self) -> list[str]:
-        return list(map(operator.itemgetter(0), self.COLUMN_INFO))
-
-    @property
-    def column_datatype(self) -> list[str]:
-        return list(map(operator.itemgetter(1), self.COLUMN_INFO))
-
-    def search(
-        self,
-        start_date: datetime.datetime,
-        end_date: datetime.datetime,
-        title_search_query: str,
-        abstract_search_query: str,
-        max_num_to_retrieve: int,
-    ) -> pd.DataFrame:
-        df = self.df_raw.copy()
-        df["date"] = pd.to_datetime(df["date"])
-
-        # Filter by date
-        df = df[(df["date"] >= start_date) & (df["date"] <= end_date)]
-        df["date"] = df["date"].dt.strftime("%Y-%m-%d")
-
-        # Filter by title
-        df = df[df["title"].str.contains(title_search_query, case=False)]
-
-        # Filter by abstract
-        if abstract_search_query:
-            results = abstract_retriever.search(abstract_search_query, k=max_num_to_retrieve)
-            remaining_ids = set(df["arxiv_id"])
-            found_id_set = set()
-            found_ids = []
-            for x in results:
-                arxiv_id = x["document_id"]
-                if arxiv_id not in remaining_ids:
-                    continue
-                if arxiv_id in found_id_set:
-                    continue
-                found_id_set.add(arxiv_id)
-                found_ids.append(arxiv_id)
-            df = df[df["arxiv_id"].isin(found_ids)].set_index("arxiv_id").reindex(index=found_ids).reset_index()
-
-        return self._prettifier(df).loc[:, self.column_names]

pyproject.toml

@@ -5,12 +5,13 @@ description = ""
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    "apscheduler>=3.11.0",
-    "datasets>=4.0.0",
-    "gradio>=5.39.0",
-    "hf-transfer>=0.1.9",
-    "ragatouille>=0.0.8.post4",
-    "setuptools>=75.6.0",
+    "datasets>=4.4.1",
+    "faiss-cpu>=1.13.1",
+    "gradio>=6.1.0",
+    "polars>=1.36.1",
+    "sentence-transformers>=5.2.0",
+    "spaces>=0.44.0",
+    "torch==2.8.0",
 ]
 
 [tool.ruff]
@@ -35,7 +36,6 @@ ignore = [
     "EM101",   # raw-string-in-exception
     "FBT001",  # boolean-type-hint-positional-argument
     "FBT002",  # boolean-default-value-positional-argument
-    "PD901",   # pandas-df-variable-name
     "PGH003",  # blanket-type-ignore
     "PLR0913", # too-many-arguments
     "PLR0915", # too-many-statements
@@ -53,3 +53,9 @@ convention = "google"
 
 [tool.ruff.format]
 docstring-code-format = true
+
+[dependency-groups]
+dev = [
+    "pre-commit>=4.5.1",
+    "ruff>=0.14.10",
+]

requirements.txt

@@ -1,141 +1,93 @@
 # This file was autogenerated by uv via the following command:
 #    uv pip compile pyproject.toml -o requirements.txt
-aiofiles==23.2.1
+aiofiles==24.1.0
     # via gradio
-aiohappyeyeballs==2.4.4
+aiohappyeyeballs==2.6.1
     # via aiohttp
-aiohttp==3.11.11
-    # via
-    #   fsspec
-    #   langchain
-    #   llama-index-core
-aiosignal==1.3.2
+aiohttp==3.13.2
+    # via fsspec
+aiosignal==1.4.0
     # via aiohttp
+annotated-doc==0.0.4
+    # via fastapi
 annotated-types==0.7.0
     # via pydantic
-anyio==4.8.0
+anyio==4.12.0
     # via
     #   gradio
     #   httpx
-    #   openai
     #   starlette
-apscheduler==3.11.0
-    # via daily-papers (pyproject.toml)
-async-timeout==4.0.3
-    # via
-    #   aiohttp
-    #   langchain
-attrs==24.3.0
+async-timeout==5.0.1
     # via aiohttp
-beautifulsoup4==4.12.3
-    # via llama-index-readers-file
-bitarray==3.0.0
-    # via colbert-ai
-blinker==1.9.0
-    # via flask
-brotli==1.1.0
+attrs==25.4.0
+    # via aiohttp
+brotli==1.2.0
     # via gradio
-catalogue==2.0.10
-    # via srsly
-certifi==2024.12.14
+certifi==2025.11.12
     # via
     #   httpcore
     #   httpx
-    #   llama-cloud
     #   requests
-charset-normalizer==3.4.1
+charset-normalizer==3.4.4
     # via requests
-click==8.1.8
+click==8.3.1
     # via
-    #   flask
-    #   llama-parse
-    #   nltk
     #   typer
     #   uvicorn
-colbert-ai==0.2.19
-    # via ragatouille
-dataclasses-json==0.6.7
-    # via llama-index-core
-datasets==4.0.0
-    # via
-    #   daily-papers (pyproject.toml)
-    #   colbert-ai
-deprecated==1.2.15
-    # via llama-index-core
-dill==0.3.8
+datasets==4.4.1
+    # via daily-papers (pyproject.toml)
+dill==0.4.0
     # via
     #   datasets
     #   multiprocess
-dirtyjson==1.0.8
-    # via llama-index-core
-distro==1.9.0
-    # via openai
-exceptiongroup==1.2.2
+exceptiongroup==1.3.1
     # via anyio
-faiss-cpu==1.9.0.post1
-    # via ragatouille
-fast-pytorch-kmeans==0.2.0.1
-    # via ragatouille
-fastapi==0.115.6
+faiss-cpu==1.13.1
+    # via daily-papers (pyproject.toml)
+fastapi==0.125.0
     # via gradio
-ffmpy==0.5.0
+ffmpy==1.0.0
     # via gradio
-filelock==3.16.1
+filelock==3.20.1
     # via
     #   datasets
     #   huggingface-hub
     #   torch
     #   transformers
-    #   triton
-filetype==1.2.0
-    # via llama-index-core
-flask==3.1.0
-    # via colbert-ai
-frozenlist==1.5.0
+frozenlist==1.8.0
     # via
     #   aiohttp
     #   aiosignal
-fsspec==2024.9.0
+fsspec==2025.10.0
     # via
     #   datasets
     #   gradio-client
     #   huggingface-hub
-    #   llama-index-core
     #   torch
-git-python==1.0.3
-    # via colbert-ai
-gitdb==4.0.12
-    # via gitpython
-gitpython==3.1.44
-    # via git-python
-gradio==5.39.0
-    # via daily-papers (pyproject.toml)
-gradio-client==1.11.0
+gradio==6.1.0
+    # via
+    #   daily-papers (pyproject.toml)
+    #   spaces
+gradio-client==2.0.1
     # via gradio
-greenlet==3.1.1
-    # via sqlalchemy
 groovy==0.1.2
     # via gradio
-h11==0.14.0
+h11==0.16.0
     # via
     #   httpcore
     #   uvicorn
-hf-transfer==0.1.9
-    # via daily-papers (pyproject.toml)
-hf-xet==1.1.5
+hf-xet==1.2.0
     # via huggingface-hub
-httpcore==1.0.7
+httpcore==1.0.9
     # via httpx
 httpx==0.28.1
     # via
+    #   datasets
     #   gradio
     #   gradio-client
-    #   langsmith
-    #   llama-cloud
-    #   llama-index-core
-    #   openai
     #   safehttpx
-huggingface-hub==0.34.3
+    #   spaces
+huggingface-hub==0.36.0
     # via
     #   datasets
     #   gradio
@@ -143,401 +95,222 @@ huggingface-hub==0.34.3
     #   sentence-transformers
     #   tokenizers
     #   transformers
-idna==3.10
+idna==3.11
     # via
     #   anyio
     #   httpx
     #   requests
     #   yarl
-itsdangerous==2.2.0
-    # via flask
-jinja2==3.1.5
+jinja2==3.1.6
     # via
-    #   flask
     #   gradio
     #   torch
-jiter==0.8.2
-    # via openai
-joblib==1.4.2
-    # via
-    #   nltk
-    #   scikit-learn
-jsonpatch==1.33
-    # via langchain-core
-jsonpointer==3.0.0
-    # via jsonpatch
-langchain==0.3.14
-    # via ragatouille
-langchain-core==0.3.29
-    # via
-    #   langchain
-    #   langchain-text-splitters
-    #   ragatouille
-langchain-text-splitters==0.3.5
-    # via langchain
-langsmith==0.2.10
-    # via
-    #   langchain
-    #   langchain-core
-llama-cloud==0.1.8
-    # via llama-index-indices-managed-llama-cloud
-llama-index==0.12.10
-    # via ragatouille
-llama-index-agent-openai==0.4.1
-    # via
-    #   llama-index
-    #   llama-index-program-openai
-llama-index-cli==0.4.0
-    # via llama-index
-llama-index-core==0.12.10.post1
-    # via
-    #   llama-index
-    #   llama-index-agent-openai
-    #   llama-index-cli
-    #   llama-index-embeddings-openai
-    #   llama-index-indices-managed-llama-cloud
-    #   llama-index-llms-openai
-    #   llama-index-multi-modal-llms-openai
-    #   llama-index-program-openai
-    #   llama-index-question-gen-openai
-    #   llama-index-readers-file
-    #   llama-index-readers-llama-parse
-    #   llama-parse
-llama-index-embeddings-openai==0.3.1
-    # via
-    #   llama-index
-    #   llama-index-cli
-llama-index-indices-managed-llama-cloud==0.6.3
-    # via llama-index
-llama-index-llms-openai==0.3.13
-    # via
-    #   llama-index
-    #   llama-index-agent-openai
-    #   llama-index-cli
-    #   llama-index-multi-modal-llms-openai
-    #   llama-index-program-openai
-    #   llama-index-question-gen-openai
-llama-index-multi-modal-llms-openai==0.4.2
-    # via llama-index
-llama-index-program-openai==0.3.1
-    # via
-    #   llama-index
-    #   llama-index-question-gen-openai
-llama-index-question-gen-openai==0.3.0
-    # via llama-index
-llama-index-readers-file==0.4.3
-    # via llama-index
-llama-index-readers-llama-parse==0.4.0
-    # via llama-index
-llama-parse==0.5.19
-    # via llama-index-readers-llama-parse
-markdown-it-py==3.0.0
+joblib==1.5.3
+    # via scikit-learn
+markdown-it-py==4.0.0
     # via rich
-markupsafe==2.1.5
+markupsafe==3.0.3
     # via
     #   gradio
     #   jinja2
-    #   werkzeug
-marshmallow==3.25.0
-    # via dataclasses-json
 mdurl==0.1.2
     # via markdown-it-py
 mpmath==1.3.0
     # via sympy
-multidict==6.1.0
+multidict==6.7.0
     # via
     #   aiohttp
     #   yarl
-multiprocess==0.70.16
+multiprocess==0.70.18
     # via datasets
-mypy-extensions==1.0.0
-    # via typing-inspect
-nest-asyncio==1.6.0
-    # via llama-index-core
 networkx==3.4.2
-    # via
-    #   llama-index-core
-    #   torch
-ninja==1.11.1.3
-    # via colbert-ai
-nltk==3.9.1
-    # via
-    #   llama-index
-    #   llama-index-core
-numpy==1.26.4
+    # via torch
+numpy==2.2.6
     # via
     #   datasets
     #   faiss-cpu
-    #   fast-pytorch-kmeans
     #   gradio
-    #   langchain
-    #   llama-index-core
-    #   onnx
     #   pandas
     #   scikit-learn
     #   scipy
-    #   sentence-transformers
     #   transformers
-    #   voyager
-nvidia-cublas-cu12==12.4.5.8
+nvidia-cublas-cu12==12.8.4.1
     # via
     #   nvidia-cudnn-cu12
     #   nvidia-cusolver-cu12
     #   torch
-nvidia-cuda-cupti-cu12==12.4.127
+nvidia-cuda-cupti-cu12==12.8.90
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.8.93
     # via torch
-nvidia-cuda-nvrtc-cu12==12.4.127
+nvidia-cuda-runtime-cu12==12.8.90
     # via torch
-nvidia-cuda-runtime-cu12==12.4.127
+nvidia-cudnn-cu12==9.10.2.21
     # via torch
-nvidia-cudnn-cu12==9.1.0.70
+nvidia-cufft-cu12==11.3.3.83
     # via torch
-nvidia-cufft-cu12==11.2.1.3
+nvidia-cufile-cu12==1.13.1.3
     # via torch
-nvidia-curand-cu12==10.3.5.147
+nvidia-curand-cu12==10.3.9.90
     # via torch
-nvidia-cusolver-cu12==11.6.1.9
+nvidia-cusolver-cu12==11.7.3.90
     # via torch
-nvidia-cusparse-cu12==12.3.1.170
+nvidia-cusparse-cu12==12.5.8.93
     # via
     #   nvidia-cusolver-cu12
     #   torch
-nvidia-ml-py==12.560.30
-    # via pynvml
-nvidia-nccl-cu12==2.21.5
+nvidia-cusparselt-cu12==0.7.1
     # via torch
-nvidia-nvjitlink-cu12==12.4.127
+nvidia-nccl-cu12==2.27.3
+    # via torch
+nvidia-nvjitlink-cu12==12.8.93
     # via
+    #   nvidia-cufft-cu12
     #   nvidia-cusolver-cu12
     #   nvidia-cusparse-cu12
     #   torch
-nvidia-nvtx-cu12==12.4.127
+nvidia-nvtx-cu12==12.8.90
     # via torch
-onnx==1.17.0
-    # via ragatouille
-openai==1.59.6
-    # via
-    #   llama-index-agent-openai
-    #   llama-index-embeddings-openai
-    #   llama-index-llms-openai
-orjson==3.10.14
-    # via
-    #   gradio
-    #   langsmith
-packaging==24.2
+orjson==3.11.5
+    # via gradio
+packaging==25.0
     # via
     #   datasets
     #   faiss-cpu
     #   gradio
     #   gradio-client
     #   huggingface-hub
-    #   langchain-core
-    #   marshmallow
+    #   spaces
     #   transformers
-pandas==2.2.3
+pandas==2.3.3
     # via
     #   datasets
     #   gradio
-    #   llama-index-readers-file
-pillow==11.1.0
-    # via
-    #   gradio
-    #   llama-index-core
-    #   sentence-transformers
-propcache==0.2.1
+pillow==12.0.0
+    # via gradio
+polars==1.36.1
+    # via daily-papers (pyproject.toml)
+polars-runtime-32==1.36.1
+    # via polars
+propcache==0.4.1
     # via
     #   aiohttp
     #   yarl
-protobuf==5.29.3
-    # via onnx
-pyarrow==18.1.0
+psutil==5.9.8
+    # via spaces
+pyarrow==22.0.0
     # via datasets
-pydantic==2.10.5
+pydantic==2.12.4
     # via
     #   fastapi
     #   gradio
-    #   langchain
-    #   langchain-core
-    #   langsmith
-    #   llama-cloud
-    #   llama-index-core
-    #   llama-parse
-    #   openai
-pydantic-core==2.27.2
+    #   spaces
+pydantic-core==2.41.5
     # via pydantic
 pydub==0.25.1
     # via gradio
-pygments==2.19.1
+pygments==2.19.2
     # via rich
-pynvml==12.0.0
-    # via fast-pytorch-kmeans
-pypdf==5.1.0
-    # via llama-index-readers-file
 python-dateutil==2.9.0.post0
     # via pandas
-python-dotenv==1.0.1
-    # via colbert-ai
-python-multipart==0.0.20
+python-multipart==0.0.21
     # via gradio
-pytz==2024.2
+pytz==2025.2
     # via pandas
-pyyaml==6.0.2
+pyyaml==6.0.3
     # via
     #   datasets
     #   gradio
     #   huggingface-hub
-    #   langchain
-    #   langchain-core
-    #   llama-index-core
     #   transformers
-ragatouille==0.0.8.post4
-    # via daily-papers (pyproject.toml)
-regex==2024.11.6
-    # via
-    #   nltk
-    #   tiktoken
-    #   transformers
-requests==2.32.3
+regex==2025.11.3
+    # via transformers
+requests==2.32.5
     # via
     #   datasets
     #   huggingface-hub
-    #   langchain
-    #   langsmith
-    #   llama-index-core
-    #   requests-toolbelt
-    #   tiktoken
+    #   spaces
     #   transformers
-requests-toolbelt==1.0.0
-    # via langsmith
-rich==13.9.4
+rich==14.2.0
     # via typer
-ruff==0.12.2
-    # via gradio
-safehttpx==0.1.6
+safehttpx==0.1.7
     # via gradio
-safetensors==0.5.2
+safetensors==0.7.0
     # via transformers
-scikit-learn==1.6.1
+scikit-learn==1.7.2
     # via sentence-transformers
-scipy==1.15.1
+scipy==1.15.3
     # via
-    #   colbert-ai
     #   scikit-learn
     #   sentence-transformers
 semantic-version==2.10.0
     # via gradio
-sentence-transformers==2.7.0
-    # via ragatouille
-setuptools==75.8.0
+sentence-transformers==5.2.0
     # via daily-papers (pyproject.toml)
+setuptools==80.9.0
+    # via triton
 shellingham==1.5.4
     # via typer
 six==1.17.0
     # via python-dateutil
-smmap==5.0.2
-    # via gitdb
-sniffio==1.3.1
-    # via
-    #   anyio
-    #   openai
-soupsieve==2.6
-    # via beautifulsoup4
-sqlalchemy==2.0.37
-    # via
-    #   langchain
-    #   llama-index-core
-srsly==2.4.8
-    # via ragatouille
-starlette==0.41.3
+spaces==0.44.0
+    # via daily-papers (pyproject.toml)
+starlette==0.50.0
     # via
     #   fastapi
     #   gradio
-striprtf==0.0.26
-    # via llama-index-readers-file
-sympy==1.13.1
+sympy==1.14.0
     # via torch
-tenacity==9.0.0
-    # via
-    #   langchain
-    #   langchain-core
-    #   llama-index-core
-threadpoolctl==3.5.0
+threadpoolctl==3.6.0
     # via scikit-learn
-tiktoken==0.8.0
-    # via llama-index-core
-tokenizers==0.21.0
+tokenizers==0.22.1
     # via transformers
-tomlkit==0.13.2
+tomlkit==0.13.3
     # via gradio
-torch==2.5.1
+torch==2.8.0
     # via
-    #   fast-pytorch-kmeans
-    #   ragatouille
+    #   daily-papers (pyproject.toml)
     #   sentence-transformers
 tqdm==4.67.1
     # via
-    #   colbert-ai
     #   datasets
     #   huggingface-hub
-    #   llama-index-core
-    #   nltk
-    #   openai
     #   sentence-transformers
     #   transformers
-transformers==4.48.0
-    # via
-    #   colbert-ai
-    #   ragatouille
-    #   sentence-transformers
-triton==3.1.0
+transformers==4.57.3
+    # via sentence-transformers
+triton==3.4.0
     # via torch
-typer==0.15.1
+typer==0.20.0
     # via gradio
-typing-extensions==4.12.2
+typing-extensions==4.15.0
     # via
+    #   aiosignal
     #   anyio
+    #   exceptiongroup
     #   fastapi
     #   gradio
     #   gradio-client
     #   huggingface-hub
-    #   langchain-core
-    #   llama-index-core
     #   multidict
-    #   openai
     #   pydantic
     #   pydantic-core
-    #   pypdf
-    #   rich
-    #   sqlalchemy
+    #   sentence-transformers
+    #   spaces
+    #   starlette
     #   torch
     #   typer
-    #   typing-inspect
+    #   typing-inspection
     #   uvicorn
-typing-inspect==0.9.0
-    # via
-    #   dataclasses-json
-    #   llama-index-core
-tzdata==2024.2
+typing-inspection==0.4.2
+    # via pydantic
+tzdata==2025.3
     # via pandas
-tzlocal==5.2
-    # via apscheduler
-ujson==5.10.0
-    # via colbert-ai
-urllib3==2.3.0
+urllib3==2.6.2
     # via requests
-uvicorn==0.34.0
+uvicorn==0.38.0
     # via gradio
-voyager==2.1.0
-    # via ragatouille
-websockets==14.1
-    # via gradio-client
-werkzeug==3.1.3
-    # via flask
-wrapt==1.17.1
-    # via
-    #   deprecated
-    #   llama-index-core
-xxhash==3.5.0
+xxhash==3.6.0
     # via datasets
-yarl==1.18.3
+yarl==1.22.0
     # via aiohttp

search.py

@@ -0,0 +1,34 @@
+import datasets
+import numpy as np
+import polars as pl
+import spaces
+from datasets import Sequence, Value
+from sentence_transformers import CrossEncoder, SentenceTransformer
+
+from table import df_orig
+
+ds = datasets.Dataset.from_polars(
+    df_orig.select(["arxiv_id", "title", "abstract", "embedding"]).filter(pl.col("embedding").is_not_null())
+).cast_column("embedding", Sequence(Value("float64")))
+ds.add_faiss_index(column="embedding")
+
+bi_model = SentenceTransformer("BAAI/bge-base-en-v1.5")
+ce_model = CrossEncoder("BAAI/bge-reranker-base")
+
+
+@spaces.GPU(duration=10)
+def search(query: str, candidate_pool_size: int = 100, retrieval_k: int = 50) -> list[dict]:
+    prefix = "Represent this sentence for searching relevant passages: "
+    q_vec = bi_model.encode(prefix + query, normalize_embeddings=True)
+
+    _, retrieved_ds = ds.get_nearest_examples("embedding", q_vec, k=candidate_pool_size)
+
+    ce_inputs = [
+        (query, f"{retrieved_ds['title'][i]} {retrieved_ds['abstract'][i]}") for i in range(len(retrieved_ds["title"]))
+    ]
+    ce_scores = ce_model.predict(ce_inputs, batch_size=16)
+
+    sorted_idx = np.argsort(ce_scores)[::-1]
+    return [
+        {"arxiv_id": retrieved_ds["arxiv_id"][i], "ce_score": float(ce_scores[i])} for i in sorted_idx[:retrieval_k]
+    ]

table.py

@@ -0,0 +1,38 @@
+import datasets
+import polars as pl
+
+BASE_REPO_ID = "hysts-bot-data/daily-papers"
+STATS_REPO_ID = "hysts-bot-data/daily-papers-stats"
+EMBEDDING_REPO_ID = "hysts-bot-data/daily-papers-abstract-index"
+
+
+df_orig = datasets.load_dataset(BASE_REPO_ID, split="train").to_polars()
+df_orig = df_orig.join(
+    datasets.load_dataset(STATS_REPO_ID, split="train").to_polars(), on="arxiv_id", how="left"
+).join(datasets.load_dataset(EMBEDDING_REPO_ID, split="train").to_polars(), on="arxiv_id", how="left")
+
+# format date
+df_orig = df_orig.with_columns(
+    pl.format(
+        "[{}](https://huggingface.co/papers/date/{})",
+        pl.col("date").dt.strftime("%Y-%m-%d"),
+        pl.col("date").dt.strftime("%Y-%m-%d"),
+    ).alias("date_md")
+)
+# format links
+df_orig = df_orig.with_columns(
+    [
+        pl.when(pl.col(col).fill_null("") != pl.lit(""))
+        .then(pl.format("[github]({})", pl.col(col)))
+        .otherwise(pl.lit(""))
+        .alias(f"{col}_md")
+        for col in ["github"]
+    ]
+)
+# format paper page link
+df_orig = df_orig.with_columns(
+    (pl.lit("https://huggingface.co/papers/") + pl.col("arxiv_id")).alias("paper_page")
+).with_columns(pl.format("[{}]({})", pl.col("arxiv_id"), pl.col("paper_page")).fill_null("").alias("paper_page_md"))
+
+# sort by date (descending) and arxiv_id (descending)
+df_orig = df_orig.sort(["date", "arxiv_id"], descending=True)