Upload 3 files

app/config.py

@@ -5,7 +5,8 @@ DATA_ROOT = Path(os.environ.get("DATA_ROOT", str(APP_ROOT / "data")))
 PARSED_ROOT = DATA_ROOT / "parsed"
 INDEX_NAME = os.environ.get("ES_INDEX", "article")
 ES_URL = os.environ.get("ES_URL", "http://127.0.0.1:9200")
-REPO_PREFIX = os.environ.get("REPO_PREFIX", "https://github.com/banned-historical-archives")
+REPO_PREFIX = os.environ.get("REPO_PREFIX", "https://github.com/anftm")
+SOURCE_REPO_OWNER = os.environ.get("SOURCE_REPO_OWNER", "anftm")
 ARCHIVE_START = int(os.environ.get("ARCHIVE_START", "0"))
 ARCHIVE_END = int(os.environ.get("ARCHIVE_END", "31"))
 RESET_INDEX = os.environ.get("RESET_INDEX", "0") == "1"

app/data_loader.py

@@ -28,6 +28,7 @@ def ensure_parsed_data(progress: Callable[[int, int], None] | None = None) -> No
         target = PARSED_ROOT / f"archives{archive_id}"
         repo = f"{REPO_PREFIX}/banned-historical-archives{archive_id}.git"
         if (target / ".git").exists():
+            run(["git", "remote", "set-url", "origin", repo], target)
             run(["git", "fetch", "--depth", "1", "origin", "parsed"], target)
             run(["git", "checkout", "parsed"], target)
             run(["git", "reset", "--hard", "origin/parsed"], target)

app/main.py

@@ -11,7 +11,7 @@ import socket
 import tempfile
 import threading
 import time
-from urllib.parse import quote, urlparse
+from urllib.parse import quote, urlparse, urlunparse
 from typing import Any
 from zipfile import ZIP_DEFLATED, ZipFile
 import httpx
@@ -24,7 +24,7 @@ from brotli_asgi import BrotliMiddleware
 from starlette.background import BackgroundTask
 from fastapi.staticfiles import StaticFiles
 from pydantic import BaseModel, Field
-from .config import APP_ROOT, DATA_ROOT, ES_URL, INDEX_NAME
+from .config import APP_ROOT, DATA_ROOT, ES_URL, INDEX_NAME, SOURCE_REPO_OWNER
 from .doc_store import available_years, get_doc, random_doc, variants_for_title, warmup_db
 from .search_store import SearchStoreError, get_search_docs, literal_match_ids
 from .data_loader import initialize_search_tokenizer, search_words
@@ -561,6 +561,7 @@ async def start_ready_data_initialization():
 @app.get("/api/ping")
 def ping():
     return Response(status_code=204, headers={"Cache-Control": "no-store"})
+
 @app.on_event("shutdown")
 
 async def close_source_client():
@@ -830,6 +831,14 @@ def validate_source_url(url: str) -> str | None:
         return "source URL IP is not allowed"
     return None
 
+def mirror_source_url(url: str) -> str:
+    parsed = urlparse(str(url))
+    parts = parsed.path.split("/")
+    if parsed.hostname in {"github.com", "raw.githubusercontent.com"} and len(parts) > 2 and parts[1] == "banned-historical-archives":
+        parts[1] = SOURCE_REPO_OWNER
+        return urlunparse(parsed._replace(path="/".join(parts)))
+    return str(url)
+
 async def validate_source_target(url: str) -> str | None:
     if error := validate_source_url(url):
         return error
@@ -916,7 +925,7 @@ async def download_source(doc_id: str):
     files = source.get("source_files") or []
     if not files:
         return JSONResponse({"error": "source file not available"}, status_code=404)
-    url = str(files[0])
+    url = mirror_source_url(files[0])
     suffix = Path(url.split("?", 1)[0]).suffix or ".bin"
     filename = safe_filename(source_filename_base(source, doc_id), suffix)
     try:
@@ -955,7 +964,8 @@ async def download_source_zip(doc_id: str):
     downloaded: list[tuple[Path, str]] = []
     total_bytes = 0
     try:
-        for index, url in enumerate(files, start=1):
+        for index, source_url in enumerate(files, start=1):
+            url = mirror_source_url(source_url)
             source_path = temp_dir / f"source-{index:03d}"
             size = await download_source_to_path(url, source_path, MAX_SOURCE_ZIP_BYTES - total_bytes)
             total_bytes += size