fix: index CW:28 body, validate API contracts, repair broken data pointer

- Convert CW/1/革命时期的爱情.txt from GB18030 to UTF-8 (203,548 -> 304,547
bytes, 102,538 characters, zero replacement characters, exact round-trip) and
drop its LFS attribute so the text ships as ordinary content.
- build_fulltext_db.py now decodes source TXT with the same strict UTF-8 then
strict GB18030 fallback as app.py. It previously used lenient UTF-8, so CW:28
was indexed as mojibake and its body could not be searched.
- /api/random accepts the canonical plural sources filter and returns 404 for an
unknown source instead of silently returning an unrelated record.
- /api/folders and /api/folders/.../contents return a JSON 404 for an unknown
source, matching /api/search/{source}.
- Unknown /api/* paths return a JSON 404 instead of the SPA HTML with HTTP 200.
- data/sources.json.gz: the LFS object has been absent from the Hub since the
first upload, so the file is rebuilt from the verified source contract in
data/sidebar/global.json and data/search_data.json.gz, and it is excluded from
the generic *.gz LFS rule so it can be stored as ordinary content.
- Corpus integrity now asserts every source TXT is strict UTF-8.

.gitattributes

@@ -34,4 +34,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 static/icons/icon-512.png filter=lfs diff=lfs merge=lfs -text
-CW/1/革命时期的爱情.txt filter=lfs diff=lfs merge=lfs -text
+# This 72-byte source list was first committed as an LFS pointer whose object
+# never existed on the Hub, so every push re-created the same broken pointer.
+# Keep it as ordinary content and exclude it from the generic *.gz rule.
+data/sources.json.gz -filter -diff -merge text

app.py

@@ -1637,16 +1637,41 @@ def api_extensions(source: Optional[str] = Query(default=None)):
 @app.get("/api/folders/{source_slug}")
 
 def api_folders(source_slug: str):
+    if source_slug not in source_counts:
+        return JSONResponse({"error": "source not found", "folders": []}, status_code=404)
     return JSONResponse(cached_payload(("folders", source_slug), lambda: folder_tree_data.get(source_slug, [])))
 @app.get("/api/folders/{source_slug}/contents")
 
 def api_folder_contents(source_slug: str, path: str = Query(default="")):
+    if source_slug not in source_counts:
+        return JSONResponse(
+            {"error": "source not found", "folders": [], "files": [], "current_path": path},
+            status_code=404,
+        )
     return JSONResponse(cached_payload(("folder_contents", source_slug, path), lambda: get_folder_contents(source_slug, path)))
 @app.get("/api/random")
 
-def api_random(source: Optional[str] = Query(default=None)):
-    if source:
-        indices = source_records_map.get(source, [])
+def api_random(
+    source: Optional[str] = Query(default=None),
+    sources: Optional[list[str]] = Query(default=None),
+):
+    # `sources` is the canonical filter, matching the search API. The singular
+    # `source` stays accepted as a compatibility alias for older cached clients.
+    # Previously only `source` was read, so `?sources=NOPE` was silently ignored
+    # and returned an unrelated record with HTTP 200.
+    requested = list(dict.fromkeys(
+        [value for value in ([source] if source else []) if value]
+        + [value for value in (sources or []) if value]
+    ))
+    if requested:
+        missing = [value for value in requested if value not in source_counts]
+        if missing:
+            return JSONResponse({"error": "source not found", "sources": missing}, status_code=404)
+        indices = [
+            index
+            for value in requested
+            for index in source_records_map.get(value, [])
+        ]
         if not indices:
             return JSONResponse({"error": "no record"}, status_code=404)
         return JSONResponse(trim_record(records[random.choice(indices)]))
@@ -1882,4 +1907,8 @@ def render_index_html(rest_of_path: str = "") -> str:
 @app.get("/{rest_of_path:path}")
 
 async def serve_spa(rest_of_path: str):
+    # Unknown API paths must not masquerade as successful SPA pages: clients that
+    # expect JSON otherwise parse HTML with HTTP 200 as a valid response.
+    if rest_of_path == "api" or rest_of_path.startswith("api/"):
+        return JSONResponse({"error": "not found", "path": "/" + rest_of_path}, status_code=404)
     return HTMLResponse(render_index_html(rest_of_path))

build_fulltext_db.py

@@ -19,6 +19,25 @@ LITERAL_PREFIX = "\0literal:"
 def load_json_gz(path: Path):
     return json.loads(gzip.decompress(path.read_bytes()).decode("utf-8"))
 
+def decode_source_text(data: bytes) -> str:
+    """Strict UTF-8 first; a complete strict GB18030 decode is the only fallback.
+
+    Keep this byte-for-byte equivalent to `app.decode_source_text`. One corpus TXT
+    (CW/1/革命时期的爱情.txt) is GB18030; lenient UTF-8 silently drops most of its
+    bytes, which made that document nearly unsearchable in the published index.
+    """
+    try:
+        return data.decode("utf-8")
+    except UnicodeDecodeError:
+        pass
+    try:
+        return data.decode("gb18030")
+    except UnicodeDecodeError:
+        return data.decode("utf-8", errors="ignore")
+
+def read_source_text(path: Path) -> str:
+    return decode_source_text(path.read_bytes())
+
 def index_tokens(text: str) -> set[str]:
     tokens = set()
     normalized = unicodedata.normalize("NFKC", text or "").casefold()
@@ -154,7 +173,7 @@ def build_source_db(source: str, source_records: list[dict]) -> None:
             if not file_path.exists():
                 missing += 1
                 continue
-            text = file_path.read_text(encoding="utf-8", errors="ignore")
+            text = read_source_text(file_path)
             number = doc_number(doc_id)
             connection.execute("INSERT INTO documents (doc_id, summary) VALUES (?, ?)", (doc_id, build_summary(text)))
             connection.execute("INSERT INTO content_fts (rowid, content) VALUES (?, ?)", (number, fts_content(text)))

tests/test_integration.py

@@ -475,12 +475,30 @@ class HttpIntegrationTests(unittest.TestCase):
         self.assertEqual(random_doc.status_code, 200)
         self.assertEqual(random_doc.json()["Source"], "CW")
         self.assertEqual(self.client.get("/api/random", params={"source": "missing"}).status_code, 404)
+        # `sources` is the canonical filter and must not be silently ignored.
+        self.assertEqual(self.client.get("/api/random", params={"sources": "missing"}).status_code, 404)
+        self.assertEqual(self.client.get("/api/random", params={"sources": ["CW"]}).json()["Source"], "CW")
+        self.assertEqual(
+            self.client.get("/api/random", params=[("sources", "CCRD"), ("sources", "CW")]).status_code, 200
+        )
         for route in ("/", "/CCRD", "/CW"):
             page = self.client.get(route)
             self.assertEqual(page.status_code, 200)
             self.assertIn("text/html", page.headers["content-type"])
             self.assertIn("initial-sidebar-data", page.text)
 
+    def test_unknown_sources_and_unknown_api_paths_return_json_404(self):
+        for path in ("/api/folders/missing", "/api/folders/missing/contents"):
+            response = self.client.get(path)
+            self.assertEqual(response.status_code, 404, path)
+            self.assertIn("application/json", response.headers["content-type"], path)
+            self.assertEqual(response.json().get("error"), "source not found", path)
+        for path in ("/api/nope", "/api/search", "/api/folders"):
+            response = self.client.get(path)
+            self.assertEqual(response.status_code, 404, path)
+            self.assertIn("application/json", response.headers["content-type"], path)
+            self.assertEqual(response.json().get("error"), "not found", path)
+
     def test_search_cache_ttl_is_longer_than_burst_idle_gap(self):
         self.assertGreaterEqual(app.SEARCH_CACHE_TTL_SECONDS, 300)
 
@@ -523,6 +541,16 @@ class CorpusIntegrityTests(unittest.TestCase):
             if not (Path(app.BASE_DIR) / record["storage_root"] / record["storage_rel_path"]).is_file()
         ]
         self.assertEqual(missing, [])
+        # Every source TXT must be strict UTF-8 after the CW:28 conversion. A GB18030
+        # file here would be indexed by the builder only through its fallback path.
+        non_utf8 = []
+        for record in records:
+            path = Path(app.BASE_DIR) / record["storage_root"] / record["storage_rel_path"]
+            try:
+                path.read_bytes().decode("utf-8")
+            except UnicodeDecodeError:
+                non_utf8.append(record["doc_id"])
+        self.assertEqual(non_utf8, [])
         self.assertEqual(sum(record["source"] == "CCRD" for record in records), 35586)
         self.assertEqual(sum(record["source"] == "CW" for record in records), 11664)
         archive_keys = [(record["source"], record["display_rel_path"], record["doc_id"]) for record in records]

tests/test_search.py

@@ -93,6 +93,46 @@ class SourceDecodingTests(unittest.TestCase):
         # Neither strict decoder accepts this; keep the old lenient behaviour.
         self.assertEqual(app.decode_source_text(b"ok\x81\xff"), "ok")
 
+    def test_builder_and_runtime_decode_source_text_identically(self):
+        samples = [
+            "手机\n文化".encode("utf-8"),
+            "　　《革命时期的爱情（序）》\r\n这是一本关于性爱的书。".encode("gb18030"),
+            b"ok\x81\xff",
+            b"",
+        ]
+        for data in samples:
+            self.assertEqual(
+                build_fulltext_db.decode_source_text(data),
+                app.decode_source_text(data),
+                data[:20],
+            )
+
+    def test_builder_indexes_gb18030_source_instead_of_dropping_it(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            legacy = "　　《革命时期的爱情（序）》\r\n这是一本关于性爱的书。".encode("gb18030")
+            source = root / "CW" / "1" / "革命时期的爱情.txt"
+            source.parent.mkdir(parents=True)
+            source.write_bytes(legacy)
+            records = [{
+                "doc_id": "CW:28", "source": "CW", "storage_root": "CW",
+                "storage_rel_path": "1/革命时期的爱情.txt",
+            }]
+            original_dir = build_fulltext_db.FULLTEXT_DIR
+            build_fulltext_db.FULLTEXT_DIR = root / "data" / "fulltext"
+            build_fulltext_db.FULLTEXT_DIR.mkdir(parents=True)
+            try:
+                with patch.object(build_fulltext_db, "BASE_DIR", root):
+                    build_fulltext_db.build_source_db("CW", records)
+                with sqlite3.connect(build_fulltext_db.FULLTEXT_DIR / "CW.sqlite3") as connection:
+                    content = connection.execute("SELECT content FROM content_fts").fetchone()[0]
+                    summary = connection.execute("SELECT summary FROM documents").fetchone()[0]
+                self.assertIn("革命时期的爱情", content)
+                self.assertIn("性爱", content)
+                self.assertIn("性爱", summary)
+            finally:
+                build_fulltext_db.FULLTEXT_DIR = original_dir
+
 
 class MetadataSearchTests(unittest.TestCase):
     def setUp(self):