Upload 3 files

app/data_loader.py

@@ -47,6 +47,49 @@ def text_from_article(article: dict[str, Any]) -> str:
     return "\n".join(values)
 
 
+def normalize_date_value(item: dict[str, Any]) -> int | None:
+    year = item.get("year")
+    if not isinstance(year, int):
+        return None
+    month = item.get("month") if isinstance(item.get("month"), int) else 1
+    day = item.get("day") if isinstance(item.get("day"), int) else 1
+    return year * 10000 + month * 100 + day
+
+
+def normalize_dates(article: dict[str, Any]) -> tuple[int | None, int | None, list[str]]:
+    raw_dates = article.get("dates") if isinstance(article.get("dates"), list) else []
+    values: list[int] = []
+    displays: list[str] = []
+    for item in raw_dates:
+        if not isinstance(item, dict):
+            continue
+        value = normalize_date_value(item)
+        if value is not None:
+            values.append(value)
+        year = item.get("year") if item.get("year") is not None else "----"
+        month = item.get("month") if item.get("month") is not None else "--"
+        day = item.get("day") if item.get("day") is not None else "--"
+        displays.append(f"{year}/{month}/{day}")
+    return (min(values) if values else None, max(values) if values else None, displays)
+
+
+def normalize_tags(tags: Any) -> tuple[list[str], list[str]]:
+    if not isinstance(tags, list):
+        return [], []
+    names: list[str] = []
+    types: list[str] = []
+    for item in tags:
+        if not isinstance(item, dict):
+            continue
+        name = item.get("name")
+        tag_type = item.get("type")
+        if name is not None:
+            names.append(str(name))
+        if tag_type is not None:
+            types.append(str(tag_type))
+    return names, types
+
+
 def iter_documents() -> Iterator[dict[str, Any]]:
     for archive_id in range(ARCHIVE_START, ARCHIVE_END + 1):
         archive_root = PARSED_ROOT / f"archives{archive_id}"
@@ -65,9 +108,11 @@ def iter_documents() -> Iterator[dict[str, Any]]:
                     continue
                 tags_path = article_path.with_suffix(".tags")
                 tags = read_json(tags_path) if tags_path.exists() else []
+                tag_names, tag_types = normalize_tags(tags)
                 authors = article.get("authors") or []
                 if not isinstance(authors, list):
                     authors = []
+                date_min, date_max, date_display = normalize_dates(article)
                 doc_id = f"{article_id}--{publication_id}"
                 yield {
                     "doc_id": doc_id,
@@ -81,7 +126,12 @@ def iter_documents() -> Iterator[dict[str, Any]]:
                     "title": str(article.get("title") or article_id),
                     "authors": [str(item) for item in authors],
                     "dates": article.get("dates") or [],
+                    "date_min": date_min,
+                    "date_max": date_max,
+                    "date_display": date_display,
                     "tags": tags if isinstance(tags, list) else [],
+                    "tag_names": tag_names,
+                    "tag_types": tag_types,
                     "content": text_from_article(article),
                     "path": str(article_path.relative_to(archive_root)),
                 }

app/indexer.py

@@ -34,6 +34,10 @@ def index_mapping() -> dict:
                 "archive_id": {"type": "integer"},
                 "title": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
                 "authors": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "tag_names": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "tag_types": {"type": "text", "fields": {"keyword": {"type": "keyword"}}},
+                "date_min": {"type": "integer"},
+                "date_max": {"type": "integer"},
                 "content": {"type": "text", "analyzer": "text_mixed"},
                 "path": {"type": "keyword"},
             }

app/main.py

@@ -25,6 +25,12 @@ class SearchRequest(BaseModel):
     page_size: int = 20
     exact: bool = True
     source: str | None = None
+    author: str | None = None
+    tag: str | None = None
+    archive_id: int | None = None
+    publication_type: str | None = None
+    year_from: int | None = None
+    year_to: int | None = None
 
 
 def has_wildcard(query: str) -> bool:
@@ -36,6 +42,18 @@ def build_query(body: SearchRequest) -> dict[str, Any]:
     filters: list[dict[str, Any]] = []
     if body.source:
         filters.append({"term": {"publication_name.keyword": body.source}})
+    if body.author:
+        filters.append({"term": {"authors.keyword": body.author}})
+    if body.tag:
+        filters.append({"term": {"tag_names.keyword": body.tag}})
+    if body.archive_id is not None:
+        filters.append({"term": {"archive_id": body.archive_id}})
+    if body.publication_type:
+        filters.append({"term": {"publication_type": body.publication_type}})
+    if body.year_from is not None:
+        filters.append({"range": {"date_max": {"gte": body.year_from * 10000}}})
+    if body.year_to is not None:
+        filters.append({"range": {"date_min": {"lte": body.year_to * 10000 + 1231}}})
 
     if not q:
         query: dict[str, Any] = {"match_all": {}}
@@ -78,6 +96,8 @@ def trim_hit(hit: dict[str, Any]) -> dict[str, Any]:
         "publication_name": source.get("publication_name"),
         "archive_id": source.get("archive_id"),
         "publication_type": source.get("publication_type"),
+        "date_display": source.get("date_display") or [],
+        "tag_names": source.get("tag_names") or [],
         "title": source.get("title"),
         "authors": source.get("authors") or [],
         "path": source.get("path"),
@@ -96,10 +116,22 @@ def sources():
     data = es.search(
         index=INDEX_NAME,
         size=0,
-        aggs={"sources": {"terms": {"field": "publication_name.keyword", "size": 500}}},
+        aggs={
+            "sources": {"terms": {"field": "publication_name.keyword", "size": 500}},
+            "authors": {"terms": {"field": "authors.keyword", "size": 300}},
+            "tags": {"terms": {"field": "tag_names.keyword", "size": 300}},
+            "archives": {"terms": {"field": "archive_id", "size": 64}},
+            "types": {"terms": {"field": "publication_type", "size": 32}},
+        },
     )
-    buckets = data.get("aggregations", {}).get("sources", {}).get("buckets", [])
-    return [{"name": item["key"], "count": item["doc_count"]} for item in buckets]
+    aggs = data.get("aggregations", {})
+    return {
+        "sources": [{"name": item["key"], "count": item["doc_count"]} for item in aggs.get("sources", {}).get("buckets", [])],
+        "authors": [{"name": item["key"], "count": item["doc_count"]} for item in aggs.get("authors", {}).get("buckets", [])],
+        "tags": [{"name": item["key"], "count": item["doc_count"]} for item in aggs.get("tags", {}).get("buckets", [])],
+        "archives": [{"name": str(item["key"]), "value": item["key"], "count": item["doc_count"]} for item in aggs.get("archives", {}).get("buckets", [])],
+        "types": [{"name": item["key"], "count": item["doc_count"]} for item in aggs.get("types", {}).get("buckets", [])],
+    }
 
 
 @app.post("/api/search")
@@ -129,6 +161,8 @@ def preview(doc_id: str):
         "title": source.get("title"),
         "publication_name": source.get("publication_name"),
         "publication_type": source.get("publication_type"),
+        "date_display": source.get("date_display") or [],
+        "tag_names": source.get("tag_names") or [],
         "authors": source.get("authors") or [],
         "content": source.get("content") or "",
         "article_id": source.get("article_id"),