build_character_data.py

"""Build per-character file index from English_JSON.

For each character in Contacts.json, produces character_data/{id}.json with:
  mentions  - direct conversation/chat files (filename matches character)
  references - other files where the character's name appears in content
"""

import json
import os
import re
from pathlib import Path

ROOT = Path(__file__).parent
JSON_DIR = ROOT / "English_JSON"
OUT_DIR = ROOT / "character_data"


def _normalise_names(info: dict) -> list[str]:
    names = info.get("name", [])
    if isinstance(names, str):
        names = [names] if names else []
    detective = info.get("detective_name") or ""
    if detective:
        names = list(names) + [detective]
    return [n for n in names if n]


def load_characters() -> dict:
    """Merge all Contacts.json files into {id: {name, search_patterns, file_ids}}."""
    chars: dict = {}
    for contacts_path in sorted(JSON_DIR.rglob("Contacts.json")):
        data = json.loads(contacts_path.read_text(encoding="utf-8"))
        for char_id, info in data.items():
            if char_id in chars:
                continue
            names = _normalise_names(info)
            chars[char_id] = {
                "name": names,
                # Patterns for content search (word-boundary for plain words, substring otherwise)
                "search_patterns": [_make_pattern(n) for n in names],
                # Tokens to look for in filenames (ID + name variants, all lowercase)
                "file_ids": {char_id.lower()} | {n.lower() for n in names},
            }
    return chars


def _make_pattern(name: str) -> re.Pattern:
    if re.match(r"^[A-Za-z]+$", name):
        return re.compile(r"\b" + re.escape(name) + r"\b", re.IGNORECASE)
    return re.compile(re.escape(name), re.IGNORECASE)


def _is_chat_file(path: Path) -> bool:
    return any(p in ("Conversations", "Filler Chats") for p in path.parts)


def _filename_matches(stem: str, file_ids: set[str]) -> bool:
    stem_lower = stem.lower()
    # Try each identifier: exact match, ends-with, or space-separated token
    tokens = set(re.split(r"[^a-z0-9]+", stem_lower))
    for fid in file_ids:
        if fid in tokens or stem_lower == fid or stem_lower.endswith(fid):
            return True
    return False


def _flatten_strings(obj) -> list[str]:
    if isinstance(obj, str):
        return [obj]
    if isinstance(obj, list):
        out = []
        for item in obj:
            out.extend(_flatten_strings(item))
        return out
    if isinstance(obj, dict):
        out = []
        for k, v in obj.items():
            out.append(k)
            out.extend(_flatten_strings(v))
        return out
    return []


def _content_matches(text: str, patterns: list[re.Pattern]) -> bool:
    return any(p.search(text) for p in patterns)


def main() -> None:
    OUT_DIR.mkdir(exist_ok=True)
    chars = load_characters()

    results = {
        char_id: {"id": char_id, "name": info["name"], "mentions": [], "references": []}
        for char_id, info in chars.items()
    }

    for json_file in sorted(JSON_DIR.rglob("*.json")):
        rel_path = str(json_file.relative_to(ROOT))
        is_chat = _is_chat_file(json_file)
        stem = json_file.stem

        try:
            data = json.loads(json_file.read_text(encoding="utf-8"))
        except Exception:
            continue

        # Flatten content once per file; reuse across all character checks
        flat_text = " ".join(_flatten_strings(data))

        for char_id, info in chars.items():
            if is_chat and _filename_matches(stem, info["file_ids"]):
                results[char_id]["mentions"].append(rel_path)
            elif info["search_patterns"] and _content_matches(flat_text, info["search_patterns"]):
                results[char_id]["references"].append(rel_path)

    for char_id, char_data in results.items():
        out_path = OUT_DIR / f"{char_id}.json"
        out_path.write_text(
            json.dumps(char_data, indent=2, ensure_ascii=False), encoding="utf-8"
        )

    print(f"Written {len(results)} character files to {OUT_DIR}/")


if __name__ == "__main__":
    main()