app.py

import re
import gradio as gr
from collections import Counter


GENRE_PATTERNS = {
    "newspaper article": [
        "gazette", "journal", "newspaper", "reported", "yesterday", "correspondent",
        "editor", "issue", "published", "news"
    ],
    "advertisement": [
        "for sale", "advertisement", "apply to", "price", "cheap", "wanted",
        "auction", "to be sold", "notice is hereby given"
    ],
    "letter": [
        "dear sir", "dear madam", "your obedient", "yours faithfully",
        "i remain", "sir,", "madam,"
    ],
    "administrative record": [
        "register", "inventory", "account", "receipt", "recorded", "dated",
        "signed", "clerk", "office", "ordinance"
    ],
    "legal notice": [
        "whereas", "hereby", "aforesaid", "witness", "court", "decree",
        "plaintiff", "defendant", "notary"
    ],
    "shipping/trade notice": [
        "ship", "vessel", "cargo", "harbour", "port", "captain",
        "freight", "imported", "exported", "arrived", "sailed"
    ],
    "obituary/death notice": [
        "died", "deceased", "funeral", "widow", "buried", "lamented",
        "aged", "death of"
    ],
}


THEME_PATTERNS = {
    "trade and commerce": ["trade", "merchant", "cargo", "price", "market", "sold", "export", "import"],
    "war and conflict": ["war", "battle", "army", "troops", "enemy", "siege", "navy", "weapon"],
    "colonial administration": ["colony", "governor", "company", "council", "fort", "settlement"],
    "religion": ["church", "god", "mission", "priest", "bishop", "sermon", "christian"],
    "health and disease": ["disease", "fever", "hospital", "death", "medicine", "plague", "illness"],
    "family and social life": ["wife", "husband", "child", "family", "marriage", "widow", "son", "daughter"],
    "law and justice": ["court", "judge", "law", "legal", "trial", "sentence", "witness"],
    "mobility and travel": ["travel", "journey", "road", "ship", "port", "arrived", "departed"],
}


def normalize_text(text: str) -> str:
    return re.sub(r"\s+", " ", text or "").strip()


def score_patterns(text: str, patterns: dict) -> list:
    text_l = text.lower()
    scores = []
    for label, keywords in patterns.items():
        hits = [kw for kw in keywords if kw in text_l]
        if hits:
            scores.append({
                "label": label,
                "score": len(hits),
                "evidence": ", ".join(hits[:6])
            })
    return sorted(scores, key=lambda x: x["score"], reverse=True)


def diagnose_ocr_noise(text: str) -> dict:
    if not text:
        return {
            "noise_level": "unknown",
            "signals": ["No text provided."]
        }

    total_chars = len(text)
    weird_chars = len(re.findall(r"[^A-Za-zÀ-ÿ0-9\s.,;:!?'\-\"()\[\]/]", text))
    many_short_tokens = len(re.findall(r"\b[A-Za-zÀ-ÿ]\b", text))
    hyphen_breaks = len(re.findall(r"-\s*\n|\-\s+[a-zà-ÿ]", text))
    digit_letter_mixes = len(re.findall(r"\b(?=\w*[A-Za-zÀ-ÿ])(?=\w*\d)\w+\b", text))
    repeated_punct = len(re.findall(r"[.,;:!?]{2,}", text))

    score = 0
    signals = []

    weird_ratio = weird_chars / max(total_chars, 1)
    if weird_ratio > 0.02:
        score += 2
        signals.append("many unusual characters")
    elif weird_ratio > 0.005:
        score += 1
        signals.append("some unusual characters")

    if many_short_tokens > 10:
        score += 1
        signals.append("many isolated one-letter tokens")

    if hyphen_breaks > 3:
        score += 1
        signals.append("possible broken hyphenation")

    if digit_letter_mixes > 3:
        score += 1
        signals.append("words mixing digits and letters")

    if repeated_punct > 3:
        score += 1
        signals.append("repeated punctuation")

    if score >= 4:
        noise = "high"
    elif score >= 2:
        noise = "medium"
    else:
        noise = "low"

    if not signals:
        signals = ["No strong OCR-noise signal detected."]

    return {
        "noise_level": noise,
        "signals": signals
    }


def extract_keywords(text: str, top_n: int = 12) -> list:
    stopwords = {
        "the", "and", "for", "that", "with", "this", "from", "were", "was", "are",
        "his", "her", "not", "but", "have", "has", "had", "you", "your", "their",
        "les", "des", "une", "dans", "pour", "avec", "qui", "que", "sur", "par",
        "der", "die", "das", "und", "mit", "von", "den", "dem", "ein", "eine",
        "het", "een", "van", "voor", "met", "dat", "aan", "zijn"
    }

    words = re.findall(r"[A-Za-zÀ-ÿ]{4,}", text.lower())
    words = [w for w in words if w not in stopwords]
    return [w for w, _ in Counter(words).most_common(top_n)]


def make_researcher_notes(text, genre_scores, theme_scores, noise):
    notes = []

    if genre_scores:
        notes.append(
            f"The strongest genre signal is **{genre_scores[0]['label']}**, based on: {genre_scores[0]['evidence']}."
        )
    else:
        notes.append("The genre is uncertain; the fragment may be too short or too noisy.")

    if theme_scores:
        top_themes = ", ".join([x["label"] for x in theme_scores[:3]])
        notes.append(f"Main thematic signals: **{top_themes}**.")

    if noise["noise_level"] in {"medium", "high"}:
        notes.append(
            "The text shows OCR/HTR noise. Manual inspection is recommended before using it for downstream NLP."
        )

    if len(text.split()) < 50:
        notes.append("The fragment is short, so the profile should be treated as tentative.")

    return "\n".join(f"- {n}" for n in notes)


def profile_document(text):
    text = text or ""
    clean = normalize_text(text)

    genre_scores = score_patterns(clean, GENRE_PATTERNS)
    theme_scores = score_patterns(clean, THEME_PATTERNS)
    noise = diagnose_ocr_noise(text)
    keywords = extract_keywords(clean)

    top_genre = genre_scores[0]["label"] if genre_scores else "uncertain"
    top_themes = [x["label"] for x in theme_scores[:5]]

    overview = f"""
### Document profile

**Probable genre:** {top_genre}

**OCR/HTR noise level:** {noise["noise_level"]}

**Length:** {len(clean.split())} words / {len(clean)} characters

**Top keywords:** {", ".join(keywords) if keywords else "None detected"}

**Detected themes:** {", ".join(top_themes) if top_themes else "None detected"}
"""

    genre_table = [
        [x["label"], x["score"], x["evidence"]]
        for x in genre_scores
    ] or [["uncertain", 0, "No genre-specific lexical evidence detected."]]

    theme_table = [
        [x["label"], x["score"], x["evidence"]]
        for x in theme_scores
    ] or [["uncertain", 0, "No strong thematic evidence detected."]]

    noise_report = "\n".join(f"- {s}" for s in noise["signals"])
    notes = make_researcher_notes(clean, genre_scores, theme_scores, noise)

    return overview, genre_table, theme_table, noise_report, notes


EXAMPLES = [
    [
        "Yesterday arrived in the harbour the ship Mercurius, Captain Jansen, with a cargo of pepper, textiles, and porcelain. The goods shall be sold by public auction next Monday."
    ],
    [
        "Notice is hereby given that the estate of the late Jan van der Meer shall be inventoried before the appointed notary and witnesses."
    ],
    [
        "Dear Sir, I received your letter concerning the illness in the settlement and remain your obedient servant."
    ],
]


with gr.Blocks(title="Historical Document Profiler") as demo:
    gr.Markdown(
        """
# 📜 Historical Document Profiler

Paste OCR/HTR text from a historical document and get a lightweight research profile:
genre, themes, OCR-noise signals, keywords, and notes for close reading.

This is a demo for historical NLP and digital humanities exploration.
"""
    )

    with gr.Row():
        text = gr.Textbox(
            label="Historical document text",
            placeholder="Paste OCR/HTR text here...",
            lines=12,
        )

    run = gr.Button("Analyze document", variant="primary")

    with gr.Tab("Overview"):
        overview = gr.Markdown()

    with gr.Tab("Genre signals"):
        genre = gr.Dataframe(
            headers=["Genre", "Score", "Evidence"],
            datatype=["str", "number", "str"],
            interactive=False,
        )

    with gr.Tab("Themes"):
        themes = gr.Dataframe(
            headers=["Theme", "Score", "Evidence"],
            datatype=["str", "number", "str"],
            interactive=False,
        )

    with gr.Tab("OCR/HTR noise"):
        noise = gr.Markdown()

    with gr.Tab("Researcher notes"):
        notes = gr.Markdown()

    gr.Examples(
        examples=EXAMPLES,
        inputs=text,
    )

    run.click(
        profile_document,
        inputs=text,
        outputs=[overview, genre, themes, noise, notes],
    )


if __name__ == "__main__":
    demo.launch()