import re import gradio as gr from collections import Counter GENRE_PATTERNS = { "newspaper article": [ "gazette", "journal", "newspaper", "reported", "yesterday", "correspondent", "editor", "issue", "published", "news" ], "advertisement": [ "for sale", "advertisement", "apply to", "price", "cheap", "wanted", "auction", "to be sold", "notice is hereby given" ], "letter": [ "dear sir", "dear madam", "your obedient", "yours faithfully", "i remain", "sir,", "madam," ], "administrative record": [ "register", "inventory", "account", "receipt", "recorded", "dated", "signed", "clerk", "office", "ordinance" ], "legal notice": [ "whereas", "hereby", "aforesaid", "witness", "court", "decree", "plaintiff", "defendant", "notary" ], "shipping/trade notice": [ "ship", "vessel", "cargo", "harbour", "port", "captain", "freight", "imported", "exported", "arrived", "sailed" ], "obituary/death notice": [ "died", "deceased", "funeral", "widow", "buried", "lamented", "aged", "death of" ], } THEME_PATTERNS = { "trade and commerce": ["trade", "merchant", "cargo", "price", "market", "sold", "export", "import"], "war and conflict": ["war", "battle", "army", "troops", "enemy", "siege", "navy", "weapon"], "colonial administration": ["colony", "governor", "company", "council", "fort", "settlement"], "religion": ["church", "god", "mission", "priest", "bishop", "sermon", "christian"], "health and disease": ["disease", "fever", "hospital", "death", "medicine", "plague", "illness"], "family and social life": ["wife", "husband", "child", "family", "marriage", "widow", "son", "daughter"], "law and justice": ["court", "judge", "law", "legal", "trial", "sentence", "witness"], "mobility and travel": ["travel", "journey", "road", "ship", "port", "arrived", "departed"], } def normalize_text(text: str) -> str: return re.sub(r"\s+", " ", text or "").strip() def score_patterns(text: str, patterns: dict) -> list: text_l = text.lower() scores = [] for label, keywords in patterns.items(): hits = [kw for kw in keywords if kw in text_l] if hits: scores.append({ "label": label, "score": len(hits), "evidence": ", ".join(hits[:6]) }) return sorted(scores, key=lambda x: x["score"], reverse=True) def diagnose_ocr_noise(text: str) -> dict: if not text: return { "noise_level": "unknown", "signals": ["No text provided."] } total_chars = len(text) weird_chars = len(re.findall(r"[^A-Za-zÀ-ÿ0-9\s.,;:!?'\-\"()\[\]/]", text)) many_short_tokens = len(re.findall(r"\b[A-Za-zÀ-ÿ]\b", text)) hyphen_breaks = len(re.findall(r"-\s*\n|\-\s+[a-zà-ÿ]", text)) digit_letter_mixes = len(re.findall(r"\b(?=\w*[A-Za-zÀ-ÿ])(?=\w*\d)\w+\b", text)) repeated_punct = len(re.findall(r"[.,;:!?]{2,}", text)) score = 0 signals = [] weird_ratio = weird_chars / max(total_chars, 1) if weird_ratio > 0.02: score += 2 signals.append("many unusual characters") elif weird_ratio > 0.005: score += 1 signals.append("some unusual characters") if many_short_tokens > 10: score += 1 signals.append("many isolated one-letter tokens") if hyphen_breaks > 3: score += 1 signals.append("possible broken hyphenation") if digit_letter_mixes > 3: score += 1 signals.append("words mixing digits and letters") if repeated_punct > 3: score += 1 signals.append("repeated punctuation") if score >= 4: noise = "high" elif score >= 2: noise = "medium" else: noise = "low" if not signals: signals = ["No strong OCR-noise signal detected."] return { "noise_level": noise, "signals": signals } def extract_keywords(text: str, top_n: int = 12) -> list: stopwords = { "the", "and", "for", "that", "with", "this", "from", "were", "was", "are", "his", "her", "not", "but", "have", "has", "had", "you", "your", "their", "les", "des", "une", "dans", "pour", "avec", "qui", "que", "sur", "par", "der", "die", "das", "und", "mit", "von", "den", "dem", "ein", "eine", "het", "een", "van", "voor", "met", "dat", "aan", "zijn" } words = re.findall(r"[A-Za-zÀ-ÿ]{4,}", text.lower()) words = [w for w in words if w not in stopwords] return [w for w, _ in Counter(words).most_common(top_n)] def make_researcher_notes(text, genre_scores, theme_scores, noise): notes = [] if genre_scores: notes.append( f"The strongest genre signal is **{genre_scores[0]['label']}**, based on: {genre_scores[0]['evidence']}." ) else: notes.append("The genre is uncertain; the fragment may be too short or too noisy.") if theme_scores: top_themes = ", ".join([x["label"] for x in theme_scores[:3]]) notes.append(f"Main thematic signals: **{top_themes}**.") if noise["noise_level"] in {"medium", "high"}: notes.append( "The text shows OCR/HTR noise. Manual inspection is recommended before using it for downstream NLP." ) if len(text.split()) < 50: notes.append("The fragment is short, so the profile should be treated as tentative.") return "\n".join(f"- {n}" for n in notes) def profile_document(text): text = text or "" clean = normalize_text(text) genre_scores = score_patterns(clean, GENRE_PATTERNS) theme_scores = score_patterns(clean, THEME_PATTERNS) noise = diagnose_ocr_noise(text) keywords = extract_keywords(clean) top_genre = genre_scores[0]["label"] if genre_scores else "uncertain" top_themes = [x["label"] for x in theme_scores[:5]] overview = f""" ### Document profile **Probable genre:** {top_genre} **OCR/HTR noise level:** {noise["noise_level"]} **Length:** {len(clean.split())} words / {len(clean)} characters **Top keywords:** {", ".join(keywords) if keywords else "None detected"} **Detected themes:** {", ".join(top_themes) if top_themes else "None detected"} """ genre_table = [ [x["label"], x["score"], x["evidence"]] for x in genre_scores ] or [["uncertain", 0, "No genre-specific lexical evidence detected."]] theme_table = [ [x["label"], x["score"], x["evidence"]] for x in theme_scores ] or [["uncertain", 0, "No strong thematic evidence detected."]] noise_report = "\n".join(f"- {s}" for s in noise["signals"]) notes = make_researcher_notes(clean, genre_scores, theme_scores, noise) return overview, genre_table, theme_table, noise_report, notes EXAMPLES = [ [ "Yesterday arrived in the harbour the ship Mercurius, Captain Jansen, with a cargo of pepper, textiles, and porcelain. The goods shall be sold by public auction next Monday." ], [ "Notice is hereby given that the estate of the late Jan van der Meer shall be inventoried before the appointed notary and witnesses." ], [ "Dear Sir, I received your letter concerning the illness in the settlement and remain your obedient servant." ], ] with gr.Blocks(title="Historical Document Profiler") as demo: gr.Markdown( """ # 📜 Historical Document Profiler Paste OCR/HTR text from a historical document and get a lightweight research profile: genre, themes, OCR-noise signals, keywords, and notes for close reading. This is a demo for historical NLP and digital humanities exploration. """ ) with gr.Row(): text = gr.Textbox( label="Historical document text", placeholder="Paste OCR/HTR text here...", lines=12, ) run = gr.Button("Analyze document", variant="primary") with gr.Tab("Overview"): overview = gr.Markdown() with gr.Tab("Genre signals"): genre = gr.Dataframe( headers=["Genre", "Score", "Evidence"], datatype=["str", "number", "str"], interactive=False, ) with gr.Tab("Themes"): themes = gr.Dataframe( headers=["Theme", "Score", "Evidence"], datatype=["str", "number", "str"], interactive=False, ) with gr.Tab("OCR/HTR noise"): noise = gr.Markdown() with gr.Tab("Researcher notes"): notes = gr.Markdown() gr.Examples( examples=EXAMPLES, inputs=text, ) run.click( profile_document, inputs=text, outputs=[overview, genre, themes, noise, notes], ) if __name__ == "__main__": demo.launch()