Spaces:
Sleeping
Sleeping
| """ | |
| Combined Scripture dashboards -- Network/Cluster explorer + Original-Languages | |
| word study. Loads precomputed embeddings and verse data (built once by | |
| export_to_space.ipynb) rather than recomputing anything at startup, so this | |
| runs comfortably on a free CPU-only Hugging Face Space. | |
| The two sentence-transformer models are still loaded here (not at the | |
| export step) because live free-text search needs to encode whatever the | |
| user types -- but encoding one short query is fast even on CPU, unlike | |
| re-embedding all ~31,000 verses. | |
| """ | |
| import json | |
| import re | |
| import tempfile | |
| import os | |
| from collections import defaultdict | |
| import numpy as np | |
| import pandas as pd | |
| import gradio as gr | |
| from sentence_transformers import SentenceTransformer | |
| print("Loading precomputed data...") | |
| with open("slim_verses.json", encoding="utf-8") as f: | |
| all_verses = json.load(f) | |
| with open("lexicon_clean.json", encoding="utf-8") as f: | |
| lexicon = json.load(f) | |
| with open("cluster_info.json", encoding="utf-8") as f: | |
| cluster_info = json.load(f) | |
| en_embeddings = np.load("en_embeddings.npy") | |
| orig_embeddings = np.load("orig_embeddings.npy") | |
| print(f"Loaded {len(all_verses)} verses, {en_embeddings.shape} English embeddings, " | |
| f"{orig_embeddings.shape} original-language embeddings, {len(cluster_info)} theme clusters.") | |
| print("Loading sentence-transformer models (for live query encoding only)...") | |
| model_en = SentenceTransformer("all-mpnet-base-v2") | |
| model_orig = SentenceTransformer("sentence-transformers/LaBSE") | |
| print("Models loaded. Ready.") | |
| ref_to_idx = {v["ref"]: i for i, v in enumerate(all_verses)} | |
| # --------------------------------------------------------------------------- | |
| # Reference normalization (handles "Psalm" vs "Psalms", abbreviations, etc.) | |
| # --------------------------------------------------------------------------- | |
| ALIASES = { | |
| "psalm": "Psalms", "psa": "Psalms", "ps": "Psalms", | |
| "song": "Song of Solomon", "song of songs": "Song of Solomon", | |
| "canticles": "Song of Solomon", "sos": "Song of Solomon", | |
| "gen": "Genesis", "exo": "Exodus", "ex": "Exodus", "lev": "Leviticus", | |
| "num": "Numbers", "deut": "Deuteronomy", "deu": "Deuteronomy", | |
| "josh": "Joshua", "jos": "Joshua", "judg": "Judges", "jdg": "Judges", | |
| "1 sam": "1 Samuel", "2 sam": "2 Samuel", "1sam": "1 Samuel", "2sam": "2 Samuel", | |
| "1 kgs": "1 Kings", "2 kgs": "2 Kings", "1 chron": "1 Chronicles", "2 chron": "2 Chronicles", | |
| "1 chr": "1 Chronicles", "2 chr": "2 Chronicles", | |
| "neh": "Nehemiah", "est": "Esther", "prov": "Proverbs", "pro": "Proverbs", | |
| "eccl": "Ecclesiastes", "ecc": "Ecclesiastes", "eccles": "Ecclesiastes", | |
| "isa": "Isaiah", "jer": "Jeremiah", "lam": "Lamentations", "ezek": "Ezekiel", | |
| "eze": "Ezekiel", "dan": "Daniel", "hos": "Hosea", "obad": "Obadiah", "oba": "Obadiah", | |
| "jon": "Jonah", "mic": "Micah", "nah": "Nahum", "hab": "Habakkuk", "zeph": "Zephaniah", | |
| "zep": "Zephaniah", "hag": "Haggai", "zech": "Zechariah", "zec": "Zechariah", "mal": "Malachi", | |
| "matt": "Matthew", "mat": "Matthew", "mt": "Matthew", "mk": "Mark", "mar": "Mark", | |
| "lk": "Luke", "luk": "Luke", "jn": "John", "jhn": "John", | |
| "act": "Acts", "rom": "Romans", | |
| "1 cor": "1 Corinthians", "2 cor": "2 Corinthians", "1cor": "1 Corinthians", "2cor": "2 Corinthians", | |
| "gal": "Galatians", "eph": "Ephesians", "phil": "Philippians", "php": "Philippians", "phl": "Philippians", | |
| "col": "Colossians", "1 thess": "1 Thessalonians", "2 thess": "2 Thessalonians", | |
| "1 thes": "1 Thessalonians", "2 thes": "2 Thessalonians", | |
| "1 tim": "1 Timothy", "2 tim": "2 Timothy", "tit": "Titus", "phlm": "Philemon", "phm": "Philemon", | |
| "heb": "Hebrews", "jas": "James", "jam": "James", | |
| "1 pet": "1 Peter", "2 pet": "2 Peter", "1pet": "1 Peter", "2pet": "2 Peter", | |
| "1 jn": "1 John", "2 jn": "2 John", "3 jn": "3 John", | |
| "rev": "Revelation", | |
| } | |
| _CANONICAL_LOWER = {v["book"].lower(): v["book"] for v in all_verses} | |
| REF_RE = re.compile(r'^\s*(.+?)\s+(\d+)\s*[:.]\s*(\d+)\s*$') | |
| def normalize_reference(user_input): | |
| m = REF_RE.match(user_input.strip()) | |
| if not m: | |
| return None | |
| book_part, chapter, verse = m.groups() | |
| book_key = book_part.strip().lower() | |
| canonical_book = _CANONICAL_LOWER.get(book_key) or ALIASES.get(book_key) | |
| if canonical_book is None: | |
| stripped = book_key.rstrip(".") | |
| canonical_book = _CANONICAL_LOWER.get(stripped) or ALIASES.get(stripped) | |
| if canonical_book is None: | |
| return None | |
| return f"{canonical_book} {int(chapter)}:{int(verse)}" | |
| # --------------------------------------------------------------------------- | |
| # Tab 1: Network / cluster explorer (English-only) | |
| # --------------------------------------------------------------------------- | |
| import plotly.graph_objects as go | |
| import networkx as nx | |
| CBC_NAVY, CBC_BLUE, CBC_TEAL = "#2F3F82", "#4A6CB5", "#49A1CF" | |
| def build_graph_figure(node_verses, node_sims=None, sim_threshold=0.55, max_edges_per_node=4): | |
| n = len(node_verses) | |
| if n == 0: | |
| fig = go.Figure() | |
| fig.update_layout(height=560, plot_bgcolor="#FAF8F3", paper_bgcolor="#FAF8F3", | |
| xaxis=dict(visible=False), yaxis=dict(visible=False)) | |
| return fig | |
| refs = [v["ref"] for v in node_verses] | |
| local_embs, valid_local_idx = [], [] | |
| for i, r in enumerate(refs): | |
| gi = ref_to_idx.get(r) | |
| if gi is not None: | |
| local_embs.append(en_embeddings[gi]) | |
| valid_local_idx.append(i) | |
| local_embs = np.array(local_embs) if local_embs else np.zeros((0, en_embeddings.shape[1])) | |
| G = nx.Graph() | |
| for i in range(n): | |
| G.add_node(i) | |
| if len(valid_local_idx) > 1: | |
| sim_matrix = local_embs @ local_embs.T | |
| for a_pos, a in enumerate(valid_local_idx): | |
| sims_row = [(valid_local_idx[b_pos], sim_matrix[a_pos, b_pos]) | |
| for b_pos in range(len(valid_local_idx)) if valid_local_idx[b_pos] != a] | |
| sims_row.sort(key=lambda x: -x[1]) | |
| for b, s in sims_row[:max_edges_per_node]: | |
| if s >= sim_threshold: | |
| G.add_edge(a, b, weight=float(s)) | |
| pos = nx.spring_layout(G, k=1.2 / max(1, n ** 0.5), seed=42, iterations=100) | |
| edge_x, edge_y = [], [] | |
| for u, v in G.edges(): | |
| x0, y0 = pos[u]; x1, y1 = pos[v] | |
| edge_x += [x0, x1, None] | |
| edge_y += [y0, y1, None] | |
| edge_trace = go.Scatter(x=edge_x, y=edge_y, mode="lines", | |
| line=dict(width=1, color=CBC_TEAL), opacity=0.35, hoverinfo="none") | |
| node_x = [pos[i][0] for i in range(n)] | |
| node_y = [pos[i][1] for i in range(n)] | |
| colors = [CBC_BLUE if v["testament"] == "OT" else CBC_TEAL for v in node_verses] | |
| sizes = [16 + (node_sims[i] * 14 if node_sims else 0) for i in range(n)] | |
| hover_text = [f"<b>{v['ref']}</b><br>{v['kjv_text'][:120]}" for v in node_verses] | |
| labels = [v["ref"] for v in node_verses] | |
| node_trace = go.Scatter( | |
| x=node_x, y=node_y, mode="markers+text", | |
| text=labels, textposition="top center", | |
| textfont=dict(size=9, color=CBC_NAVY, family="Georgia, serif"), | |
| marker=dict(size=sizes, color=colors, line=dict(width=1.5, color="white")), | |
| hovertext=hover_text, hoverinfo="text", | |
| ) | |
| fig = go.Figure(data=[edge_trace, node_trace]) | |
| fig.update_layout(showlegend=False, height=560, plot_bgcolor="#FAF8F3", paper_bgcolor="#FAF8F3", | |
| xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), | |
| yaxis=dict(showgrid=False, zeroline=False, showticklabels=False), | |
| margin=dict(l=10, r=10, t=10, b=10)) | |
| return fig | |
| def verses_to_dataframe(node_verses, node_sims=None): | |
| rows = [] | |
| for i, v in enumerate(node_verses): | |
| match_pct = f"{node_sims[i]*100:.0f}%" if node_sims else "" | |
| rows.append({"Match": match_pct, "Reference": v["ref"], "Testament": v["testament"], "Text": v["kjv_text"]}) | |
| return pd.DataFrame(rows) | |
| def search_verses_network(query_text, top_k=40): | |
| if not query_text or not query_text.strip(): | |
| return [], [] | |
| q_emb = model_en.encode([query_text], normalize_embeddings=True)[0] | |
| sims = en_embeddings @ q_emb | |
| top_idx = np.argsort(-sims)[:top_k] | |
| result_verses = [{"ref": all_verses[i]["ref"], "kjv_text": all_verses[i]["kjv_text"], | |
| "testament": all_verses[i]["testament"]} for i in top_idx] | |
| result_sims = [float(sims[i]) for i in top_idx] | |
| return result_verses, result_sims | |
| def get_cluster_by_theme(theme_name): | |
| for c in cluster_info: | |
| if c["theme"] == theme_name: | |
| return [{"ref": v["ref"], "kjv_text": v["text"], "testament": v["testament"]} for v in c["verses"]] | |
| return [] | |
| theme_choices = [c["theme"] for c in cluster_info] | |
| def on_theme_select(theme_name): | |
| if not theme_name: | |
| return build_graph_figure([]), pd.DataFrame(), "Pick a theme from the dropdown." | |
| node_verses = get_cluster_by_theme(theme_name) | |
| fig = build_graph_figure(node_verses) | |
| df = verses_to_dataframe(node_verses) | |
| return fig, df, f"**{theme_name}** -- showing {len(node_verses)} verses. Click a row below for related verses." | |
| def on_search_network(query_text): | |
| if not query_text or not query_text.strip(): | |
| return build_graph_figure([]), pd.DataFrame(), "Type a theme or phrase above and press Search." | |
| node_verses, sims = search_verses_network(query_text, top_k=40) | |
| fig = build_graph_figure(node_verses, node_sims=sims) | |
| df = verses_to_dataframe(node_verses, sims) | |
| return fig, df, f'Search: "{query_text}" -- showing top {len(node_verses)} matches.' | |
| def on_row_select_network(evt: gr.SelectData): | |
| if evt.row_value is None: | |
| return "Click a row in the table to inspect that verse." | |
| ref = evt.row_value[1] | |
| gi = ref_to_idx.get(ref) | |
| if gi is None: | |
| return f"Couldn't find {ref}." | |
| v = all_verses[gi] | |
| sims_all = en_embeddings @ en_embeddings[gi] | |
| sims_all[gi] = -1 | |
| top_neighbors = np.argsort(-sims_all)[:8] | |
| detail = f"### {v['ref']} ({v['testament']})\n\n> {v['kjv_text']}\n\n**Closest related verses:**\n\n" | |
| for ni in top_neighbors: | |
| nv = all_verses[ni] | |
| detail += f"- **{nv['ref']}** ({sims_all[ni]*100:.0f}%) -- {nv['kjv_text'][:90]}\n" | |
| return detail | |
| # --------------------------------------------------------------------------- | |
| # Tab 2: Original-languages word study | |
| # --------------------------------------------------------------------------- | |
| def get_word_study_table(ref): | |
| normalized = normalize_reference(ref) | |
| lookup_ref = normalized if normalized in ref_to_idx else (ref if ref in ref_to_idx else None) | |
| if lookup_ref is None: | |
| hint = "" | |
| if normalized and normalized not in ref_to_idx: | |
| hint = f" (parsed as '{normalized}', which also isn't in this dataset)" | |
| return pd.DataFrame(), ( | |
| f"Couldn't find that reference{hint}. Try a format like 'Genesis 16:13', " | |
| "'Psalm 27:14', or 'John 20:27'." | |
| ) | |
| v = all_verses[ref_to_idx[lookup_ref]] | |
| rows = [] | |
| for w in v["words"]: | |
| lex = lexicon.get(w["strongs"], {}) | |
| gloss = lex.get("gloss", "") | |
| rows.append({ | |
| "English": w["english"], "Strong's": w["strongs"], | |
| "Original": lex.get("original_word", ""), | |
| "Transliteration": lex.get("transliteration", ""), | |
| "Meaning": gloss[:150] + ("..." if len(gloss) > 150 else ""), | |
| }) | |
| df = pd.DataFrame(rows) | |
| note = "" | |
| if not v["has_original"]: | |
| note = (" *Note: no aligned original-language text for this verse in this dataset " | |
| "(a known Hebrew/English versification difference, or a textus-receptus-only " | |
| "passage absent from the critical Greek text used here.)*") | |
| return df, f"**{v['ref']}** ({v['testament']})\n\n{v['kjv_text']}\n\n*Original:* {v['original_text']}{note}" | |
| def search_dual(ref_or_query, top_k=10): | |
| normalized = normalize_reference(ref_or_query) | |
| resolved_ref = normalized if normalized in ref_to_idx else (ref_or_query if ref_or_query in ref_to_idx else None) | |
| if resolved_ref is not None: | |
| idx = ref_to_idx[resolved_ref] | |
| q_en = en_embeddings[idx] | |
| v = all_verses[idx] | |
| has_orig_query = v["has_original"] | |
| q_orig = orig_embeddings[idx] if has_orig_query else None | |
| else: | |
| q_en = model_en.encode([ref_or_query], normalize_embeddings=True)[0] | |
| has_orig_query = False | |
| q_orig = None | |
| sims_en = en_embeddings @ q_en | |
| top_en = np.argsort(-sims_en)[:top_k + 1] | |
| rows_en = [] | |
| for i in top_en: | |
| if resolved_ref is not None and i == ref_to_idx[resolved_ref]: | |
| continue | |
| v = all_verses[i] | |
| rows_en.append({"Match": f"{sims_en[i]*100:.0f}%", "Reference": v["ref"], "Text (English)": v["kjv_text"]}) | |
| df_en = pd.DataFrame(rows_en[:top_k]) | |
| df_orig = pd.DataFrame() | |
| if has_orig_query: | |
| sims_orig = orig_embeddings @ q_orig | |
| has_orig_mask = np.array([v["has_original"] for v in all_verses]) | |
| sims_orig = np.where(has_orig_mask, sims_orig, -1) | |
| top_orig = np.argsort(-sims_orig)[:top_k + 1] | |
| rows_orig = [] | |
| for i in top_orig: | |
| if i == ref_to_idx[resolved_ref]: | |
| continue | |
| v = all_verses[i] | |
| rows_orig.append({"Match": f"{sims_orig[i]*100:.0f}%", "Reference": v["ref"], "Text (original)": v["original_text"][:60]}) | |
| df_orig = pd.DataFrame(rows_orig[:top_k]) | |
| return df_en, df_orig | |
| def on_lookup(ref_or_query): | |
| word_df, display_text = get_word_study_table(ref_or_query) | |
| en_df, orig_df = search_dual(ref_or_query) | |
| return word_df, display_text, en_df, orig_df | |
| # --------------------------------------------------------------------------- | |
| # Tab 3: Cross-Testament Typology Finder | |
| # --------------------------------------------------------------------------- | |
| TYPOLOGY_PAIRS = [ | |
| {"name": "Passover Lamb → Lamb of God", | |
| "ot_anchor": "Exodus 12:13", "nt_anchor": "John 1:29"}, | |
| {"name": "Suffering Servant → Passion", | |
| "ot_anchor": "Isaiah 53:5", "nt_anchor": "1 Peter 2:24"}, | |
| {"name": "Bronze Serpent → Christ Lifted Up", | |
| "ot_anchor": "Numbers 21:9", "nt_anchor": "John 3:14"}, | |
| {"name": "Jonah Three Days → Resurrection Sign", | |
| "ot_anchor": "Jonah 1:17", "nt_anchor": "Matthew 12:40"}, | |
| {"name": "Melchizedek → Eternal Priesthood", | |
| "ot_anchor": "Genesis 14:18", "nt_anchor": "Hebrews 7:3"}, | |
| {"name": "Davidic King → Messiah Enthroned", | |
| "ot_anchor": "Psalms 2:7", "nt_anchor": "Acts 13:33"}, | |
| {"name": "New Covenant Promised → Enacted", | |
| "ot_anchor": "Jeremiah 31:31", "nt_anchor": "Hebrews 8:8"}, | |
| {"name": "Isaac as Offering → Son Given", | |
| "ot_anchor": "Genesis 22:2", "nt_anchor": "John 3:16"}, | |
| ] | |
| def _resolve_to_idx(ref_input): | |
| normalized = normalize_reference(ref_input) | |
| if normalized and normalized in ref_to_idx: | |
| return ref_to_idx[normalized] | |
| if ref_input in ref_to_idx: | |
| return ref_to_idx[ref_input] | |
| return None | |
| def typology_search_verse(ref_input, top_k=15): | |
| idx = _resolve_to_idx(ref_input) | |
| if idx is None: | |
| return "Could not find that verse. Try a format like 'Isaiah 53:5' or 'John 1:29'.", pd.DataFrame() | |
| verse = all_verses[idx] | |
| opposite = "NT" if verse["testament"] == "OT" else "OT" | |
| opposite_mask = np.array([v["testament"] == opposite for v in all_verses]) | |
| q_emb = en_embeddings[idx] | |
| sims = en_embeddings @ q_emb | |
| sims_masked = np.where(opposite_mask, sims, -1) | |
| top_idx = np.argsort(-sims_masked)[:top_k] | |
| header = ( | |
| f"**{verse['ref']}** ({verse['testament']}) — searching {opposite} for typological echoes\n\n" | |
| f"> {verse['kjv_text']}" | |
| ) | |
| rows = [] | |
| for i in top_idx: | |
| v = all_verses[i] | |
| rows.append({"Match %": f"{sims_masked[i]*100:.0f}%", "Reference": v["ref"], "Text": v["kjv_text"]}) | |
| return header, pd.DataFrame(rows) | |
| def typology_browse_pair(pair_name, top_k=10): | |
| pair = next((p for p in TYPOLOGY_PAIRS if p["name"] == pair_name), None) | |
| if pair is None: | |
| return "Select a pair from the dropdown.", pd.DataFrame() | |
| ot_idx = _resolve_to_idx(pair["ot_anchor"]) | |
| nt_idx = _resolve_to_idx(pair["nt_anchor"]) | |
| ot_verse = all_verses[ot_idx] if ot_idx is not None else None | |
| nt_verse = all_verses[nt_idx] if nt_idx is not None else None | |
| ot_text = f"**{pair['ot_anchor']}** (OT)\n> {ot_verse['kjv_text']}" if ot_verse else f"**{pair['ot_anchor']}** — not found in dataset" | |
| nt_text = f"**{pair['nt_anchor']}** (NT)\n> {nt_verse['kjv_text']}" if nt_verse else f"**{pair['nt_anchor']}** — not found in dataset" | |
| display = f"## {pair_name}\n\n{ot_text}\n\n{nt_text}\n\n---\n\n**Verses near the thematic midpoint:**" | |
| rows = [] | |
| if ot_idx is not None and nt_idx is not None: | |
| midpoint = en_embeddings[ot_idx] + en_embeddings[nt_idx] | |
| norm = np.linalg.norm(midpoint) | |
| if norm > 0: | |
| midpoint = midpoint / norm | |
| sims = en_embeddings @ midpoint | |
| # exclude the two anchors | |
| sims[ot_idx] = -1 | |
| sims[nt_idx] = -1 | |
| top_idx = np.argsort(-sims)[:top_k] | |
| for i in top_idx: | |
| v = all_verses[i] | |
| rows.append({"Match %": f"{sims[i]*100:.0f}%", "Reference": v["ref"], "Testament": v["testament"], "Text": v["kjv_text"]}) | |
| return display, pd.DataFrame(rows) | |
| def on_typology_search(ref_input): | |
| header, df = typology_search_verse(ref_input) | |
| return header, df | |
| def on_typology_browse(pair_name): | |
| display, df = typology_browse_pair(pair_name) | |
| return display, df | |
| # --------------------------------------------------------------------------- | |
| # Tab 4: Preaching Calendar | |
| # --------------------------------------------------------------------------- | |
| RANGE_RE = re.compile(r'^(.+?)\s+(\d+)\s*[:.]\s*(\d+)\s*[-–]\s*(\d+)\s*$') | |
| def parse_reference_or_range(text): | |
| text = text.strip() | |
| # try range first | |
| m = RANGE_RE.match(text) | |
| if m: | |
| book_part, chapter, start_v, end_v = m.groups() | |
| book_key = book_part.strip().lower() | |
| canonical_book = _CANONICAL_LOWER.get(book_key) or ALIASES.get(book_key) | |
| if canonical_book is None: | |
| stripped = book_key.rstrip(".") | |
| canonical_book = _CANONICAL_LOWER.get(stripped) or ALIASES.get(stripped) | |
| if canonical_book is None: | |
| return [] | |
| chapter, start_v, end_v = int(chapter), int(start_v), int(end_v) | |
| if end_v < start_v: | |
| return [] | |
| return [(canonical_book, chapter, v) for v in range(start_v, end_v + 1)] | |
| # try single verse | |
| normalized = normalize_reference(text) | |
| if normalized: | |
| m2 = REF_RE.match(normalized) | |
| if m2: | |
| book_part, chapter, verse = m2.groups() | |
| book_key = book_part.strip().lower() | |
| canonical_book = _CANONICAL_LOWER.get(book_key) or ALIASES.get(book_key) | |
| if canonical_book is None: | |
| canonical_book = book_part # already canonical from normalize_reference | |
| # re-parse the normalized form directly | |
| parts = normalized.rsplit(" ", 1) | |
| book_ref = parts[0] | |
| ch_v = parts[1].split(":") | |
| return [(book_ref, int(ch_v[0]), int(ch_v[1]))] | |
| return [] | |
| def _lookup_verse_by_tuple(book, chapter, verse): | |
| ref = f"{book} {chapter}:{verse}" | |
| idx = ref_to_idx.get(ref) | |
| return idx | |
| # Compute cluster centroids at startup from the member verses stored in cluster_info. | |
| # cluster_info only stores up to 30 verses per cluster, so these are approximate | |
| # centroids, but good enough for nearest-theme matching. | |
| _cluster_centroids = [] | |
| for _c in cluster_info: | |
| _member_idxs = [ref_to_idx[v["ref"]] for v in _c["verses"] if v["ref"] in ref_to_idx] | |
| if _member_idxs: | |
| _centroid = en_embeddings[_member_idxs].mean(axis=0) | |
| _norm = np.linalg.norm(_centroid) | |
| if _norm > 0: | |
| _centroid = _centroid / _norm | |
| else: | |
| _centroid = np.zeros(en_embeddings.shape[1], dtype=np.float32) | |
| _cluster_centroids.append(_centroid) | |
| _cluster_centroids = np.array(_cluster_centroids) # (num_clusters, 768) | |
| def _find_closest_theme(idx): | |
| q = en_embeddings[idx] | |
| sims = _cluster_centroids @ q | |
| best = int(np.argmax(sims)) | |
| return cluster_info[best]["theme"] | |
| def generate_calendar(input_text): | |
| if not input_text or not input_text.strip(): | |
| return "Enter verse references above and click Generate.", None | |
| lines = input_text.strip().splitlines() | |
| sections = [] | |
| for raw_line in lines: | |
| raw_line = raw_line.strip() | |
| if not raw_line: | |
| continue | |
| if "|" in raw_line: | |
| label, ref_part = raw_line.split("|", 1) | |
| label = label.strip() | |
| ref_part = ref_part.strip() | |
| else: | |
| label = None | |
| ref_part = raw_line.strip() | |
| tuples = parse_reference_or_range(ref_part) | |
| if not tuples: | |
| header = f"## {label}: {ref_part}" if label else f"## {ref_part}" | |
| sections.append(f"{header}\n\n⚠️ Could not find: {ref_part}\n\n---\n") | |
| continue | |
| # use first verse as representative for embeddings | |
| rep_book, rep_ch, rep_v = tuples[0] | |
| rep_idx = _lookup_verse_by_tuple(rep_book, rep_ch, rep_v) | |
| # collect full text across range | |
| full_texts = [] | |
| for (book, ch, v) in tuples: | |
| vi = _lookup_verse_by_tuple(book, ch, v) | |
| if vi is not None: | |
| full_texts.append(f"[{ch}:{v}] {all_verses[vi]['kjv_text']}") | |
| display_ref = ref_part | |
| header = f"## {label}: {display_ref}" if label else f"## {display_ref}" | |
| if rep_idx is None: | |
| sections.append(f"{header}\n\n⚠️ Could not find: {ref_part}\n\n---\n") | |
| continue | |
| verse_text = "\n".join(full_texts) if full_texts else all_verses[rep_idx]["kjv_text"] | |
| closest_theme = _find_closest_theme(rep_idx) | |
| # top 5 related verses | |
| q = en_embeddings[rep_idx] | |
| sims = en_embeddings @ q | |
| sims[rep_idx] = -1 | |
| top5 = np.argsort(-sims)[:5] | |
| related_lines = [] | |
| for ri in top5: | |
| rv = all_verses[ri] | |
| related_lines.append(f"- **{rv['ref']}** ({sims[ri]*100:.0f}%) — {rv['kjv_text'][:100]}") | |
| section = ( | |
| f"{header}\n\n" | |
| f"> {verse_text}\n\n" | |
| f"**Closest theme:** {closest_theme}\n\n" | |
| f"**Related verses:**\n\n" + "\n".join(related_lines) + "\n\n---\n" | |
| ) | |
| sections.append(section) | |
| full_md = "\n".join(sections) | |
| # write temp file for download | |
| tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".md", delete=False, encoding="utf-8") | |
| tmp.write(full_md) | |
| tmp.close() | |
| return full_md, tmp.name | |
| # --------------------------------------------------------------------------- | |
| # Tab 5: Concordance Frequency View | |
| # --------------------------------------------------------------------------- | |
| print("Building concordance index...") | |
| strongs_to_verses = defaultdict(list) | |
| for i, v in enumerate(all_verses): | |
| for w in v.get("words", []): | |
| strongs_to_verses[w["strongs"]].append(i) | |
| # deduplicate (a verse may repeat the same word multiple times) | |
| strongs_to_verses = {k: list(dict.fromkeys(v)) for k, v in strongs_to_verses.items()} | |
| word_to_strongs = defaultdict(set) | |
| for strongs_num, entry in lexicon.items(): | |
| translit = entry.get("transliteration", "").lower().strip() | |
| if translit: | |
| word_to_strongs[translit].add(strongs_num) | |
| gloss = entry.get("gloss", "") | |
| for word in re.split(r'[\s,;/]+', gloss.lower()): | |
| word = word.strip() | |
| if word: | |
| word_to_strongs[word].add(strongs_num) | |
| print("Concordance index ready.") | |
| # --------------------------------------------------------------------------- | |
| # Tab 6: Semantic Verse Timeline — PCA 2D projection | |
| # --------------------------------------------------------------------------- | |
| print("Computing PCA projection for semantic timeline...") | |
| from sklearn.decomposition import PCA as _PCA | |
| _pca = _PCA(n_components=2, random_state=42) | |
| _coords = _pca.fit_transform(en_embeddings).astype(np.float32) # (31102, 2) | |
| print(f"PCA done. Explained variance: {_pca.explained_variance_ratio_.sum()*100:.1f}%") | |
| # Build per-book color map for timeline (cycle through a palette) | |
| _BOOK_COLORS = [ | |
| "#E63946","#457B9D","#2A9D8F","#E9C46A","#F4A261","#264653","#6D6875", | |
| "#B5838D","#A8DADC","#95D5B2","#52B788","#1B4332","#D62828","#F77F00", | |
| "#FCBF49","#EAE2B7","#003049","#606C38","#BC6C25","#DDA15E", | |
| ] | |
| _unique_books = list(dict.fromkeys(v["book"] for v in all_verses)) # insertion order = canonical | |
| _book_color_map = {b: _BOOK_COLORS[i % len(_BOOK_COLORS)] for i, b in enumerate(_unique_books)} | |
| _SAMPLE_SIZE = 5000 # default scatter sample for "All books" | |
| _rng = np.random.default_rng(42) | |
| _sample_idx = _rng.choice(len(all_verses), size=_SAMPLE_SIZE, replace=False) | |
| def _timeline_figure(filter_book=None, color_by="Testament"): | |
| if filter_book and filter_book != "All (sampled)": | |
| idxs = [i for i, v in enumerate(all_verses) if v["book"] == filter_book] | |
| else: | |
| idxs = list(_sample_idx) | |
| xs = _coords[idxs, 0] | |
| ys = _coords[idxs, 1] | |
| if color_by == "Testament": | |
| colors = [CBC_BLUE if all_verses[i]["testament"] == "OT" else CBC_TEAL for i in idxs] | |
| colorscale_used = None | |
| else: | |
| colors = [_book_color_map.get(all_verses[i]["book"], "#888888") for i in idxs] | |
| colorscale_used = None | |
| hover = [ | |
| f"<b>{all_verses[i]['ref']}</b><br>{all_verses[i]['kjv_text'][:100]}..." | |
| for i in idxs | |
| ] | |
| customdata = [all_verses[i]["ref"] for i in idxs] | |
| trace = go.Scattergl( | |
| x=xs, y=ys, | |
| mode="markers", | |
| marker=dict(size=5, color=colors, opacity=0.65, line=dict(width=0)), | |
| hovertext=hover, hoverinfo="text", | |
| customdata=customdata, | |
| ) | |
| fig = go.Figure(trace) | |
| title_suffix = f" — {filter_book}" if (filter_book and filter_book != "All (sampled)") else " — random sample of 5,000" | |
| fig.update_layout( | |
| height=580, | |
| plot_bgcolor="#FAF8F3", paper_bgcolor="#FAF8F3", | |
| xaxis=dict(showgrid=False, zeroline=False, showticklabels=False, title="PC 1"), | |
| yaxis=dict(showgrid=False, zeroline=False, showticklabels=False, title="PC 2"), | |
| margin=dict(l=10, r=10, t=40, b=10), | |
| title=dict(text=f"Semantic landscape{title_suffix}", font=dict(size=13, color=CBC_NAVY)), | |
| ) | |
| return fig | |
| def on_timeline_update(filter_book, color_by): | |
| return _timeline_figure(filter_book, color_by) | |
| STRONGS_RE = re.compile(r'^[HG]\d+$', re.IGNORECASE) | |
| CANONICAL_BOOK_ORDER = [ | |
| "Genesis","Exodus","Leviticus","Numbers","Deuteronomy","Joshua","Judges","Ruth", | |
| "1 Samuel","2 Samuel","1 Kings","2 Kings","1 Chronicles","2 Chronicles", | |
| "Ezra","Nehemiah","Esther","Job","Psalms","Proverbs","Ecclesiastes", | |
| "Song of Solomon","Isaiah","Jeremiah","Lamentations","Ezekiel","Daniel", | |
| "Hosea","Joel","Amos","Obadiah","Jonah","Micah","Nahum","Habakkuk", | |
| "Zephaniah","Haggai","Zechariah","Malachi", | |
| "Matthew","Mark","Luke","John","Acts","Romans", | |
| "1 Corinthians","2 Corinthians","Galatians","Ephesians","Philippians", | |
| "Colossians","1 Thessalonians","2 Thessalonians","1 Timothy","2 Timothy", | |
| "Titus","Philemon","Hebrews","James","1 Peter","2 Peter", | |
| "1 John","2 John","3 John","Jude","Revelation", | |
| ] | |
| _BOOK_ORDER = {b: i for i, b in enumerate(CANONICAL_BOOK_ORDER)} | |
| _BOOK_TESTAMENT = {b: ("OT" if i < 39 else "NT") for i, b in enumerate(CANONICAL_BOOK_ORDER)} | |
| def _concordance_chart(strongs_num): | |
| idxs = strongs_to_verses.get(strongs_num.upper(), []) | |
| if not idxs: | |
| return go.Figure() | |
| book_counts = defaultdict(int) | |
| for i in idxs: | |
| book_counts[all_verses[i]["book"]] += 1 | |
| books_sorted = sorted(book_counts.keys(), key=lambda b: _BOOK_ORDER.get(b, 999)) | |
| counts = [book_counts[b] for b in books_sorted] | |
| colors = [CBC_BLUE if _BOOK_TESTAMENT.get(b) == "OT" else CBC_TEAL for b in books_sorted] | |
| fig = go.Figure(go.Bar(x=books_sorted, y=counts, marker_color=colors)) | |
| fig.update_layout( | |
| height=320, | |
| plot_bgcolor="#FAF8F3", paper_bgcolor="#FAF8F3", | |
| xaxis=dict(tickangle=-45, tickfont=dict(size=9)), | |
| yaxis=dict(title="Occurrences"), | |
| margin=dict(l=40, r=10, t=30, b=100), | |
| title=dict(text=f"{strongs_num} — occurrences by book", font=dict(size=13, color=CBC_NAVY)), | |
| ) | |
| return fig | |
| def concordance_lookup(query): | |
| query = query.strip() | |
| if not query: | |
| return "Enter a Strong's number (e.g. H7462) or an English/Greek/Hebrew word.", gr.update(value=None, visible=False), pd.DataFrame(), gr.update(choices=[], value=None, visible=False) | |
| # direct Strong's number | |
| if STRONGS_RE.match(query): | |
| return _show_strongs(query.upper()) | |
| # word/transliteration lookup | |
| matches = word_to_strongs.get(query.lower(), set()) | |
| if not matches: | |
| return f"No matches found for **{query}**. Try a Strong's number like H7462, or a different spelling.", gr.update(value=None, visible=False), pd.DataFrame(), gr.update(choices=[], value=None, visible=False) | |
| if len(matches) == 1: | |
| return _show_strongs(next(iter(matches))) | |
| # multiple matches — show picker | |
| choices = [] | |
| for sn in sorted(matches): | |
| lex = lexicon.get(sn, {}) | |
| label = f"{sn} — {lex.get('transliteration', '')} ({lex.get('gloss', '')[:40]})" | |
| choices.append(label) | |
| return ( | |
| f"Found {len(matches)} Strong's numbers matching **{query}**. Pick one below:", | |
| gr.update(value=None, visible=False), | |
| pd.DataFrame(), | |
| gr.update(choices=choices, value=None, visible=True), | |
| ) | |
| def _show_strongs(strongs_num): | |
| lex = lexicon.get(strongs_num, {}) | |
| idxs = strongs_to_verses.get(strongs_num, []) | |
| if not lex and not idxs: | |
| return f"No data found for **{strongs_num}**.", gr.update(value=None, visible=False), pd.DataFrame(), gr.update(visible=False) | |
| lex_md = ( | |
| f"### {strongs_num} — {lex.get('original_word', '')} (*{lex.get('transliteration', '')}*)\n\n" | |
| f"**Gloss:** {lex.get('gloss', '(none)')}\n\n" | |
| f"**Definition:** {lex.get('definition', '(none)')}\n\n" | |
| f"**Total occurrences:** {len(idxs)}" | |
| ) | |
| chart = _concordance_chart(strongs_num) | |
| rows = [] | |
| for i in idxs: | |
| v = all_verses[i] | |
| rows.append({"Reference": v["ref"], "Testament": v["testament"], "Text": v["kjv_text"]}) | |
| df = pd.DataFrame(rows) | |
| return lex_md, gr.update(value=chart, visible=True), df, gr.update(visible=False) | |
| def on_concordance_search(query): | |
| return concordance_lookup(query) | |
| def on_concordance_pick(choice): | |
| if not choice: | |
| return "", gr.update(value=None, visible=False), pd.DataFrame(), gr.update(visible=True) | |
| strongs_num = choice.split(" — ")[0].strip() | |
| return _show_strongs(strongs_num) | |
| # --------------------------------------------------------------------------- | |
| # Combined app | |
| # --------------------------------------------------------------------------- | |
| with gr.Blocks(title="Scripture Dashboards") as demo: | |
| gr.Markdown("# Scripture Dashboards") | |
| with gr.Tab("Network / Theme Explorer"): | |
| with gr.Row(): | |
| with gr.Column(scale=1): | |
| theme_dropdown = gr.Dropdown(choices=theme_choices, label="Theological cluster", value=None) | |
| gr.Markdown("**-- or --**") | |
| search_box_net = gr.Textbox(label="Search a theme or phrase", | |
| placeholder='e.g. "wounds in his hands and side"') | |
| search_btn_net = gr.Button("Search", variant="primary") | |
| status_md_net = gr.Markdown("") | |
| with gr.Column(scale=2): | |
| graph_plot = gr.Plot(label="Scripture network (visual)") | |
| gr.Markdown("### Verses shown above -- click a row to inspect") | |
| verse_table = gr.Dataframe(headers=["Match", "Reference", "Testament", "Text"], interactive=False, wrap=True) | |
| detail_md_net = gr.Markdown("*Click any row to see the verse and its closest matches.*") | |
| theme_dropdown.change(on_theme_select, inputs=[theme_dropdown], outputs=[graph_plot, verse_table, status_md_net]) | |
| search_btn_net.click(on_search_network, inputs=[search_box_net], outputs=[graph_plot, verse_table, status_md_net]) | |
| search_box_net.submit(on_search_network, inputs=[search_box_net], outputs=[graph_plot, verse_table, status_md_net]) | |
| verse_table.select(on_row_select_network, inputs=None, outputs=[detail_md_net]) | |
| with gr.Tab("Original Languages Word Study"): | |
| with gr.Row(): | |
| ref_input = gr.Textbox(label="Verse reference or search phrase", | |
| placeholder="e.g. 'Genesis 16:13' or 'Psalm 27:14' or 'wounds in his hands'") | |
| lookup_btn = gr.Button("Look up", variant="primary") | |
| verse_display = gr.Markdown("") | |
| gr.Markdown("### Word-by-word breakdown") | |
| word_table = gr.Dataframe(headers=["English", "Strong's", "Original", "Transliteration", "Meaning"], | |
| interactive=False, wrap=True) | |
| gr.Markdown("### Similar verses") | |
| with gr.Row(): | |
| with gr.Column(): | |
| gr.Markdown("**By English meaning**") | |
| en_results_table = gr.Dataframe(headers=["Match", "Reference", "Text (English)"], interactive=False, wrap=True) | |
| with gr.Column(): | |
| gr.Markdown("**By original-language meaning** (only for verse-reference input)") | |
| orig_results_table = gr.Dataframe(headers=["Match", "Reference", "Text (original)"], interactive=False, wrap=True) | |
| lookup_btn.click(on_lookup, inputs=[ref_input], outputs=[word_table, verse_display, en_results_table, orig_results_table]) | |
| ref_input.submit(on_lookup, inputs=[ref_input], outputs=[word_table, verse_display, en_results_table, orig_results_table]) | |
| with gr.Tab("Typology Finder"): | |
| gr.Markdown( | |
| "Find cross-testament echoes: search any OT verse to find its NT parallels, " | |
| "or browse curated typology pairs." | |
| ) | |
| typo_mode = gr.Radio( | |
| choices=["Search my own verse", "Browse curated pairs"], | |
| value="Search my own verse", | |
| label="Mode", | |
| ) | |
| # Search sub-mode | |
| with gr.Row(visible=True) as typo_search_row: | |
| typo_ref_input = gr.Textbox( | |
| label="Verse reference", | |
| placeholder="e.g. 'Isaiah 53:5' or 'Exodus 12:13'", | |
| ) | |
| typo_search_btn = gr.Button("Find echoes", variant="primary") | |
| typo_search_display = gr.Markdown("", visible=True) | |
| typo_search_table = gr.Dataframe( | |
| headers=["Match %", "Reference", "Text"], interactive=False, wrap=True, visible=True | |
| ) | |
| # Browse sub-mode | |
| typo_pair_dropdown = gr.Dropdown( | |
| choices=[p["name"] for p in TYPOLOGY_PAIRS], | |
| label="Curated typology pair", | |
| value=None, | |
| visible=False, | |
| ) | |
| typo_browse_display = gr.Markdown("", visible=False) | |
| typo_browse_table = gr.Dataframe( | |
| headers=["Match %", "Reference", "Testament", "Text"], interactive=False, wrap=True, visible=False | |
| ) | |
| def on_typo_mode(mode): | |
| search = mode == "Search my own verse" | |
| return ( | |
| gr.update(visible=search), gr.update(visible=search), gr.update(visible=search), | |
| gr.update(visible=not search), gr.update(visible=not search), gr.update(visible=not search), | |
| ) | |
| typo_mode.change( | |
| on_typo_mode, inputs=[typo_mode], | |
| outputs=[typo_search_row, typo_search_display, typo_search_table, | |
| typo_pair_dropdown, typo_browse_display, typo_browse_table], | |
| ) | |
| typo_search_btn.click(on_typology_search, inputs=[typo_ref_input], outputs=[typo_search_display, typo_search_table]) | |
| typo_ref_input.submit(on_typology_search, inputs=[typo_ref_input], outputs=[typo_search_display, typo_search_table]) | |
| typo_pair_dropdown.change(on_typology_browse, inputs=[typo_pair_dropdown], outputs=[typo_browse_display, typo_browse_table]) | |
| with gr.Tab("Preaching Calendar"): | |
| gr.Markdown( | |
| "Enter one verse or range per line. Format: `Label | Reference` (label is optional).\n\n" | |
| "Example:\n```\nAdvent 1 | Isaiah 9:6\nAdvent 2 | Luke 1:26-38\nChristmas | John 1:1-14\n```" | |
| ) | |
| cal_input = gr.Textbox( | |
| label="Sermon series / lectionary", | |
| placeholder="Advent 1 | Isaiah 9:6\nAdvent 2 | Luke 1:26-38\nChristmas | John 1:1-14", | |
| lines=8, | |
| ) | |
| cal_btn = gr.Button("Generate", variant="primary") | |
| cal_output = gr.Markdown("") | |
| cal_download = gr.File(label="Download as Markdown", visible=False) | |
| def on_generate_calendar(text): | |
| md, filepath = generate_calendar(text) | |
| return md, gr.update(value=filepath, visible=filepath is not None) | |
| cal_btn.click(on_generate_calendar, inputs=[cal_input], outputs=[cal_output, cal_download]) | |
| with gr.Tab("Concordance"): | |
| gr.Markdown( | |
| "Look up every verse where a specific original-language word appears.\n\n" | |
| "Enter a **Strong's number** (e.g. `H7462`, `G3056`) or an **English/transliterated word** (e.g. `shepherd`, `logos`)." | |
| ) | |
| with gr.Row(): | |
| conc_input = gr.Textbox(label="Strong's number or word", placeholder="e.g. H7462 or shepherd") | |
| conc_btn = gr.Button("Search", variant="primary") | |
| conc_status = gr.Markdown("") | |
| conc_picker = gr.Dropdown(label="Multiple matches — pick one:", choices=[], visible=False) | |
| conc_chart = gr.Plot(label="Occurrences by book", visible=False) | |
| conc_table = gr.Dataframe(headers=["Reference", "Testament", "Text"], interactive=False, wrap=True) | |
| conc_btn.click( | |
| on_concordance_search, | |
| inputs=[conc_input], | |
| outputs=[conc_status, conc_chart, conc_table, conc_picker], | |
| ) | |
| conc_input.submit( | |
| on_concordance_search, | |
| inputs=[conc_input], | |
| outputs=[conc_status, conc_chart, conc_table, conc_picker], | |
| ) | |
| conc_picker.change( | |
| on_concordance_pick, | |
| inputs=[conc_picker], | |
| outputs=[conc_status, conc_chart, conc_table, conc_picker], | |
| ) | |
| with gr.Tab("Semantic Timeline"): | |
| gr.Markdown( | |
| "Every verse plotted as a point in 2D semantic space (PCA of the English embeddings). " | |
| "Points close together share similar themes. **Blue = OT, teal = NT** (or color by book). " | |
| "Click any point to see the verse and its nearest neighbors." | |
| ) | |
| with gr.Row(): | |
| timeline_book = gr.Dropdown( | |
| choices=["All (sampled)"] + _unique_books, | |
| value="All (sampled)", | |
| label="Filter by book", | |
| scale=2, | |
| ) | |
| timeline_color = gr.Radio( | |
| choices=["Testament", "Book"], | |
| value="Testament", | |
| label="Color by", | |
| scale=1, | |
| ) | |
| timeline_btn = gr.Button("Update", variant="primary", scale=1) | |
| gr.Markdown("*Hover over any point to see the verse. Use the book filter to zoom into a single book.*") | |
| timeline_plot = gr.Plot(label="Semantic landscape", value=_timeline_figure()) | |
| timeline_btn.click(on_timeline_update, inputs=[timeline_book, timeline_color], outputs=[timeline_plot]) | |
| timeline_book.change(on_timeline_update, inputs=[timeline_book, timeline_color], outputs=[timeline_plot]) | |
| timeline_color.change(on_timeline_update, inputs=[timeline_book, timeline_color], outputs=[timeline_plot]) | |
| if __name__ == "__main__": | |
| demo.launch() | |