Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+twl_concordancer.db filter=lfs diff=lfs merge=lfs -text

AGENT.md

@@ -0,0 +1,1014 @@
+# TWL Concordancer — Build Instructions
+
+## Project Overview
+
+Build a bilingual concordancer web app for Taiwan Law (TWL) that allows users to search for keywords/regex across aligned Chinese-English legal texts, with expandable context at paragraph and article levels.
+
+## Directory Structure
+
+```
+/Users/rubentsui/NLP/TWL/
+├── 2026-03-27/
+│   ├── TWL.2026-03-27.json              # Raw bilingual corpus (123MB)
+│   └── TWL.2026-03-27.aligned.json      # Aligned corpus (to be generated)
+├── .opencode/skills/twl-align-corpus/
+│   ├── SKILL.md
+│   └── scripts/
+│       └── align_corpus.py              # Alignment script
+└── twl-concordancer/
+    ├── concordancer.py                  # Streamlit app (to be built)
+    ├── db.py                            # Database query layer (to be built)
+    ├── build_db.py                      # Ingestion script (to be built)
+    └── twl_concordancer.db              # SQLite database (to be generated)
+```
+
+---
+
+## Step 1: Understand the Input Corpus
+
+### Source File: `2026-03-27/TWL.2026-03-27.json`
+
+**Structure:**
+```json
+{
+  "metadata": {
+    "release_date": "2026-3-27",
+    "description": "Taiwan Law Chinese-English Bilingual Corpus"
+  },
+  "laws": [...],
+  "orders": [...]
+}
+```
+
+**Each law/order has this structure:**
+```json
+{
+  "law_id": "A0000001",
+  "zh": {
+    "LawLevel": "憲法",
+    "LawName": "中華民國憲法",
+    "LawURL": "https://law.moj.gov.tw/LawClass/LawAll.aspx?pcode=A0000001",
+    "LawCategory": "憲法",
+    "LawModifiedDate": "19470101",
+    "LawEffectiveDate": "",
+    "LawHasEngVersion": "Y",
+    "EngLawName": "Constitution of the Republic of China (Taiwan)",
+    "LawForeword": "中華民國國民大會受全體國民之付託...",
+    "LawArticles": [
+      {
+        "ArticleType": "C",
+        "ArticleNo": "",
+        "ArticleContent": "第 一 章 總綱"
+      },
+      {
+        "ArticleType": "A",
+        "ArticleNo": "第 1 條",
+        "ArticleContent": "中華民國基於三民主義，為民有民治民享之民主共和國。"
+      }
+    ]
+  },
+  "en": {
+    "LawLevel": "憲法",
+    "EngLawName": "Constitution of the Republic of China (Taiwan)",
+    "LawName": "中華民國憲法",
+    "EngLawURL": "https://law.moj.gov.tw/ENG/LawClass/LawAll.aspx?pcode=A0000001",
+    "EngLawForeword": "The National Assembly of the Republic of China...",
+    "EngLawArticles": [
+      {
+        "EngArticleType": "C",
+        "EngArticleNo": "",
+        "EngArticleContent": "   Chapter I. General Provisions"
+      },
+      {
+        "EngArticleType": "A",
+        "EngArticleNo": "Article 1",
+        "EngArticleContent": "The Republic of China, founded on the Three Principles of the People, shall be a democratic republic of the people, to be governed by the people and for the people."
+      }
+    ]
+  }
+}
+```
+
+**Key details:**
+- `ArticleType`: `"A"` = numbered article, `"C"` = chapter/section header
+- `ArticleContent` may contain `\n` for paragraph breaks within an article
+- English articles use `EngArticleNo` (e.g., `"Article 1"`) vs Chinese `ArticleNo` (e.g., `"第 1 條"`)
+- Chinese article numbers use Chinese numerals: 一, 二, 三, etc.
+- The `zh` object has `LawArticles` key; the `en` object has `EngLawArticles` key
+- Some English articles may be empty or missing
+
+---
+
+## Step 2: Align the Corpus
+
+### Prerequisites
+- Python 3.10+ with: `vecalign`, `sentence-transformers`, `torch`, `numpy`, `regex`, `sentence_splitter`, `tqdm`
+- VecAlignMulti2 directory at `/Users/rubentsui/NLP/VecAlignMulti2/` containing `dp_utils.py`, `vecalign.py`, `score.py`
+- GPU: CUDA (recommended for full corpus) or MPS (Apple Silicon)
+
+### Run Alignment
+
+```bash
+python3 .opencode/skills/twl-align-corpus/scripts/align_corpus.py \
+    2026-03-27/TWL.2026-03-27.json \
+    2026-03-27/TWL.2026-03-27.aligned.json \
+    --device cuda \
+    --alignment-max-size 7 \
+    --model LaBSE
+```
+
+**Options:**
+- `--device`: `cuda`, `mps`, or `cpu` (auto-detects if not specified)
+- `--resume`: Resume from existing output file if interrupted
+- `--law-ids A0000001 A0000002`: Process specific laws only
+- `--model`: Embedding model (default: `LaBSE`)
+
+### Alignment Output Structure
+
+The aligned JSON has this structure:
+```json
+{
+  "metadata": {
+    "release_date": "2026-3-27",
+    "description": "Taiwan Law Aligned Corpus (article, paragraph, sentence)",
+    "alignment_model": "LaBSE",
+    "alignment_max_size": 7,
+    "device": "cuda",
+    "created_at": "2026-04-03T..."
+  },
+  "laws": [
+    {
+      "law_id": "A0000001",
+      "zh_name": "中華民國憲法",
+      "en_name": "Constitution of the Republic of China (Taiwan)",
+      "category": "憲法",
+      "foreword_alignment": [
+        {
+          "score": 0.1658,
+          "zh_indices": [0],
+          "en_indices": [0],
+          "zh": "中華民國國民大會受全體國民之付託...",
+          "en": "The National Assembly of the Republic of China..."
+        }
+      ],
+      "articles": [
+        {
+          "article_no_zh": "第 1 條",
+          "article_no_en": "Article 1",
+          "article_type": "A",
+          "paragraphs": [
+            {
+              "zh_indices": [0],
+              "en_indices": [0],
+              "zh": "中華民國基於三民主義...",
+              "en": "The Republic of China, founded on...",
+              "score": 0.278,
+              "sentences": [
+                {
+                  "score": 0.278,
+                  "zh_indices": [0],
+                  "en_indices": [0],
+                  "zh": "中華民國基於三民主義，為民有民治民享之民主共和國。",
+                  "en": "The Republic of China, founded on the Three Principles of the People, shall be a democratic republic of the people, to be governed by the people and for the people."
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "orders": [...]
+}
+```
+
+**Notes:**
+- Articles are matched by number (第 X 條 ↔ Article X); unmatched articles are paired sequentially
+- Paragraphs are created by splitting `ArticleContent` on `\n`
+- Sentences within each paragraph are aligned using vecalign with LaBSE embeddings
+- `score` is cosine distance (lower = better alignment)
+
+---
+
+## Step 3: Build SQLite Database
+
+### Schema (Lean — No Duplication)
+
+Text is stored **only at sentence level**. Paragraph and article text are reconstructed from sentences on demand.
+
+```sql
+CREATE TABLE laws (
+    id INTEGER PRIMARY KEY,
+    law_id TEXT UNIQUE,           -- A0000001
+    type TEXT,                    -- 'law' or 'order'
+    zh_name TEXT,
+    en_name TEXT,
+    category TEXT,
+    effective_date TEXT,
+    modified_date TEXT
+);
+
+CREATE TABLE articles (
+    id INTEGER PRIMARY KEY,
+    law_id INTEGER REFERENCES laws(id),
+    article_no_zh TEXT,
+    article_no_en TEXT,
+    article_type TEXT,            -- 'A' = article, 'C' = chapter
+    article_index INTEGER         -- ordering within law
+);
+
+CREATE TABLE paragraphs (
+    id INTEGER PRIMARY KEY,
+    article_id INTEGER REFERENCES articles(id),
+    law_id INTEGER REFERENCES laws(id),
+    paragraph_index INTEGER       -- ordering within article
+);
+
+CREATE TABLE sentences (
+    id INTEGER PRIMARY KEY,
+    paragraph_id INTEGER REFERENCES paragraphs(id),
+    article_id INTEGER REFERENCES articles(id),
+    law_id INTEGER REFERENCES laws(id),
+    zh_text TEXT NOT NULL,
+    en_text TEXT NOT NULL,
+    alignment_score REAL,
+    zh_sentence_idx INTEGER,      -- position within zh paragraph
+    en_sentence_idx INTEGER       -- position within en paragraph
+);
+
+-- Indexes for context expansion
+CREATE INDEX idx_sentences_paragraph ON sentences(paragraph_id);
+CREATE INDEX idx_sentences_article ON sentences(article_id);
+CREATE INDEX idx_sentences_law ON sentences(law_id);
+CREATE INDEX idx_paragraphs_article ON paragraphs(article_id);
+CREATE INDEX idx_articles_law ON articles(law_id);
+```
+
+### Ingestion Script (`build_db.py`)
+
+```python
+#!/usr/bin/env python3
+"""Build SQLite database from TWL aligned corpus JSON."""
+
+import sys
+import json
+import sqlite3
+import argparse
+from pathlib import Path
+
+DDL = """
+CREATE TABLE IF NOT EXISTS laws (
+    id INTEGER PRIMARY KEY,
+    law_id TEXT UNIQUE,
+    type TEXT,
+    zh_name TEXT,
+    en_name TEXT,
+    category TEXT,
+    effective_date TEXT,
+    modified_date TEXT
+);
+
+CREATE TABLE IF NOT EXISTS articles (
+    id INTEGER PRIMARY KEY,
+    law_id INTEGER REFERENCES laws(id),
+    article_no_zh TEXT,
+    article_no_en TEXT,
+    article_type TEXT,
+    article_index INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS paragraphs (
+    id INTEGER PRIMARY KEY,
+    article_id INTEGER REFERENCES articles(id),
+    law_id INTEGER REFERENCES laws(id),
+    paragraph_index INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS sentences (
+    id INTEGER PRIMARY KEY,
+    paragraph_id INTEGER REFERENCES paragraphs(id),
+    article_id INTEGER REFERENCES articles(id),
+    law_id INTEGER REFERENCES laws(id),
+    zh_text TEXT NOT NULL,
+    en_text TEXT NOT NULL,
+    alignment_score REAL,
+    zh_sentence_idx INTEGER,
+    en_sentence_idx INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS idx_sentences_paragraph ON sentences(paragraph_id);
+CREATE INDEX IF NOT EXISTS idx_sentences_article ON sentences(article_id);
+CREATE INDEX IF NOT EXISTS idx_sentences_law ON sentences(law_id);
+CREATE INDEX IF NOT EXISTS idx_paragraphs_article ON paragraphs(article_id);
+CREATE INDEX IF NOT EXISTS idx_articles_law ON articles(law_id);
+"""
+
+
+def build_db(input_file, db_file, append=False):
+    conn = sqlite3.connect(db_file)
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute("PRAGMA foreign_keys=ON")
+    cur = conn.cursor()
+
+    if not append:
+        cur.executescript("""
+            DROP TABLE IF EXISTS sentences;
+            DROP TABLE IF EXISTS paragraphs;
+            DROP TABLE IF EXISTS articles;
+            DROP TABLE IF EXISTS laws;
+        """)
+
+    cur.executescript(DDL)
+
+    with open(input_file, encoding="utf-8") as f:
+        corpus = json.load(f)
+
+    law_count = article_count = paragraph_count = sentence_count = 0
+
+    for entry_type, key in [("law", "laws"), ("order", "orders")]:
+        items = corpus.get(key, [])
+        for item in items:
+            law_id = item.get("law_id") or item.get("order_id", "")
+            zh_name = item.get("zh_name", "")
+            en_name = item.get("en_name", "")
+            category = item.get("category", "")
+
+            try:
+                cur.execute(
+                    "INSERT INTO laws (law_id, type, zh_name, en_name, category) VALUES (?, ?, ?, ?, ?)",
+                    (law_id, entry_type, zh_name, en_name, category),
+                )
+            except sqlite3.IntegrityError:
+                if append:
+                    cur.execute("SELECT id FROM laws WHERE law_id = ?", (law_id,))
+                    row = cur.fetchone()
+                    if row:
+                        cur.execute(
+                            "UPDATE laws SET zh_name=?, en_name=?, category=? WHERE id=?",
+                            (zh_name, en_name, category, row[0]),
+                        )
+                        law_db_id = row[0]
+                    else:
+                        continue
+                else:
+                    raise
+            else:
+                law_db_id = cur.lastrowid
+
+            law_count += 1
+            articles = item.get("articles", [])
+
+            for art_idx, art in enumerate(articles):
+                article_no_zh = art.get("article_no_zh", "")
+                article_no_en = art.get("article_no_en", "")
+                article_type = art.get("article_type", "")
+
+                cur.execute(
+                    "INSERT INTO articles (law_id, article_no_zh, article_no_en, article_type, article_index) VALUES (?, ?, ?, ?, ?)",
+                    (law_db_id, article_no_zh, article_no_en, article_type, art_idx),
+                )
+                article_db_id = cur.lastrowid
+                article_count += 1
+
+                paragraphs = art.get("paragraphs", [])
+                for para_idx, para in enumerate(paragraphs):
+                    cur.execute(
+                        "INSERT INTO paragraphs (article_id, law_id, paragraph_index) VALUES (?, ?, ?)",
+                        (article_db_id, law_db_id, para_idx),
+                    )
+                    paragraph_db_id = cur.lastrowid
+                    paragraph_count += 1
+
+                    aligned_sentences = para.get("sentences", [])
+                    for sent_idx, sent in enumerate(aligned_sentences):
+                        zh_text = sent.get("zh", "")
+                        en_text = sent.get("en", "")
+                        score = sent.get("score", 0.0)
+                        zh_sidx = sent.get("zh_indices", [sent_idx])[0] if sent.get("zh_indices") else sent_idx
+                        en_sidx = sent.get("en_indices", [sent_idx])[0] if sent.get("en_indices") else sent_idx
+
+                        if zh_text or en_text:
+                            cur.execute(
+                                "INSERT INTO sentences (paragraph_id, article_id, law_id, zh_text, en_text, alignment_score, zh_sentence_idx, en_sentence_idx) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                                (paragraph_db_id, article_db_id, law_db_id, zh_text, en_text, score, zh_sidx, en_sidx),
+                            )
+                            sentence_count += 1
+
+            if law_count % 100 == 0:
+                conn.commit()
+                print(f"  Processed {law_count} {entry_type}s, {sentence_count} sentences...")
+
+    conn.commit()
+    print(f"\nDatabase built: {db_file}")
+    print(f"  Laws/orders: {law_count}")
+    print(f"  Articles: {article_count}")
+    print(f"  Paragraphs: {paragraph_count}")
+    print(f"  Sentences: {sentence_count}")
+    conn.close()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input_file")
+    parser.add_argument("db_file")
+    parser.add_argument("--append", action="store_true")
+    args = parser.parse_args()
+    build_db(args.input_file, args.db_file, append=args.append)
+```
+
+**Run:**
+```bash
+python3 twl-concordancer/build_db.py \
+    2026-03-27/TWL.2026-03-27.aligned.json \
+    twl-concordancer/twl_concordancer.db
+```
+
+---
+
+## Step 4: Database Query Layer (`db.py`)
+
+```python
+"""Database query layer for TWL Concordancer."""
+
+import sqlite3
+import regex
+from pathlib import Path
+
+DEFAULT_DB = Path(__file__).parent / "twl_concordancer.db"
+
+
+def get_conn(db_path=None):
+    if db_path is None:
+        db_path = DEFAULT_DB
+    conn = sqlite3.connect(str(db_path))
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA journal_mode=WAL")
+    return conn
+
+
+def search_sentences(conn, query, use_regex=False, law_id=None, article_no=None,
+                     max_score=None, lang="both", limit=100, offset=0):
+    if use_regex:
+        return _search_regex(conn, query, law_id, article_no, max_score, lang, limit, offset)
+    return _search_like(conn, query, law_id, article_no, max_score, lang, limit, offset)
+
+
+def _search_like(conn, query, law_id, article_no, max_score, lang, limit, offset):
+    """LIKE-based search. Works for both Chinese and English."""
+    terms = query.strip()
+    if not terms:
+        return [], 0
+
+    where = []
+    params = []
+
+    if lang == "zh":
+        where.append("s.zh_text LIKE ?")
+        params.append(f"%{terms}%")
+    elif lang == "en":
+        where.append("s.en_text LIKE ?")
+        params.append(f"%{terms}%")
+    else:
+        where.append("(s.zh_text LIKE ? OR s.en_text LIKE ?)")
+        params.extend([f"%{terms}%", f"%{terms}%"])
+
+    if max_score is not None:
+        where.append("s.alignment_score <= ?")
+        params.append(max_score)
+    if law_id:
+        where.append("l.law_id = ?")
+        params.append(law_id)
+    if article_no:
+        pat = f"%{article_no}%"
+        where.append("(a.article_no_zh LIKE ? OR a.article_no_en LIKE ?)")
+        params.extend([pat, pat])
+
+    where_clause = " AND ".join(where)
+
+    count_sql = f"""
+        SELECT count(*) FROM sentences s
+        JOIN laws l ON s.law_id = l.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE {where_clause}
+    """
+
+    data_sql = f"""
+        SELECT s.id, s.zh_text, s.en_text, s.alignment_score,
+               l.law_id, l.zh_name, l.en_name, l.type,
+               a.article_no_zh, a.article_no_en, a.article_type,
+               s.zh_sentence_idx, s.en_sentence_idx
+        FROM sentences s
+        JOIN laws l ON s.law_id = l.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE {where_clause}
+        ORDER BY s.alignment_score
+        LIMIT ? OFFSET ?
+    """
+    data_params = params + [limit, offset]
+
+    cur = conn.execute(count_sql, params)
+    total = cur.fetchone()[0]
+
+    cur = conn.execute(data_sql, data_params)
+    rows = [dict(r) for r in cur.fetchall()]
+    return rows, total
+
+
+def _search_regex(conn, pattern, law_id, article_no, max_score, lang, limit, offset):
+    """Regex search using custom REGEXP function."""
+    try:
+        regex.compile(pattern)
+    except regex.error:
+        return [], 0
+
+    where = "1=1"
+    params = []
+
+    if lang == "zh":
+        where += " AND s.zh_text REGEXP ?"
+        params.append(pattern)
+    elif lang == "en":
+        where += " AND s.en_text REGEXP ?"
+        params.append(pattern)
+    else:
+        where += " AND (s.zh_text REGEXP ? OR s.en_text REGEXP ?)"
+        params.extend([pattern, pattern])
+
+    if max_score is not None:
+        where += " AND s.alignment_score <= ?"
+        params.append(max_score)
+    if law_id:
+        where += " AND l.law_id = ?"
+        params.append(law_id)
+    if article_no:
+        where += " AND (a.article_no_zh LIKE ? OR a.article_no_en LIKE ?)"
+        pat = f"%{article_no}%"
+        params.extend([pat, pat])
+
+    count_sql = f"""
+        SELECT count(*) FROM sentences s
+        JOIN laws l ON s.law_id = l.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE {where}
+    """
+
+    data_sql = f"""
+        SELECT s.id, s.zh_text, s.en_text, s.alignment_score,
+               l.law_id, l.zh_name, l.en_name, l.type,
+               a.article_no_zh, a.article_no_en, a.article_type,
+               s.zh_sentence_idx, s.en_sentence_idx
+        FROM sentences s
+        JOIN laws l ON s.law_id = l.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE {where}
+        ORDER BY s.alignment_score
+        LIMIT ? OFFSET ?
+    """
+    data_params = params + [limit, offset]
+
+    # Register REGEXP function
+    conn.create_function("REGEXP", 2, lambda pat, txt: bool(regex.search(pat, txt)) if txt else False)
+
+    cur = conn.execute(count_sql, params)
+    total = cur.fetchone()[0]
+
+    cur = conn.execute(data_sql, data_params)
+    rows = [dict(r) for r in cur.fetchall()]
+    return rows, total
+
+
+def get_paragraph(conn, sentence_id):
+    """Get all sentences in the same paragraph as the given sentence.
+    Returns BOTH zh and en text for every sentence."""
+    cur = conn.execute(
+        """
+        SELECT s.id, s.zh_text, s.en_text, s.alignment_score,
+               s.zh_sentence_idx, s.en_sentence_idx,
+               p.paragraph_index, a.article_no_zh, a.article_no_en
+        FROM sentences s
+        JOIN paragraphs p ON s.paragraph_id = p.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE s.paragraph_id = (SELECT paragraph_id FROM sentences WHERE id = ?)
+        ORDER BY s.zh_sentence_idx
+        """,
+        (sentence_id,),
+    )
+    rows = [dict(r) for r in cur.fetchall()]
+    if not rows:
+        return None
+    return {
+        "paragraph_index": rows[0]["paragraph_index"],
+        "article_no_zh": rows[0]["article_no_zh"],
+        "article_no_en": rows[0]["article_no_en"],
+        "sentences": rows,
+    }
+
+
+def get_article(conn, sentence_id):
+    """Get all sentences in the same article as the given sentence.
+    Returns BOTH zh and en text for every sentence, grouped by paragraph."""
+    cur = conn.execute(
+        """
+        SELECT s.id, s.zh_text, s.en_text, s.alignment_score,
+               s.zh_sentence_idx, s.en_sentence_idx,
+               p.paragraph_index, p.id as paragraph_id,
+               a.article_no_zh, a.article_no_en, a.article_type
+        FROM sentences s
+        JOIN paragraphs p ON s.paragraph_id = p.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE s.article_id = (SELECT article_id FROM sentences WHERE id = ?)
+        ORDER BY p.paragraph_index, s.zh_sentence_idx
+        """,
+        (sentence_id,),
+    )
+    rows = [dict(r) for r in cur.fetchall()]
+    if not rows:
+        return None
+
+    paragraphs = {}
+    for r in rows:
+        pidx = r["paragraph_index"]
+        if pidx not in paragraphs:
+            paragraphs[pidx] = {
+                "paragraph_index": pidx,
+                "paragraph_id": r["paragraph_id"],
+                "sentences": [],
+            }
+        paragraphs[pidx]["sentences"].append({
+            "id": r["id"],
+            "zh_text": r["zh_text"],
+            "en_text": r["en_text"],
+            "alignment_score": r["alignment_score"],
+            "zh_sentence_idx": r["zh_sentence_idx"],
+            "en_sentence_idx": r["en_sentence_idx"],
+        })
+
+    return {
+        "article_no_zh": rows[0]["article_no_zh"],
+        "article_no_en": rows[0]["article_no_en"],
+        "article_type": rows[0]["article_type"],
+        "paragraphs": [paragraphs[k] for k in sorted(paragraphs.keys())],
+    }
+
+
+def list_laws(conn, law_type=None, category=None):
+    where = []
+    params = []
+    if law_type:
+        where.append("type = ?")
+        params.append(law_type)
+    if category:
+        where.append("category = ?")
+        params.append(category)
+
+    where_clause = " AND ".join(where) if where else "1=1"
+    cur = conn.execute(
+        f"SELECT law_id, type, zh_name, en_name, category FROM laws WHERE {where_clause} ORDER BY law_id",
+        params,
+    )
+    return [dict(r) for r in cur.fetchall()]
+
+
+def list_categories(conn, law_type=None):
+    if law_type:
+        where = "WHERE type = ? AND category IS NOT NULL AND category != ''"
+        params = [law_type]
+    else:
+        where = "WHERE category IS NOT NULL AND category != ''"
+        params = []
+    cur = conn.execute(
+        f"SELECT DISTINCT category FROM laws {where} ORDER BY category",
+        params,
+    )
+    return [r["category"] for r in cur.fetchall()]
+
+
+def get_law_articles(conn, law_id):
+    cur = conn.execute(
+        """
+        SELECT article_no_zh, article_no_en, article_type, article_index
+        FROM articles
+        WHERE law_id = (SELECT id FROM laws WHERE law_id = ?)
+        ORDER BY article_index
+        """,
+        (law_id,),
+    )
+    return [dict(r) for r in cur.fetchall()]
+```
+
+---
+
+## Step 5: Streamlit App (`concordancer.py`)
+
+### Critical Requirements
+
+1. **Use `regex` library, NOT `re`** — The `re` module has issues with some Unicode patterns. Use `import regex` instead.
+
+2. **Use `st.html()` NOT `st.markdown(..., unsafe_allow_html=True)`** for expanded paragraph/article views. `st.markdown()` strips text before the first HTML tag. `st.html()` renders raw HTML correctly.
+
+3. **Always HTML-escape text before inserting into HTML** — Use `html.escape()` to prevent `<`, `>`, `&` in legal text from being interpreted as HTML.
+
+4. **Match detection uses sentence `id`, NOT indices** — Compare `s["id"] == sid` to highlight the matched sentence in expanded views. Do NOT use `zh_sentence_idx` comparison — it breaks when searching English-only.
+
+### Full Implementation
+
+```python
+"""TWL Bilingual Concordancer — Streamlit App."""
+
+import html
+import regex
+import streamlit as st
+import db
+from pathlib import Path
+
+st.set_page_config(page_title="TWL Concordancer", page_icon="⚖️", layout="wide")
+
+DB_PATH = Path(__file__).parent / "twl_concordancer.db"
+
+st.markdown(
+    """
+<style>
+.zh-text, .en-text {
+    line-height: 1.8;
+    padding: 6px 10px;
+    border-radius: 4px;
+    font-size: 0.95rem;
+    white-space: pre-wrap;
+}
+.zh-text {
+    font-family: "Noto Serif TC", "Source Han Serif TC", "MingLiU", serif;
+}
+.en-text {
+    font-family: "Source Sans 3", "Segoe UI", sans-serif;
+}
+.zh-text.match, .en-text.match {
+    background-color: #fff9e6;
+    border-left: 3px solid #f5c518;
+}
+</style>
+""",
+    unsafe_allow_html=True,
+)
+
+
+def _highlight(text, query):
+    """Highlight query in text. Always HTML-escapes text first."""
+    if not text or not query:
+        return html.escape(text)
+    escaped_text = html.escape(text)
+    escaped_query = html.escape(query)
+    return regex.sub(
+        rf"({regex.escape(escaped_query)})",
+        r'<mark style="background:#fef08a;padding:1px 2px;border-radius:2px">\1</mark>',
+        escaped_text,
+        flags=regex.IGNORECASE | regex.V1,
+    )
+
+
+if "page" not in st.session_state:
+    st.session_state.page = 0
+if "expanded" not in st.session_state:
+    st.session_state.expanded = {}
+
+st.title("⚖️ TWL Bilingual Concordancer")
+st.caption("Taiwan Law Chinese–English Aligned Corpus")
+
+conn = db.get_conn(DB_PATH)
+
+# ── Sidebar Filters ──────────────────────────────────────────
+with st.sidebar:
+    st.header("Filters")
+
+    law_types = ["All", "law", "order"]
+    selected_type = st.selectbox("Type", law_types, index=0)
+    type_filter = None if selected_type == "All" else selected_type
+
+    categories = db.list_categories(conn, type_filter)
+    selected_cat = st.selectbox("Category", ["All"] + categories, index=0)
+    cat_filter = None if selected_cat == "All" else selected_cat
+
+    laws = db.list_laws(conn, type_filter, cat_filter)
+    law_options = ["All"] + [f"{l['law_id']} — {l['zh_name']}" for l in laws]
+    selected_law = st.selectbox("Law / Order", law_options, index=0)
+    law_id_filter = None
+    if selected_law != "All":
+        law_id_filter = selected_law.split(" — ")[0]
+
+    max_score = st.slider("Max alignment score (lower = better)", 0.0, 1.0, 1.0, 0.05)
+    max_score_filter = None if max_score >= 1.0 else max_score
+
+    lang_options = ["Both", "Chinese only", "English only"]
+    selected_lang = st.radio("Search language", lang_options, index=0)
+    lang_filter = {"Both": "both", "Chinese only": "zh", "English only": "en"}[selected_lang]
+
+    st.divider()
+    st.caption(f"{len(laws)} laws/orders in database")
+
+st.divider()
+
+# ── Search Bar ───────────────────────────────────────────────
+col1, col2, col3 = st.columns([4, 1, 1])
+with col1:
+    query = st.text_input("Search", placeholder="Enter keyword or regex…", key="search_query")
+with col2:
+    use_regex = st.checkbox("Regex", value=False)
+with col3:
+    per_page = st.selectbox("Per page", [10, 25, 50, 100], index=1)
+
+# Article filter (only shown when a law is selected)
+article_filter = None
+if law_id_filter:
+    articles = db.get_law_articles(conn, law_id_filter)
+    art_options = ["All"] + [
+        f"{a['article_no_zh']} / {a['article_no_en']}"
+        for a in articles if a["article_no_zh"] or a["article_no_en"]
+    ]
+    selected_art = st.selectbox("Article", art_options, index=0)
+    if selected_art != "All":
+        article_filter = selected_art.split(" / ")[0]
+
+# ── Search & Display ─────────────────────────────────────────
+if query:
+    results, total = db.search_sentences(
+        conn, query, use_regex=use_regex,
+        law_id=law_id_filter, article_no=article_filter,
+        max_score=max_score_filter, lang=lang_filter,
+        limit=per_page, offset=st.session_state.page * per_page,
+    )
+
+    st.write(f"**{total}** sentence pair{'s' if total != 1 else ''} found")
+
+    # Pagination
+    if total > per_page:
+        total_pages = (total + per_page - 1) // per_page
+        cols = st.columns([1, 4, 1])
+        with cols[0]:
+            if st.button("← Previous", disabled=st.session_state.page == 0, use_container_width=True):
+                st.session_state.page -= 1
+                st.session_state.expanded = {}
+        with cols[1]:
+            st.write(f"Page {st.session_state.page + 1} of {total_pages}")
+        with cols[2]:
+            if st.button("Next →", disabled=(st.session_state.page + 1) * per_page >= total, use_container_width=True):
+                st.session_state.page += 1
+                st.session_state.expanded = {}
+
+    for row in results:
+        sid = row["id"]
+        score = row["alignment_score"]
+        law_ref = f"{row['law_id']} {row['zh_name']}"
+        art_ref = f"{row['article_no_zh']} / {row['article_no_en']}" if row["article_no_zh"] or row["article_no_en"] else ""
+
+        with st.container(border=True):
+            st.markdown(f"`{law_ref}`{' | ' + art_ref if art_ref else ''}  |  Score: `{score:.4f}`")
+
+            # Sentence-level display (always bilingual)
+            zh_text = row["zh_text"] or ""
+            en_text = row["en_text"] or ""
+
+            if query and not use_regex:
+                zh_display = _highlight(zh_text, query)
+                en_display = _highlight(en_text, query)
+            else:
+                zh_display = html.escape(zh_text)
+                en_display = html.escape(en_text)
+
+            col_zh, col_en = st.columns(2)
+            with col_zh:
+                st.html(f'<div class="zh-text">{zh_display}</div>')
+            with col_en:
+                st.html(f'<div class="en-text">{en_display}</div>')
+
+            # Expand buttons
+            exp_col1, exp_col2 = st.columns(2)
+            with exp_col1:
+                if st.button("▸ Paragraph", key=f"para_{sid}"):
+                    st.session_state.expanded[f"para_{sid}"] = not st.session_state.expanded.get(f"para_{sid}", False)
+            with exp_col2:
+                if st.button("▸ Article", key=f"art_{sid}"):
+                    st.session_state.expanded[f"art_{sid}"] = not st.session_state.expanded.get(f"art_{sid}", False)
+
+            # ── Expanded Paragraph View ──────────────────────
+            if st.session_state.expanded.get(f"para_{sid}"):
+                para = db.get_paragraph(conn, sid)
+                if para:
+                    with st.container(border=True):
+                        st.markdown(f"**Paragraph** ({para['article_no_zh']} / {para['article_no_en']})")
+                        for s in para["sentences"]:
+                            p_zh = s["zh_text"] or ""
+                            p_en = s["en_text"] or ""
+                            if query and not use_regex:
+                                p_zh_display = _highlight(p_zh, query)
+                                p_en_display = _highlight(p_en, query)
+                            else:
+                                p_zh_display = html.escape(p_zh)
+                                p_en_display = html.escape(p_en)
+
+                            # CRITICAL: Match by sentence id, NOT by index
+                            is_match = s["id"] == sid
+                            marker = "◀" if is_match else ""
+
+                            c1, c2 = st.columns(2)
+                            with c1:
+                                st.html(f'<div class="zh-text{" match" if is_match else ""}>{marker} {p_zh_display}</div>')
+                            with c2:
+                                st.html(f'<div class="en-text{" match" if is_match else ""}>{marker} {p_en_display}</div>')
+
+            # ── Expanded Article View ────────────────────────
+            if st.session_state.expanded.get(f"art_{sid}"):
+                article = db.get_article(conn, sid)
+                if article:
+                    with st.container(border=True):
+                        st.markdown(f"**Article** ({article['article_no_zh']} / {article['article_no_en']})")
+                        for pi, para in enumerate(article["paragraphs"]):
+                            st.markdown(f"*Paragraph {pi + 1}*")
+                            for s in para["sentences"]:
+                                a_zh = s["zh_text"] or ""
+                                a_en = s["en_text"] or ""
+                                if query and not use_regex:
+                                    a_zh_display = _highlight(a_zh, query)
+                                    a_en_display = _highlight(a_en, query)
+                                else:
+                                    a_zh_display = html.escape(a_zh)
+                                    a_en_display = html.escape(a_en)
+
+                                # CRITICAL: Match by sentence id, NOT by index
+                                is_match = s["id"] == sid
+                                marker = "◀" if is_match else ""
+
+                                c1, c2 = st.columns(2)
+                                with c1:
+                                    st.html(f'<div class="zh-text{" match" if is_match else ""}>{marker} {a_zh_display}</div>')
+                                with c2:
+                                    st.html(f'<div class="en-text{" match" if is_match else ""}>{marker} {a_en_display}</div>')
+
+elif not query:
+    st.info("Enter a search term above to find aligned sentence pairs.")
+
+st.divider()
+st.caption("TWL Concordancer | Taiwan Law Bilingual Corpus")
+
+conn.close()
+```
+
+---
+
+## Step 6: Run
+
+```bash
+# 1. Align the corpus (on CUDA machine)
+python3 .opencode/skills/twl-align-corpus/scripts/align_corpus.py \
+    2026-03-27/TWL.2026-03-27.json \
+    2026-03-27/TWL.2026-03-27.aligned.json \
+    --device cuda --resume
+
+# 2. Build SQLite database
+python3 twl-concordancer/build_db.py \
+    2026-03-27/TWL.2026-03-27.aligned.json \
+    twl-concordancer/twl_concordancer.db
+
+# 3. Launch Streamlit app
+streamlit run twl-concordancer/concordancer.py
+```
+
+---
+
+## Known Pitfalls & Fixes
+
+### 1. `list_categories` SQL syntax error
+**Bug:** `SELECT DISTINCT category FROM laws AND category IS NOT NULL` — missing `WHERE` when `law_type` is `None`.
+**Fix:** Always include `WHERE` clause.
+
+### 2. Paragraph/Article shows only text AFTER the match
+**Bug:** `st.markdown(..., unsafe_allow_html=True)` strips text before the first HTML tag (`<mark>`).
+**Fix:** Use `st.html()` instead of `st.markdown()` for all HTML rendering.
+
+### 3. Match highlighting highlights wrong sentence
+**Bug:** Comparing `s["zh_sentence_idx"] == row["zh_sentence_idx"]` fails when searching English-only (the matched sentence's Chinese index may differ).
+**Fix:** Compare by sentence `id`: `is_match = s["id"] == sid`.
+
+### 4. HTML injection from legal text
+**Bug:** Legal text contains `<`, `>`, `&` characters that break HTML rendering.
+**Fix:** Always `html.escape()` text BEFORE inserting `<mark>` tags.
+
+### 5. `get_article()` missing `id` key
+**Bug:** The dict builder in `get_article()` dropped the `s.id` column from the SQL result.
+**Fix:** Include `"id": r["id"]` in the sentence dict.
+
+### 6. Regex library
+**Requirement:** Use `regex` (PyPI), NOT `re` (stdlib). The `regex` library handles Unicode better and provides `regex.V1` flag for proper escape behavior.
+
+---
+
+## Expected Database Size
+
+| Metric | Full Corpus (est.) |
+|--------|-------------------|
+| Laws | ~967 |
+| Orders | ~2,193 |
+| Articles | ~50,000+ |
+| Sentences | ~300,000+ |
+| DB Size | ~200-400MB |
+
+LIKE search on 300K rows is fast enough (<100ms) with proper indexes. No FTS5 needed.

README.md

@@ -1,20 +1,23 @@
 ---
-title: TWL
-emoji: 🚀
-colorFrom: red
-colorTo: red
-sdk: docker
-app_port: 8501
-tags:
-- streamlit
+title: TWL Concordancer
+emoji: ⚖️
+colorFrom: blue
+colorTo: gray
+sdk: streamlit
+sdk_version: 1.39.0
+app_file: app.py
 pinned: false
-short_description: Taiwan Law Database
 license: mit
 ---
 
-# Welcome to Streamlit!
+# TWL Concordancer
 
-Edit `/src/streamlit_app.py` to customize this app to your heart's desire. :heart:
+Streamlit bilingual concordancer for the Taiwan Law Chinese-English aligned corpus.
 
-If you have any questions, checkout our [documentation](https://docs.streamlit.io) and [community
-forums](https://discuss.streamlit.io).
+This Space expects the SQLite database file `twl_concordancer.db` to be present at the repository root.
+
+## Local Development
+
+```bash
+streamlit run app.py
+```

app.py

@@ -0,0 +1,3 @@
+"""Hugging Face Spaces entrypoint for the TWL concordancer."""
+
+import concordancer  # noqa: F401

build_db.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+"""
+Build SQLite database from TWL aligned corpus JSON.
+
+Usage:
+    python3 build_db.py 2026-03-27/TWL.2026-03-27.aligned.json twl_concordancer.db
+    python3 build_db.py 2026-03-27/A0000001.aligned.json twl_concordancer.db --append
+"""
+
+import sys
+import json
+import sqlite3
+import argparse
+from pathlib import Path
+
+
+DDL = """
+CREATE TABLE IF NOT EXISTS laws (
+    id INTEGER PRIMARY KEY,
+    law_id TEXT UNIQUE,
+    type TEXT,
+    zh_name TEXT,
+    en_name TEXT,
+    category TEXT,
+    effective_date TEXT,
+    modified_date TEXT
+);
+
+CREATE TABLE IF NOT EXISTS articles (
+    id INTEGER PRIMARY KEY,
+    law_id INTEGER REFERENCES laws(id),
+    article_no_zh TEXT,
+    article_no_en TEXT,
+    article_type TEXT,
+    article_index INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS paragraphs (
+    id INTEGER PRIMARY KEY,
+    article_id INTEGER REFERENCES articles(id),
+    law_id INTEGER REFERENCES laws(id),
+    paragraph_index INTEGER
+);
+
+CREATE TABLE IF NOT EXISTS sentences (
+    id INTEGER PRIMARY KEY,
+    paragraph_id INTEGER REFERENCES paragraphs(id),
+    article_id INTEGER REFERENCES articles(id),
+    law_id INTEGER REFERENCES laws(id),
+    zh_text TEXT NOT NULL,
+    en_text TEXT NOT NULL,
+    alignment_score REAL,
+    zh_sentence_idx INTEGER,
+    en_sentence_idx INTEGER
+);
+
+CREATE INDEX IF NOT EXISTS idx_sentences_paragraph ON sentences(paragraph_id);
+CREATE INDEX IF NOT EXISTS idx_sentences_article ON sentences(article_id);
+CREATE INDEX IF NOT EXISTS idx_sentences_law ON sentences(law_id);
+CREATE INDEX IF NOT EXISTS idx_paragraphs_article ON paragraphs(article_id);
+CREATE INDEX IF NOT EXISTS idx_articles_law ON articles(law_id);
+"""
+
+TRIGGERS = ""
+
+
+def build_db(input_file, db_file, append=False):
+    conn = sqlite3.connect(db_file)
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.execute("PRAGMA foreign_keys=ON")
+    cur = conn.cursor()
+
+    if not append:
+        cur.executescript(
+            "DROP TABLE IF EXISTS sentences; DROP TABLE IF EXISTS paragraphs; DROP TABLE IF EXISTS articles; DROP TABLE IF EXISTS laws; DROP TABLE IF EXISTS sentence_fts;"
+        )
+
+    cur.executescript(DDL)
+    cur.executescript(TRIGGERS)
+
+    with open(input_file, encoding="utf-8") as f:
+        corpus = json.load(f)
+
+    law_count = 0
+    article_count = 0
+    paragraph_count = 0
+    sentence_count = 0
+
+    for entry_type, key in [("law", "laws"), ("order", "orders")]:
+        items = corpus.get(key, [])
+        for item in items:
+            law_id = item.get("law_id") or item.get("order_id", "")
+            zh_name = item.get("zh_name", "")
+            en_name = item.get("en_name", "")
+            category = item.get("category", "")
+
+            try:
+                cur.execute(
+                    "INSERT INTO laws (law_id, type, zh_name, en_name, category) VALUES (?, ?, ?, ?, ?)",
+                    (law_id, entry_type, zh_name, en_name, category),
+                )
+            except sqlite3.IntegrityError:
+                if append:
+                    cur.execute("SELECT id FROM laws WHERE law_id = ?", (law_id,))
+                    row = cur.fetchone()
+                    if row:
+                        cur.execute(
+                            "UPDATE laws SET zh_name = ?, en_name = ?, category = ? WHERE id = ?",
+                            (zh_name, en_name, category, row[0]),
+                        )
+                        law_db_id = row[0]
+                    else:
+                        cur.execute(
+                            "INSERT INTO laws (law_id, type, zh_name, en_name, category) VALUES (?, ?, ?, ?, ?)",
+                            (law_id, entry_type, zh_name, en_name, category),
+                        )
+                        law_db_id = cur.lastrowid
+                else:
+                    raise
+            else:
+                law_db_id = cur.lastrowid
+
+            law_count += 1
+
+            articles = item.get("articles", [])
+            for art_idx, art in enumerate(articles):
+                article_no_zh = art.get("article_no_zh", "")
+                article_no_en = art.get("article_no_en", "")
+                article_type = art.get("article_type", "")
+
+                cur.execute(
+                    "INSERT INTO articles (law_id, article_no_zh, article_no_en, article_type, article_index) VALUES (?, ?, ?, ?, ?)",
+                    (law_db_id, article_no_zh, article_no_en, article_type, art_idx),
+                )
+                article_db_id = cur.lastrowid
+                article_count += 1
+
+                paragraphs = art.get("paragraphs", [])
+                for para_idx, para in enumerate(paragraphs):
+                    cur.execute(
+                        "INSERT INTO paragraphs (article_id, law_id, paragraph_index) VALUES (?, ?, ?)",
+                        (article_db_id, law_db_id, para_idx),
+                    )
+                    paragraph_db_id = cur.lastrowid
+                    paragraph_count += 1
+
+                    aligned_sentences = para.get("sentences", [])
+                    for sent_idx, sent in enumerate(aligned_sentences):
+                        zh_text = sent.get("zh", "")
+                        en_text = sent.get("en", "")
+                        score = sent.get("score", 0.0)
+                        zh_sidx = (
+                            sent.get("zh_indices", [sent_idx])[0]
+                            if sent.get("zh_indices")
+                            else sent_idx
+                        )
+                        en_sidx = (
+                            sent.get("en_indices", [sent_idx])[0]
+                            if sent.get("en_indices")
+                            else sent_idx
+                        )
+
+                        if zh_text or en_text:
+                            cur.execute(
+                                "INSERT INTO sentences (paragraph_id, article_id, law_id, zh_text, en_text, alignment_score, zh_sentence_idx, en_sentence_idx) VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
+                                (
+                                    paragraph_db_id,
+                                    article_db_id,
+                                    law_db_id,
+                                    zh_text,
+                                    en_text,
+                                    score,
+                                    zh_sidx,
+                                    en_sidx,
+                                ),
+                            )
+                            sentence_count += 1
+
+            if law_count % 100 == 0:
+                conn.commit()
+                print(
+                    f"  Processed {law_count} {entry_type}s, {sentence_count} sentences..."
+                )
+
+    conn.commit()
+
+    print(f"\nDatabase built: {db_file}")
+    print(f"  Laws/orders: {law_count}")
+    print(f"  Articles: {article_count}")
+    print(f"  Paragraphs: {paragraph_count}")
+    print(f"  Sentences: {sentence_count}")
+
+    cur.execute("SELECT count(*) FROM sentences")
+    sent_count = cur.fetchone()[0]
+    print(f"  Sentence records: {sent_count}")
+
+    conn.close()
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Build SQLite concordancer DB from TWL aligned JSON"
+    )
+    parser.add_argument("input_file", help="Path to aligned corpus JSON")
+    parser.add_argument("db_file", help="Output SQLite database path")
+    parser.add_argument(
+        "--append",
+        action="store_true",
+        help="Append to existing database instead of replacing",
+    )
+    args = parser.parse_args()
+
+    build_db(args.input_file, args.db_file, append=args.append)

concordancer.py

@@ -0,0 +1,369 @@
+"""TWL Bilingual Concordancer — Streamlit App."""
+
+import html
+from pathlib import Path
+
+import regex
+import streamlit as st
+
+import db
+
+st.set_page_config(page_title="TWL Concordancer", page_icon="⚖️", layout="wide")
+
+DB_PATH = Path(__file__).parent / "twl_concordancer.db"
+
+st.markdown(
+    """
+<style>
+section[data-testid="stSidebar"] > div:first-child {
+    top: 0;
+    height: 100vh;
+}
+section[data-testid="stSidebar"] div[data-testid="stSidebarContent"] {
+    padding-top: 0rem !important;
+    margin-top: 0rem !important;
+}
+section[data-testid="stSidebar"] div[data-testid="stSidebarHeader"] {
+    min-height: 0rem !important;
+    height: 0.25rem !important;
+    padding-top: 0rem !important;
+    padding-bottom: 0rem !important;
+    margin-bottom: 0rem !important;
+}
+section[data-testid="stSidebar"] div[data-testid="stSidebarUserContent"] {
+    padding-top: 0rem !important;
+    margin-top: 0rem !important;
+}
+div[data-testid="stMainBlockContainer"],
+.main .block-container {
+    padding-top: 1.2rem;
+}
+.zh-text, .en-text {
+    line-height: 1.8;
+    padding: 6px 10px;
+    border-radius: 4px;
+    white-space: pre-wrap;
+    color: var(--text-color);
+    word-break: break-word;
+}
+.zh-text {
+    font-family: "Microsoft JhengHei", "Source Han Sans", "Noto Sans CJK TC Regular", "Hiragino Sans CNS", "LantingHei TC", "Source Han Serif", sans-serif;
+    font-size: 20px;
+    letter-spacing: 0.01em;
+}
+.en-text {
+    font-family: "Source Pro", Consolas, "LingWai TC", Menlo, "Courier New", Arial, sans-serif;
+    font-size: 15px;
+}
+.zh-text.match, .en-text.match {
+    background-color: color-mix(in srgb, var(--primary-color) 12%, var(--background-color));
+    border-left: 3px solid #f5c518;
+    color: var(--text-color);
+}
+mark {
+    background: #f5c518;
+    color: #111827;
+    padding: 1px 2px;
+    border-radius: 2px;
+}
+</style>
+""",
+    unsafe_allow_html=True,
+)
+
+
+def _highlight(text, query):
+    if not text or not query:
+        return html.escape(text)
+    escaped_text = html.escape(text)
+    escaped_query = html.escape(query)
+    return regex.sub(
+        rf"({regex.escape(escaped_query)})",
+        r'<mark style="background:#fef08a;padding:1px 2px;border-radius:2px">\1</mark>',
+        escaped_text,
+        flags=regex.IGNORECASE | regex.V1,
+    )
+
+
+def _highlight_regex(text, pattern):
+    if not text or not pattern:
+        return html.escape(text)
+    try:
+        compiled = regex.compile(pattern, flags=regex.IGNORECASE | regex.V1)
+    except regex.error:
+        return html.escape(text)
+
+    parts = []
+    last_end = 0
+    for match in compiled.finditer(text):
+        start, end = match.span()
+        if start == end:
+            continue
+        parts.append(html.escape(text[last_end:start]))
+        parts.append(
+            f'<mark style="background:#fef08a;padding:1px 2px;border-radius:2px">{html.escape(text[start:end])}</mark>'
+        )
+        last_end = end
+    parts.append(html.escape(text[last_end:]))
+    return "".join(parts)
+
+
+def _join_sentences(sentences, lang):
+    parts = [
+        (s.get("zh_text", "") if lang == "zh" else s.get("en_text", "")).strip()
+        for s in sentences
+    ]
+    parts = [p for p in parts if p]
+    if not parts:
+        return ""
+    if lang == "zh":
+        return "".join(parts)
+    return " ".join(parts)
+
+
+if "page" not in st.session_state:
+    st.session_state.page = 0
+if "expanded" not in st.session_state:
+    st.session_state.expanded = {}
+if "search_signature" not in st.session_state:
+    st.session_state.search_signature = None
+
+st.title("⚖️ 全國法規資料庫 華英檢索系統")
+st.caption("Taiwan Law (TWL) Chinese–English Aligned Corpus - Bilingual Concordancer")
+
+conn = db.get_conn(DB_PATH)
+
+with st.sidebar:
+    st.header("搜尋範圍過濾 Filters")
+
+    law_types = ["All", "law", "order"]
+    selected_type = st.selectbox("法規/命令 Type", law_types, index=0)
+    type_filter = None if selected_type == "All" else selected_type
+
+    categories = db.list_categories(conn, type_filter)
+    selected_cat = st.selectbox("機關 Category", ["All"] + categories, index=0)
+    cat_filter = None if selected_cat == "All" else selected_cat
+
+    laws = db.list_laws(conn, type_filter, cat_filter)
+    law_options = ["All"] + [f"{l['law_id']} — {l['zh_name']}" for l in laws]
+    selected_law = st.selectbox("單一法規/命令 Law/Order", law_options, index=0)
+    law_id_filter = None
+    if selected_law != "All":
+        law_id_filter = selected_law.split(" — ")[0]
+
+    max_score = st.slider("Max alignment score (lower = better)", 0.0, 1.0, 1.0, 0.05)
+    max_score_filter = None if max_score >= 1.0 else max_score
+
+    lang_options = {
+        "中英 / Both": "both",
+        "中文 / Chinese": "zh",
+        "英文 / English": "en",
+    }
+    selected_lang = st.radio("搜尋語言 / Search language", list(lang_options), index=0)
+    lang_filter = lang_options[selected_lang]
+
+    st.divider()
+    st.caption(f"{len(laws)} laws/orders in database")
+
+with st.form("search_form", clear_on_submit=False):
+    col1, col2, col3 = st.columns([4, 1, 1])
+    with col1:
+        query = st.text_input(
+            "Search", placeholder="Enter keyword or regex…", key="search_query"
+        )
+    with col2:
+        use_regex = st.checkbox("Regex", value=False)
+        submitted = st.form_submit_button("Submit", use_container_width=True)
+    with col3:
+        per_page = st.selectbox("Per page", [10, 25, 50, 100], index=1)
+
+article_filter = None
+if law_id_filter:
+    articles = db.get_law_articles(conn, law_id_filter)
+    art_options = ["All"] + [
+        f"{a['article_no_zh']} / {a['article_no_en']}"
+        for a in articles
+        if a["article_no_zh"] or a["article_no_en"]
+    ]
+    selected_art = st.selectbox("Article", art_options, index=0)
+    if selected_art != "All":
+        parts = selected_art.split(" / ")
+        article_filter = parts[0] if parts else None
+
+search_signature = (
+    query,
+    use_regex,
+    per_page,
+    cat_filter,
+    law_id_filter,
+    article_filter,
+    max_score_filter,
+    lang_filter,
+)
+if st.session_state.search_signature != search_signature:
+    st.session_state.page = 0
+    st.session_state.expanded = {}
+    st.session_state.search_signature = search_signature
+
+if query:
+    results, total = db.search_sentences(
+        conn,
+        query,
+        use_regex=use_regex,
+        law_id=law_id_filter,
+        category=cat_filter,
+        article_no=article_filter,
+        max_score=max_score_filter,
+        lang=lang_filter,
+        limit=per_page,
+        offset=st.session_state.page * per_page,
+    )
+
+    st.write(f"**{total}** sentence pair{'s' if total != 1 else ''} found")
+
+    if total > per_page:
+        total_pages = (total + per_page - 1) // per_page
+        cols = st.columns([1, 4, 1])
+        with cols[0]:
+            if st.button(
+                "← Previous",
+                disabled=st.session_state.page == 0,
+                use_container_width=True,
+            ):
+                st.session_state.page -= 1
+                st.session_state.expanded = {}
+                st.rerun()
+        with cols[1]:
+            st.write(f"Page {st.session_state.page + 1} of {total_pages}")
+        with cols[2]:
+            if st.button(
+                "Next →",
+                disabled=(st.session_state.page + 1) * per_page >= total,
+                use_container_width=True,
+            ):
+                st.session_state.page += 1
+                st.session_state.expanded = {}
+                st.rerun()
+
+    for row in results:
+        sid = row["id"]
+        score = row["alignment_score"]
+        law_ref = f"{row['law_id']} {row['zh_name']}"
+        art_ref = (
+            f"{row['article_no_zh']} / {row['article_no_en']}"
+            if row["article_no_zh"] or row["article_no_en"]
+            else ""
+        )
+
+        with st.container(border=True):
+            st.markdown(
+                f"`{law_ref}`{' | ' + art_ref if art_ref else ''}  |  Score: `{score:.4f}`"
+            )
+
+            zh_text = row["zh_text"] or ""
+            en_text = row["en_text"] or ""
+
+            if query and use_regex:
+                zh_display = _highlight_regex(zh_text, query)
+                en_display = _highlight_regex(en_text, query)
+            elif query and not use_regex:
+                zh_display = _highlight(zh_text, query)
+                en_display = _highlight(en_text, query)
+            else:
+                zh_display = html.escape(zh_text)
+                en_display = html.escape(en_text)
+
+            col_zh, col_en = st.columns([2, 3])
+            with col_zh:
+                st.markdown(
+                    f'<div class="zh-text">{zh_display}</div>', unsafe_allow_html=True
+                )
+            with col_en:
+                st.markdown(
+                    f'<div class="en-text">{en_display}</div>', unsafe_allow_html=True
+                )
+
+            exp_col1, exp_col2 = st.columns(2)
+            with exp_col1:
+                if st.button("▸ Paragraph", key=f"para_{sid}"):
+                    st.session_state.expanded[
+                        f"para_{sid}"
+                    ] = not st.session_state.expanded.get(f"para_{sid}", False)
+            with exp_col2:
+                if st.button("▸ Article", key=f"art_{sid}"):
+                    st.session_state.expanded[
+                        f"art_{sid}"
+                    ] = not st.session_state.expanded.get(f"art_{sid}", False)
+
+            if st.session_state.expanded.get(f"para_{sid}"):
+                para = db.get_paragraph(conn, sid)
+                if para:
+                    with st.container(border=True):
+                        st.markdown(
+                            f"**Paragraph** ({para['article_no_zh']} / {para['article_no_en']})"
+                        )
+                        para_zh = _join_sentences(para["sentences"], "zh")
+                        para_en = _join_sentences(para["sentences"], "en")
+                        if query and use_regex:
+                            para_zh_display = _highlight_regex(para_zh, query)
+                            para_en_display = _highlight_regex(para_en, query)
+                        elif query and not use_regex:
+                            para_zh_display = _highlight(para_zh, query)
+                            para_en_display = _highlight(para_en, query)
+                        else:
+                            para_zh_display = html.escape(para_zh)
+                            para_en_display = html.escape(para_en)
+                        c1, c2 = st.columns([2, 3])
+                        with c1:
+                            st.markdown(
+                                f'<div class="zh-text match">{para_zh_display}</div>',
+                                unsafe_allow_html=True,
+                            )
+                        with c2:
+                            st.markdown(
+                                f'<div class="en-text match">{para_en_display}</div>',
+                                unsafe_allow_html=True,
+                            )
+
+            if st.session_state.expanded.get(f"art_{sid}"):
+                article = db.get_article(conn, sid)
+                if article:
+                    with st.container(border=True):
+                        st.markdown(
+                            f"**Article** ({article['article_no_zh']} / {article['article_no_en']})"
+                        )
+                        for pi, para in enumerate(article["paragraphs"]):
+                            st.markdown(f"*Paragraph {pi + 1}*")
+                            art_zh = _join_sentences(para["sentences"], "zh")
+                            art_en = _join_sentences(para["sentences"], "en")
+                            if query and use_regex:
+                                art_zh_display = _highlight_regex(art_zh, query)
+                                art_en_display = _highlight_regex(art_en, query)
+                            elif query and not use_regex:
+                                art_zh_display = _highlight(art_zh, query)
+                                art_en_display = _highlight(art_en, query)
+                            else:
+                                art_zh_display = html.escape(art_zh)
+                                art_en_display = html.escape(art_en)
+                            contains_match = any(
+                                s["id"] == sid for s in para["sentences"]
+                            )
+                            c1, c2 = st.columns([2, 3])
+                            with c1:
+                                st.markdown(
+                                    f'<div class="zh-text{" match" if contains_match else ""}">{art_zh_display}</div>',
+                                    unsafe_allow_html=True,
+                                )
+                            with c2:
+                                st.markdown(
+                                    f'<div class="en-text{" match" if contains_match else ""}">{art_en_display}</div>',
+                                    unsafe_allow_html=True,
+                                )
+
+elif not query:
+    st.info("Enter a search term above to find aligned sentence pairs.")
+
+st.divider()
+st.caption("TWL Concordancer | Taiwan Law Bilingual Corpus")
+
+conn.close()

db.py

@@ -0,0 +1,385 @@
+"""Database query layer for TWL Concordancer."""
+
+import sqlite3
+from pathlib import Path
+from typing import Iterable
+
+import regex
+
+DEFAULT_DB = Path(__file__).parent / "twl_concordancer.db"
+
+
+def get_conn(db_path=None):
+    if db_path is None:
+        db_path = DEFAULT_DB
+    conn = sqlite3.connect(str(db_path))
+    conn.row_factory = sqlite3.Row
+    conn.execute("PRAGMA busy_timeout=5000")
+    try:
+        conn.execute("PRAGMA journal_mode=WAL")
+    except sqlite3.OperationalError:
+        # Some hosted environments are fine with plain read access but do not
+        # allow switching journal mode.
+        pass
+    return conn
+
+
+def _expand_category_options(categories: Iterable[str]):
+    expanded = set()
+    for category in categories:
+        category = (category or "").strip()
+        if not category:
+            continue
+        expanded.add(category)
+        parts = [part.strip() for part in category.split("＞") if part.strip()]
+        if len(parts) <= 1:
+            continue
+        for depth in range(1, len(parts)):
+            expanded.add("＞".join(parts[:depth]) + "＞")
+    return sorted(expanded, key=_category_sort_key)
+
+
+def _category_sort_key(category: str):
+    category = (category or "").strip()
+    is_repealed = category.startswith("廢止法規＞")
+    return (1 if is_repealed else 0, category)
+
+
+def _build_order_by(lang):
+    if lang == "zh":
+        return """
+        ORDER BY
+            CASE
+                WHEN s.en_text IS NOT NULL AND trim(s.en_text) != '' THEN 0
+                ELSE 1
+            END,
+            s.alignment_score,
+            s.id
+        """
+    if lang == "en":
+        return """
+        ORDER BY
+            CASE
+                WHEN s.zh_text IS NOT NULL AND trim(s.zh_text) != '' THEN 0
+                ELSE 1
+            END,
+            s.alignment_score,
+            s.id
+        """
+    return """
+    ORDER BY
+        CASE
+            WHEN s.zh_text IS NOT NULL AND trim(s.zh_text) != ''
+             AND s.en_text IS NOT NULL AND trim(s.en_text) != '' THEN 0
+            ELSE 1
+        END,
+        s.alignment_score,
+        s.id
+    """
+
+
+def search_sentences(
+    conn,
+    query,
+    use_regex=False,
+    law_id=None,
+    category=None,
+    article_no=None,
+    max_score=None,
+    lang="both",
+    limit=100,
+    offset=0,
+):
+    if use_regex:
+        return _search_regex(
+            conn, query, law_id, category, article_no, max_score, lang, limit, offset
+        )
+    return _search_like(
+        conn, query, law_id, category, article_no, max_score, lang, limit, offset
+    )
+
+
+def _search_like(
+    conn, query, law_id, category, article_no, max_score, lang, limit, offset
+):
+    terms = query.strip()
+    if not terms:
+        return [], 0
+
+    where = []
+    params = []
+
+    if lang == "zh":
+        where.append("s.zh_text LIKE ?")
+        params.append(f"%{terms}%")
+    elif lang == "en":
+        where.append("s.en_text LIKE ?")
+        params.append(f"%{terms}%")
+    else:
+        where.append("(s.zh_text LIKE ? OR s.en_text LIKE ?)")
+        params.extend([f"%{terms}%", f"%{terms}%"])
+
+    if max_score is not None:
+        where.append("s.alignment_score <= ?")
+        params.append(max_score)
+    if law_id:
+        where.append("l.law_id = ?")
+        params.append(law_id)
+    if category:
+        if category.endswith("＞"):
+            where.append("l.category LIKE ?")
+            params.append(f"{category}%")
+        else:
+            where.append("l.category = ?")
+            params.append(category)
+    if article_no:
+        pat = f"%{article_no}%"
+        where.append("(a.article_no_zh LIKE ? OR a.article_no_en LIKE ?)")
+        params.extend([pat, pat])
+
+    where_clause = " AND ".join(where)
+
+    count_sql = f"""
+        SELECT count(*) FROM sentences s
+        JOIN laws l ON s.law_id = l.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE {where_clause}
+    """
+
+    order_by = _build_order_by(lang)
+
+    data_sql = f"""
+        SELECT s.id, s.zh_text, s.en_text, s.alignment_score,
+               l.law_id, l.zh_name, l.en_name, l.type,
+               a.article_no_zh, a.article_no_en, a.article_type,
+               s.zh_sentence_idx, s.en_sentence_idx
+        FROM sentences s
+        JOIN laws l ON s.law_id = l.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE {where_clause}
+        {order_by}
+        LIMIT ? OFFSET ?
+    """
+    data_params = params + [limit, offset]
+
+    cur = conn.execute(count_sql, params)
+    total = cur.fetchone()[0]
+
+    cur = conn.execute(data_sql, data_params)
+    rows = [dict(r) for r in cur.fetchall()]
+
+    return rows, total
+
+
+def _search_regex(
+    conn, pattern, law_id, category, article_no, max_score, lang, limit, offset
+):
+    try:
+        regex.compile(pattern)
+    except regex.error:
+        return [], 0
+
+    where = "1=1"
+    params = []
+
+    if lang == "zh":
+        where += " AND s.zh_text REGEXP ?"
+        params.append(pattern)
+    elif lang == "en":
+        where += " AND s.en_text REGEXP ?"
+        params.append(pattern)
+    else:
+        where += " AND (s.zh_text REGEXP ? OR s.en_text REGEXP ?)"
+        params.extend([pattern, pattern])
+
+    if max_score is not None:
+        where += " AND s.alignment_score <= ?"
+        params.append(max_score)
+    if law_id:
+        where += " AND l.law_id = ?"
+        params.append(law_id)
+    if category:
+        if category.endswith("＞"):
+            where += " AND l.category LIKE ?"
+            params.append(f"{category}%")
+        else:
+            where += " AND l.category = ?"
+            params.append(category)
+    if article_no:
+        where += " AND (a.article_no_zh LIKE ? OR a.article_no_en LIKE ?)"
+        pat = f"%{article_no}%"
+        params.extend([pat, pat])
+
+    count_sql = f"""
+        SELECT count(*) FROM sentences s
+        JOIN laws l ON s.law_id = l.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE {where}
+    """
+
+    order_by = _build_order_by(lang)
+
+    data_sql = f"""
+        SELECT s.id, s.zh_text, s.en_text, s.alignment_score,
+               l.law_id, l.zh_name, l.en_name, l.type,
+               a.article_no_zh, a.article_no_en, a.article_type,
+               s.zh_sentence_idx, s.en_sentence_idx
+        FROM sentences s
+        JOIN laws l ON s.law_id = l.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE {where}
+        {order_by}
+        LIMIT ? OFFSET ?
+    """
+    data_params = params + [limit, offset]
+
+    conn.create_function(
+        "REGEXP",
+        2,
+        lambda pat, txt: bool(regex.search(pat, txt, flags=regex.V1)) if txt else False,
+    )
+
+    cur = conn.execute(count_sql, params)
+    total = cur.fetchone()[0]
+
+    cur = conn.execute(data_sql, data_params)
+    rows = [dict(r) for r in cur.fetchall()]
+
+    return rows, total
+
+
+def get_paragraph(conn, sentence_id):
+    cur = conn.execute(
+        """
+        SELECT s.id, s.zh_text, s.en_text, s.alignment_score, s.zh_sentence_idx, s.en_sentence_idx,
+               p.paragraph_index, a.article_no_zh, a.article_no_en
+        FROM sentences s
+        JOIN paragraphs p ON s.paragraph_id = p.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE s.paragraph_id = (SELECT paragraph_id FROM sentences WHERE id = ?)
+        ORDER BY s.zh_sentence_idx
+        """,
+        (sentence_id,),
+    )
+    rows = [dict(r) for r in cur.fetchall()]
+    if not rows:
+        return None
+    return {
+        "paragraph_index": rows[0]["paragraph_index"],
+        "article_no_zh": rows[0]["article_no_zh"],
+        "article_no_en": rows[0]["article_no_en"],
+        "sentences": rows,
+    }
+
+
+def get_article(conn, sentence_id):
+    cur = conn.execute(
+        """
+        SELECT s.id, s.zh_text, s.en_text, s.alignment_score, s.zh_sentence_idx, s.en_sentence_idx,
+               p.paragraph_index, p.id as paragraph_id,
+               a.article_no_zh, a.article_no_en, a.article_type
+        FROM sentences s
+        JOIN paragraphs p ON s.paragraph_id = p.id
+        JOIN articles a ON s.article_id = a.id
+        WHERE s.article_id = (SELECT article_id FROM sentences WHERE id = ?)
+        ORDER BY p.paragraph_index, s.zh_sentence_idx
+        """,
+        (sentence_id,),
+    )
+    rows = [dict(r) for r in cur.fetchall()]
+    if not rows:
+        return None
+
+    paragraphs = {}
+    for r in rows:
+        pidx = r["paragraph_index"]
+        if pidx not in paragraphs:
+            paragraphs[pidx] = {
+                "paragraph_index": pidx,
+                "paragraph_id": r["paragraph_id"],
+                "sentences": [],
+            }
+        paragraphs[pidx]["sentences"].append(
+            {
+                "id": r["id"],
+                "zh_text": r["zh_text"],
+                "en_text": r["en_text"],
+                "alignment_score": r["alignment_score"],
+                "zh_sentence_idx": r["zh_sentence_idx"],
+                "en_sentence_idx": r["en_sentence_idx"],
+            }
+        )
+
+    return {
+        "article_no_zh": rows[0]["article_no_zh"],
+        "article_no_en": rows[0]["article_no_en"],
+        "article_type": rows[0]["article_type"],
+        "paragraphs": [paragraphs[k] for k in sorted(paragraphs.keys())],
+    }
+
+
+def list_laws(conn, law_type=None, category=None):
+    where = []
+    params = []
+    if law_type:
+        where.append("type = ?")
+        params.append(law_type)
+    if category:
+        if category.endswith("＞"):
+            where.append("category LIKE ?")
+            params.append(f"{category}%")
+        else:
+            where.append("category = ?")
+            params.append(category)
+
+    where_clause = " AND ".join(where) if where else "1=1"
+    cur = conn.execute(
+        f"SELECT law_id, type, zh_name, en_name, category FROM laws WHERE {where_clause} ORDER BY law_id",
+        params,
+    )
+    return [dict(r) for r in cur.fetchall()]
+
+
+def list_categories(conn, law_type=None):
+    if law_type:
+        where = "WHERE type = ? AND category IS NOT NULL AND category != ''"
+        params = [law_type]
+    else:
+        where = "WHERE category IS NOT NULL AND category != ''"
+        params = []
+    cur = conn.execute(
+        f"SELECT DISTINCT category FROM laws {where} ORDER BY category",
+        params,
+    )
+    return _expand_category_options(r["category"] for r in cur.fetchall())
+
+
+def get_law_articles(conn, law_id):
+    cur = conn.execute(
+        """
+        SELECT article_no_zh, article_no_en, article_type, article_index
+        FROM articles
+        WHERE law_id = (SELECT id FROM laws WHERE law_id = ?)
+        ORDER BY article_index
+        """,
+        (law_id,),
+    )
+    return [dict(r) for r in cur.fetchall()]
+
+
+def get_law_full_text(conn, law_id):
+    cur = conn.execute(
+        """
+        SELECT s.zh_text, s.en_text, s.alignment_score,
+               a.article_no_zh, a.article_no_en, a.article_type, a.article_index,
+               p.paragraph_index
+        FROM sentences s
+        JOIN paragraphs p ON s.paragraph_id = p.id
+        JOIN articles a ON s.article_id = a.id
+        JOIN laws l ON s.law_id = l.id
+        WHERE l.law_id = ?
+        ORDER BY a.article_index, p.paragraph_index, s.zh_sentence_idx
+        """,
+        (law_id,),
+    )
+    return [dict(r) for r in cur.fetchall()]

list_unmatched_sentences.sql

@@ -0,0 +1,34 @@
+SELECT
+    l.law_id AS "Law/Order ID",
+    CASE
+        WHEN COALESCE(TRIM(a.article_no_zh), '') != ''
+         AND COALESCE(TRIM(a.article_no_en), '') != ''
+            THEN a.article_no_zh || ' / ' || a.article_no_en
+        ELSE COALESCE(NULLIF(TRIM(a.article_no_zh), ''), TRIM(a.article_no_en))
+    END AS "Article No.",
+    CASE
+        WHEN TRIM(COALESCE(s.zh_text, '')) != ''
+         AND TRIM(COALESCE(s.en_text, '')) = ''
+            THEN TRIM(s.zh_text)
+        ELSE ''
+    END AS "Source sentence(s) with no matched Target sentence(s)",
+    CASE
+        WHEN TRIM(COALESCE(s.en_text, '')) != ''
+         AND TRIM(COALESCE(s.zh_text, '')) = ''
+            THEN TRIM(s.en_text)
+        ELSE ''
+    END AS "Target sentence(s) with no matched Source sentence(s)"
+FROM sentences s
+JOIN laws l ON s.law_id = l.id
+JOIN articles a ON s.article_id = a.id
+WHERE s.alignment_score = 0.0
+  AND (
+        (TRIM(COALESCE(s.zh_text, '')) != '' AND TRIM(COALESCE(s.en_text, '')) = '')
+     OR (TRIM(COALESCE(s.en_text, '')) != '' AND TRIM(COALESCE(s.zh_text, '')) = '')
+  )
+ORDER BY
+    l.law_id,
+    a.id,
+    COALESCE(s.zh_sentence_idx, 999999),
+    COALESCE(s.en_sentence_idx, 999999),
+    s.id;

requirements.txt

@@ -1,3 +1 @@
-altair
-pandas
-streamlit
+regex>=2024.0.0

twl_concordancer.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f19d8a6a060ab2d1a71aa2c2a77d86ed698b1b977e043827c9b0f62cca72731b
+size 129490944