features: independant from docfinder, eol creds, input ts mode

app.py

@@ -9,6 +9,7 @@ Three-step UI:
   4. DONE/ERROR — download ZIP of results
 """
 
+import hashlib
 import io
 import json
 import os
@@ -18,11 +19,19 @@ import threading
 import time
 import uuid
 import zipfile
+from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
 
 import streamlit as st
 
+# ── Load .env from the same directory as app.py ───────────────────────────────
+try:
+    from dotenv import load_dotenv
+    load_dotenv(Path(__file__).parent / ".env")
+except ImportError:
+    pass  # python-dotenv not installed; rely on environment variables
+
 # ── EOL credential verification ───────────────────────────────────────────────
 
 def verify_eol_credentials(username: str, password: str) -> bool:
@@ -108,6 +117,14 @@ def new_state(sid: str) -> dict:
         "started_at": None,
         "completed_at": None,
         "return_code": None,
+        # TS mode fields
+        "mode":         "contributor",   # "contributor" | "ts"
+        "excel_hash":   "",
+        "hf_repo":      "OrganizedProgrammers/CR_Index",
+        "index_log":    "",
+        "ts_id":        "",
+        # Logs for the current pipeline run (main + retry); reset on new run
+        "run_log_paths": [],
     }
 
 
@@ -235,6 +252,18 @@ def make_zip(output_dir: Path) -> bytes:
     return buf.read()
 
 
+def load_hf_index_cached(hf_token: str, hf_repo: str) -> list[dict]:
+    """Load HF CR index, caching result in session_state to avoid redundant fetches."""
+    key = f"hf_index_{hf_repo}"
+    if key not in st.session_state:
+        from hf_cr_index import load_hf_index
+        try:
+            st.session_state[key] = load_hf_index(hf_token, hf_repo)
+        except Exception:
+            st.session_state[key] = []
+    return st.session_state[key]
+
+
 # ── Page config ───────────────────────────────────────────────────────────────
 st.set_page_config(
     page_title="CR Application Tool",
@@ -326,31 +355,264 @@ if status == "login":
 elif status == "upload":
     st.subheader("Step 1 — Upload contribution list")
 
-    uploaded = st.file_uploader(
-        "Excel contribution list (.xlsx or .xls)",
-        type=["xlsx", "xls"],
-    )
-    person_name = st.text_input(
-        "Contributor name (must match SubmittedBy column)",
-        value=state.get("person_name", "Ly Thanh PHAN"),
+    mode_label = st.radio(
+        "Pipeline mode",
+        ["By contributor name", "By TS (all CRs for a spec)"],
+        key="mode_radio",
     )
+    pipeline_mode = "contributor" if mode_label.startswith("By contributor") else "ts"
+    state["mode"] = pipeline_mode
+
+    # ── Resolve active Excel (saved from a previous run, or freshly uploaded) ──
+    saved_name = state.get("excel_filename")
+    saved_path = session_dir(sid) / saved_name if saved_name else None
+    has_saved  = saved_path is not None and saved_path.exists()
+
+    if has_saved:
+        st.success(f"Using: **{saved_name}**")
+        with st.expander("Replace Excel file"):
+            uploaded = st.file_uploader(
+                "Upload a different Excel file (.xlsx or .xls)",
+                type=["xlsx", "xls"],
+                key="excel_replace",
+            )
+    else:
+        uploaded = st.file_uploader(
+            "Excel contribution list (.xlsx or .xls)",
+            type=["xlsx", "xls"],
+        )
 
-    if uploaded and st.button("Parse CR list →", type="primary"):
-        excel_path = session_dir(sid) / uploaded.name
-        excel_path.write_bytes(uploaded.getbuffer())
+    # When a new file is dropped, save it immediately and update state
+    if uploaded:
+        active_path  = session_dir(sid) / uploaded.name
+        active_bytes = bytes(uploaded.getbuffer())
+        active_path.write_bytes(active_bytes)
+        state["excel_filename"] = uploaded.name
+        state["excel_hash"]     = hashlib.sha256(active_bytes).hexdigest()[:16]
+        save_state(sid, state)
+    elif has_saved:
+        active_path = saved_path
+    else:
+        active_path = None
 
-        with st.spinner("Parsing Excel…"):
-            try:
-                from fetch_crs import parse_excel
-                cr_list = parse_excel(str(excel_path), person_name)
-                state["status"] = "preview"
-                state["excel_filename"] = uploaded.name
-                state["person_name"] = person_name
-                state["cr_list"] = [list(row) for row in cr_list]
-                save_state(sid, state)
-                st.rerun()
-            except Exception as exc:
-                st.error(f"Failed to parse Excel: {exc}")
+    if pipeline_mode == "contributor":
+        person_name = st.text_input(
+            "Contributor name (must match SubmittedBy column)",
+            value=state.get("person_name", "Ly Thanh PHAN"),
+        )
+
+        if active_path and st.button("Parse CR list →", type="primary"):
+            with st.spinner("Parsing Excel…"):
+                try:
+                    from fetch_crs import parse_excel
+                    cr_list = parse_excel(str(active_path), person_name)
+                    state["status"]     = "preview"
+                    state["person_name"] = person_name
+                    state["cr_list"]    = [list(row) for row in cr_list]
+                    save_state(sid, state)
+                    st.rerun()
+                except Exception as exc:
+                    st.error(f"Failed to parse Excel: {exc}")
+
+    else:  # TS mode
+        if active_path:
+            excel_hash = state.get("excel_hash") or hashlib.sha256(active_path.read_bytes()).hexdigest()[:16]
+            state["excel_hash"] = excel_hash
+
+            hf_token = os.environ.get("HF_TOKEN", "")
+            hf_repo  = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
+
+            # Check whether this Excel is already indexed
+            existing       = load_hf_index_cached(hf_token, hf_repo)
+            already_indexed = any(r.get("excel_hash") == excel_hash for r in existing)
+
+            if already_indexed:
+                st.success(f"This Excel (`{excel_hash}`) is already indexed in HF.")
+                if st.button("Select TS →", type="primary"):
+                    state["status"] = "ts_select"
+                    save_state(sid, state)
+                    st.rerun()
+            else:
+                st.info(f"Excel hash: `{excel_hash}` — not yet indexed.")
+                if st.button("Build CR Index", type="primary"):
+                    # CRs downloaded during indexing go to the session-level cache
+                    cr_cache_dir = session_dir(sid) / "CRs"
+                    cr_cache_dir.mkdir(parents=True, exist_ok=True)
+                    index_log = str(session_dir(sid) / "index.log")
+                    rc_path   = _rc_path(sid)
+                    rc_path.unlink(missing_ok=True)
+
+                    cmd = [
+                        sys.executable,
+                        str(SCRIPTS_DIR / "build_cr_index.py"),
+                        str(active_path),
+                        "--output-dir", str(session_dir(sid)),
+                        "--hf-repo", hf_repo,
+                    ]
+                    env = os.environ.copy()
+                    env["EOL_USER"]     = st.session_state.eol_user
+                    env["EOL_PASSWORD"] = st.session_state.eol_password
+                    # HF_TOKEN is already in env via os.environ
+
+                    log_file = open(index_log, "w")
+                    proc = subprocess.Popen(
+                        cmd,
+                        stdout=log_file,
+                        stderr=subprocess.STDOUT,
+                        env=env,
+                    )
+                    log_file.close()
+
+                    threading.Thread(
+                        target=_run_and_save_rc,
+                        args=(proc, rc_path),
+                        daemon=True,
+                    ).start()
+                    st.session_state.proc = proc
+
+                    state["status"]     = "indexing"
+                    state["pid"]        = proc.pid
+                    state["index_log"]  = index_log
+                    state["output_dir"] = ""   # no pipeline output yet
+                    state["started_at"] = datetime.now().isoformat()
+                    save_state(sid, state)
+                    st.rerun()
+
+# ════════════════════════════════════════════════════════════════════════════
+# INDEXING  (build_cr_index.py running)
+# ════════════════════════════════════════════════════════════════════════════
+elif status == "indexing":
+    pid = state["pid"]
+    index_log = state.get("index_log", "")
+
+    proc = st.session_state.get("proc")
+    alive = False
+    if proc is not None:
+        alive = proc.poll() is None
+    else:
+        rc = read_return_code(sid)
+        if rc is None:
+            alive = is_process_alive(pid)
+
+    if alive:
+        st.subheader("⏳ Building CR Index…")
+        st.info(f"PID {pid} — started {state.get('started_at', '')[:19]}")
+        log_text = tail_log(index_log, 50)
+        st.text_area("Live log (last 50 lines)", value=log_text, height=400)
+        time.sleep(2)
+        st.rerun()
+    else:
+        rc = read_return_code(sid)
+        if rc is None and proc is not None:
+            rc = proc.returncode
+        state["return_code"] = rc
+        state["completed_at"] = datetime.now().isoformat()
+        if rc == 0:
+            # Invalidate cached HF index so ts_select gets fresh data
+            st.session_state.pop(f"hf_index_{state.get('hf_repo', '')}", None)
+            state["status"] = "ts_select"
+        else:
+            # Expose the index log as log_path so the error state can display it
+            state["log_path"] = state.get("index_log", "")
+            state["status"] = "error"
+        save_state(sid, state)
+        st.rerun()
+
+# ════════════════════════════════════════════════════════════════════════════
+# TS_SELECT  (index ready — pick a spec to process)
+# ════════════════════════════════════════════════════════════════════════════
+elif status == "ts_select":
+    st.subheader("Step 2 — Select target TS spec")
+
+    hf_token = os.environ.get("HF_TOKEN", "")
+    hf_repo  = state.get("hf_repo", "OrganizedProgrammers/CR_Index")
+    excel_hash = state.get("excel_hash", "")
+
+    with st.spinner("Loading CR index from HuggingFace…"):
+        records = load_hf_index_cached(hf_token, hf_repo)
+
+    records_for_excel = [r for r in records if r.get("excel_hash") == excel_hash]
+
+    if not records_for_excel:
+        st.error(f"No records found for excel_hash `{excel_hash}`. Try rebuilding the index.")
+        if st.button("← Back to upload"):
+            state["status"] = "upload"
+            save_state(sid, state)
+            st.rerun()
+    else:
+        by_spec = defaultdict(list)
+        for r in records_for_excel:
+            by_spec[r["spec_number"]].append(r)
+
+        spec_options = sorted(by_spec.keys())
+        selected_spec = st.selectbox("Select target TS spec", spec_options)
+
+        if selected_spec:
+            versions = defaultdict(list)
+            for r in by_spec[selected_spec]:
+                versions[r["version"]].append(r["uid"])
+            st.write("**Versions found:**")
+            for ver, uids in sorted(versions.items()):
+                st.write(f"  v{ver}: {len(uids)} CR(s) — {', '.join(uids)}")
+
+            col1, col2 = st.columns(2)
+            with col1:
+                if st.button("← Back"):
+                    state["status"] = "upload"
+                    save_state(sid, state)
+                    st.rerun()
+            with col2:
+                if st.button("▶ Apply CRs for this TS", type="primary"):
+                    # Per-run directory so each pipeline's outputs are isolated
+                    run_id = int(time.time())
+                    output_dir = session_dir(sid) / f"run_{run_id}"
+                    output_dir.mkdir(parents=True, exist_ok=True)
+                    cr_cache_dir = session_dir(sid) / "CRs"
+                    cr_cache_dir.mkdir(parents=True, exist_ok=True)
+                    log_path = session_dir(sid) / f"pipeline_{run_id}.log"
+                    rc_path = _rc_path(sid)
+                    rc_path.unlink(missing_ok=True)
+
+                    cmd = [
+                        sys.executable,
+                        str(SCRIPTS_DIR / "orchestrate_cr.py"),
+                        "--output-dir", str(output_dir),
+                        "--cr-cache-dir", str(cr_cache_dir),
+                        "--ts-mode",
+                        "--ts-id", selected_spec,
+                        "--excel-hash", excel_hash,
+                        "--hf-repo", hf_repo,
+                    ]
+                    env = os.environ.copy()
+                    env["EOL_USER"]     = st.session_state.eol_user
+                    env["EOL_PASSWORD"] = st.session_state.eol_password
+                    # HF_TOKEN already in env via os.environ
+
+                    log_file = open(str(log_path), "w")
+                    proc = subprocess.Popen(
+                        cmd,
+                        stdout=log_file,
+                        stderr=subprocess.STDOUT,
+                        env=env,
+                    )
+                    log_file.close()
+
+                    threading.Thread(
+                        target=_run_and_save_rc,
+                        args=(proc, rc_path),
+                        daemon=True,
+                    ).start()
+                    st.session_state.proc = proc
+
+                    state["ts_id"]          = selected_spec
+                    state["status"]         = "running"
+                    state["pid"]            = proc.pid
+                    state["output_dir"]     = str(output_dir)
+                    state["log_path"]       = str(log_path)
+                    state["run_log_paths"]  = [str(log_path)]
+                    state["started_at"]     = datetime.now().isoformat()
+                    save_state(sid, state)
+                    st.rerun()
 
 # ═════════════════════════════════════════════════════════════════════════���══
 # PREVIEW
@@ -386,10 +648,15 @@ elif status == "preview":
     with col2:
         if cr_list and st.button("▶ Start Pipeline", type="primary"):
             excel_path = session_dir(sid) / state["excel_filename"]
-            output_dir = session_dir(sid) / "output"
+            # Per-run directory so each pipeline's outputs are isolated
+            run_id = int(time.time())
+            output_dir = session_dir(sid) / f"run_{run_id}"
             output_dir.mkdir(parents=True, exist_ok=True)
-            log_path = session_dir(sid) / "pipeline.log"
+            cr_cache_dir = session_dir(sid) / "CRs"
+            cr_cache_dir.mkdir(parents=True, exist_ok=True)
+            log_path = session_dir(sid) / f"pipeline_{run_id}.log"
             rc_path = _rc_path(sid)
+            rc_path.unlink(missing_ok=True)
 
             cmd = [
                 sys.executable,
@@ -397,6 +664,7 @@ elif status == "preview":
                 str(excel_path),
                 state["person_name"],
                 "--output-dir", str(output_dir),
+                "--cr-cache-dir", str(cr_cache_dir),
             ]
 
             env = os.environ.copy()
@@ -425,6 +693,7 @@ elif status == "preview":
             state["pid"] = proc.pid
             state["output_dir"] = str(output_dir)
             state["log_path"] = str(log_path)
+            state["run_log_paths"] = [str(log_path)]
             state["started_at"] = datetime.now().isoformat()
             save_state(sid, state)
             st.rerun()
@@ -478,12 +747,11 @@ elif status in ("done", "error"):
     else:
         st.error(f"❌ Pipeline finished with errors (return code: {rc})")
 
-    # Per-TS results table — merge all pipeline logs so retry results don't
-    # replace original ones; later logs (pipeline_retry.log) supersede earlier
-    # ones (pipeline.log) for the same TS key.
+    # Per-TS results table — merge this run's logs so retry results supersede
+    # the original failed entries for the same TS key.
     _merged: dict[str, dict] = {}
-    for _lf in sorted(session_dir(sid).glob("pipeline*.log")):
-        for _r in parse_log_results(str(_lf)):
+    for _lf in state.get("run_log_paths", []):
+        for _r in parse_log_results(_lf):
             _merged[_r["TS"]] = _r
     results = list(_merged.values())
     if results:
@@ -604,7 +872,7 @@ elif status in ("done", "error"):
                     st.success(f"All {len(ready_entries)} TS(s) ready.")
 
                 if st.button("▶ Apply CRs to recovered TSs", type="primary"):
-                    retry_log = str(session_dir(sid) / "pipeline_retry.log")
+                    retry_log = str(session_dir(sid) / f"pipeline_{int(time.time())}_retry.log")
                     _rc_path(sid).unlink(missing_ok=True)   # clear old returncode
 
                     cmd = [
@@ -630,23 +898,41 @@ elif status in ("done", "error"):
                     ).start()
                     st.session_state.proc = proc
 
-                    state["status"]     = "running"
-                    state["pid"]        = proc.pid
-                    state["log_path"]   = retry_log
-                    state["started_at"] = datetime.now().isoformat()
+                    state["status"]          = "running"
+                    state["pid"]             = proc.pid
+                    state["log_path"]        = retry_log
+                    state["run_log_paths"]   = state.get("run_log_paths", []) + [retry_log]
+                    state["started_at"]      = datetime.now().isoformat()
                     save_state(sid, state)
                     st.rerun()
             else:
                 st.warning("No TSs available yet — retry download or upload DOCX files above.")
 
-    # Start new session
+    # Navigation
     st.divider()
-    if st.button("Start new session"):
-        new_sid = str(uuid.uuid4())
-        st.session_state.sid = new_sid
-        st.session_state.state = new_state(new_sid)
-        if "proc" in st.session_state:
-            del st.session_state.proc
-        st.query_params["sid"] = new_sid
-        save_state(new_sid, st.session_state.state)
-        st.rerun()
+    col_restart, col_new = st.columns(2)
+    with col_restart:
+        if st.button("← Run another pipeline", type="primary"):
+            # Reset pipeline fields, keep session ID and credentials
+            for _k in ("cr_list", "pid", "output_dir", "log_path", "index_log",
+                       "started_at", "completed_at", "return_code", "ts_id"):
+                state[_k] = None if _k != "cr_list" else []
+            # excel_filename, excel_hash and person_name are intentionally kept
+            # so the user does not have to re-upload on the next run.
+            state["run_log_paths"] = []
+            state["status"] = "upload"
+            if "proc" in st.session_state:
+                del st.session_state.proc
+            _rc_path(sid).unlink(missing_ok=True)
+            save_state(sid, state)
+            st.rerun()
+    with col_new:
+        if st.button("Start new session"):
+            new_sid = str(uuid.uuid4())
+            st.session_state.sid = new_sid
+            st.session_state.state = new_state(new_sid)
+            if "proc" in st.session_state:
+                del st.session_state.proc
+            st.query_params["sid"] = new_sid
+            save_state(new_sid, st.session_state.state)
+            st.rerun()

scripts/build_cr_index.py

@@ -0,0 +1,169 @@
+#!/usr/bin/env python3
+"""
+build_cr_index.py — Build and push a CR index to a HuggingFace dataset.
+
+Parses all Accepted CRs from the Excel, downloads each one (reusing cached
+files), parses the cover page to get (spec_number, version), then pushes a
+JSONL index to the HF dataset.
+
+Usage:
+    python3 build_cr_index.py <excel_path> --output-dir DIR --hf-repo ORG/REPO
+
+Environment variables:
+    EOL_USER, EOL_PASSWORD  — ETSI EOL credentials for download_cr
+    HF_TOKEN                — HuggingFace token (write access to hf-repo)
+"""
+
+import argparse
+import datetime
+import hashlib
+import os
+import sys
+import time
+from pathlib import Path
+
+# ── sys.path setup ────────────────────────────────────────────────────────────
+SCRIPT_DIR = Path(__file__).parent
+sys.path.insert(0, str(SCRIPT_DIR))
+
+from fetch_crs import parse_excel_all_accepted, download_cr, parse_cr_cover, wsl_path
+from hf_cr_index import load_hf_index, push_hf_index
+
+
+def main():
+    ap = argparse.ArgumentParser(
+        description="Build and push CR index to HuggingFace dataset.",
+    )
+    ap.add_argument("excel_path", help="Path to .xls or .xlsx contribution list")
+    ap.add_argument(
+        "--output-dir",
+        default=str(Path.home() / "CR_Processing"),
+        help="Base output directory (default: ~/CR_Processing)",
+    )
+    ap.add_argument(
+        "--hf-repo",
+        default="OrganizedProgrammers/CR_Index",
+        help="HuggingFace dataset repo (default: OrganizedProgrammers/CR_Index)",
+    )
+    args = ap.parse_args()
+
+    eol_user = os.environ.get("EOL_USER", "")
+    eol_password = os.environ.get("EOL_PASSWORD", "")
+    hf_token = os.environ.get("HF_TOKEN", "")
+
+    if not eol_user or not eol_password:
+        sys.exit("ERROR: EOL_USER and EOL_PASSWORD must be set")
+    if not hf_token:
+        sys.exit("ERROR: HF_TOKEN must be set")
+
+    excel_path = Path(wsl_path(args.excel_path))
+    if not excel_path.exists():
+        sys.exit(f"ERROR: Excel file not found: {excel_path}")
+
+    output_dir = Path(wsl_path(args.output_dir)).expanduser()
+    cr_dir = output_dir / "CRs"
+    cr_dir.mkdir(parents=True, exist_ok=True)
+
+    # ── 1. Compute Excel hash ─────────────────────────────────────────────────
+    excel_hash = hashlib.sha256(excel_path.read_bytes()).hexdigest()[:16]
+    meeting_label = excel_path.stem
+    print(f"Excel:        {excel_path.name}")
+    print(f"Excel hash:   {excel_hash}")
+    print(f"Meeting:      {meeting_label}")
+    print(f"HF repo:      {args.hf_repo}")
+    print()
+
+    # ── 2. Parse all Accepted CRs ─────────────────────────────────────────────
+    print("Parsing Excel for all Accepted CRs...")
+    try:
+        cr_list = parse_excel_all_accepted(str(excel_path))
+    except Exception as e:
+        sys.exit(f"ERROR parsing Excel: {e}")
+    print(f"Found {len(cr_list)} Accepted CR(s)\n")
+
+    if not cr_list:
+        print("Nothing to index.")
+        sys.exit(0)
+
+    # ── 3. Load existing HF index ─────────────────────────────────────────────
+    print("Loading existing HF index...")
+    try:
+        existing = load_hf_index(hf_token, args.hf_repo)
+    except Exception as e:
+        print(f"  WARNING: could not load existing index: {e}")
+        existing = []
+    existing_keys = {(r["excel_hash"], r["uid"]) for r in existing}
+    print(f"  {len(existing)} existing record(s), {len(existing_keys)} unique keys\n")
+
+    # ── 4. Download and parse each new CR ─────────────────────────────────────
+    new_records = []
+    skipped = 0
+    failed = []
+
+    print("Processing CRs...")
+    for uid, title, submitted_by in cr_list:
+        if (excel_hash, uid) in existing_keys:
+            print(f"  [{uid}] already indexed — skipping")
+            skipped += 1
+            continue
+
+        # Retry loop (3 attempts)
+        docx_path = None
+        note = ""
+        for attempt in range(1, 4):
+            docx_path, note = download_cr(uid, cr_dir, eol_user, eol_password)
+            if docx_path:
+                break
+            if attempt < 3:
+                print(f"  [{uid}] attempt {attempt}/3 failed ({note}) — retrying in 5s")
+                time.sleep(5)
+
+        if not docx_path:
+            print(f"  [{uid}] FAILED — {note}")
+            failed.append((uid, note))
+            continue
+
+        spec_number, version = parse_cr_cover(docx_path)
+        if not spec_number or not version:
+            print(f"  [{uid}] WARNING: could not parse cover page — skipping")
+            failed.append((uid, "cover page parse failed"))
+            continue
+
+        print(f"  [{uid}] -> TS {spec_number} v{version}")
+        new_records.append({
+            "excel_hash":    excel_hash,
+            "meeting_label": meeting_label,
+            "uid":           uid,
+            "title":         title,
+            "submitted_by":  submitted_by,
+            "spec_number":   spec_number,
+            "version":       version,
+            "parsed_at":     datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"),
+        })
+
+    print()
+    print(f"Summary: {len(new_records)} new, {skipped} skipped, {len(failed)} failed")
+    if failed:
+        print("Failed CRs:")
+        for uid, reason in failed:
+            print(f"  [{uid}] {reason}")
+    print()
+
+    # ── 5. Merge and push ─────────────────────────────────────────────────────
+    if new_records:
+        all_records = existing + new_records
+        print(f"Pushing {len(all_records)} record(s) to {args.hf_repo}...")
+        try:
+            push_hf_index(all_records, hf_token, args.hf_repo)
+            print("  Push successful")
+        except Exception as e:
+            sys.exit(f"ERROR pushing to HF: {e}")
+    else:
+        print("No new records to push.")
+
+    # ── 6. Sentinel line (watched by app.py) ──────────────────────────────────
+    print(f"INDEX_COMPLETE excel_hash={excel_hash}")
+
+
+if __name__ == "__main__":
+    main()

scripts/fetch_crs.py

@@ -57,6 +57,23 @@ def parse_excel(excel_path: str, person_name: str):
         raise ValueError(f"Unsupported file extension: {ext!r}. Expected .xls or .xlsx")
 
 
+def parse_excel_all_accepted(excel_path: str):
+    """
+    Return list of (uid, title, submitted_by) for ALL Accepted CRs
+    regardless of who submitted them. Used by build_cr_index.py.
+    Handles both .xls and .xlsx.
+    """
+    path = Path(wsl_path(excel_path))
+    ext = path.suffix.lower()
+
+    if ext == ".xls":
+        return _parse_xls_all(path)
+    elif ext == ".xlsx":
+        return _parse_xlsx_all(path)
+    else:
+        raise ValueError(f"Unsupported file extension: {ext!r}. Expected .xls or .xlsx")
+
+
 def _name_pattern(name: str) -> re.Pattern:
     return re.compile(r"\b" + re.escape(name) + r"\b", re.IGNORECASE)
 
@@ -169,6 +186,103 @@ def _parse_xlsx(path: Path, person_name: str):
     return results
 
 
+def _parse_xls_all(path: Path):
+    """Return (uid, title, submitted_by) for all Accepted CRs (no person filter)."""
+    try:
+        import xlrd
+    except ImportError:
+        sys.exit("ERROR: xlrd is not installed. Run: pip install xlrd")
+
+    wb = xlrd.open_workbook(str(path))
+    try:
+        ws = wb.sheet_by_name("Contributions")
+    except xlrd.XLRDError:
+        ws = wb.sheet_by_index(0)
+
+    headers = [str(ws.cell_value(0, c)).strip() for c in range(ws.ncols)]
+    col = {h: i for i, h in enumerate(headers)}
+
+    uid_col = col.get("Uid") or col.get("UID") or col.get("uid")
+    type_col = col.get("Type") or col.get("type")
+    status_col = col.get("Status") or col.get("status")
+    by_col = col.get("SubmittedBy") or col.get("Submitted By") or col.get("submittedby")
+    title_col = col.get("Title") or col.get("title")
+
+    for name, c in [("Uid", uid_col), ("Type", type_col),
+                    ("Status", status_col), ("SubmittedBy", by_col)]:
+        if c is None:
+            raise ValueError(f"Column {name!r} not found. Available: {list(col.keys())}")
+
+    results = []
+    for r in range(2, ws.nrows):
+        uid = str(ws.cell_value(r, uid_col)).strip()
+        doc_type = str(ws.cell_value(r, type_col)).strip()
+        status = str(ws.cell_value(r, status_col)).strip()
+        submitted_by = str(ws.cell_value(r, by_col)).strip()
+        title = str(ws.cell_value(r, title_col)).strip() if title_col is not None else ""
+
+        if doc_type != "CR":
+            continue
+        if status != "Accepted":
+            continue
+
+        results.append((uid, title, submitted_by))
+
+    return results
+
+
+def _parse_xlsx_all(path: Path):
+    """Return (uid, title, submitted_by) for all Accepted CRs (no person filter)."""
+    try:
+        import openpyxl
+    except ImportError:
+        sys.exit("ERROR: openpyxl is not installed. Run: pip install openpyxl")
+
+    wb = openpyxl.load_workbook(str(path), read_only=True, data_only=True)
+    ws = wb["Contributions"] if "Contributions" in wb.sheetnames else wb.active
+
+    rows = iter(ws.iter_rows(values_only=True))
+    header_row = next(rows)
+    headers = [str(h).strip() if h is not None else "" for h in header_row]
+    col = {h: i for i, h in enumerate(headers)}
+
+    next(rows, None)  # skip empty duplicate row
+
+    uid_col = col.get("Uid") or col.get("UID") or col.get("uid")
+    type_col = col.get("Type") or col.get("type")
+    status_col = col.get("Status") or col.get("status")
+    by_col = col.get("SubmittedBy") or col.get("Submitted By") or col.get("submittedby")
+    title_col = col.get("Title") or col.get("title")
+
+    for name, c in [("Uid", uid_col), ("Type", type_col),
+                    ("Status", status_col), ("SubmittedBy", by_col)]:
+        if c is None:
+            raise ValueError(f"Column {name!r} not found. Available: {list(col.keys())}")
+
+    results = []
+    for row in rows:
+        def cell(c):
+            v = row[c] if c < len(row) else None
+            return str(v).strip() if v is not None else ""
+
+        uid = cell(uid_col)
+        doc_type = cell(type_col)
+        status = cell(status_col)
+        submitted_by = cell(by_col)
+        title = cell(title_col) if title_col is not None else ""
+
+        if not uid:
+            continue
+        if doc_type != "CR":
+            continue
+        if status != "Accepted":
+            continue
+
+        results.append((uid, title, submitted_by))
+
+    return results
+
+
 # ---------------------------------------------------------------------------
 # Step 2 — Download CR DOCXs
 # ---------------------------------------------------------------------------

scripts/hf_cr_index.py

@@ -0,0 +1,58 @@
+"""
+hf_cr_index.py — Shared HuggingFace dataset index helpers.
+
+Used by both build_cr_index.py and orchestrate_cr.py.
+Depends only on huggingface_hub (pip install huggingface_hub).
+"""
+
+import json
+from pathlib import Path
+
+
+def load_hf_index(hf_token: str, hf_repo: str) -> list[dict]:
+    """
+    Download cr_index.jsonl from the HF dataset and return parsed records.
+    Returns an empty list if the file does not exist yet.
+    """
+    from huggingface_hub import hf_hub_download
+    from huggingface_hub.errors import EntryNotFoundError
+
+    try:
+        path = hf_hub_download(
+            repo_id=hf_repo,
+            filename="cr_index.jsonl",
+            repo_type="dataset",
+            token=hf_token,
+        )
+        text = Path(path).read_text(encoding="utf-8")
+        return [json.loads(line) for line in text.splitlines() if line.strip()]
+    except EntryNotFoundError:
+        return []
+
+
+def push_hf_index(records: list[dict], hf_token: str, hf_repo: str) -> None:
+    """
+    Push all records as cr_index.jsonl to the HF dataset repo.
+    Creates the repo if it does not exist.
+    """
+    from huggingface_hub import HfApi
+
+    api = HfApi()
+    api.create_repo(
+        repo_id=hf_repo,
+        repo_type="dataset",
+        exist_ok=True,
+        token=hf_token,
+    )
+    tmp = Path("/tmp/cr_index.jsonl")
+    tmp.write_text(
+        "\n".join(json.dumps(r, ensure_ascii=False) for r in records),
+        encoding="utf-8",
+    )
+    api.upload_file(
+        path_or_fileobj=str(tmp),
+        path_in_repo="cr_index.jsonl",
+        repo_id=hf_repo,
+        repo_type="dataset",
+        token=hf_token,
+    )

scripts/orchestrate_cr.py

@@ -16,6 +16,11 @@ Arguments:
 Options:
     --output-dir  Base output folder (default: ~/CR_Processing)
     --author      Tracked-change author name (default: "CR Application")
+    --retry-mode  Skip steps 1-4; apply CRs listed in failed_ts.json
+    --ts-mode     Apply all CRs for a given spec number across all versions
+    --ts-id       Spec number to process in ts-mode (e.g. "102 267")
+    --excel-hash  Excel hash used to filter the HF index in ts-mode
+    --hf-repo     HuggingFace dataset repo containing the CR index
 """
 
 import argparse
@@ -25,6 +30,7 @@ import io
 import json
 import os
 import re
+import shutil
 import sys
 import time
 from pathlib import Path
@@ -75,6 +81,249 @@ class _TeeWriter:
         self._real.flush()
 
 
+# ── Shared Steps 2, 4, 5, 6 ──────────────────────────────────────────────────
+
+def _run_steps_2_to_6(cr_list, ts_groups, output_dir, cr_dir, ts_dir,
+                       eol_user, eol_password, author, tc_date):
+    """
+    Execute steps 2 (download CRs), 4 (download TSs), 5 & 6 (apply + finalise).
+
+    cr_list    : list of (uid, title)
+    ts_groups  : dict {(spec_number, version): [uid, ...]} — may be pre-built
+                 (ts-mode) or None to trigger Step 3 (cover page parse).
+    """
+    # ── Step 2: Download CR DOCXs ─────────────────────────────────────────────
+    _section('Step 2 — Downloading CR DOCXs')
+    cr_paths = {}  # uid -> Path
+
+    for uid, _ in cr_list:
+        docx_path, note = download_cr(uid, cr_dir, eol_user, eol_password)
+        if docx_path:
+            cr_paths[uid] = docx_path
+            print(f'  [{uid}] OK ({note}) — {docx_path.name}')
+        else:
+            print(f'  [{uid}] FAILED — {note}')
+
+    n_cr_failed = len(cr_list) - len(cr_paths)
+    if n_cr_failed:
+        print(f'  {len(cr_paths)}/{len(cr_list)} downloaded ({n_cr_failed} failed)')
+    else:
+        print(f'  All {len(cr_list)} CR(s) downloaded successfully')
+
+    # ── Step 3: Parse cover pages (only when ts_groups not pre-built) ─────────
+    if ts_groups is None:
+        _section('Step 3 — Parsing CR cover pages')
+        ts_groups = {}
+        uid_cover_failed = []
+
+        for uid in cr_paths:
+            spec_number, version = parse_cr_cover(cr_paths[uid])
+            if spec_number and version:
+                key = (spec_number, version)
+                ts_groups.setdefault(key, []).append(uid)
+                print(f'  [{uid}] -> TS {spec_number}  v{version}')
+            else:
+                uid_cover_failed.append(uid)
+                print(f'  [{uid}] WARNING: could not parse cover page — skipping')
+
+        if not ts_groups:
+            print('\nNo TSs identified. Nothing to apply.')
+            return [], {}, {}, {}
+
+    # ── Step 4: Download TSs ──────────────────────────────────────────────────
+    _section('Step 4 — Downloading TSs')
+    ts_paths = {}   # (spec_number, version) -> Path
+    spec_dirs = {}  # (spec_number, version) -> Path (per-spec subfolder)
+
+    for (spec_number, version) in ts_groups:
+        spec_compact = spec_number.replace(' ', '')
+        spec_dir = ts_dir / spec_compact
+        spec_dir.mkdir(parents=True, exist_ok=True)
+        spec_dirs[(spec_number, version)] = spec_dir
+
+        print(f'  [TS {spec_number} v{version}] ', end='', flush=True)
+        filename, note = None, "not attempted"
+        for attempt in range(1, 4):
+            filename, note = download_ts(spec_number, version, spec_dir, eol_user, eol_password)
+            if filename:
+                break
+            if attempt < 3:
+                print(f'\n    [attempt {attempt}/3 failed — retrying in 5s: {note}]', flush=True)
+                print(f'  [TS {spec_number} v{version}] ', end='', flush=True)
+                time.sleep(5)
+            else:
+                print(f'\n    [all 3 attempts failed]', flush=True)
+        if filename:
+            ts_paths[(spec_number, version)] = spec_dir / filename
+            print(f'OK ({note}) — {spec_compact}/{filename}')
+        else:
+            print(f'FAILED — {note}')
+
+    # Write failed_ts.json
+    failed_ts_entries = [
+        {
+            "spec_number":       spec_number,
+            "version":           version,
+            "spec_compact":      spec_number.replace(' ', ''),
+            "spec_dir":          str(spec_dirs[(spec_number, version)]),
+            "expected_filename": f"ts_{spec_number.replace(' ', '')}_v{version}.docx",
+            "cr_uids":           ts_groups[(spec_number, version)],
+            "cr_dir":            str(cr_dir),
+        }
+        for (spec_number, version) in ts_groups
+        if (spec_number, version) not in ts_paths
+    ]
+    (output_dir / "failed_ts.json").write_text(
+        json.dumps(failed_ts_entries, indent=2)
+    )
+
+    # ── Steps 5 & 6: Apply CRs + Finalise each TS ────────────────────────────
+    _section('Steps 5 & 6 — Applying CRs and Finalising Metadata')
+    report = []  # (ts_key, n_ok, n_skip, n_crs, out_path, log_path, errors)
+
+    for (spec_number, version), uids in ts_groups.items():
+        ts_key = f'TS {spec_number} v{version}'
+        spec_compact = spec_number.replace(' ', '')
+        spec_dir = spec_dirs.get((spec_number, version), ts_dir / spec_compact)
+        spec_dir.mkdir(parents=True, exist_ok=True)
+
+        new_v = derive_new_version(version)
+        stem = f'ts_{spec_compact}_v{new_v}_was_v{version}'
+        ts_applied = spec_dir / f'ts_{spec_compact}_v{version}_applied.docx'
+        ts_final   = spec_dir / f'{stem}.docx'
+        log_path   = spec_dir / f'{stem}.log'
+        errors = []
+
+        print(f'\n-- {ts_key} ({len(uids)} CR(s): {", ".join(uids)}) --')
+
+        if (spec_number, version) not in ts_paths:
+            msg = 'TS download failed — skipping'
+            print(f'  SKIP: {msg}')
+            report.append((ts_key, 0, 0, len(uids), None, log_path, [msg]))
+            continue
+
+        ts_in = ts_paths[(spec_number, version)]
+
+        log_buf = io.StringIO()
+        tee = _TeeWriter(sys.stdout, log_buf)
+
+        with contextlib.redirect_stdout(tee):
+            log_header = (
+                f'Pipeline Log\n'
+                f'TS:   {spec_number}  v{version} -> v{new_v}\n'
+                f'CRs:  {", ".join(uids)}\n'
+                f'Date: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n'
+                f'{"=" * 60}\n'
+            )
+            print(log_header, end='')
+
+            combined_manifest = []
+            participating_uids = []
+
+            for uid in uids:
+                if uid not in cr_paths:
+                    errors.append(f'[{uid}] CR download had failed — skipped')
+                    continue
+                print(f'  Parsing {uid}... ', end='', flush=True)
+                try:
+                    changes = parse_cr(cr_paths[uid])
+                    combined_manifest.extend(changes)
+                    participating_uids.append(uid)
+                    print(f'{len(changes)} change(s)')
+                except Exception as e:
+                    errors.append(f'[{uid}] parse ERROR: {e}')
+                    print(f'ERROR: {e}')
+
+            if not combined_manifest:
+                print('  No changes parsed — skipping apply step.')
+                report.append((ts_key, 0, 0, len(uids), None, log_path,
+                                errors + ['No changes parsed']))
+                log_path.write_text(log_buf.getvalue(), encoding='utf-8')
+                continue
+
+            print(f'  Applying {len(combined_manifest)} change(s) to {ts_in.name}...')
+            try:
+                n_ok, n_skip, log_lines = apply_manifest(
+                    ts_in, combined_manifest, ts_applied, author=author, date=tc_date
+                )
+            except Exception as e:
+                errors.append(f'apply_manifest ERROR: {e}')
+                print(f'  ERROR: {e}')
+                report.append((ts_key, 0, 0, len(uids), None, log_path, errors))
+                log_path.write_text(log_buf.getvalue(), encoding='utf-8')
+                continue
+
+            for line in log_lines:
+                print(f'  {line}')
+            for line in log_lines:
+                if line.strip().startswith('ERROR'):
+                    errors.append(line.strip())
+            print(f'  -> Applied: {n_ok}  Skipped: {n_skip}')
+
+            print('  Finalising metadata...')
+            try:
+                ts_doc = docx_lib.Document(str(ts_applied))
+                rev = RevCounter(ts_doc)
+
+                pub_ym, pub_month_year = compute_pub_date()
+                old_v = version
+
+                title_text = ts_doc.paragraphs[0].text
+                date_match = re.search(r'\((\d{4}-\d{2})\)', title_text)
+                old_date_str = date_match.group(1) if date_match else ''
+
+                print(f'    Version:     {old_v} -> {new_v}')
+                print(f'    Publication: {pub_month_year} ({pub_ym})')
+
+                for uid in participating_uids:
+                    try:
+                        meta = extract_cr_metadata(str(cr_paths[uid]))
+                        ch_cells = update_change_history_table(
+                            ts_doc, meta, pub_ym, old_v, new_v, rev, author, tc_date
+                        )
+                        print(f'    [Change History] {uid}: {ch_cells}')
+                    except NoChangeHistoryTable:
+                        print(f'    [Change History] {uid}: NOT PRESENT — this document has no Change History table (History table only)')
+                    except Exception as e:
+                        errors.append(f'[{uid}] Change History ERROR: {e}')
+                        print(f'    [Change History] {uid}: ERROR — {e}')
+
+                try:
+                    h_cells = update_history_table(
+                        ts_doc, new_v, pub_month_year, rev, author, tc_date
+                    )
+                    print(f'    [History] {h_cells}')
+                except Exception as e:
+                    errors.append(f'History table ERROR: {e}')
+                    print(f'    [History] ERROR — {e}')
+
+                if old_date_str:
+                    try:
+                        update_title_para(
+                            ts_doc, old_v, new_v, old_date_str, pub_ym, rev, author, tc_date
+                        )
+                        print(f'    [Title] V{old_v} -> V{new_v}, ({old_date_str}) -> ({pub_ym})')
+                    except Exception as e:
+                        errors.append(f'Title update ERROR: {e}')
+                        print(f'    [Title] ERROR — {e}')
+                else:
+                    print(f'    [Title] SKIP — no (YYYY-MM) pattern in: {title_text!r}')
+
+                ts_doc.save(str(ts_final))
+                print(f'  Saved: {spec_compact}/{ts_final.name}')
+                print(f'  Log:   {spec_compact}/{log_path.name}')
+                report.append((ts_key, n_ok, n_skip, len(uids), ts_final, log_path, errors))
+
+            except Exception as e:
+                errors.append(f'Finalisation ERROR: {e}')
+                print(f'  Finalisation ERROR: {e}')
+                report.append((ts_key, n_ok, n_skip, len(uids), ts_applied, log_path, errors))
+
+        log_path.write_text(log_buf.getvalue(), encoding='utf-8')
+
+    return report, cr_paths, ts_paths, spec_dirs
+
+
 # ── Main ──────────────────────────────────────────────────────────────────────
 
 def main():
@@ -86,7 +335,7 @@ def main():
         'excel_path',
         nargs='?',
         default=None,
-        help='Path to .xls or .xlsx contribution list (not required in --retry-mode)',
+        help='Path to .xls or .xlsx contribution list (not required in --retry-mode or --ts-mode)',
     )
     ap.add_argument(
         'person_name',
@@ -109,10 +358,38 @@ def main():
         action='store_true',
         help='Skip steps 1-4; apply CRs to TSs listed in failed_ts.json that now have their DOCX on disk',
     )
+    ap.add_argument(
+        '--ts-mode',
+        action='store_true',
+        help='Apply all CRs for a given spec number across all versions (uses HF index)',
+    )
+    ap.add_argument(
+        '--ts-id',
+        default='',
+        help='Spec number to process in ts-mode, e.g. "102 267"',
+    )
+    ap.add_argument(
+        '--excel-hash',
+        default='',
+        help='Excel hash used to filter the HF index in ts-mode',
+    )
+    ap.add_argument(
+        '--hf-repo',
+        default='OrganizedProgrammers/CR_Index',
+        help='HuggingFace dataset repo containing the CR index',
+    )
+    ap.add_argument(
+        '--cr-cache-dir',
+        default='',
+        help='Shared directory for caching downloaded CR DOCXs across runs '
+             '(default: <output-dir>/CRs)',
+    )
     args = ap.parse_args()
 
-    if not args.retry_mode and not args.excel_path:
-        ap.error('excel_path is required when not in --retry-mode')
+    if args.ts_mode and not args.ts_id:
+        ap.error('--ts-id is required when using --ts-mode')
+    if not args.ts_mode and not args.retry_mode and not args.excel_path:
+        ap.error('excel_path is required when not in --retry-mode or --ts-mode')
 
     eol_user = os.environ.get("EOL_USER", "")
     eol_password = os.environ.get("EOL_PASSWORD", "")
@@ -120,8 +397,9 @@ def main():
         sys.exit("ERROR: EOL_USER and EOL_PASSWORD must be set")
 
     output_dir = Path(wsl_path(args.output_dir)).expanduser()
-    cr_dir = output_dir / 'CRs'
-    ts_dir = output_dir / 'TS'   # spec subfolders created per-TS below
+    cr_cache = args.cr_cache_dir.strip()
+    cr_dir = Path(wsl_path(cr_cache)).expanduser() if cr_cache else output_dir / 'CRs'
+    ts_dir = output_dir / 'TS'
     cr_dir.mkdir(parents=True, exist_ok=True)
     ts_dir.mkdir(parents=True, exist_ok=True)
 
@@ -159,7 +437,6 @@ def main():
                 print(f'  [TS {spec_number} v{version}] DOCX found — will apply')
             else:
                 print(f'  [TS {spec_number} v{version}] DOCX missing — skipping')
-            # Reconstruct cr_paths for each UID
             cr_entry_dir = Path(entry['cr_dir'])
             for uid in entry['cr_uids']:
                 extracted = cr_entry_dir / f'{uid}_extracted.docx'
@@ -169,7 +446,7 @@ def main():
                 elif plain.exists():
                     cr_paths[uid] = plain
 
-        # ── Steps 5 & 6 (retry mode falls through to shared loop below) ──────
+        # ── Steps 5 & 6 (retry mode) ─────────────────────────────────────────
         report = []
 
         for (spec_number, version), uids in ts_groups.items():
@@ -246,7 +523,6 @@ def main():
 
                 for line in log_lines:
                     print(f'  {line}')
-                # Bubble every un-applied change into the warnings list
                 for line in log_lines:
                     if line.strip().startswith('ERROR'):
                         errors.append(line.strip())
@@ -332,6 +608,87 @@ def main():
                 print(f'         ! {err}')
         return
 
+    # ── TS mode — load HF index, skip Steps 1 & 3 ────────────────────────────
+    if args.ts_mode:
+        hf_token = os.environ.get("HF_TOKEN", "")
+        if not hf_token:
+            sys.exit("ERROR: HF_TOKEN must be set in ts-mode")
+
+        from hf_cr_index import load_hf_index
+
+        _section(f'TS mode — spec {args.ts_id!r}')
+        print(f'Loading HF index from {args.hf_repo}...')
+        try:
+            all_records = load_hf_index(hf_token, args.hf_repo)
+        except Exception as e:
+            sys.exit(f'ERROR loading HF index: {e}')
+
+        records = [
+            r for r in all_records
+            if r.get("excel_hash") == args.excel_hash
+            and r.get("spec_number") == args.ts_id
+        ]
+
+        if not records:
+            sys.exit(
+                f'ERROR: no indexed CRs found for spec {args.ts_id!r} '
+                f'with excel_hash={args.excel_hash!r}'
+            )
+
+        # Build ts_groups from index (bypasses Step 3)
+        ts_groups = {}
+        for r in records:
+            key = (r["spec_number"], r["version"])
+            ts_groups.setdefault(key, []).append(r["uid"])
+
+        # Build cr_list for Step 2 download
+        cr_list = [(r["uid"], r["title"]) for r in records]
+
+        print(f'Found {len(records)} CR(s) across {len(ts_groups)} version(s):')
+        for (spec, ver), uids in ts_groups.items():
+            print(f'  TS {spec} v{ver}: {", ".join(uids)}')
+
+        report, cr_paths, ts_paths, spec_dirs = _run_steps_2_to_6(
+            cr_list, ts_groups, output_dir, cr_dir, ts_dir,
+            eol_user, eol_password, author, tc_date,
+        )
+
+        # Copy the CRs actually applied into the run output dir so the ZIP
+        # contains exactly the CRs used for this TS (only needed when using
+        # a shared CR cache that lives outside output_dir).
+        _run_cr_dir = output_dir / 'CRs'
+        if cr_dir.resolve() != _run_cr_dir.resolve():
+            _run_cr_dir.mkdir(parents=True, exist_ok=True)
+            for _p in cr_paths.values():
+                if _p.exists():
+                    shutil.copy2(_p, _run_cr_dir / _p.name)
+
+        _section('Final Report (TS mode)')
+        n_success = sum(1 for r in report if r[4] is not None and not r[6])
+        n_partial  = sum(1 for r in report if r[4] is not None and r[6])
+        n_failed   = sum(1 for r in report if r[4] is None)
+
+        print(f'Spec:         {args.ts_id}')
+        print(f'CRs found:    {len(cr_list)}')
+        print(f'TSs updated:  {n_success} fully OK, {n_partial} with warnings, {n_failed} failed')
+        print()
+
+        for ts_key, n_ok, n_skip, n_crs, out_path, log_path, errors in report:
+            status = 'OK' if out_path and not errors else ('WARN' if out_path else 'FAIL')
+            print(f'  [{status}] {ts_key}')
+            print(f'         CRs: {n_crs}  |  Body changes applied: {n_ok}  |  Skipped: {n_skip}')
+            if out_path:
+                print(f'         Output: {out_path.parent.name}/{out_path.name}')
+            if log_path and log_path.exists():
+                print(f'         Log:    {log_path.parent.name}/{log_path.name}')
+            for err in errors:
+                print(f'         ! {err}')
+
+        print()
+        print(f'Output directory: {output_dir}/')
+        return
+
+    # ── Normal mode ───────────────────────────────────────────────────────────
     excel_path = wsl_path(args.excel_path)
 
     # ── Step 1: Parse Excel ───────────────────────────────────────────────────
@@ -350,242 +707,21 @@ def main():
         print('Nothing to process.')
         return
 
-    # ── Step 2: Download CR DOCXs ─────────────────────────────────────────────
-    _section('Step 2 — Downloading CR DOCXs')
-    cr_paths = {}  # uid -> Path
-
-    for uid, _ in cr_list:
-        docx_path, note = download_cr(uid, cr_dir, eol_user, eol_password)
-        if docx_path:
-            cr_paths[uid] = docx_path
-            print(f'  [{uid}] OK ({note}) — {docx_path.name}')
-
-    n_cr_failed = len(cr_list) - len(cr_paths)
-    if n_cr_failed:
-        print(f'  {len(cr_paths)}/{len(cr_list)} downloaded ({n_cr_failed} failed — details in warnings)')
-    else:
-        print(f'  All {len(cr_list)} CR(s) downloaded successfully')
-
-    # ── Step 3: Parse cover pages → group by target TS ───────────────────────
-    _section('Step 3 — Parsing CR cover pages')
-    ts_groups = {}  # (spec_number, version) -> [uid, ...]
-    uid_cover_failed = []
-
-    for uid in cr_paths:
-        spec_number, version = parse_cr_cover(cr_paths[uid])
-        if spec_number and version:
-            key = (spec_number, version)
-            ts_groups.setdefault(key, []).append(uid)
-            print(f'  [{uid}] -> TS {spec_number}  v{version}')
-        else:
-            uid_cover_failed.append(uid)
-            print(f'  [{uid}] WARNING: could not parse cover page — skipping')
-
-    if not ts_groups:
-        print('\nNo TSs identified. Nothing to apply.')
-        return
-
-    # ── Step 4: Download TSs ──────────────────────────────────────────────────
-    _section('Step 4 — Downloading TSs')
-    ts_paths = {}   # (spec_number, version) -> Path
-    spec_dirs = {}  # (spec_number, version) -> Path (per-spec subfolder)
-
-    for (spec_number, version) in ts_groups:
-        spec_compact = spec_number.replace(' ', '')
-        spec_dir = ts_dir / spec_compact
-        spec_dir.mkdir(parents=True, exist_ok=True)
-        spec_dirs[(spec_number, version)] = spec_dir
-
-        print(f'  [TS {spec_number} v{version}] ', end='', flush=True)
-        filename, note = None, "not attempted"
-        for attempt in range(1, 4):
-            filename, note = download_ts(spec_number, version, spec_dir, eol_user, eol_password)
-            if filename:
-                break
-            if attempt < 3:
-                print(f'\n    [attempt {attempt}/3 failed — retrying in 5s: {note}]', flush=True)
-                print(f'  [TS {spec_number} v{version}] ', end='', flush=True)
-                time.sleep(5)
-            else:
-                print(f'\n    [all 3 attempts failed]', flush=True)
-        if filename:
-            ts_paths[(spec_number, version)] = spec_dir / filename
-            print(f'OK ({note}) — {spec_compact}/{filename}')
-        else:
-            print(f'FAILED — {note}')
-
-    # Write failed_ts.json (even when empty so app.py can detect "no failures")
-    failed_ts_entries = [
-        {
-            "spec_number":       spec_number,
-            "version":           version,
-            "spec_compact":      spec_number.replace(' ', ''),
-            "spec_dir":          str(spec_dirs[(spec_number, version)]),
-            "expected_filename": f"ts_{spec_number.replace(' ', '')}_v{version}.docx",
-            "cr_uids":           ts_groups[(spec_number, version)],
-            "cr_dir":            str(cr_dir),
-        }
-        for (spec_number, version) in ts_groups
-        if (spec_number, version) not in ts_paths
-    ]
-    (output_dir / "failed_ts.json").write_text(
-        json.dumps(failed_ts_entries, indent=2)
+    # Steps 2, 3 (cover page parse), 4, 5, 6
+    report, cr_paths, ts_paths, spec_dirs = _run_steps_2_to_6(
+        cr_list, None, output_dir, cr_dir, ts_dir,
+        eol_user, eol_password, author, tc_date,
     )
 
-    # ── Steps 5 & 6: Apply CRs + Finalise each TS ────────────────────────────
-    _section('Steps 5 & 6 — Applying CRs and Finalising Metadata')
-    report = []  # (ts_key, n_ok, n_skip, n_crs, out_path, log_path, errors)
-
-    for (spec_number, version), uids in ts_groups.items():
-        ts_key = f'TS {spec_number} v{version}'
-        spec_compact = spec_number.replace(' ', '')
-        spec_dir = spec_dirs.get((spec_number, version), ts_dir / spec_compact)
-        spec_dir.mkdir(parents=True, exist_ok=True)
-
-        # Derive new version early so filenames are known upfront
-        new_v = derive_new_version(version)
-        stem = f'ts_{spec_compact}_v{new_v}_was_v{version}'
-        ts_applied = spec_dir / f'ts_{spec_compact}_v{version}_applied.docx'
-        ts_final   = spec_dir / f'{stem}.docx'
-        log_path   = spec_dir / f'{stem}.log'
-        errors = []
-
-        print(f'\n-- {ts_key} ({len(uids)} CR(s): {", ".join(uids)}) --')
-
-        if (spec_number, version) not in ts_paths:
-            msg = 'TS download failed — skipping'
-            print(f'  SKIP: {msg}')
-            report.append((ts_key, 0, 0, len(uids), None, log_path, [msg]))
-            continue
-
-        ts_in = ts_paths[(spec_number, version)]
-
-        # All per-TS output is captured to log_buf (tee: stdout + file)
-        log_buf = io.StringIO()
-        tee = _TeeWriter(sys.stdout, log_buf)
-
-        with contextlib.redirect_stdout(tee):
-            log_header = (
-                f'Pipeline Log\n'
-                f'TS:   {spec_number}  v{version} -> v{new_v}\n'
-                f'CRs:  {", ".join(uids)}\n'
-                f'Date: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")}\n'
-                f'{"=" * 60}\n'
-            )
-            print(log_header, end='')
-
-            # 5a. Parse all CR manifests and combine
-            combined_manifest = []
-            participating_uids = []
-
-            for uid in uids:
-                if uid not in cr_paths:
-                    errors.append(f'[{uid}] CR download had failed — skipped')
-                    continue
-                print(f'  Parsing {uid}... ', end='', flush=True)
-                try:
-                    changes = parse_cr(cr_paths[uid])
-                    combined_manifest.extend(changes)
-                    participating_uids.append(uid)
-                    print(f'{len(changes)} change(s)')
-                except Exception as e:
-                    errors.append(f'[{uid}] parse ERROR: {e}')
-                    print(f'ERROR: {e}')
-
-            if not combined_manifest:
-                print('  No changes parsed — skipping apply step.')
-                report.append((ts_key, 0, 0, len(uids), None, log_path,
-                                errors + ['No changes parsed']))
-                log_path.write_text(log_buf.getvalue(), encoding='utf-8')
-                continue
-
-            # 5b. Apply manifest to TS
-            print(f'  Applying {len(combined_manifest)} change(s) to {ts_in.name}...')
-            try:
-                n_ok, n_skip, log_lines = apply_manifest(
-                    ts_in, combined_manifest, ts_applied, author=author, date=tc_date
-                )
-            except Exception as e:
-                errors.append(f'apply_manifest ERROR: {e}')
-                print(f'  ERROR: {e}')
-                report.append((ts_key, 0, 0, len(uids), None, log_path, errors))
-                log_path.write_text(log_buf.getvalue(), encoding='utf-8')
-                continue
-
-            for line in log_lines:
-                print(f'  {line}')
-            # Bubble every un-applied change into the warnings list
-            for line in log_lines:
-                if line.strip().startswith('ERROR'):
-                    errors.append(line.strip())
-            print(f'  -> Applied: {n_ok}  Skipped: {n_skip}')
-
-            # 6. Finalise metadata (Change History, History, title paragraph)
-            print('  Finalising metadata...')
-            try:
-                ts_doc = docx_lib.Document(str(ts_applied))
-                rev = RevCounter(ts_doc)
-
-                pub_ym, pub_month_year = compute_pub_date()
-                old_v = version
-
-                # Extract old date string from first paragraph
-                title_text = ts_doc.paragraphs[0].text
-                date_match = re.search(r'\((\d{4}-\d{2})\)', title_text)
-                old_date_str = date_match.group(1) if date_match else ''
-
-                print(f'    Version:     {old_v} -> {new_v}')
-                print(f'    Publication: {pub_month_year} ({pub_ym})')
-
-                # One Change History row per CR
-                for uid in participating_uids:
-                    try:
-                        meta = extract_cr_metadata(str(cr_paths[uid]))
-                        ch_cells = update_change_history_table(
-                            ts_doc, meta, pub_ym, old_v, new_v, rev, author, tc_date
-                        )
-                        print(f'    [Change History] {uid}: {ch_cells}')
-                    except NoChangeHistoryTable:
-                        print(f'    [Change History] {uid}: NOT PRESENT — this document has no Change History table (History table only)')
-                    except Exception as e:
-                        errors.append(f'[{uid}] Change History ERROR: {e}')
-                        print(f'    [Change History] {uid}: ERROR — {e}')
-
-                # One History row for the whole TS
-                try:
-                    h_cells = update_history_table(
-                        ts_doc, new_v, pub_month_year, rev, author, tc_date
-                    )
-                    print(f'    [History] {h_cells}')
-                except Exception as e:
-                    errors.append(f'History table ERROR: {e}')
-                    print(f'    [History] ERROR — {e}')
-
-                # Title paragraph version + date
-                if old_date_str:
-                    try:
-                        update_title_para(
-                            ts_doc, old_v, new_v, old_date_str, pub_ym, rev, author, tc_date
-                        )
-                        print(f'    [Title] V{old_v} -> V{new_v}, ({old_date_str}) -> ({pub_ym})')
-                    except Exception as e:
-                        errors.append(f'Title update ERROR: {e}')
-                        print(f'    [Title] ERROR — {e}')
-                else:
-                    print(f'    [Title] SKIP — no (YYYY-MM) pattern in: {title_text!r}')
-
-                ts_doc.save(str(ts_final))
-                print(f'  Saved: {spec_compact}/{ts_final.name}')
-                print(f'  Log:   {spec_compact}/{log_path.name}')
-                report.append((ts_key, n_ok, n_skip, len(uids), ts_final, log_path, errors))
-
-            except Exception as e:
-                errors.append(f'Finalisation ERROR: {e}')
-                print(f'  Finalisation ERROR: {e}')
-                report.append((ts_key, n_ok, n_skip, len(uids), ts_applied, log_path, errors))
-
-        # Write log file after the tee context exits
-        log_path.write_text(log_buf.getvalue(), encoding='utf-8')
+    # Copy the CRs actually applied into the run output dir so the ZIP
+    # contains exactly the CRs used for this run (only needed when using
+    # a shared CR cache that lives outside output_dir).
+    _run_cr_dir = output_dir / 'CRs'
+    if cr_dir.resolve() != _run_cr_dir.resolve():
+        _run_cr_dir.mkdir(parents=True, exist_ok=True)
+        for _p in cr_paths.values():
+            if _p.exists():
+                shutil.copy2(_p, _run_cr_dir / _p.name)
 
     # ── Final Report ──────────────────────────────────────────────────────────
     _section('Final Report')