Deploy Rachana Data Studio

data_studio/api.py

@@ -19,7 +19,7 @@ from data_studio.review import (
     ReviewConflict,
     ReviewValidationError,
     apply_review_action,
-    claim_next_sample,
+    claim_next_sample_with_enrichment,
     queue_overview,
     release_claim,
 )
@@ -152,7 +152,7 @@ def overview(user: dict[str, Any] = Depends(current_user)) -> dict[str, Any]:
 
 @app.post("/api/review/claim/{queue_name}")
 def claim(queue_name: str, user: dict[str, Any] = Depends(require_permission("review"))) -> dict[str, Any]:
-    return {"sample": claim_next_sample(db, queue_name, user["username"])}
+    return {"sample": claim_next_sample_with_enrichment(db, settings, queue_name, user["username"])}
 
 
 @app.post("/api/review/{sample_id}/release")
@@ -166,7 +166,7 @@ def review(
     sample_id: str,
     payload: ReviewRequest,
     user: dict[str, Any] = Depends(require_permission("review")),
-) -> dict[str, str]:
+) -> dict[str, Any]:
     try:
         resolved_action = apply_review_action(
             db=db,
@@ -187,7 +187,7 @@ def review(
         raise HTTPException(status_code=status.HTTP_409_CONFLICT, detail=str(exc)) from exc
     except ReviewValidationError as exc:
         raise HTTPException(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, detail=str(exc)) from exc
-    return {"action": resolved_action}
+    return resolved_action
 
 
 @app.post("/api/admin/sources/seed")

data_studio/config.py

@@ -22,10 +22,12 @@ class StudioSettings:
     hf_raw_dataset_repo: str
     hf_clean_dataset_repo: str
     hf_audit_dataset_repo: str
+    hf_live_review_dataset_repo: str
     custom_pure_telugu_hf_dataset: str
     custom_pure_telugu_source_name: str
     active_tokenizer_version: str
     active_tokenizer_model_path: str
+    active_ner_model_id: str
     target_clean_tokens: int
     studio_env: str
     deployment_version: str
@@ -47,6 +49,7 @@ class StudioSettings:
             hf_raw_dataset_repo=os.getenv("HF_RAW_DATASET_REPO", "").strip(),
             hf_clean_dataset_repo=os.getenv("HF_CLEAN_DATASET_REPO", "").strip(),
             hf_audit_dataset_repo=os.getenv("HF_AUDIT_DATASET_REPO", "").strip(),
+            hf_live_review_dataset_repo=os.getenv("HF_LIVE_REVIEW_DATASET_REPO", "").strip(),
             custom_pure_telugu_hf_dataset=os.getenv("CUSTOM_PURE_TELUGU_HF_DATASET", "").strip(),
             custom_pure_telugu_source_name=os.getenv(
                 "CUSTOM_PURE_TELUGU_SOURCE_NAME",
@@ -60,6 +63,7 @@ class StudioSettings:
                 str(Path("tokenizer") / "rachana_bpe32k.model"),
             ).strip()
             or str(Path("tokenizer") / "rachana_bpe32k.model"),
+            active_ner_model_id=os.getenv("ACTIVE_NER_MODEL_ID", "ai4bharat/IndicNER").strip() or "ai4bharat/IndicNER",
             target_clean_tokens=int(os.getenv("TARGET_CLEAN_TOKENS", "0").strip() or "0"),
             studio_env=os.getenv("STUDIO_ENV", "development").strip().lower(),
             deployment_version=os.getenv("DEPLOYMENT_VERSION", "local").strip() or "local",
@@ -104,6 +108,7 @@ class StudioSettings:
             "legacy_import_export_enabled": self.allow_legacy_import_export,
             "custom_pure_telugu_hf_dataset": self.custom_pure_telugu_hf_dataset or None,
             "active_tokenizer_version": self.active_tokenizer_version,
+            "active_ner_model_id": self.active_ner_model_id,
             "target_clean_tokens": self.target_clean_tokens,
             "single_user_mode": self.single_user_mode,
             "single_user_username": self.single_user_username or None,

data_studio/importer.py

@@ -57,18 +57,19 @@ def _document_sample(source: dict[str, Any], row: dict[str, Any]) -> dict[str, A
     tags = [payload["doc_type"]] if payload["doc_type"] and payload["doc_type"] != "unknown" else []
     return {
         "payload": payload,
-        "review": {
-            "current_text": raw_text,
-            "current_pair": None,
-            "current_transliteration": None,
-            "tags": tags,
-            "quality_tags": [],
-            "task_tags": [],
-            "notes": "",
-            "last_reviewed_by": None,
-            "last_reviewed_at": None,
-            "edit_count": 0,
-        },
+        "review": {
+            "current_text": raw_text,
+            "current_pair": None,
+            "current_transliteration": None,
+            "tags": tags,
+            "quality_tags": [],
+            "task_tags": [],
+            "ner": None,
+            "notes": "",
+            "last_reviewed_by": None,
+            "last_reviewed_at": None,
+            "edit_count": 0,
+        },
         "identity_parts": [raw_text],
     }
 
@@ -90,21 +91,22 @@ def _translation_sample(source: dict[str, Any], row: dict[str, Any]) -> dict[str
     }
     return {
         "payload": payload,
-        "review": {
-            "current_text": None,
-            "current_pair": {
-                "source_text": source_text,
-                "target_text": target_text,
-            },
-            "current_transliteration": None,
-            "tags": [],
-            "quality_tags": [],
-            "task_tags": ["translation"],
-            "notes": "",
-            "last_reviewed_by": None,
-            "last_reviewed_at": None,
-            "edit_count": 0,
-        },
+        "review": {
+            "current_text": None,
+            "current_pair": {
+                "source_text": source_text,
+                "target_text": target_text,
+            },
+            "current_transliteration": None,
+            "tags": [],
+            "quality_tags": [],
+            "task_tags": ["translation"],
+            "ner": None,
+            "notes": "",
+            "last_reviewed_by": None,
+            "last_reviewed_at": None,
+            "edit_count": 0,
+        },
         "identity_parts": [source_text, target_text],
     }
 
@@ -127,21 +129,22 @@ def _transliteration_sample(source: dict[str, Any], row: dict[str, Any]) -> dict
     }
     return {
         "payload": payload,
-        "review": {
-            "current_text": None,
-            "current_pair": None,
-            "current_transliteration": {
-                "native_text": native_text,
-                "latin_text": latin_text,
-            },
-            "tags": [],
-            "quality_tags": [],
-            "task_tags": ["transliteration"],
-            "notes": "",
-            "last_reviewed_by": None,
-            "last_reviewed_at": None,
-            "edit_count": 0,
-        },
+        "review": {
+            "current_text": None,
+            "current_pair": None,
+            "current_transliteration": {
+                "native_text": native_text,
+                "latin_text": latin_text,
+            },
+            "tags": [],
+            "quality_tags": [],
+            "task_tags": ["transliteration"],
+            "ner": None,
+            "notes": "",
+            "last_reviewed_by": None,
+            "last_reviewed_at": None,
+            "edit_count": 0,
+        },
         "identity_parts": [native_text, latin_text],
     }
 
@@ -183,14 +186,21 @@ def _sample_template(
             "script": _script_for_queue(source["queue_name"]),
             "language": _language_for_queue(source["queue_name"]),
         },
-        "export": {
-            "eligible": False,
-            "dataset_version": None,
-            "exported_at": None,
-        },
-        "created_at": now,
-        "updated_at": now,
-    }
+        "export": {
+            "eligible": False,
+            "dataset_version": None,
+            "exported_at": None,
+        },
+        "live_sync": {
+            "repo_id": None,
+            "path_in_repo": None,
+            "status": "pending",
+            "last_synced_at": None,
+            "last_error": None,
+        },
+        "created_at": now,
+        "updated_at": now,
+    }
 
 
 def build_sample(source: dict[str, Any], row: dict[str, Any]) -> dict[str, Any] | None:

data_studio/live_review_sync.py

@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import json
+import tempfile
+from pathlib import Path
+from typing import Any
+
+from huggingface_hub import HfApi
+
+from data_studio.config import StudioSettings
+
+
+def target_repo(settings: StudioSettings) -> str:
+    repo = settings.hf_live_review_dataset_repo or settings.hf_clean_dataset_repo
+    if not repo:
+        raise ValueError("HF_LIVE_REVIEW_DATASET_REPO or HF_CLEAN_DATASET_REPO is required for live review sync.")
+    return repo
+
+
+def reviewed_sample_row(sample: dict[str, Any], review_action: str) -> dict[str, Any]:
+    return {
+        "sample_id": sample["sample_id"],
+        "queue_name": sample["queue_name"],
+        "sample_type": sample["sample_type"],
+        "source": sample["source"],
+        "original": sample["payload"],
+        "cleaned": sample["review"],
+        "status": sample["status"],
+        "metrics": sample.get("metrics", {}),
+        "review_action": review_action,
+    }
+
+
+def sync_reviewed_sample(settings: StudioSettings, sample: dict[str, Any], review_action: str) -> dict[str, Any]:
+    repo_id = target_repo(settings)
+    queue_name = sample["queue_name"]
+    sample_path = f"live_reviews/{queue_name}/{sample['sample_id']}.json"
+    api = HfApi(token=settings.hf_token)
+    api.create_repo(repo_id, repo_type="dataset", private=True, exist_ok=True)
+    payload = reviewed_sample_row(sample, review_action)
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_file = Path(temp_dir) / "sample.json"
+        temp_file.write_text(json.dumps(payload, ensure_ascii=False, indent=2, default=str), encoding="utf-8")
+        api.upload_file(
+            path_or_fileobj=str(temp_file),
+            path_in_repo=sample_path,
+            repo_id=repo_id,
+            repo_type="dataset",
+            commit_message=f"Sync reviewed sample {sample['sample_id']}",
+        )
+    return {"repo_id": repo_id, "path_in_repo": sample_path, "status": "synced"}

data_studio/ner.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Any
+
+from transformers import AutoModelForTokenClassification, AutoTokenizer, pipeline
+
+from data_studio.config import StudioSettings
+from data_studio.utils import utc_now_iso
+
+
+def _normalize_entity_label(label: str) -> str:
+    normalized = label.upper().replace("B-", "").replace("I-", "")
+    return normalized
+
+
+@lru_cache(maxsize=2)
+def _ner_pipeline(model_id: str, token: str):
+    tokenizer = AutoTokenizer.from_pretrained(model_id, token=token)
+    model = AutoModelForTokenClassification.from_pretrained(model_id, token=token)
+    return pipeline(
+        "token-classification",
+        model=model,
+        tokenizer=tokenizer,
+        aggregation_strategy="simple",
+        device=-1,
+    )
+
+
+def should_run_ner(sample: dict[str, Any]) -> bool:
+    return sample.get("queue_name") == "pure_telugu" and sample.get("sample_type") == "document"
+
+
+def build_ner_suggestions(settings: StudioSettings, text: str) -> dict[str, Any]:
+    ner_pipe = _ner_pipeline(settings.active_ner_model_id, settings.hf_token)
+    raw_entities = ner_pipe(text)
+    entities: list[dict[str, Any]] = []
+    suggested_tags: list[str] = []
+    seen = set()
+    for item in raw_entities:
+        label = _normalize_entity_label(str(item.get("entity_group", "")))
+        word = str(item.get("word", "")).strip()
+        if not word or not label:
+            continue
+        key = (label, word)
+        if key in seen:
+            continue
+        seen.add(key)
+        entities.append(
+            {
+                "text": word,
+                "label": label,
+                "score": round(float(item.get("score", 0.0)), 4),
+                "start": item.get("start"),
+                "end": item.get("end"),
+            }
+        )
+        suggested_tags.append(word)
+    return {
+        "model_version": settings.active_ner_model_id,
+        "generated_at": utc_now_iso(),
+        "entities": entities,
+        "suggested_tags": suggested_tags[:24],
+        "status": "ready",
+        "error": None,
+    }

data_studio/review.py

@@ -10,6 +10,8 @@ from pymongo import ReturnDocument
 from pymongo.database import Database
 
 from data_studio.config import StudioSettings
+from data_studio.live_review_sync import sync_reviewed_sample
+from data_studio.ner import build_ner_suggestions, should_run_ner
 from data_studio.token_accounting import token_progress, token_stats_for_review
 from data_studio.utils import normalize_multiline_text, normalize_text, utc_now
 
@@ -78,6 +80,35 @@ def claim_next_sample(db: Database, queue_name: str, reviewer: str) -> dict[str,
     return _serialize_sample(sample)
 
 
+def claim_next_sample_with_enrichment(
+    db: Database,
+    settings: StudioSettings,
+    queue_name: str,
+    reviewer: str,
+) -> dict[str, Any] | None:
+    sample = claim_next_sample(db, queue_name, reviewer)
+    if sample is None:
+        return None
+    if sample.get("review", {}).get("ner") is None and should_run_ner(sample):
+        try:
+            ner = build_ner_suggestions(settings, str(sample["review"].get("current_text", "")))
+        except Exception as exc:
+            ner = {
+                "model_version": settings.active_ner_model_id,
+                "generated_at": None,
+                "entities": [],
+                "suggested_tags": [],
+                "status": "failed",
+                "error": str(exc),
+            }
+        db["samples"].update_one(
+            {"sample_id": sample["sample_id"]},
+            {"$set": {"review.ner": ner, "updated_at": utc_now()}},
+        )
+        sample["review"]["ner"] = ner
+    return sample
+
+
 def next_sample(db: Database, queue_name: str) -> dict[str, Any] | None:
     """Legacy Streamlit compatibility; the API uses claim_next_sample."""
     return _serialize_sample(db["samples"].find_one({"queue_name": queue_name, "status": "pending"}, sort=[("created_at", 1)]))
@@ -144,7 +175,7 @@ def apply_review_action(
     notes: str,
     confidence: int,
     reason: str,
-) -> str:
+) -> dict[str, Any]:
     if action not in {"accept", "edit", "reject", "skip"}:
         raise ReviewValidationError(f"Unsupported review action: {action}")
     if not 1 <= int(confidence) <= 5:
@@ -153,6 +184,9 @@ def apply_review_action(
         raise ReviewValidationError("A rejection reason is required.")
 
     client = db.client
+    updated_sample: dict[str, Any] | None = None
+    status = "pending"
+    resolved_action = action
     with client.start_session() as session:
         with session.start_transaction():
             sample = db["samples"].find_one(
@@ -219,4 +253,40 @@ def apply_review_action(
                 },
                 session=session,
             )
-    return resolved_action
+            updated_sample = db["samples"].find_one({"sample_id": sample_id}, session=session)
+
+    live_sync: dict[str, Any] | None = None
+    if updated_sample is not None and status in {"accepted", "edited"}:
+        try:
+            live_sync = sync_reviewed_sample(settings, updated_sample, resolved_action)
+            db["samples"].update_one(
+                {"sample_id": sample_id},
+                {
+                    "$set": {
+                        "live_sync.repo_id": live_sync["repo_id"],
+                        "live_sync.path_in_repo": live_sync["path_in_repo"],
+                        "live_sync.status": "synced",
+                        "live_sync.last_synced_at": utc_now(),
+                        "live_sync.last_error": None,
+                    }
+                },
+            )
+        except Exception as exc:
+            live_sync = {"status": "failed", "error": str(exc)}
+            db["samples"].update_one(
+                {"sample_id": sample_id},
+                {
+                    "$set": {
+                        "live_sync.status": "failed",
+                        "live_sync.last_synced_at": None,
+                        "live_sync.last_error": str(exc),
+                    }
+                },
+            )
+
+    return {
+        "action": resolved_action,
+        "status": status,
+        "token_stats": updated_sample["review"]["token_stats"] if updated_sample else None,
+        "live_sync": live_sync,
+    }

frontend/app/globals.css

@@ -1 +1,2 @@
 :root{--ink:#17221d;--muted:#637169;--paper:#f4f5f0;--card:#fff;--line:#dfe4dd;--green:#176b4d;--green-soft:#e8f2ed;--gold:#d2a547;--red:#a23d3d;--shadow:0 18px 50px rgba(30,48,39,.08)}*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}button,input,textarea,select{font:inherit}button{cursor:pointer}.shell{min-height:100vh;display:grid;grid-template-columns:280px 1fr}.sidebar{background:#102a20;color:#f5f8f5;padding:26px 18px;display:flex;flex-direction:column;gap:28px;position:sticky;top:0;height:100vh}.brand{display:flex;gap:12px;align-items:center}.brandMark{width:42px;height:42px;display:grid;place-items:center;border-radius:12px;background:var(--gold);color:#183427;font-size:25px;font-weight:800}.brand div{display:grid}.brand strong{font-size:18px}.brand small{color:#b8c9c0}.sidebar nav{display:grid;gap:8px}.sidebar nav button{border:0;background:transparent;color:#c7d4cd;text-align:left;padding:12px 14px;border-radius:9px}.sidebar nav button.active,.sidebar nav button:hover{background:#214638;color:#fff}.tokenPanel{border:1px solid #345044;border-radius:14px;padding:16px;background:#163126;display:grid;gap:8px}.tokenPanel strong{font-size:20px}.tokenPanel small{color:#b8c9c0}.tokenPanelStats{display:grid;grid-template-columns:1fr 1fr;gap:10px;margin-top:6px}.tokenPanelStats div{background:#214638;border-radius:10px;padding:10px;display:grid;gap:3px}.tokenPanelStats span{font-size:11px;color:#b8c9c0;text-transform:uppercase;letter-spacing:.06em}.progressBar{height:10px;border-radius:999px;background:#284136;overflow:hidden}.progressBar span{display:block;height:100%;background:linear-gradient(90deg,#d2a547,#47a977)}.profile{margin-top:auto;display:grid;grid-template-columns:34px 1fr;gap:8px;align-items:center;border-top:1px solid #345044;padding-top:18px}.profile>span{width:32px;height:32px;border-radius:50%;background:#dceade;color:#183427;display:grid;place-items:center;font-weight:700}.profile div{display:grid}.profile small{color:#a8bbb1;text-transform:capitalize}.profile button{grid-column:1/-1;border:0;color:#bdcec5;background:transparent;text-align:left;padding:8px 0}main{padding:40px;max-width:1600px;width:100%;margin:auto}.pageHeader{display:flex;justify-content:space-between;align-items:start;margin-bottom:28px;gap:18px}.pageHeader h1{font-family:Georgia,serif;font-size:36px;line-height:1.08;margin:7px 0}.pageHeader p{color:var(--muted);margin:0}.eyebrow{text-transform:uppercase;letter-spacing:.13em;color:var(--green);font-size:12px;font-weight:800}.lease{background:var(--green-soft);color:var(--green);border:1px solid #c9dfd3;padding:9px 13px;border-radius:20px;font-size:13px}.queueTabs{display:grid;grid-template-columns:repeat(3,1fr);gap:10px;margin-bottom:22px}.queueTabs button{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:14px;text-align:left;display:grid;gap:4px;color:var(--ink)}.queueTabs button span{font-size:12px;color:var(--muted)}.queueTabs button.active{border-color:var(--green);box-shadow:inset 0 -3px var(--green)}.reviewGrid{display:grid;grid-template-columns:minmax(0,1fr) 360px;gap:18px}.workCard,.decisionCard,.tableCard,.operationCard,.metric,.emptyState{background:var(--card);border:1px solid var(--line);border-radius:14px;box-shadow:var(--shadow)}.cardHeader{padding:16px 20px;border-bottom:1px solid var(--line);display:flex;justify-content:space-between;gap:12px}.cardHeader div{display:grid;grid-template-columns:auto 1fr;gap:2px 8px}.cardHeader small{grid-column:2;color:var(--muted)}.statusDot{width:9px;height:9px;border-radius:50%;background:#47a977;grid-row:1/3;align-self:center}.sampleType{font-size:12px;background:#f0f2ee;border-radius:20px;padding:6px 10px;height:max-content}.lineColumnHeader{display:flex;justify-content:space-between;align-items:center;gap:12px;margin-bottom:12px}.lineColumnHeader div{display:grid;gap:3px}.lineColumnHeader small{color:var(--muted);font-size:12px}.decisionCard{padding:20px;height:max-content}.decisionCard h2,.operationCard h2,.tableCard h2{margin:0 0 6px;font-family:Georgia,serif}.decisionCard p,.operationCard p{color:var(--muted);font-size:13px;margin:0 0 18px}.decisionCard label,.operationCard label,.loginCard label{display:grid;gap:6px;font-size:12px;font-weight:700;margin:12px 0}.decisionCard input,.decisionCard textarea,.operationCard input,.operationCard select,.loginCard input,.editorTextarea{border:1px solid var(--line);border-radius:10px;padding:12px;background:#fff;color:var(--ink)}.editorPanel{display:grid;grid-template-columns:1fr 1fr;min-height:580px}.editorColumn{padding:18px;min-width:0}.editorColumn+.editorColumn{border-left:1px solid var(--line)}.editableColumn{background:#fafbf9}.editorTextarea{width:100%;min-height:460px;resize:vertical;line-height:1.6}.referenceBlock{border:1px solid var(--line);border-radius:10px;background:#f6f8f5;padding:14px;white-space:pre-wrap;line-height:1.7;min-height:460px;overflow:auto}.statsStrip{display:grid;grid-template-columns:1fr 1fr;gap:10px;margin:14px 0 6px}.statsStrip div{border:1px solid var(--line);border-radius:10px;padding:12px;background:#f7f8f4;display:grid;gap:4px}.statsStrip span{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.06em}.statsStrip strong{font-family:Georgia,serif;font-size:24px}.actions{display:grid;grid-template-columns:1fr 1fr;gap:8px;margin-top:18px}button{border:1px solid var(--line);background:#fff;border-radius:8px;padding:10px 14px;color:var(--ink)}button.primary{background:var(--green);border-color:var(--green);color:#fff;font-weight:700}button.danger{color:var(--red);border-color:#e6caca}button:disabled{opacity:.55;cursor:not-allowed}.emptyState{text-align:center;padding:80px 20px}.emptyState div{width:56px;height:56px;border-radius:50%;background:var(--green-soft);color:var(--green);display:grid;place-items:center;margin:auto;font-size:24px}.emptyState h2{font-family:Georgia,serif;margin-bottom:5px}.emptyState p{color:var(--muted);margin-bottom:22px}.notice{position:fixed;top:18px;right:22px;z-index:5;background:#18372a;color:#fff;padding:12px 14px;border-radius:9px;box-shadow:var(--shadow);display:flex;gap:20px;align-items:center}.notice button{background:transparent;color:#d6e3dc;border:0;padding:0}.metrics{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin-bottom:20px}.metric{padding:20px;display:grid;gap:8px}.metric span{text-transform:capitalize;color:var(--muted)}.metric strong{font-size:32px;font-family:Georgia,serif}.tableCard{padding:22px;overflow:auto;margin-bottom:20px}table{border-collapse:collapse;width:100%;margin-top:16px}th,td{text-align:left;border-bottom:1px solid var(--line);padding:13px 10px;font-size:13px;vertical-align:top}th{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.08em}td small{display:block;color:var(--muted);margin-top:3px}.adminGrid{display:grid;grid-template-columns:repeat(3,1fr);gap:16px}.operationCard{padding:20px}.operationCard button{margin-top:10px}.result{white-space:pre-wrap;background:#102a20;color:#dce9e2;padding:18px;border-radius:10px;margin-top:20px;overflow:auto}.loginPage{min-height:100vh;display:grid;grid-template-columns:1.2fr .8fr;background:#102a20}.loginStory{color:#fff;padding:10vw;display:flex;flex-direction:column;justify-content:center;background:radial-gradient(circle at 20% 20%,#285c46 0,transparent 35%),#102a20}.loginStory .eyebrow{color:#d9b666}.loginStory h1{font:54px/1.05 Georgia,serif;max-width:680px;margin:18px 0}.loginStory p{color:#bed0c7;font-size:18px;max-width:540px}.teluguSample{font:27px/1.7 Georgia,serif;color:#e6cf98;margin-top:50px}.loginCard{background:#fff;margin:auto;width:min(420px,85%);border-radius:16px;padding:34px;box-shadow:var(--shadow)}.loginCard h2{font:32px Georgia,serif;margin:38px 0 5px}.loginCard p{color:var(--muted);margin:0 0 22px}.loginCard button{width:100%;margin-top:14px}.fieldError{background:#fff0f0;color:var(--red);padding:10px;border-radius:8px;font-size:13px}@media(max-width:1150px){.reviewGrid{grid-template-columns:1fr}.decisionCard{width:100%}.adminGrid{grid-template-columns:1fr}.loginPage{grid-template-columns:1fr}.loginStory{display:none}}@media(max-width:760px){.shell{display:block}.sidebar{position:static;height:auto;padding:16px}.sidebar nav{grid-template-columns:repeat(3,1fr)}.profile{display:none}main{padding:22px 14px}.queueTabs,.metrics,.statsStrip,.tokenPanelStats{grid-template-columns:1fr}.editorPanel{grid-template-columns:1fr}.editorColumn+.editorColumn{border-left:0;border-top:1px solid var(--line)}.pageHeader{display:grid}.pageHeader h1{font-size:30px}}
+:root{--ink:#17221d;--muted:#637169;--paper:#f4f5f0;--card:#fff;--line:#dfe4dd;--green:#176b4d;--green-soft:#e8f2ed;--gold:#d2a547;--red:#a23d3d;--shadow:0 18px 50px rgba(30,48,39,.08)}*{box-sizing:border-box}body{margin:0;background:var(--paper);color:var(--ink);font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}button,input,textarea,select{font:inherit}button{cursor:pointer}.shell{min-height:100vh;display:grid;grid-template-columns:280px 1fr}.sidebar{background:#102a20;color:#f5f8f5;padding:26px 18px;display:flex;flex-direction:column;gap:28px;position:sticky;top:0;height:100vh}.brand{display:flex;gap:12px;align-items:center}.brandMark{width:42px;height:42px;display:grid;place-items:center;border-radius:12px;background:var(--gold);color:#183427;font-size:25px;font-weight:800}.brand div{display:grid}.brand strong{font-size:18px}.brand small{color:#b8c9c0}.sidebar nav{display:grid;gap:8px}.sidebar nav button{border:0;background:transparent;color:#c7d4cd;text-align:left;padding:12px 14px;border-radius:9px}.sidebar nav button.active,.sidebar nav button:hover{background:#214638;color:#fff}.tokenPanel{border:1px solid #345044;border-radius:14px;padding:16px;background:#163126;display:grid;gap:8px}.tokenPanel strong{font-size:20px}.tokenPanel small{color:#b8c9c0}.tokenPanelStats{display:grid;grid-template-columns:1fr 1fr;gap:10px;margin-top:6px}.tokenPanelStats div{background:#214638;border-radius:10px;padding:10px;display:grid;gap:3px}.tokenPanelStats span{font-size:11px;color:#b8c9c0;text-transform:uppercase;letter-spacing:.06em}.progressBar{height:10px;border-radius:999px;background:#284136;overflow:hidden}.progressBar span{display:block;height:100%;background:linear-gradient(90deg,#d2a547,#47a977)}.profile{margin-top:auto;display:grid;grid-template-columns:34px 1fr;gap:8px;align-items:center;border-top:1px solid #345044;padding-top:18px}.profile>span{width:32px;height:32px;border-radius:50%;background:#dceade;color:#183427;display:grid;place-items:center;font-weight:700}.profile div{display:grid}.profile small{color:#a8bbb1;text-transform:capitalize}.profile button{grid-column:1/-1;border:0;color:#bdcec5;background:transparent;text-align:left;padding:8px 0}main{padding:40px;max-width:1600px;width:100%;margin:auto}.pageHeader{display:flex;justify-content:space-between;align-items:start;margin-bottom:28px;gap:18px}.pageHeader h1{font-family:Georgia,serif;font-size:36px;line-height:1.08;margin:7px 0}.pageHeader p{color:var(--muted);margin:0}.eyebrow{text-transform:uppercase;letter-spacing:.13em;color:var(--green);font-size:12px;font-weight:800}.lease{background:var(--green-soft);color:var(--green);border:1px solid #c9dfd3;padding:9px 13px;border-radius:20px;font-size:13px}.queueTabs{display:grid;grid-template-columns:repeat(3,1fr);gap:10px;margin-bottom:22px}.queueTabs button{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:14px;text-align:left;display:grid;gap:4px;color:var(--ink)}.queueTabs button span{font-size:12px;color:var(--muted)}.queueTabs button.active{border-color:var(--green);box-shadow:inset 0 -3px var(--green)}.reviewGrid{display:grid;grid-template-columns:minmax(0,1fr) 360px;gap:18px}.workCard,.decisionCard,.tableCard,.operationCard,.metric,.emptyState{background:var(--card);border:1px solid var(--line);border-radius:14px;box-shadow:var(--shadow)}.cardHeader{padding:16px 20px;border-bottom:1px solid var(--line);display:flex;justify-content:space-between;gap:12px}.cardHeader div{display:grid;grid-template-columns:auto 1fr;gap:2px 8px}.cardHeader small{grid-column:2;color:var(--muted)}.statusDot{width:9px;height:9px;border-radius:50%;background:#47a977;grid-row:1/3;align-self:center}.sampleType{font-size:12px;background:#f0f2ee;border-radius:20px;padding:6px 10px;height:max-content}.lineColumnHeader{display:flex;justify-content:space-between;align-items:center;gap:12px;margin-bottom:12px}.lineColumnHeader div{display:grid;gap:3px}.lineColumnHeader small{color:var(--muted);font-size:12px}.decisionCard{padding:20px;height:max-content}.decisionCard h2,.operationCard h2,.tableCard h2{margin:0 0 6px;font-family:Georgia,serif}.decisionCard p,.operationCard p{color:var(--muted);font-size:13px;margin:0 0 18px}.decisionCard label,.operationCard label,.loginCard label{display:grid;gap:6px;font-size:12px;font-weight:700;margin:12px 0}.decisionCard input,.decisionCard textarea,.operationCard input,.operationCard select,.loginCard input,.editorTextarea,.lineInput{border:1px solid var(--line);border-radius:10px;padding:12px;background:#fff;color:var(--ink)}.editorPanel{display:grid;grid-template-columns:1fr 1fr;min-height:580px}.editorColumn{padding:18px;min-width:0}.editorColumn+.editorColumn{border-left:1px solid var(--line)}.editableColumn{background:#fafbf9}.editorTextarea{width:100%;min-height:460px;resize:vertical;line-height:1.6}.referenceBlock{border:1px solid var(--line);border-radius:10px;background:#f6f8f5;padding:14px;white-space:pre-wrap;line-height:1.7;min-height:460px;overflow:auto}.lineEditorPanel{min-height:auto}.lineRowList{display:grid;gap:10px;max-height:640px;overflow:auto;padding-right:4px}.lineRow{display:grid;grid-template-columns:28px minmax(0,1fr) auto;gap:10px;align-items:start;border:1px solid var(--line);border-radius:10px;background:#fff;padding:10px}.lineRow>span{width:24px;height:24px;border-radius:50%;display:grid;place-items:center;background:#edf3ef;color:var(--green);font-size:11px;font-weight:700}.lineInput{width:100%;min-height:74px;resize:vertical;line-height:1.5}.referenceInline{border:1px solid var(--line);border-radius:10px;background:#f6f8f5;padding:10px;white-space:pre-wrap;line-height:1.6;min-height:74px}.deleteLine{padding:8px 10px;border-color:#ead0d0;color:var(--red);background:#fff7f7;font-size:12px}.statsStrip{display:grid;grid-template-columns:1fr 1fr;gap:10px;margin:14px 0 6px}.statsStrip div{border:1px solid var(--line);border-radius:10px;padding:12px;background:#f7f8f4;display:grid;gap:4px}.statsStrip span{font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.06em}.statsStrip strong{font-family:Georgia,serif;font-size:24px}.suggestionBlock{border:1px solid var(--line);border-radius:10px;background:#f7f8f4;padding:12px;margin:10px 0 14px}.suggestionBlock h3{margin:0 0 8px;font-size:14px}.chipRow,.entityList{display:flex;flex-wrap:wrap;gap:8px}.chipButton,.entityChip{border:1px solid #c9dfd3;border-radius:999px;padding:6px 10px;background:#fff;color:var(--green);font-size:12px}.chipButton{cursor:pointer}.syncNote{border:1px solid var(--line);border-radius:10px;background:#f7f8f4;padding:10px 12px;margin:10px 0 14px;font-size:12px}.actions{display:grid;grid-template-columns:1fr 1fr;gap:8px;margin-top:18px}button{border:1px solid var(--line);background:#fff;border-radius:8px;padding:10px 14px;color:var(--ink)}button.primary{background:var(--green);border-color:var(--green);color:#fff;font-weight:700}button.danger{color:var(--red);border-color:#e6caca}button:disabled{opacity:.55;cursor:not-allowed}.emptyState{text-align:center;padding:80px 20px}.emptyState div{width:56px;height:56px;border-radius:50%;background:var(--green-soft);color:var(--green);display:grid;place-items:center;margin:auto;font-size:24px}.emptyState h2{font-family:Georgia,serif;margin-bottom:5px}.emptyState p{color:var(--muted);margin-bottom:22px}.notice{position:fixed;top:18px;right:22px;z-index:5;background:#18372a;color:#fff;padding:12px 14px;border-radius:9px;box-shadow:var(--shadow);display:flex;gap:20px;align-items:center}.notice button{background:transparent;color:#d6e3dc;border:0;padding:0}.metrics{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin-bottom:20px}.metric{padding:20px;display:grid;gap:8px}.metric span{text-transform:capitalize;color:var(--muted)}.metric strong{font-size:32px;font-family:Georgia,serif}.tableCard{padding:22px;overflow:auto;margin-bottom:20px}table{border-collapse:collapse;width:100%;margin-top:16px}th,td{text-align:left;border-bottom:1px solid var(--line);padding:13px 10px;font-size:13px;vertical-align:top}th{color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.08em}td small{display:block;color:var(--muted);margin-top:3px}.adminGrid{display:grid;grid-template-columns:repeat(3,1fr);gap:16px}.operationCard{padding:20px}.operationCard button{margin-top:10px}.result{white-space:pre-wrap;background:#102a20;color:#dce9e2;padding:18px;border-radius:10px;margin-top:20px;overflow:auto}.loginPage{min-height:100vh;display:grid;grid-template-columns:1.2fr .8fr;background:#102a20}.loginStory{color:#fff;padding:10vw;display:flex;flex-direction:column;justify-content:center;background:radial-gradient(circle at 20% 20%,#285c46 0,transparent 35%),#102a20}.loginStory .eyebrow{color:#d9b666}.loginStory h1{font:54px/1.05 Georgia,serif;max-width:680px;margin:18px 0}.loginStory p{color:#bed0c7;font-size:18px;max-width:540px}.teluguSample{font:27px/1.7 Georgia,serif;color:#e6cf98;margin-top:50px}.loginCard{background:#fff;margin:auto;width:min(420px,85%);border-radius:16px;padding:34px;box-shadow:var(--shadow)}.loginCard h2{font:32px Georgia,serif;margin:38px 0 5px}.loginCard p{color:var(--muted);margin:0 0 22px}.loginCard button{width:100%;margin-top:14px}.fieldError{background:#fff0f0;color:var(--red);padding:10px;border-radius:8px;font-size:13px}@media(max-width:1150px){.reviewGrid{grid-template-columns:1fr}.decisionCard{width:100%}.adminGrid{grid-template-columns:1fr}.loginPage{grid-template-columns:1fr}.loginStory{display:none}}@media(max-width:760px){.shell{display:block}.sidebar{position:static;height:auto;padding:16px}.sidebar nav{grid-template-columns:repeat(3,1fr)}.profile{display:none}main{padding:22px 14px}.queueTabs,.metrics,.statsStrip,.tokenPanelStats{grid-template-columns:1fr}.editorPanel{grid-template-columns:1fr}.editorColumn+.editorColumn{border-left:0;border-top:1px solid var(--line)}.pageHeader{display:grid}.pageHeader h1{font-size:30px}.lineRow{grid-template-columns:24px minmax(0,1fr)}.deleteLine{grid-column:2}}

frontend/app/page.tsx

@@ -30,8 +30,17 @@ type Sample = {
     task_tags: string[];
     notes: string;
     token_stats?: { tokenizer_version: string; clean_char_count: number; clean_token_count: number };
+    ner?: {
+      model_version: string;
+      generated_at: string | null;
+      entities: Array<{ text: string; label: string; score: number }>;
+      suggested_tags: string[];
+      status: string;
+      error: string | null;
+    } | null;
   };
   source: { dataset_key: string; source_record_id?: string; source_title?: string; source_url?: string };
+  live_sync?: { status: string; repo_id?: string | null; path_in_repo?: string | null; last_error?: string | null };
   lease?: { expires_at: string };
 };
 
@@ -120,7 +129,7 @@ export default function Home() {
     setBusy(true);
     setMessage("");
     try {
-      const result = await api<{ action: string }>(`/api/review/${encodeURIComponent(sample.sample_id)}`, token, {
+      const result = await api<{ action: string; live_sync?: { status: string; error?: string } | null }>(`/api/review/${encodeURIComponent(sample.sample_id)}`, token, {
         method: "POST",
         body: JSON.stringify({
           action,
@@ -134,7 +143,8 @@ export default function Home() {
           confidence: editor.confidence,
         }),
       });
-      setMessage(result.action === "edit" && action === "accept" ? "Changes were safely recorded as an edit." : `Review recorded: ${result.action}.`);
+      const syncSuffix = result.live_sync?.status === "failed" ? " Hugging Face sync failed; MongoDB audit was saved." : result.live_sync?.status === "synced" ? " Saved to Hugging Face." : "";
+      setMessage((result.action === "edit" && action === "accept" ? "Changes were safely recorded as an edit." : `Review recorded: ${result.action}.`) + syncSuffix);
       setSample(null);
       await Promise.all([refreshOverview(token), claim(queue)]);
     } catch (error) {
@@ -201,6 +211,8 @@ function Login({ onLogin }: { onLogin: (token: string, user: User) => void }) {
 
 type EditorState = { cleaned: Record<string, unknown>; tags: string; qualityTags: string; taskTags: string; notes: string; reason: string; confidence: number };
 
+type ReviewLine = { id: string; left: string; right: string };
+
 function ReviewWorkspace({ queue, queues, sample, busy, onQueue, onClaim, onSubmit }: { queue: string; queues: QueueCounts; sample: Sample | null; busy: boolean; onQueue: (q: string) => void; onClaim: () => void; onSubmit: (action: string, editor: EditorState) => void }) {
   return <><header className="pageHeader"><div><span className="eyebrow">Review workspace</span><h1>Make one clear decision at a time.</h1><p>Work through the audit queue, clean the text directly, and keep the token target moving.</p></div>{sample?.lease && <div className="lease">Reserved until {new Date(sample.lease.expires_at).toLocaleTimeString([], { hour: "2-digit", minute: "2-digit" })}</div>}</header><div className="queueTabs">{QUEUES.map((item) => <button key={item.id} className={queue === item.id ? "active" : ""} onClick={() => onQueue(item.id)}><strong>{item.label}</strong><span>{queues[item.id]?.pending || 0} pending</span></button>)}</div>{sample ? <ReviewEditor key={sample.sample_id} sample={sample} busy={busy} onSubmit={onSubmit} /> : <div className="emptyState"><div>✓</div><h2>Ready for the next item?</h2><p>{QUEUES.find((q) => q.id === queue)?.description}</p><button className="primary" onClick={onClaim} disabled={busy}>{busy ? "Finding an item..." : "Claim next sample"}</button></div>}</>;
 }
@@ -218,28 +230,72 @@ function ReviewEditor({ sample, busy, onSubmit }: { sample: Sample; busy: boolea
   const cleanedText = reviewText(sample.sample_type, cleaned);
   const cleanChars = countChars(cleanedText);
   const lastTokens = sample.review.token_stats?.clean_token_count;
+  const ner = sample.review.ner;
+  const addSuggestedTag = (tag: string) => {
+    const current = new Set(csv(tags));
+    current.add(tag);
+    setTags(Array.from(current).join(", "));
+  };
 
-  return <div className="reviewGrid"><section className="workCard"><div className="cardHeader"><div><span className="statusDot"></span><strong>{sample.source.dataset_key}</strong><small>{sample.source.source_title || `Record ${sample.source.source_record_id || "unavailable"}`}</small></div><span className="sampleType">{sample.sample_type.replaceAll("_", " ")}</span></div><Editor sample={sample} cleaned={cleaned} setCleaned={setCleaned} /></section><aside className="decisionCard"><h2>Review decision</h2><p>Keep fixes fast. Reject obvious junk, edit when the cleanup is worthwhile, and use the token view to track progress.</p><div className="statsStrip"><div><span>Current chars</span><strong>{cleanChars.toLocaleString()}</strong></div><div><span>Last saved tokens</span><strong>{lastTokens?.toLocaleString() || "n/a"}</strong></div></div><label>Content tags<input value={tags} onChange={(e) => setTags(e.target.value)} placeholder="education, science" /></label><label>Quality tags<input value={qualityTags} onChange={(e) => setQualityTags(e.target.value)} placeholder="high_quality, needs_review" /></label><label>Task tags<input value={taskTags} onChange={(e) => setTaskTags(e.target.value)} placeholder="translation, reasoning" /></label><label>Notes<textarea rows={4} value={notes} onChange={(e) => setNotes(e.target.value)} placeholder="Optional notes for later filtering or export." /></label><label>Action reason<textarea rows={3} value={reason} onChange={(e) => setReason(e.target.value)} placeholder="Required when rejecting. Useful for skip decisions too." /></label><label>Confidence <strong>{confidence}/5</strong><input type="range" min="1" max="5" value={confidence} onChange={(e) => setConfidence(Number(e.target.value))} /></label><div className="actions"><button className="primary" disabled={busy} onClick={() => onSubmit("accept", state)}>Accept</button><button disabled={busy} onClick={() => onSubmit("edit", state)}>Save edit</button><button className="danger" disabled={busy} onClick={() => onSubmit("reject", state)}>Reject</button><button disabled={busy} onClick={() => onSubmit("skip", state)}>Skip</button></div></aside></div>;
+  return <div className="reviewGrid"><section className="workCard"><div className="cardHeader"><div><span className="statusDot"></span><strong>{sample.source.dataset_key}</strong><small>{sample.source.source_title || `Record ${sample.source.source_record_id || "unavailable"}`}</small></div><span className="sampleType">{sample.sample_type.replaceAll("_", " ")}</span></div><Editor sample={sample} cleaned={cleaned} setCleaned={setCleaned} /></section><aside className="decisionCard"><h2>Review decision</h2><p>Keep fixes fast. Reject obvious junk, edit when the cleanup is worthwhile, and use the token view to track progress.</p><div className="statsStrip"><div><span>Current chars</span><strong>{cleanChars.toLocaleString()}</strong></div><div><span>Last saved tokens</span><strong>{lastTokens?.toLocaleString() || "n/a"}</strong></div></div>{sample.live_sync?.status && <div className="syncNote"><strong>Live sync:</strong> {sample.live_sync.status}{sample.live_sync.last_error ? ` - ${sample.live_sync.last_error}` : ""}</div>}{ner && <section className="suggestionBlock"><h3>NER suggestions</h3><p>{ner.status === "ready" ? "Use suggestions as optional tags. Review decisions remain manual." : ner.error || "NER suggestions unavailable."}</p>{ner.status === "ready" && <><div className="chipRow">{ner.suggested_tags.map((tag) => <button type="button" key={tag} className="chipButton" onClick={() => addSuggestedTag(tag)}>{tag}</button>)}</div><div className="entityList">{ner.entities.slice(0, 12).map((entity) => <span key={`${entity.label}-${entity.text}`} className="entityChip">{entity.label}: {entity.text}</span>)}</div></>}</section>}<label>Content tags<input value={tags} onChange={(e) => setTags(e.target.value)} placeholder="education, science" /></label><label>Quality tags<input value={qualityTags} onChange={(e) => setQualityTags(e.target.value)} placeholder="high_quality, needs_review" /></label><label>Task tags<input value={taskTags} onChange={(e) => setTaskTags(e.target.value)} placeholder="translation, reasoning" /></label><label>Notes<textarea rows={4} value={notes} onChange={(e) => setNotes(e.target.value)} placeholder="Optional notes for later filtering or export." /></label><label>Action reason<textarea rows={3} value={reason} onChange={(e) => setReason(e.target.value)} placeholder="Required when rejecting. Useful for skip decisions too." /></label><label>Confidence <strong>{confidence}/5</strong><input type="range" min="1" max="5" value={confidence} onChange={(e) => setConfidence(Number(e.target.value))} /></label><div className="actions"><button className="primary" disabled={busy} onClick={() => onSubmit("accept", state)}>Accept</button><button disabled={busy} onClick={() => onSubmit("edit", state)}>Save edit</button><button className="danger" disabled={busy} onClick={() => onSubmit("reject", state)}>Reject</button><button disabled={busy} onClick={() => onSubmit("skip", state)}>Skip</button></div></aside></div>;
 }
 
 function Editor({ sample, cleaned, setCleaned }: { sample: Sample; cleaned: Record<string, unknown>; setCleaned: (v: Record<string, unknown>) => void }) {
   if (sample.sample_type === "document") {
-    return <DocumentEditor originalText={sample.payload.text || ""} currentText={String(cleaned.current_text || "")} onChange={(value) => setCleaned({ current_text: value })} />;
+    return <LineEditor leftTitle="Original lines" rightTitle="Cleaned lines" leftText={sample.payload.text || ""} rightText={String(cleaned.current_text || "")} lockLeft onChange={(leftValue, rightValue) => setCleaned({ current_text: rightValue })} />;
   }
   if (sample.sample_type === "translation_pair") {
     const pair = cleaned.current_pair as { source_text: string; target_text: string };
-    return <PairEditor leftTitle={`Source text (${sample.payload.source_lang})`} rightTitle={`Target text (${sample.payload.target_lang})`} leftValue={pair.source_text} rightValue={pair.target_text} editableRight onChange={(leftValue, rightValue) => setCleaned({ current_pair: { source_text: leftValue, target_text: rightValue } })} />;
+    return <LineEditor leftTitle={`Source lines (${sample.payload.source_lang})`} rightTitle={`Target lines (${sample.payload.target_lang})`} leftText={pair.source_text} rightText={pair.target_text} onChange={(leftValue, rightValue) => setCleaned({ current_pair: { source_text: leftValue, target_text: rightValue } })} />;
   }
   const pair = cleaned.current_transliteration as { native_text: string; latin_text: string };
-  return <PairEditor leftTitle="Telugu script" rightTitle="Latin script" leftValue={pair.native_text} rightValue={pair.latin_text} editableRight onChange={(leftValue, rightValue) => setCleaned({ current_transliteration: { native_text: leftValue, latin_text: rightValue } })} />;
+  return <LineEditor leftTitle="Telugu script lines" rightTitle="Latin script lines" leftText={pair.native_text} rightText={pair.latin_text} onChange={(leftValue, rightValue) => setCleaned({ current_transliteration: { native_text: leftValue, latin_text: rightValue } })} />;
+}
+
+function splitReviewLines(value: string) {
+  const normalized = value.replace(/\r\n?/g, "\n").trim();
+  if (!normalized) return [""];
+  return normalized
+    .split(/\n+|(?<=[.!?।॥])\s+/u)
+    .map((line) => line.trim())
+    .filter(Boolean);
 }
 
-function DocumentEditor({ originalText, currentText, onChange }: { originalText: string; currentText: string; onChange: (value: string) => void }) {
-  return <div className="editorPanel"><section className="editorColumn"><div className="lineColumnHeader"><div><strong>Original text</strong><small>Reference only</small></div></div><div className="referenceBlock">{originalText}</div></section><section className="editorColumn editableColumn"><div className="lineColumnHeader"><div><strong>Cleaned text</strong><small>Edit directly. Keep only what should survive export.</small></div></div><textarea className="editorTextarea" value={currentText} onChange={(e) => onChange(e.target.value)} rows={22} /></section></div>;
+function buildLineRows(leftText: string, rightText: string) {
+  const leftLines = splitReviewLines(leftText);
+  const rightLines = splitReviewLines(rightText);
+  const size = Math.max(leftLines.length, rightLines.length, 1);
+  return Array.from({ length: size }, (_, index) => ({
+    id: `${index}-${leftLines[index] || ""}-${rightLines[index] || ""}`,
+    left: leftLines[index] || "",
+    right: rightLines[index] || "",
+  }));
 }
 
-function PairEditor({ leftTitle, rightTitle, leftValue, rightValue, editableRight, onChange }: { leftTitle: string; rightTitle: string; leftValue: string; rightValue: string; editableRight: boolean; onChange: (leftValue: string, rightValue: string) => void }) {
-  return <div className="editorPanel"><section className="editorColumn"><div className="lineColumnHeader"><div><strong>{leftTitle}</strong><small>Editable reference field</small></div></div><textarea className="editorTextarea" value={leftValue} onChange={(e) => onChange(e.target.value, rightValue)} rows={18} /></section><section className="editorColumn editableColumn"><div className="lineColumnHeader"><div><strong>{rightTitle}</strong><small>{editableRight ? "Edit directly." : "Reference only."}</small></div></div>{editableRight ? <textarea className="editorTextarea" value={rightValue} onChange={(e) => onChange(leftValue, e.target.value)} rows={18} /> : <div className="referenceBlock">{rightValue}</div>}</section></div>;
+function LineEditor({ leftTitle, rightTitle, leftText, rightText, onChange, lockLeft = false }: { leftTitle: string; rightTitle: string; leftText: string; rightText: string; onChange: (leftValue: string, rightValue: string) => void; lockLeft?: boolean }) {
+  const [rows, setRows] = useState<ReviewLine[]>(() => buildLineRows(leftText, rightText));
+
+  const publish = (nextRows: ReviewLine[]) => {
+    setRows(nextRows);
+    onChange(
+      nextRows.map((row) => row.left).filter((value) => value.trim()).join("\n"),
+      nextRows.map((row) => row.right).filter((value) => value.trim()).join("\n"),
+    );
+  };
+
+  const updateRow = (index: number, side: "left" | "right", value: string) => {
+    const nextRows = rows.map((row, rowIndex) => rowIndex === index ? { ...row, [side]: value } : row);
+    publish(nextRows);
+  };
+
+  const removeRow = (index: number) => {
+    const nextRows = rows.filter((_, rowIndex) => rowIndex !== index);
+    publish(nextRows.length ? nextRows : [{ id: "empty", left: "", right: "" }]);
+  };
+
+  const addRow = () => publish([...rows, { id: `new-${rows.length}`, left: "", right: "" }]);
+
+  return <div className="editorPanel lineEditorPanel"><section className="editorColumn"><div className="lineColumnHeader"><div><strong>{leftTitle}</strong><small>{lockLeft ? "Reference column" : "Editable column"}</small></div><button type="button" onClick={addRow}>Add row</button></div><div className="lineRowList">{rows.map((row, index) => <div className="lineRow" key={row.id}><span>{index + 1}</span>{lockLeft ? <div className="referenceInline">{row.left}</div> : <textarea className="lineInput" rows={2} value={row.left} onChange={(e) => updateRow(index, "left", e.target.value)} />}</div>)}</div></section><section className="editorColumn editableColumn"><div className="lineColumnHeader"><div><strong>{rightTitle}</strong><small>Inline edit and delete rows directly.</small></div></div><div className="lineRowList">{rows.map((row, index) => <div className="lineRow" key={row.id}><span>{index + 1}</span><textarea className="lineInput" rows={2} value={row.right} onChange={(e) => updateRow(index, "right", e.target.value)} /><button type="button" className="deleteLine" onClick={() => removeRow(index)}>Delete</button></div>)}</div></section></div>;
 }
 
 function TokenProgressPanel({ tokenProgress, queue }: { tokenProgress: TokenProgress; queue: string }) {

requirements-studio.txt

@@ -2,6 +2,8 @@ datasets
 fastapi==0.115.12
 huggingface_hub
 sentencepiece
+torch
+transformers
 pydantic==2.11.5
 pymongo
 python-dotenv