Upload 8 files

.gitignore

@@ -0,0 +1,3 @@
+*.json
+.env
+__pycache__/

app.py

@@ -0,0 +1,492 @@
+"""
+Gradio Interface for Clinical CRF Filling Pipeline
+Deployable on HuggingFace Spaces (GPU or CPU)
+
+Usage:
+  Local:  python app.py
+  HF:     Set as main file in Space settings
+"""
+
+import os
+import json
+import asyncio
+import tempfile
+import pandas as pd
+import gradio as gr
+
+# Pipeline imports
+from src.preprocess.wtts_builder import WTTSBuilder
+from src.utils.data_loader import DataLoader
+
+import google.generativeai as genai
+
+# ---------------------------------------------------------------------------
+#  CONFIG — reads from environment variables (set via HF Secrets or .env)
+# ---------------------------------------------------------------------------
+API_KEY = os.environ.get("GOOGLE_API_KEY", "")
+DEFAULT_MODEL = os.environ.get("GEMINI_MODEL", "gemini-1.5-pro")
+
+# RAG imports (optional — only if dependencies installed)
+RAG_AVAILABLE = False
+try:
+    from src.rag.embedder import WTTSEmbedder
+    from src.rag.rag_pipeline import RAGCRFExtractor
+    RAG_AVAILABLE = True
+except ImportError:
+    pass
+
+
+# ---------------------------------------------------------------------------
+#  PROMPTS (same as main.py)
+# ---------------------------------------------------------------------------
+SKELETON_PROMPT = """
+You are a Clinical Data Specialist.
+Convert the Weighted Time Series (WTTS) below into a "Clinical Chronology Skeleton".
+
+INPUT (WTTS):
+{wtts_string}
+
+INSTRUCTIONS:
+1. Create a strict chronological timeline (Admission to Discharge).
+2. IMPORTANT: You MUST retain the [S_xx] ID for every event you list.
+3. Filter out "Routine" (Weight 0.1) events unless they indicate a status change.
+4. Keep exact values (e.g., "BP 90/60", "Temp 102.5").
+
+OUTPUT FORMAT:
+[Date] [S_xx]: Event details
+[Date] [S_xx]: Event details
+...
+"""
+
+EXTRACTION_PROMPT = """
+You are a Clinical Coding Expert.
+Review the Patient Skeleton and the Valid Options for the requested items.
+
+PATIENT SKELETON:
+{skeleton}
+
+TASK:
+For each Clinical Item listed below, determine the value AND the supporting Sentence ID.
+1. **Value**: Must come strictly from the "Valid Options" provided.
+2. **Evidence**: Must be the specific [S_xx] ID from the skeleton that proves the value.
+
+ITEMS TO EXTRACT & THEIR OPTIONS:
+{chunk_schema_json}
+
+OUTPUT FORMAT (JSON Object):
+{{
+  "item_name": {{
+    "value": "Selected Option",
+    "evidence": "S_xx",
+    "reasoning": "Brief explanation"
+  }},
+  ...
+}}
+"""
+
+
+# ---------------------------------------------------------------------------
+#  LLM call
+# ---------------------------------------------------------------------------
+async def generate_async(prompt, model, max_retries=3, initial_delay=1):
+    loop = asyncio.get_event_loop()
+    for attempt in range(max_retries):
+        try:
+            response = await loop.run_in_executor(
+                None,
+                lambda: model.generate_content(
+                    contents=prompt,
+                    generation_config=genai.GenerationConfig(
+                        response_mime_type="application/json"
+                    ),
+                ),
+            )
+            try:
+                return json.loads(response.text)
+            except json.JSONDecodeError:
+                continue
+        except Exception as e:
+            error_message = str(e)
+            if "429" in error_message or "500" in error_message:
+                if attempt < max_retries - 1:
+                    delay = initial_delay * (2 ** attempt)
+                    await asyncio.sleep(delay)
+                else:
+                    return {"error": f"Max retries reached - {error_message}"}
+            else:
+                return {"error": error_message}
+    return {"error": "Failed to generate valid JSON"}
+
+
+def chunk_data(data, size):
+    for i in range(0, len(data), size):
+        yield data[i:i + size]
+
+
+# ---------------------------------------------------------------------------
+#  Core processing function
+# ---------------------------------------------------------------------------
+def process_clinical_text(
+    clinical_text: str,
+    api_key: str,
+    model_name: str,
+    use_rag: bool,
+    admission_time: str,
+    discharge_time: str,
+    progress=gr.Progress(),
+):
+    """Main Gradio handler — processes clinical text and returns CRF predictions."""
+
+    if not clinical_text.strip():
+        return "⚠️ Please paste clinical text or upload a file.", "", ""
+
+    if not api_key.strip():
+        return "⚠️ Please provide a Google API key.", "", ""
+
+    # Configure Gemini
+    genai.configure(api_key=api_key.strip())
+    model = genai.GenerativeModel(model_name)
+
+    # Build a synthetic patient data dict
+    patient_data = {
+        "document_id": "gradio_patient",
+        "admission_time": admission_time or "2026-01-01",
+        "discharge_time": discharge_time or "2026-01-15",
+        "notes": [
+            {
+                "timestamp": admission_time or "2026-01-01",
+                "text": clinical_text,
+                "source": "user_input",
+            }
+        ],
+    }
+
+    builder = WTTSBuilder()
+
+    progress(0.1, desc="Building WTTS tuples...")
+    wtts_string = builder.build_wtts_string(patient_data)
+
+    if not wtts_string.strip():
+        return "⚠️ Could not extract any events from the text.", "", ""
+
+    # ---- RAG Pipeline ----
+    if use_rag and RAG_AVAILABLE:
+        progress(0.3, desc="RAG: Embedding tuples...")
+        embedder = WTTSEmbedder(model_name="all-MiniLM-L6-v2", device="cpu")
+        extractor = RAGCRFExtractor(
+            embedder=embedder,
+            generate_fn=generate_async,
+            top_k=15,
+        )
+
+        # We don't have target_items/valid_options from the UI,
+        # so use a default set of common CRF items
+        target_items = _get_default_target_items()
+        valid_options = _get_default_valid_options()
+
+        semaphore = asyncio.Semaphore(3)
+
+        progress(0.5, desc="RAG: Retrieving & extracting...")
+        result = asyncio.run(
+            extractor.extract_patient(
+                patient_data, builder, target_items, valid_options, semaphore, model
+            )
+        )
+
+        if result and result.get("predictions"):
+            predictions = result["predictions"]
+        else:
+            return "⚠️ RAG extraction returned no results.", wtts_string, ""
+
+    # ---- Original Two-Pass Pipeline ----
+    else:
+        progress(0.3, desc="Pass 1: Generating skeleton...")
+        skeleton_input = SKELETON_PROMPT.format(wtts_string=wtts_string)
+        skeleton_resp = asyncio.run(generate_async(skeleton_input, model))
+
+        skeleton_text = str(skeleton_resp)
+        if isinstance(skeleton_resp, dict):
+            skeleton_text = json.dumps(skeleton_resp, indent=2)
+
+        target_items = _get_default_target_items()
+        valid_options = _get_default_valid_options()
+
+        progress(0.6, desc="Pass 2: Extracting CRF items...")
+        predictions = {}
+        item_chunks = list(chunk_data(target_items, 10))
+
+        for i, chunk_items in enumerate(item_chunks):
+            progress(0.6 + 0.3 * (i / max(len(item_chunks), 1)),
+                     desc=f"Extracting batch {i+1}/{len(item_chunks)}...")
+
+            chunk_schema = {
+                item: valid_options.get(item, ["y", "n", "unknown"])
+                for item in chunk_items
+            }
+            extract_input = EXTRACTION_PROMPT.format(
+                skeleton=skeleton_text,
+                chunk_schema_json=json.dumps(chunk_schema),
+            )
+            chunk_resp = asyncio.run(generate_async(extract_input, model))
+
+            if isinstance(chunk_resp, dict) and "error" not in chunk_resp:
+                predictions.update(chunk_resp)
+
+    progress(0.95, desc="Formatting results...")
+
+    # Format predictions for display
+    results_md = _format_predictions_markdown(predictions)
+    predictions_json = json.dumps(predictions, indent=2)
+
+    progress(1.0, desc="Done!")
+    return results_md, wtts_string, predictions_json
+
+
+# ---------------------------------------------------------------------------
+#  File upload handler
+# ---------------------------------------------------------------------------
+def load_from_file(file):
+    """Read uploaded file (txt or parquet) and return the text content."""
+    if file is None:
+        return ""
+
+    filepath = file.name if hasattr(file, "name") else str(file)
+
+    if filepath.endswith(".parquet"):
+        df = pd.read_parquet(filepath)
+        text_col = next(
+            (c for c in df.columns if c.lower() in ["clinical_note", "text", "body"]),
+            df.columns[0],
+        )
+        return "\n\n---\n\n".join(df[text_col].dropna().astype(str).tolist())
+
+    elif filepath.endswith(".jsonl"):
+        texts = []
+        with open(filepath, "r", encoding="utf-8") as f:
+            for line in f:
+                rec = json.loads(line.strip())
+                if "text" in rec:
+                    texts.append(rec["text"])
+        return "\n\n---\n\n".join(texts) if texts else ""
+
+    else:
+        with open(filepath, "r", encoding="utf-8") as f:
+            return f.read()
+
+
+# ---------------------------------------------------------------------------
+#  Default CRF items (common dyspnea CRF items from the challenge)
+# ---------------------------------------------------------------------------
+def _get_default_target_items():
+    return [
+        "chronic pulmonary disease", "chronic respiratory failure",
+        "chronic cardiac failure", "chronic renal failure",
+        "presence of dyspnea", "improvement of dyspnea",
+        "heart rate", "blood pressure", "body temperature",
+        "respiratory rate", "spo2", "level of consciousness",
+        "hemoglobin", "platelets", "leukocytes", "c-reactive protein",
+        "creatinine", "troponin", "d-dimer",
+        "ecg, any abnormality", "chest rx, any abnormalities",
+        "brain ct scan, any abnormality",
+        "administration of diuretics", "administration of steroids",
+        "administration of bronchodilators",
+        "administration of oxygen/ventilation",
+        "heart failure", "pneumonia", "copd exacerbation",
+        "respiratory failure", "pulmonary embolism",
+        "acute coronary syndrome", "arrhythmia",
+    ]
+
+
+def _get_default_valid_options():
+    return {item: ["y", "n", "unknown"] for item in _get_default_target_items()}
+
+
+# ---------------------------------------------------------------------------
+#  Format predictions as Markdown table
+# ---------------------------------------------------------------------------
+def _format_predictions_markdown(predictions):
+    if not predictions:
+        return "No predictions generated."
+
+    lines = [
+        "## 📋 CRF Predictions\n",
+        "| # | CRF Item | Value | Evidence | Reasoning |",
+        "|---|----------|-------|----------|-----------|",
+    ]
+
+    for i, (item, val) in enumerate(predictions.items(), 1):
+        if isinstance(val, dict):
+            value = val.get("value", "—")
+            evidence = val.get("evidence", "—")
+            reasoning = val.get("reasoning", "—")
+        else:
+            value = str(val)
+            evidence = "—"
+            reasoning = "—"
+
+        # Color code the value
+        if value.lower() == "y":
+            value = "✅ Yes"
+        elif value.lower() == "n":
+            value = "❌ No"
+        elif value.lower() == "unknown":
+            value = "❓ Unknown"
+
+        lines.append(f"| {i} | {item} | {value} | {evidence} | {reasoning} |")
+
+    return "\n".join(lines)
+
+
+# ---------------------------------------------------------------------------
+#  Gradio UI
+# ---------------------------------------------------------------------------
+def create_app():
+    with gr.Blocks(
+        title="Clinical CRF Filling — RAG Pipeline",
+        theme=gr.themes.Soft(
+            primary_hue="blue",
+            secondary_hue="cyan",
+            neutral_hue="slate",
+        ),
+        css="""
+        .main-header { text-align: center; margin-bottom: 1rem; }
+        .results-box { min-height: 300px; }
+        footer { display: none !important; }
+        """,
+    ) as app:
+
+        # Header
+        gr.HTML("""
+        <div class="main-header">
+            <h1>🏥 Clinical CRF Filling</h1>
+            <p style="color: #666; font-size: 1.1em;">
+                RAG-Enhanced Pipeline for CL4Health 2026 Challenge
+            </p>
+        </div>
+        """)
+
+        with gr.Row():
+            # ---- Left column: Input ----
+            with gr.Column(scale=1):
+                gr.Markdown("### 📝 Input")
+
+                api_key_input = gr.Textbox(
+                    label="Google API Key",
+                    type="password",
+                    placeholder="Enter your Gemini API key...",
+                    value=API_KEY,
+                )
+
+                model_dropdown = gr.Dropdown(
+                    label="Model",
+                    choices=[
+                        "gemini-1.5-pro",
+                        "gemini-1.5-flash",
+                        "gemini-2.0-flash",
+                    ],
+                    value=DEFAULT_MODEL,
+                )
+
+                use_rag_checkbox = gr.Checkbox(
+                    label="🔍 Use RAG Pipeline",
+                    value=RAG_AVAILABLE,
+                    info="Retrieves relevant tuples per CRF item (requires sentence-transformers + faiss)",
+                    interactive=RAG_AVAILABLE,
+                )
+
+                with gr.Row():
+                    admission_input = gr.Textbox(
+                        label="Admission Time",
+                        placeholder="2026-01-01",
+                        value="2026-01-01",
+                    )
+                    discharge_input = gr.Textbox(
+                        label="Discharge Time",
+                        placeholder="2026-01-15",
+                        value="2026-01-15",
+                    )
+
+                clinical_text_input = gr.Textbox(
+                    label="Clinical Notes",
+                    placeholder="Paste clinical text here...",
+                    lines=12,
+                    max_lines=30,
+                )
+
+                file_upload = gr.File(
+                    label="Or Upload File (.txt, .parquet, .jsonl)",
+                    file_types=[".txt", ".parquet", ".jsonl"],
+                )
+
+                submit_btn = gr.Button(
+                    "🚀 Extract CRF Items",
+                    variant="primary",
+                    size="lg",
+                )
+
+            # ---- Right column: Output ----
+            with gr.Column(scale=1):
+                gr.Markdown("### 📊 Results")
+
+                results_output = gr.Markdown(
+                    label="CRF Predictions",
+                    elem_classes=["results-box"],
+                )
+
+                with gr.Accordion("🔧 WTTS Tuples (Debug)", open=False):
+                    wtts_output = gr.Textbox(
+                        label="Generated WTTS String",
+                        lines=8,
+                        interactive=False,
+                    )
+
+                with gr.Accordion("📦 Raw JSON Output", open=False):
+                    json_output = gr.Code(
+                        label="Predictions JSON",
+                        language="json",
+                    )
+
+        # ---- Event handlers ----
+        file_upload.change(
+            fn=load_from_file,
+            inputs=[file_upload],
+            outputs=[clinical_text_input],
+        )
+
+        submit_btn.click(
+            fn=process_clinical_text,
+            inputs=[
+                clinical_text_input,
+                api_key_input,
+                model_dropdown,
+                use_rag_checkbox,
+                admission_input,
+                discharge_input,
+            ],
+            outputs=[results_output, wtts_output, json_output],
+        )
+
+        # Footer
+        gr.Markdown("""
+        ---
+        <center>
+        <small>
+        CL4Health 2026 • CRF Filling Challenge • MIMIC-III Dataset<br>
+        Built with WTTS + RAG Pipeline
+        </small>
+        </center>
+        """)
+
+    return app
+
+
+# ---------------------------------------------------------------------------
+#  Launch
+# ---------------------------------------------------------------------------
+if __name__ == "__main__":
+    app = create_app()
+    app.launch(
+        server_name="0.0.0.0",
+        server_port=7860,
+        share=False,
+    )

check_submission_format.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""
+Validate a Codabench result-only submission JSONL and produce a clean ZIP.
+
+This script checks that the JSONL file follows the expected structure:
+- Each line is a JSON object
+- Required keys: "document_id" (string), "predictions" (list)
+- Each element in "predictions" is a dict with keys: "item" (string), "prediction" (string)
+
+If valid, it creates a ZIP with the JSONL file at the archive root, ready to upload.
+
+Usage:
+  python check_submission_format.py <path/to/mock_data_dev_codabench.jsonl> [--out submission_clean.zip]
+
+Exit codes:
+  0 on success, non-zero on validation failure or I/O error
+"""
+
+import argparse
+import json
+import os
+import sys
+import zipfile
+from typing import Any, Dict, List
+
+REQUIRED_FILE_NAME = "mock_data_dev_codabench.jsonl"
+
+
+def read_jsonl(path: str) -> List[Dict[str, Any]]:
+    data: List[Dict[str, Any]] = []
+    with open(path, "r", encoding="utf-8") as f:
+        for idx, line in enumerate(f, start=1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                obj = json.loads(line)
+            except json.JSONDecodeError as e:
+                raise ValueError(f"Line {idx}: invalid JSON: {e}")
+            if not isinstance(obj, dict):
+                raise ValueError(f"Line {idx}: JSON object must be a dict")
+            data.append(obj)
+    if not data:
+        raise ValueError("File contains no valid JSONL records")
+    return data
+
+
+def validate_record(rec: Dict[str, Any], index: int) -> None:
+    if "document_id" not in rec:
+        raise ValueError(f"Record {index}: missing 'document_id'")
+    if not isinstance(rec["document_id"], str) or not rec["document_id"].strip():
+        raise ValueError(f"Record {index}: 'document_id' must be a non-empty string")
+
+    if "predictions" not in rec:
+        raise ValueError(f"Record {index}: missing 'predictions'")
+    preds = rec["predictions"]
+    if not isinstance(preds, list):
+        raise ValueError(f"Record {index}: 'predictions' must be a list")
+    if len(preds) == 0:
+        raise ValueError(f"Record {index}: 'predictions' list must not be empty")
+
+    for j, p in enumerate(preds):
+        if not isinstance(p, dict):
+            raise ValueError(f"Record {index} prediction {j}: must be a dict")
+        if "item" not in p:
+            raise ValueError(f"Record {index} prediction {j}: missing 'item'")
+        if "prediction" not in p:
+            raise ValueError(f"Record {index} prediction {j}: missing 'prediction'")
+        if not isinstance(p["item"], str) or not p["item"].strip():
+            raise ValueError(f"Record {index} prediction {j}: 'item' must be a non-empty string")
+        if not isinstance(p["prediction"], str):
+            raise ValueError(f"Record {index} prediction {j}: 'prediction' must be a string")
+
+
+def validate_jsonl_structure(records: List[Dict[str, Any]]) -> None:
+    for i, rec in enumerate(records):
+        validate_record(rec, i)
+
+
+def make_clean_zip(jsonl_path: str, zip_out: str) -> str:
+    os.makedirs(os.path.dirname(zip_out) or ".", exist_ok=True)
+    with zipfile.ZipFile(zip_out, "w", compression=zipfile.ZIP_DEFLATED) as zf:
+        zf.write(jsonl_path, arcname=REQUIRED_FILE_NAME)
+    return zip_out
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Validate and zip Codabench submission JSONL")
+    parser.add_argument("jsonl", help="Path to mock_data_dev_codabench.jsonl (or similarly structured file)")
+    parser.add_argument("--out", dest="out_zip", default="submission_clean.zip", help="Output ZIP file path")
+    args = parser.parse_args()
+
+    jsonl_path = os.path.abspath(args.jsonl)
+    if not os.path.exists(jsonl_path):
+        print(f"Error: file not found: {jsonl_path}", file=sys.stderr)
+        return 2
+
+    # Warn if the filename differs from REQUIRED_FILE_NAME
+    valid_name = True
+    base = os.path.basename(jsonl_path)
+    if base != REQUIRED_FILE_NAME:
+        valid_name = False
+        print(f"Warning: input filename is '{base}'. Codabench expects '{REQUIRED_FILE_NAME}'. Converting it for submission.\n", file=sys.stderr)
+
+    try:
+        records = read_jsonl(jsonl_path)
+        validate_jsonl_structure(records)
+    except Exception as e:
+        print(f"Validation failed: {e}", file=sys.stderr)
+        return 3
+
+    try:
+        out_zip = os.path.abspath(args.out_zip)
+        if valid_name:
+            make_clean_zip(jsonl_path, out_zip)
+        else:
+            import tempfile
+            with tempfile.TemporaryDirectory() as tmpdir:
+                temp_jsonl_path = os.path.join(tmpdir, REQUIRED_FILE_NAME)
+                with open(temp_jsonl_path, "w", encoding="utf-8") as f_out, open(jsonl_path, "r", encoding="utf-8") as f_in:
+                    f_out.write(f_in.read())
+                make_clean_zip(temp_jsonl_path, out_zip)
+    except Exception as e:
+        print(f"Failed to create ZIP: {e}", file=sys.stderr)
+        return 4
+
+    print("Validation passed. ZIP created:", out_zip)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

convert_to_submission.py

@@ -0,0 +1,217 @@
+"""
+Convert pipeline output to official CL4Health Codabench submission format.
+
+Our pipeline outputs:
+  [{"patient_id": "1014081", "predictions": {"item_name": {"value": "y", ...}}, ...}, ...]
+
+Codabench expects (one JSON per line in JSONL):
+  {"document_id": "1014081_en", "predictions": [{"item": "item_name", "prediction": "y"}, ...]}
+
+This script:
+  1. Reads pipeline output (JSON) and ground truth (JSONL)
+  2. Maps predictions to match GT annotation order exactly
+  3. Writes a Codabench-compatible submission JSONL
+  4. Optionally runs local scoring and creates the upload ZIP
+
+Usage:
+  python convert_to_submission.py
+  python convert_to_submission.py --pipeline_output results.json --language en --score --zip
+"""
+
+import json
+import os
+import argparse
+import subprocess
+import sys
+
+
+def load_ground_truth(gt_path: str) -> list:
+    """Load dev_gt.jsonl to get document IDs and annotation item order."""
+    records = []
+    with open(gt_path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                records.append(json.loads(line))
+    return records
+
+
+def load_pipeline_output(output_path: str) -> dict:
+    """
+    Load our pipeline's output JSON.
+    Returns a dict mapping patient_id → predictions dict.
+    """
+    with open(output_path, "r", encoding="utf-8") as f:
+        results = json.load(f)
+
+    lookup = {}
+    for r in results:
+        pid = str(r.get("patient_id", "unknown"))
+        preds = r.get("predictions", {})
+        # Flatten: handle both {"item": {"value": "y"}} and {"item": "y"}
+        flat = {}
+        for item_name, item_val in preds.items():
+            if isinstance(item_val, dict):
+                flat[item_name] = str(item_val.get("value", "unknown"))
+            else:
+                flat[item_name] = str(item_val)
+        lookup[pid] = flat
+    return lookup
+
+
+def convert(gt_records: list, pipeline_lookup: dict, language: str) -> list:
+    """
+    Convert pipeline predictions to Codabench format.
+
+    For each GT patient:
+      - Create a submission record with document_id = "{id}_{language}"
+      - For each annotation item (in GT order), look up the prediction
+      - Default to "unknown" if no prediction exists
+    """
+    submission = []
+    matched = 0
+    unmatched = 0
+
+    for gt_rec in gt_records:
+        doc_id = str(gt_rec["document_id"])
+        annotations = gt_rec.get("annotations", [])
+
+        # Look up our predictions for this patient
+        preds = pipeline_lookup.get(doc_id, {})
+
+        if preds:
+            matched += 1
+        else:
+            unmatched += 1
+
+        # Build predictions list in the SAME ORDER as GT annotations
+        pred_list = []
+        for ann in annotations:
+            item_name = ann["item"]
+            predicted_value = preds.get(item_name, "unknown")
+
+            # Normalize: strip whitespace, lowercase
+            predicted_value = predicted_value.strip().lower() if predicted_value else "unknown"
+            if not predicted_value:
+                predicted_value = "unknown"
+
+            pred_list.append({
+                "item": item_name,
+                "prediction": predicted_value,
+            })
+
+        submission.append({
+            "document_id": f"{doc_id}_{language}",
+            "predictions": pred_list,
+        })
+
+    print(f"  Matched: {matched} patients")
+    print(f"  Unmatched (defaulting to 'unknown'): {unmatched} patients")
+    return submission
+
+
+def write_submission_jsonl(submission: list, output_path: str):
+    """Write one JSON object per line."""
+    os.makedirs(os.path.dirname(output_path) or ".", exist_ok=True)
+    with open(output_path, "w", encoding="utf-8") as f:
+        for record in submission:
+            f.write(json.dumps(record, ensure_ascii=False) + "\n")
+    print(f"  Written to: {output_path}")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Convert pipeline output to Codabench submission format"
+    )
+    parser.add_argument(
+        "--pipeline_output",
+        default="data/processed/materialized_ehr/submission.json",
+        help="Path to pipeline output JSON",
+    )
+    parser.add_argument(
+        "--gt_file",
+        default="data/raw/dev_gt.jsonl",
+        help="Path to ground truth JSONL",
+    )
+    parser.add_argument(
+        "--language",
+        default="en",
+        choices=["en", "it"],
+        help="Language suffix for document IDs",
+    )
+    parser.add_argument(
+        "--output",
+        default="submission/mock_data_dev_codabench.jsonl",
+        help="Output submission JSONL path",
+    )
+    parser.add_argument(
+        "--score",
+        action="store_true",
+        help="Run local scoring after conversion",
+    )
+    parser.add_argument(
+        "--zip",
+        action="store_true",
+        help="Create Codabench ZIP after conversion",
+    )
+    args = parser.parse_args()
+
+    print("\n=== Converting Pipeline Output to Codabench Format ===")
+
+    # 1. Load data
+    print(f"\n1. Loading ground truth from: {args.gt_file}")
+    gt_records = load_ground_truth(args.gt_file)
+    print(f"   {len(gt_records)} patients in GT")
+
+    print(f"\n2. Loading pipeline output from: {args.pipeline_output}")
+    pipeline_lookup = load_pipeline_output(args.pipeline_output)
+    print(f"   {len(pipeline_lookup)} patients in pipeline output")
+
+    # 2. Convert
+    print(f"\n3. Converting to Codabench format (language={args.language})...")
+    submission = convert(gt_records, pipeline_lookup, args.language)
+
+    # 3. Write
+    print(f"\n4. Writing submission JSONL...")
+    write_submission_jsonl(submission, args.output)
+
+    # 4. Optionally score locally
+    if args.score:
+        print(f"\n5. Running local scoring...")
+        # The official scorer expects dev_gt.jsonl at development_data/dev_gt.jsonl
+        # Create symlink/copy if needed
+        dev_data_dir = "development_data"
+        dev_gt_target = os.path.join(dev_data_dir, "dev_gt.jsonl")
+        if not os.path.exists(dev_gt_target):
+            os.makedirs(dev_data_dir, exist_ok=True)
+            import shutil
+            shutil.copy2(args.gt_file, dev_gt_target)
+            print(f"   Copied GT to {dev_gt_target}")
+
+        cmd = [
+            sys.executable, "scoring.py",
+            "--submission_path", args.output,
+            "--language", args.language,
+        ]
+        print(f"   Running: {' '.join(cmd)}")
+        subprocess.run(cmd, check=False)
+
+    # 5. Optionally create ZIP
+    if args.zip:
+        zip_output = os.path.join("submission", "submission_clean.zip")
+        print(f"\n6. Creating Codabench ZIP...")
+        cmd = [
+            sys.executable, "check_submission_format.py",
+            args.output,
+            "--out", zip_output,
+        ]
+        print(f"   Running: {' '.join(cmd)}")
+        subprocess.run(cmd, check=False)
+        print(f"\n   Upload {zip_output} to Codabench:")
+        print(f"   https://www.codabench.org/competitions/11984/#/participate-tab")
+
+    print("\n=== Done ===\n")
+
+
+if __name__ == "__main__":
+    main()

environment.yml

@@ -0,0 +1,21 @@
+name: clinical_summ
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.10
+  - pandas
+  - numpy
+  - pip
+  - pip:
+    - google-cloud-aiplatform
+    - google-cloud-bigquery
+    - tiktoken
+    - transformers
+    - tqdm
+    - python-dateutil
+    - pyarrow
+    - google-generativeai
+    - sentence-transformers
+    - faiss-cpu
+    

main.py

@@ -0,0 +1,416 @@
+import asyncio
+import json
+import argparse
+import os
+import math
+
+# Imports from your specific file structure
+from src.preprocess.wtts_builder import WTTSBuilder
+from src.utils.data_loader import DataLoader
+
+# RAG pipeline imports (lazy — only used when --use_rag is set)
+RAG_AVAILABLE = False
+try:
+    from src.rag.embedder import WTTSEmbedder
+    from src.rag.rag_pipeline import RAGCRFExtractor
+    RAG_AVAILABLE = True
+except ImportError:
+    pass
+
+import google.generativeai as genai
+
+# --- CONFIG ---
+API_KEY = "AIzaSyAkgKha4IxsCjRXbeirhyoygT9Qmr4qYzU"
+
+# --- PROMPTS ---
+
+SKELETON_PROMPT = """
+You are a Clinical Data Specialist.
+Convert the Weighted Time Series (WTTS) below into a "Clinical Chronology Skeleton".
+
+INPUT (WTTS):
+{wtts_string}
+
+INSTRUCTIONS:
+1. Create a strict chronological timeline (Admission to Discharge).
+2. IMPORTANT: You MUST retain the [S_xx] ID for every event you list.
+3. Filter out "Routine" (Weight 0.1) events unless they indicate a status change.
+4. Keep exact values (e.g., "BP 90/60", "Temp 102.5").
+
+OUTPUT FORMAT:
+[Date] [S_xx]: Event details
+[Date] [S_xx]: Event details
+...
+"""
+
+EXTRACTION_PROMPT = """
+You are a Clinical Coding Expert.
+Review the Patient Skeleton and the Valid Options for the requested items.
+
+PATIENT SKELETON:
+{skeleton}
+
+TASK:
+For each Clinical Item listed below, determine the value AND the supporting Sentence ID.
+1. **Value**: Must come strictly from the "Valid Options" provided.
+2. **Evidence**: Must be the specific [S_xx] ID from the skeleton that proves the value.
+
+ITEMS TO EXTRACT & THEIR OPTIONS:
+{chunk_schema_json}
+
+OUTPUT FORMAT (JSON Object):
+{{
+  "item_name": {{
+    "value": "Selected Option",
+    "evidence": "S_xx",
+    "reasoning": "Brief explanation"
+  }},
+  ...
+}}
+"""
+
+def chunk_data(data, size):
+    """Yield successive n-sized chunks from list."""
+    for i in range(0, len(data), size):
+        yield data[i:i + size]
+
+
+async def generate_async(prompt, model, max_retries=3, initial_delay=1):
+    """Call Gemini via google-generativeai SDK (async-safe)."""
+    loop = asyncio.get_event_loop()
+    for attempt in range(max_retries):
+        try:
+            response = await loop.run_in_executor(
+                None,
+                lambda: model.generate_content(
+                    contents=prompt,
+                    generation_config=genai.GenerationConfig(
+                        response_mime_type="application/json"
+                    ),
+                )
+            )
+            try:
+                json_response = json.loads(response.text)
+                return json_response
+            except json.JSONDecodeError:
+                print(f"Generated content is not valid JSON. Retrying...")
+                continue
+
+        except Exception as e:
+            error_message = str(e)
+            if "429" in error_message or "500" in error_message:
+                if attempt < max_retries - 1:
+                    delay = initial_delay * (2 ** attempt)
+                    print(f"Rate limit / server error. Retrying in {delay}s...")
+                    await asyncio.sleep(delay)
+                else:
+                    print(f"Max retries reached.")
+                    return {"error": f"Max retries reached - {error_message}"}
+            else:
+                print(f"Error in generate_async: {error_message}")
+                return {"error": error_message}
+
+    return {"error": "Failed to generate valid JSON after multiple attempts"}
+
+
+async def process_patient(model, builder, patient_data, target_items, valid_options, semaphore):
+    """Executes the Two-Pass Pipeline for a single patient."""
+    async with semaphore:
+        pid = str(patient_data.get('document_id') or patient_data.get('patient_id')
+                  or patient_data.get('hadm_id') or 'unknown')
+
+        try:
+            # --- PHASE 1: WTTS Construction ---
+            wtts_string = builder.build_wtts_string(patient_data)
+
+            # --- PHASE 2: Skeleton Generation (Pass 1) ---
+            skeleton_input = SKELETON_PROMPT.format(wtts_string=wtts_string)
+            skeleton_resp = await generate_async(skeleton_input, model)
+
+            skeleton_text = str(skeleton_resp)
+            if isinstance(skeleton_resp, dict):
+                skeleton_text = json.dumps(skeleton_resp)
+
+            # --- PHASE 3: Extraction (Pass 2) ---
+            final_predictions = {}
+            item_chunks = list(chunk_data(target_items, 10))
+
+            for chunk_items in item_chunks:
+                chunk_schema = {
+                    item: valid_options.get(item, ["Yes", "No", "Unknown"])
+                    for item in chunk_items
+                }
+
+                extract_input = EXTRACTION_PROMPT.format(
+                    skeleton=skeleton_text,
+                    chunk_schema_json=json.dumps(chunk_schema)
+                )
+
+                chunk_resp = await generate_async(extract_input, model)
+
+                if isinstance(chunk_resp, dict):
+                    if 'error' in chunk_resp:
+                        print(f"  [WARN] LLM error for {pid}, chunk {chunk_items[:3]}...: {chunk_resp['error']}")
+                    else:
+                        final_predictions.update(chunk_resp)
+
+            return {
+                "patient_id": pid,
+                "skeleton_debug": skeleton_text[:500] + "...",
+                "predictions": final_predictions
+            }
+
+        except Exception as e:
+            print(f"Error processing {pid}: {e}")
+            return None
+
+
+# ---------------------------------------------------------------------------
+#  EVALUATION -- Accuracy & F1 Scoring
+# ---------------------------------------------------------------------------
+
+def _normalise(value):
+    """Lowercase + strip for fair comparison."""
+    if value is None:
+        return ""
+    return str(value).strip().lower()
+
+
+def evaluate_predictions(results, gt_path):
+    """
+    Compare pipeline results against dev_gt.jsonl.
+    Prints accuracy, macro-F1, per-item breakdown, and sample errors.
+    Returns (overall_dict, per_item_dict).
+    """
+    # --- Load GT ---
+    gt = {}
+    with open(gt_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            rec = json.loads(line)
+            doc_id = str(rec['document_id'])
+            gt[doc_id] = {a['item']: a['ground_truth'] for a in rec.get('annotations', [])}
+
+    # --- Build prediction lookup ---
+    preds = {}
+    for r in results:
+        doc_id = str(r.get('patient_id', 'unknown'))
+        items = {}
+        for item_name, item_val in r.get('predictions', {}).items():
+            if isinstance(item_val, dict):
+                items[item_name] = item_val.get('value', str(item_val))
+            else:
+                items[item_name] = str(item_val)
+        preds[doc_id] = items
+
+    # --- Collect all unique items ---
+    all_items = set()
+    for doc_items in gt.values():
+        all_items.update(doc_items.keys())
+
+    # --- Score ---
+    item_stats = {item: {'tp': 0, 'fp': 0, 'fn': 0, 'total': 0, 'correct': 0}
+                  for item in all_items}
+    total_comparisons = 0
+    total_correct = 0
+    matched_patients = 0
+    errors = []
+
+    for doc_id, gt_items in gt.items():
+        pred_items = preds.get(doc_id, {})
+        if pred_items:
+            matched_patients += 1
+
+        for item_name, gt_val in gt_items.items():
+            gt_norm = _normalise(gt_val)
+            pred_val = pred_items.get(item_name)
+            pred_norm = _normalise(pred_val) if pred_val is not None else ""
+
+            total_comparisons += 1
+            item_stats[item_name]['total'] += 1
+
+            if gt_norm == pred_norm:
+                total_correct += 1
+                item_stats[item_name]['correct'] += 1
+                item_stats[item_name]['tp'] += 1
+            else:
+                item_stats[item_name]['fn'] += 1
+                if pred_norm:
+                    item_stats[item_name]['fp'] += 1
+                errors.append((doc_id, item_name, gt_val,
+                               pred_val if pred_val is not None else '<MISSING>'))
+
+    accuracy = total_correct / total_comparisons if total_comparisons > 0 else 0.0
+
+    # --- Per-item P/R/F1 ---
+    f1s = []
+    per_item = {}
+    for item_name in sorted(all_items):
+        s = item_stats[item_name]
+        tp, fp, fn = s['tp'], s['fp'], s['fn']
+        prec = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+        rec  = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+        f1   = 2 * prec * rec / (prec + rec) if (prec + rec) > 0 else 0.0
+        item_acc = s['correct'] / s['total'] if s['total'] > 0 else 0.0
+        per_item[item_name] = {'accuracy': item_acc, 'precision': prec,
+                               'recall': rec, 'f1': f1, 'total': s['total']}
+        f1s.append(f1)
+
+    macro_f1 = sum(f1s) / len(f1s) if f1s else 0.0
+
+    # --- Print report ---
+    print("\n" + "=" * 70)
+    print("  CL4Health CRF Filling -- Evaluation Report")
+    print("=" * 70)
+    print(f"\n  GT Patients:       {len(gt)}")
+    print(f"  Pred Patients:     {len(preds)}")
+    print(f"  Matched Patients:  {matched_patients}")
+    print(f"\n  Total Comparisons: {total_comparisons}")
+    print(f"  Total Correct:     {total_correct}")
+    print(f"\n  {'Accuracy':>20s}: {accuracy:.4f}")
+    print(f"  {'Macro F1':>20s}: {macro_f1:.4f}")
+
+    # Top / bottom items
+    sorted_items = sorted(per_item.items(), key=lambda x: x[1]['f1'], reverse=True)
+    n_show = min(15, len(sorted_items))
+
+    print(f"\n  Top {n_show} Items by F1:")
+    print(f"  {'Item':<45s} {'Acc':>6s} {'P':>6s} {'R':>6s} {'F1':>6s}")
+    print(f"  {'-'*45} {'---':>6s} {'---':>6s} {'---':>6s} {'---':>6s}")
+    for name, s in sorted_items[:n_show]:
+        print(f"  {name:<45s} {s['accuracy']:>6.2f} {s['precision']:>6.2f} {s['recall']:>6.2f} {s['f1']:>6.2f}")
+
+    print(f"\n  Bottom {n_show} Items by F1:")
+    print(f"  {'Item':<45s} {'Acc':>6s} {'P':>6s} {'R':>6s} {'F1':>6s}")
+    print(f"  {'-'*45} {'---':>6s} {'---':>6s} {'---':>6s} {'---':>6s}")
+    for name, s in sorted_items[-n_show:]:
+        print(f"  {name:<45s} {s['accuracy']:>6.2f} {s['precision']:>6.2f} {s['recall']:>6.2f} {s['f1']:>6.2f}")
+
+    # Sample errors
+    if errors:
+        n_err = min(15, len(errors))
+        print(f"\n  Sample Mismatches ({n_err} of {len(errors)}):")
+        print(f"  {'DocID':<12s} {'Item':<40s} {'GT':<20s} {'Pred':<20s}")
+        print(f"  {'-'*12} {'-'*40} {'-'*20} {'-'*20}")
+        for doc_id, item, gt_v, pred_v in errors[:n_err]:
+            print(f"  {doc_id:<12s} {item:<40s} {str(gt_v):<20s} {str(pred_v):<20s}")
+
+    print("=" * 70)
+
+    return {'accuracy': round(accuracy, 4), 'macro_f1': round(macro_f1, 4)}, per_item
+
+
+# ---------------------------------------------------------------------------
+#  MAIN
+# ---------------------------------------------------------------------------
+
+async def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--api_key", default=API_KEY,
+                        help="Google AI Studio API key")
+    parser.add_argument("--model_name", default="gemini-1.5-pro",
+                        help="Gemini model name")
+    parser.add_argument("--data_folders", nargs="+",
+                        default=[
+                            r"C:\Users\sai78\Desktop\Clinical_CRF_filling\data\raw\dyspnea-clinical-notes",
+                            r"C:\Users\sai78\Desktop\Clinical_CRF_filling\data\raw\dyspnea-crf-development",
+                        ],
+                        help="Directories containing .parquet shards (searched recursively)")
+    parser.add_argument("--gt_file",
+                        default=r"C:\Users\sai78\Desktop\Clinical_CRF_filling\data\raw\dev_gt.jsonl")
+    parser.add_argument("--options_folder",
+                        default=r"C:\Users\sai78\Desktop\Clinical_CRF_filling\data\raw\dyspnea-valid-options\dyspnea-valid-options\data")
+    parser.add_argument("--output_file",
+                        default="data/processed/materialized_ehr/submission.json")
+    parser.add_argument("--skip_eval", action="store_true",
+                        help="Skip evaluation after generating predictions")
+    parser.add_argument("--concurrency", type=int, default=5,
+                        help="Max concurrent LLM calls (free tier: keep at 5)")
+    # --- RAG options ---
+    parser.add_argument("--use_rag", action="store_true",
+                        help="Use RAG-guided extraction (retrieves relevant tuples per CRF item)")
+    parser.add_argument("--rag_top_k", type=int, default=15,
+                        help="Number of WTTS tuples to retrieve per CRF item group (RAG mode)")
+    parser.add_argument("--rag_model", type=str, default="all-MiniLM-L6-v2",
+                        help="SentenceTransformer model for embeddings (swap to clinical model on GPU)")
+    parser.add_argument("--rag_device", type=str, default="cpu",
+                        help="Device for embedding model: 'cpu' or 'cuda'")
+    args = parser.parse_args()
+
+    # 1. Setup — Configure Gemini API
+    genai.configure(api_key=args.api_key)
+    model = genai.GenerativeModel(args.model_name)
+    print(f"Using model: {args.model_name} (Google AI Studio)")
+
+    # Limit concurrency (free tier = 15 RPM, so keep low)
+    semaphore = asyncio.Semaphore(args.concurrency)
+
+    # 2. Load Data
+    loader = DataLoader(data_folders=args.data_folders, gt_path=args.gt_file)
+
+    target_items = loader.get_target_schema()
+    valid_options = loader.load_valid_options(args.options_folder)
+
+    merged_data = loader.load_and_merge()
+
+    if not merged_data:
+        print("No data found. Exiting.")
+        return
+
+    # 3. Process
+    builder = WTTSBuilder()
+    print(f"Starting pipeline for {len(merged_data)} patients...")
+    print(f"Schema: {len(target_items)} items per patient.")
+
+    if args.use_rag:
+        # --- RAG Pipeline ---
+        if not RAG_AVAILABLE:
+            print("ERROR: RAG dependencies not installed. Run:")
+            print("  pip install sentence-transformers faiss-cpu")
+            return
+
+        print(f"\n  [RAG MODE] Embedding model: {args.rag_model}")
+        print(f"  [RAG MODE] Device: {args.rag_device}")
+        print(f"  [RAG MODE] Top-k: {args.rag_top_k}\n")
+
+        embedder = WTTSEmbedder(model_name=args.rag_model, device=args.rag_device)
+        extractor = RAGCRFExtractor(
+            embedder=embedder,
+            generate_fn=generate_async,
+            top_k=args.rag_top_k,
+        )
+
+        tasks = [
+            extractor.extract_patient(
+                p, builder, target_items, valid_options, semaphore, model
+            )
+            for p in merged_data
+        ]
+    else:
+        # --- Original Two-Pass Pipeline ---
+        tasks = [
+            process_patient(model, builder, p, target_items, valid_options, semaphore)
+            for p in merged_data
+        ]
+
+    results = await asyncio.gather(*tasks)
+    results = [r for r in results if r is not None]
+
+    # 4. Save
+    os.makedirs(os.path.dirname(args.output_file), exist_ok=True)
+    with open(args.output_file, 'w') as f:
+        json.dump(results, f, indent=2)
+
+    print(f"\nDone! {len(results)} results saved to {args.output_file}")
+
+    # 5. Evaluate against GT
+    if not args.skip_eval:
+        print("\nRunning evaluation against ground truth...")
+        overall, _ = evaluate_predictions(results, args.gt_file)
+        print(f"\n  >>> Final Accuracy: {overall['accuracy']:.4f}  |  Macro F1: {overall['macro_f1']:.4f}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

run_preprocess.bat

@@ -0,0 +1,5 @@
+python -m src.preprocess.wtts_builder ^
+    --input_dir data/raw/dyspnea-clinical-notes ^
+    --gt_file data/raw/dev_gt.jsonl ^
+    --output_dir data/processed/materialized_ehr
+pause

scoring.py

@@ -0,0 +1,131 @@
+import os
+import sys
+import json
+from sklearn.metrics import f1_score
+import argparse
+
+
+def load_jsonl(path):
+    """Load a JSONL file as a list of dicts."""
+    data = []
+    with open(path, "r", encoding="utf-8") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            data.append(json.loads(line))
+    return data
+
+
+class Scorer:
+    def __init__(self, not_available_string: str, language:str):
+        self.not_available_string = not_available_string
+        self.return_value_for_zero_division = 0
+        if language not in ["en", "it"]:
+            raise ValueError(f"Unsupported language: {language}. Supported languages are 'en' and 'it'.")
+        self.language = language
+
+    def calculate_score(self, reference, submission):
+        scores = []
+        self.TP = 0
+        self.FP = 0
+        self.FN = 0        
+        for ref_one_patient, sub_one_patient in zip(reference, submission):
+            sub_one_patient_id, lang = sub_one_patient["document_id"].split("_", 1)
+            if ref_one_patient["document_id"] != sub_one_patient_id:
+                raise ValueError(
+                    f"Document ID mismatch: reference {ref_one_patient['document_id']} vs submission {sub_one_patient['document_id']}"
+                )
+            if lang != self.language:
+                raise ValueError(
+                    f"Language mismatch: expected {self.language} but got {lang} in submission"
+                )
+            score_one_patient = self.calculate_score_one_patient(
+                ref_one_patient,
+                sub_one_patient,
+            )
+            scores.append(score_one_patient)
+
+        if not scores:
+            return 0.0
+        print(f"TP: {self.TP}, FP: {self.FP}, FN: {self.FN}")   
+
+        return sum(scores) / len(scores)
+
+    def calculate_score_one_patient(self, reference_one_patient, submission_one_patient):
+        # Expected structure:
+        # reference_one_patient["annotations"] = [{"ground_truth": ...}, ...]
+        # submission_one_patient["predictions"] = [{"prediction": ...}, ...]
+        y_true = [item["ground_truth"] for item in reference_one_patient["annotations"]]
+        y_pred = [item["prediction"] for item in submission_one_patient["predictions"]]
+
+        for i, t, p in zip(range(len(y_true)), y_true, y_pred):
+            if t != self.not_available_string or p != self.not_available_string:
+                if t == p:
+                    self.TP += 1
+                elif t == self.not_available_string and p != self.not_available_string:
+                    self.FP += 1
+                elif t != p and p == self.not_available_string:
+                    self.FN += 1
+        f1 = f1_score(
+            y_true,
+            y_pred,
+            average="macro",
+        )
+        return f1
+
+
+def main(your_submission_path: str, language: str, test_or_dev: str) -> None:
+    print("\n=== Scoring program starting ===")
+    output_dir = "your_sumbmission_scores"
+
+    if test_or_dev == "test":
+        ref_path = 'development_data/dev_gt.jsonl'
+    elif test_or_dev == "development":
+        ref_path = 'development_data/dev_gt.jsonl'
+    else:
+        raise ValueError("test_or_dev must be either 'test' or 'development'")
+
+    sub_path = your_submission_path
+
+    if not os.path.exists(ref_path):
+        raise FileNotFoundError(f"Reference file not found at {ref_path}")
+    if not os.path.exists(sub_path):
+        raise FileNotFoundError(f"Submission predictions not found at {sub_path}")
+
+    print(f"Loading reference from {ref_path}")
+    try:
+        reference = load_jsonl(ref_path)
+    except:
+        if test_or_dev == "test":
+            raise ValueError(f"Test data has not been released yet.")
+
+    print(f"Loading submission from {sub_path}")
+    submission = load_jsonl(sub_path)
+
+    scorer = Scorer(not_available_string="unknown", language=language)
+    score = scorer.calculate_score(reference, submission)
+
+    print(f"Final macro-F1 = {score}")
+
+    os.makedirs(output_dir, exist_ok=True)
+
+    # Codabench reads scores.json (or scores.txt). Let's use JSON:
+    scores_path = os.path.join(output_dir, "scores.json")
+    with open(scores_path, "w", encoding="utf-8") as f:
+        json.dump({"f1_macro": float(score)}, f)
+
+    print(f"Scores written to {scores_path}")
+    print("=== Scoring program finished successfully ===\n")
+
+
+if __name__ == "__main__":
+    # get from argparse the agumetns pred, ref, output, language
+    argparse = argparse.ArgumentParser(description="Score submission")
+    argparse.add_argument("--submission_path", type=str, help="Path to the submission JSONL")
+    argparse.add_argument("--language", type=str, help="Language of the submission (en or it)")
+    args = argparse.parse_args()
+
+    your_submission_path = args.submission_path
+    language = args.language
+    main(your_submission_path, language, test_or_dev="development")