{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e78262c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "def process_and_save_json(file_paths, annotator_names):\n",
    "    all_data = []\n",
    "    \n",
    "    # Load and categorize data\n",
    "    for path, name in zip(file_paths, annotator_names):\n",
    "        with open(path, 'r') as f:\n",
    "            df = pd.DataFrame(json.load(f))\n",
    "        \n",
    "        # Map ratings to health literacy categories\n",
    "        def map_rating(r):\n",
    "            if r in [1, 2]: return \"low_health_literacy\"\n",
    "            if r == 3: return \"intermediate_health_literacy\"\n",
    "            if r in [4, 5]: return \"proficient_health_literacy\"\n",
    "            return None\n",
    "\n",
    "        df['human_category'] = df['rating'].apply(map_rating)\n",
    "        df = df[['doc_id', 'label', 'rating', 'human_category']]\n",
    "        \n",
    "        # Rename columns to distinguish between annotators\n",
    "        df = df.rename(columns={\n",
    "            'rating': f'rating_{name}',\n",
    "            'human_category': f'category_{name}',\n",
    "            'label': 'ai_label'\n",
    "        })\n",
    "        all_data.append(df)\n",
    "\n",
    "    # Merge all three dataframes\n",
    "    merged = all_data[0]\n",
    "    for next_df in all_data[1:]:\n",
    "        merged = pd.merge(merged, next_df, on=['doc_id', 'ai_label'])\n",
    "\n",
    "    # Determine agreement count\n",
    "    cat_cols = [f'category_{name}' for name in annotator_names]\n",
    "    merged['agreement_count'] = merged.apply(\n",
    "        lambda row: sum(1 for col in cat_cols if row[col] == row['ai_label']), axis=1\n",
    "    )\n",
    "\n",
    "    # Filter into two groups\n",
    "    agreement_data = merged[merged['agreement_count'] >= 2]\n",
    "    correction_needed = merged[merged['agreement_count'] < 2]\n",
    "\n",
    "    # Export to JSON\n",
    "    agreement_data.to_json(\"/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/code/annotator_agreement.json\", orient=\"records\", indent=4)\n",
    "    correction_needed.to_json(\"/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/code/needs_correction.json\", orient=\"records\", indent=4)\n",
    "    \n",
    "    print(f\"Success! {len(agreement_data)} items agreed, {len(correction_needed)} need correction.\")\n",
    "\n",
    "# Usage\n",
    "paths = [\n",
    "    \"/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/Plaban Das/annotation_results.json\",\n",
    "    \"/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/mahi/annotation_results.json\",\n",
    "    \"/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/Shama/annotation_results.json\"\n",
    "]\n",
    "names = [\"plaban\", \"mahi\", \"shama\"]\n",
    "\n",
    "process_and_save_json(paths, names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab336faf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "def create_correction_evaluation_file(source_path, agreement_results_path, output_path):\n",
    "    # 1. Load the source full data\n",
    "    with open(source_path, 'r') as f:\n",
    "        source_data = json.load(f)\n",
    "    source_df = pd.DataFrame(source_data)\n",
    "    \n",
    "    # 2. Load the \"needs correction\" data generated from previous step\n",
    "    with open(agreement_results_path, 'r') as f:\n",
    "        correction_df = pd.DataFrame(json.load(f))\n",
    "    \n",
    "    # 3. Merge based on doc_id (annotation) == index (source)\n",
    "    # We only keep the rows that exist in the correction list\n",
    "    enriched_correction = pd.merge(\n",
    "        correction_df, \n",
    "        source_df[['index', 'fulltext', 'diff_label_texts']], \n",
    "        left_on='doc_id', \n",
    "        right_on='index', \n",
    "        how='left'\n",
    "    )\n",
    "    \n",
    "    # Optional: Clean up by dropping the redundant 'index' column\n",
    "    if 'index' in enriched_correction.columns:\n",
    "        enriched_correction = enriched_correction.drop(columns=['index'])\n",
    "        \n",
    "    # 4. Save to a new JSON file\n",
    "    enriched_correction.to_json(output_path, orient=\"records\", indent=4)\n",
    "    \n",
    "    print(f\"Evaluation file created: {output_path}\")\n",
    "    print(f\"Total entries for re-evaluation: {len(enriched_correction)}\")\n",
    "\n",
    "# Paths\n",
    "source_file = '/home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_en_0_80_full.json'\n",
    "correction_list = '/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/code/needs_correction.json' # The file created from the previous script\n",
    "final_output = '/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/code/correction_evaluation_full_text.json'\n",
    "\n",
    "create_correction_evaluation_file(source_file, correction_list, final_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "93f3ae03",
   "metadata": {},
   "outputs": [],
   "source": [
    "# /home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_en_0_80_full_updated.json\n",
    "import json\n",
    "with open(\"/home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_en_0_80_full_updated.json\", 'r') as f:\n",
    "    data = json.load(f)\n",
    "text_map={}\n",
    "for item in data:\n",
    "    for label in list(item['diff_label_texts'].keys()):\n",
    "        key=f\"{item['index']}_{label}\"\n",
    "        text_map[key] = {\n",
    "            'fulltext': item['fulltext'],\n",
    "            \"diff_label_texts\": item['diff_label_texts'][label],\n",
    "            'summary': item.get('summary')\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c8d64fdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# /home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/correction_data/final_corrected_anu.json\n",
    "with open(\"/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/correction_data/final_corrected_anu.json\", 'r') as f:\n",
    "    annotator_corrections = json.load(f)\n",
    "new_data = []\n",
    "for item in annotator_corrections:\n",
    "    key = f\"{item['doc_id']}_{item['ai_label']}\"\n",
    "    final_text=item['final_text']\n",
    "    new_data.append({\n",
    "        'doc_id': item['doc_id'],\n",
    "        'label': item['ai_label'],\n",
    "        'fulltext': text_map[key]['fulltext'],\n",
    "        'diff_label_texts': final_text,\n",
    "        'summary': text_map[key]['summary']\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9521411f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# /home/mshahidul/readctrl/data/factual_testing/full_details_evaluation_0_80_qwen3-30B_v2.json\n",
    "with open(\"/home/mshahidul/readctrl/data/factual_testing/full_details_evaluation_0_80_qwen3-30B_v2.json\", 'r') as f:\n",
    "    factual_data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7628ac8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "un",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}