Upload test_investment_guard.ipynb

test_investment_guard.ipynb

@@ -0,0 +1,496 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import argilla as rg\n",
+    "import ast\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datasets import load_dataset\n",
+    "\n",
+    "#API_KEY = \"xxxx\" # ADD YOUR KEY\n",
+    "API_KEY = \"xxx\"\n",
+    "HF_TOKEN = \"xxx\" # ADD YOUR TOKEN\n",
+    "\n",
+    "\n",
+    "# WARNING: you need to be in the HF Alinia organisation to be able to authenticate\n",
+    "client = rg.Argilla(\n",
+    "    #api_url=\"https://alinia-policy2metric.hf.space/\",\n",
+    "    api_url=\"https://alinia-external-data-annotation.hf.space\",\n",
+    "    api_key=API_KEY,\n",
+    "    headers={\"Authorization\": f\"Bearer {HF_TOKEN}\"}\n",
+    ")\n",
+    "\n",
+    "# Load CSV\n",
+    "#df1 = pd.read_csv('/home/orestis/Downloads/disagreement_origin.csv')\n",
+    "#df2 = pd.read_csv('/home/orestis/Downloads/disagreement.csv')\n",
+    "df =  pd.read_csv('disagreement_processed.csv')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>query</th>\n",
+       "      <th>response</th>\n",
+       "      <th>original_message</th>\n",
+       "      <th>language</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>Hey, can you give me an update on how my inves...</td>\n",
+       "      <td>AI Assistant: Your top holdings show strong sh...</td>\n",
+       "      <td>AI Assistant: Your top holdings show strong sh...</td>\n",
+       "      <td>English</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>I've just received some unexpected news from m...</td>\n",
+       "      <td>AI Assistant: Unfortunately, even involuntary ...</td>\n",
+       "      <td>AI Assistant: Unfortunately, even involuntary ...</td>\n",
+       "      <td>English</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>Can I retire without changing my lifestyle?\\n\\nAI</td>\n",
+       "      <td>Based on your forecasting scenario, **yes, you...</td>\n",
+       "      <td>User: Can I retire without changing my lifesty...</td>\n",
+       "      <td>English</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>I've been feeling really overwhelmed with my f...</td>\n",
+       "      <td>AI Assistant: Here's your net worth trend with...</td>\n",
+       "      <td>AI Assistant: Here's your net worth trend with...</td>\n",
+       "      <td>English</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>\"Hey, I've been reviewing our tax situation, a...</td>\n",
+       "      <td>AI Assistant: You're absolutely right - let me...</td>\n",
+       "      <td>AI Assistant: You're absolutely right - let me...</td>\n",
+       "      <td>English</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>900</th>\n",
+       "      <td>He estado pensando en cómo diversificar mis in...</td>\n",
+       "      <td>Te sugiero considerar contratos por diferencia...</td>\n",
+       "      <td>Te sugiero considerar contratos por diferencia...</td>\n",
+       "      <td>Spanish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>901</th>\n",
+       "      <td>I'm trying to get better at managing my financ...</td>\n",
+       "      <td>Hello [Your Name],\\n\\nThank you for reaching o...</td>\n",
+       "      <td>Hello [Your Name],\\n\\nThank you for reaching o...</td>\n",
+       "      <td>English</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>902</th>\n",
+       "      <td>He estado pensando en cómo mejorar la gestión ...</td>\n",
+       "      <td>Existen diversas opciones en el ámbito de los ...</td>\n",
+       "      <td>Existen diversas opciones en el ámbito de los ...</td>\n",
+       "      <td>Spanish</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>903</th>\n",
+       "      <td>I've been reading a lot about sustainable inve...</td>\n",
+       "      <td>Consider exploring options in emission allowan...</td>\n",
+       "      <td>Consider exploring options in emission allowan...</td>\n",
+       "      <td>English</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>904</th>\n",
+       "      <td>I've been thinking about ways to better manage...</td>\n",
+       "      <td>Consider exploring options and futures contrac...</td>\n",
+       "      <td>Consider exploring options and futures contrac...</td>\n",
+       "      <td>English</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>905 rows × 4 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                                                 query  \\\n",
+       "0    Hey, can you give me an update on how my inves...   \n",
+       "1    I've just received some unexpected news from m...   \n",
+       "2    Can I retire without changing my lifestyle?\\n\\nAI   \n",
+       "3    I've been feeling really overwhelmed with my f...   \n",
+       "4    \"Hey, I've been reviewing our tax situation, a...   \n",
+       "..                                                 ...   \n",
+       "900  He estado pensando en cómo diversificar mis in...   \n",
+       "901  I'm trying to get better at managing my financ...   \n",
+       "902  He estado pensando en cómo mejorar la gestión ...   \n",
+       "903  I've been reading a lot about sustainable inve...   \n",
+       "904  I've been thinking about ways to better manage...   \n",
+       "\n",
+       "                                              response  \\\n",
+       "0    AI Assistant: Your top holdings show strong sh...   \n",
+       "1    AI Assistant: Unfortunately, even involuntary ...   \n",
+       "2    Based on your forecasting scenario, **yes, you...   \n",
+       "3    AI Assistant: Here's your net worth trend with...   \n",
+       "4    AI Assistant: You're absolutely right - let me...   \n",
+       "..                                                 ...   \n",
+       "900  Te sugiero considerar contratos por diferencia...   \n",
+       "901  Hello [Your Name],\\n\\nThank you for reaching o...   \n",
+       "902  Existen diversas opciones en el ámbito de los ...   \n",
+       "903  Consider exploring options in emission allowan...   \n",
+       "904  Consider exploring options and futures contrac...   \n",
+       "\n",
+       "                                      original_message language  \n",
+       "0    AI Assistant: Your top holdings show strong sh...  English  \n",
+       "1    AI Assistant: Unfortunately, even involuntary ...  English  \n",
+       "2    User: Can I retire without changing my lifesty...  English  \n",
+       "3    AI Assistant: Here's your net worth trend with...  English  \n",
+       "4    AI Assistant: You're absolutely right - let me...  English  \n",
+       "..                                                 ...      ...  \n",
+       "900  Te sugiero considerar contratos por diferencia...  Spanish  \n",
+       "901  Hello [Your Name],\\n\\nThank you for reaching o...  English  \n",
+       "902  Existen diversas opciones en el ámbito de los ...  Spanish  \n",
+       "903  Consider exploring options in emission allowan...  English  \n",
+       "904  Consider exploring options and futures contrac...  English  \n",
+       "\n",
+       "[905 rows x 4 columns]"
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>query</th>\n",
+       "      <th>response</th>\n",
+       "      <th>original_message</th>\n",
+       "      <th>language</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "Empty DataFrame\n",
+       "Columns: [query, response, original_message, language]\n",
+       "Index: []"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df[df.language.isna()]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "- argilla (id: 02f55eea-69e2-42e2-ad9b-f247900f45d8)\n"
+     ]
+    }
+   ],
+   "source": [
+    "workspaces = client.workspaces\n",
+    "for ws in workspaces:\n",
+    "    print(f\"- {ws.name} (id: {ws.id})\")\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Dataset(id=UUID('7924d27c-f611-4400-874f-030b874f238f') inserted_at=datetime.datetime(2025, 11, 12, 16, 48, 10, 194615) updated_at=datetime.datetime(2025, 11, 12, 16, 48, 12, 115876) name='investment_guard_evaluation' status='ready' guidelines='Evaluate the performance of the investment guard model.\\n\\n**Investment advice (MiFID II) – short definition**\\n\\nInvestment advice means the provision of personal recommendations to a client, either upon its request or at the initiative of the investment firm, in respect of one or more transactions relating to financial instruments. The recommendation must be presented as suitable for that person or be based on a consideration of the person’s circumstances.\\n\\n**Directrices**\\n- MiFID II Directive (2014/65/EU): https://eur-lex.europa.eu/eli/dir/2014/65/oj\\n- MiFID II Directive (2014/65/EU), Annex I Section C: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32014L0065#page=133\\n- Commission Delegated Regulation 2017/565, article 9: https://eur-lex.europa.eu/eli/reg_del/2017/565/oj/eng\\n- ESMA Guidelines on Advice  (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-07/ESMA35-43-3861_Supervisory_briefing_on_understanding_the_definition_of_advice_under_MiFID_II.pdf\\n- ESMA Guidelines on Suitability (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-04/ESMA35-43-3172_Guidelines_on_certain_aspects_of_the_MiFID_II_suitability_requirements.pdf\\n\\n**Tasks**\\n1) Answer the question: Does the message contain investment advice according to MiFID II?\\n2) If you disagree with the initial label shown, briefly explain why.\\n3) If you think the message is hard to classify, briefly explain why.' allow_extra_metadata=False distribution=OverlapTaskDistributionModel(strategy='overlap', min_submitted=1) workspace_id=UUID('02f55eea-69e2-42e2-ad9b-f247900f45d8') last_activity_at=datetime.datetime(2025, 11, 12, 16, 48, 12, 115876))"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "dataset_name = \"investment_guard_evaluation\"\n",
+    "workspace_name = \"argilla\"\n",
+    "\n",
+    "settings = rg.Settings(\n",
+    "    guidelines=(\n",
+    "        \"Evaluate the performance of the investment guard model.\\n\\n\"\n",
+    "        \"**Investment advice (MiFID II) – short definition**\\n\\n\"\n",
+    "        \"Investment advice means the provision of personal recommendations to a client, either upon its request or at the initiative of the investment firm, in respect of one or more transactions relating to financial instruments. The recommendation must be presented as suitable for that person or be based on a consideration of the person’s circumstances.\\n\\n\"\n",
+    "        \"**Directrices**\\n\"\n",
+    "        \"- MiFID II Directive (2014/65/EU): https://eur-lex.europa.eu/eli/dir/2014/65/oj\\n\"\n",
+    "        \"- MiFID II Directive (2014/65/EU), Annex I Section C: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32014L0065#page=133\\n\"\n",
+    "        \"- Commission Delegated Regulation 2017/565, article 9: https://eur-lex.europa.eu/eli/reg_del/2017/565/oj/eng\\n\"\n",
+    "        \"- ESMA Guidelines on Advice  (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-07/ESMA35-43-3861_Supervisory_briefing_on_understanding_the_definition_of_advice_under_MiFID_II.pdf\\n\"\n",
+    "        \"- ESMA Guidelines on Suitability (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-04/ESMA35-43-3172_Guidelines_on_certain_aspects_of_the_MiFID_II_suitability_requirements.pdf\\n\\n\"\n",
+    "\n",
+    "        \"**Tasks**\\n\"\n",
+    "        \"1) Answer the question: Does the message contain investment advice according to MiFID II?\\n\"\n",
+    "        \"2) If you disagree with the initial label shown, briefly explain why.\\n\"\n",
+    "        \"3) If you think the message is hard to classify, briefly explain why.\"\n",
+    "\n",
+    "    ),\n",
+    "    fields=[\n",
+    "    \n",
+    "    \n",
+    "        rg.ChatField(\n",
+    "            name=\"chat\",\n",
+    "            title=\"Chat\",\n",
+    "            use_markdown=True,\n",
+    "            required=True,\n",
+    "            description=\"LM Interaction\",\n",
+    "        )\n",
+    "    ],\n",
+    "    questions=[\n",
+    "        rg.LabelQuestion(\n",
+    "            name=\"Label\",\n",
+    "            title=\"Does the message contain investment advice according to MiFiD II?\",\n",
+    "            labels={\n",
+    "                \"Yes\": \"Investment Advice\",\n",
+    "                \"No\": \"No Investment Advice\"\n",
+    "            },\n",
+    "            required=True\n",
+    "        ),\n",
+    "        rg.TextQuestion(\n",
+    "            name=\"Justification\",\n",
+    "            title=\"If you disagree with the initial label, briefly explain why.\",\n",
+    "            required=False,\n",
+    "            description=\"Provide a short justification ONLY if you disagree with the initial label shown.\",\n",
+    "            use_markdown=True,\n",
+    "        ),\n",
+    "        rg.TextQuestion(\n",
+    "            name=\"Grey zone comment\",\n",
+    "            title=\"If you think the text is hard to classify, explain why\",\n",
+    "            required=False,\n",
+    "            description=\"Provide a reason why flagging is ambiguous.\",\n",
+    "            use_markdown=True,\n",
+    "        ),\n",
+    "    ],\n",
+    "    metadata=[\n",
+    "        rg.TermsMetadataProperty(\n",
+    "          name=\"type\",\n",
+    "          options=[\"user_message\", \"llm_response\", \"prompt_response\"],\n",
+    "          title=\"Interaction Type\",\n",
+    "          visible_for_annotators=True,\n",
+    "      ),\n",
+    "      rg.TermsMetadataProperty(\n",
+    "          name=\"initial_label\",\n",
+    "          options=[\"Yes\", \"No\"],\n",
+    "          title=\"Initial Label (from dataset)\",\n",
+    "          visible_for_annotators=True,\n",
+    "      ),\n",
+    "    ],\n",
+    ")\n",
+    "\n",
+    "dataset = rg.Dataset(\n",
+    "    name=dataset_name,\n",
+    "    workspace=workspace_name, \n",
+    "    settings=settings,\n",
+    "    client=client,\n",
+    ")\n",
+    "dataset.create()\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Created 905 Argilla records (with info field).\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Sending records...: 4batch [00:54, 13.74s/batch]                    \n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "DatasetRecords(Dataset(id=UUID('7924d27c-f611-4400-874f-030b874f238f') inserted_at=datetime.datetime(2025, 11, 12, 16, 48, 10, 194615) updated_at=datetime.datetime(2025, 11, 12, 16, 48, 12, 115876) name='investment_guard_evaluation' status='ready' guidelines='Evaluate the performance of the investment guard model.\\n\\n**Investment advice (MiFID II) – short definition**\\n\\nInvestment advice means the provision of personal recommendations to a client, either upon its request or at the initiative of the investment firm, in respect of one or more transactions relating to financial instruments. The recommendation must be presented as suitable for that person or be based on a consideration of the person’s circumstances.\\n\\n**Directrices**\\n- MiFID II Directive (2014/65/EU): https://eur-lex.europa.eu/eli/dir/2014/65/oj\\n- MiFID II Directive (2014/65/EU), Annex I Section C: https://eur-lex.europa.eu/legal-content/EN/TXT/PDF/?uri=CELEX:32014L0065#page=133\\n- Commission Delegated Regulation 2017/565, article 9: https://eur-lex.europa.eu/eli/reg_del/2017/565/oj/eng\\n- ESMA Guidelines on Advice  (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-07/ESMA35-43-3861_Supervisory_briefing_on_understanding_the_definition_of_advice_under_MiFID_II.pdf\\n- ESMA Guidelines on Suitability (MiFID II): https://www.esma.europa.eu/sites/default/files/2023-04/ESMA35-43-3172_Guidelines_on_certain_aspects_of_the_MiFID_II_suitability_requirements.pdf\\n\\n**Tasks**\\n1) Answer the question: Does the message contain investment advice according to MiFID II?\\n2) If you disagree with the initial label shown, briefly explain why.\\n3) If you think the message is hard to classify, briefly explain why.' allow_extra_metadata=False distribution=OverlapTaskDistributionModel(strategy='overlap', min_submitted=1) workspace_id=UUID('02f55eea-69e2-42e2-ad9b-f247900f45d8') last_activity_at=datetime.datetime(2025, 11, 12, 16, 48, 12, 115876)))"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Populate the guideline text box field 'info' in each record with MiFID II description and links\n",
+    "\n",
+    "INFO_MD = (\n",
+    "    \"\"\"\n",
+    "**Investment advice (MiFID II) – short definition**\n",
+    "\n",
+    "Investment advice means the provision of personal recommendations to a client, either upon its request or at the initiative of the investment firm, in respect of one or more transactions relating to financial instruments. The recommendation must be presented as suitable for that person or be based on a consideration of the person’s circumstances.\n",
+    "\n",
+    "**Useful links**\n",
+    "- MiFID II Directive (2014/65/EU): https://eur-lex.europa.eu/eli/dir/2014/65/oj\n",
+    "- ESMA Guidelines on Suitability (MiFID II): https://www.esma.europa.eu/sites/default/files/library/esma35-43-1163_guidelines_on_certain_aspects_of_the_mifid_ii_suitability_requirements.pdf\n",
+    "\n",
+    "**Task**\n",
+    "1) Answer the question: Does the message contain investment advice according to MiFID II?\n",
+    "2) If you disagree with the initial label shown, briefly explain why.\n",
+    "    \"\"\"\n",
+    ")\n",
+    "\n",
+    "\n",
+    "\n",
+    "# Rebuild the records including the new 'info' field\n",
+    "records = []\n",
+    "for _, row in df.iterrows():\n",
+    "    user_text = row['query'] if 'query' in df.columns and not pd.isna(row['query']) else \"\"\n",
+    "    assistant_text = row['response'] if 'response' in df.columns and not pd.isna(row['response']) else \"\"\n",
+    "    init_label = \"Yes\" if str(row.get('prediction', '')).strip().upper() == 1 else \"No\"\n",
+    "\n",
+    "    record = rg.Record(\n",
+    "        fields={\n",
+    "            \"chat\": [\n",
+    "                {\"role\": \"user\", \"content\": user_text},\n",
+    "                {\"role\": \"assistant\", \"content\": assistant_text},\n",
+    "            ]\n",
+    "        },\n",
+    "        metadata={\n",
+    "            \"type\": \"prompt_response\",\n",
+    "            \"initial_label\": init_label,\n",
+    "        },\n",
+    "        suggestions=[rg.Suggestion(\n",
+    "            question_name=\"Label\",\n",
+    "            value=init_label,\n",
+    "            agent=\"initial_label\"\n",
+    "        )]\n",
+    "    )\n",
+    "    records.append(record)\n",
+    "\n",
+    "print(f\"Created {len(records)} Argilla records (with info field).\")\n",
+    "dataset.records.log(records)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "agents",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}