{ "cells": [ { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# from openpyxl import load_workbook\n", "# wb = load_workbook(\"agllm-data/Responses V3 Fixed Arti.xlsm\", data_only=True)\n", "# ws = wb['Researcher-Documents-insects']\n", "# cell = ws['A1']\n", "# value = cell.value\n", "# print(value)" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import ast\n", "def safe_literal_eval(val):\n", " if pd.isna(val): # Check if the value is NaN or None\n", " return [] # or return None, depending on what you prefer for empty/missing values\n", " try:\n", " return ast.literal_eval(val)\n", " except (ValueError, SyntaxError):\n", " return val # Return the original value if it can't be evaluated" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " Q/GT Gemini GPT LLAMA Claude \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 3 1 1 1 3 \n", " 3 2 2 2 3 3 \n", "\n", " ground_truth_answer \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Start scouting when 10% of migrating larvae ar... \n", " 3 To minimize feeding injury risk in high-risk f... \n", "\n", " answer_Gemini-1.5 Pro \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:The provided scientific literature focu... \n", " 3 Answer:\\nDelay planting to minimize injury, an... \n", "\n", " answer_GPT-4 \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:\\nThe documents does not provide inform... \n", " 3 Answer:\\nDelay planting, use insecticide seed ... \n", "\n", " answer_GPT-3.5 \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 The documents does not provide information abo... \n", " 3 Answer: No-till fields are less attractive to ... \n", "\n", " answer_Llama-3 70B \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer: The document does not provide informat... \n", " 3 Answer: To minimize the risk of feeding injury... \n", "\n", " answer_Llama-3 8B \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer: Scouting for black cutworm larvae in c... \n", " 3 Answer: No-till fields are less attractive to ... \n", "\n", " answer_Claude 3 Opus \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:\\nScouting for stalk borer larvae shoul... \n", " 3 Answer:\\nTo minimize feeding injury risk from ... \n", "\n", " source \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 [agllm-data/agllm-data-isu-field-insects-all-s... \n", " 3 [agllm-data/agllm-data-isu-field-insects-all-s... \n", "\n", " relevant_docs \n", "Sheet Row \n", "Researcher-IsuField-insects 2 [agllm-data/agllm-data-isu-field-insects-all-s... \n", " 3 [agllm-data/agllm-data-isu-field-insects-all-s... \n", "\n", "Processing complete.\n", " Q/GT Gemini GPT LLAMA Claude \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 3 1 1 1 3 \n", " 3 3 3 2 2 2 \n", "\n", " ground_truth_answer \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Start scouting when 10% of migrating larvae ar... \n", " 3 To minimize feeding injury risk in high-risk f... \n", "\n", " answer_Gemini-1.5 Pro \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:The provided scientific literature focu... \n", " 3 Answer:\\nDelay planting to minimize injury, an... \n", "\n", " answer_GPT-4 \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:\\nThe documents does not provide inform... \n", " 3 Answer:\\nDelay planting, use insecticide seed ... \n", "\n", " answer_GPT-3.5 \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 The documents does not provide information abo... \n", " 3 Answer: No-till fields are less attractive to ... \n", "\n", " answer_Llama-3 70B \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer: The document does not provide informat... \n", " 3 Answer: To minimize the risk of feeding injury... \n", "\n", " answer_Llama-3 8B \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer: Scouting for black cutworm larvae in c... \n", " 3 Answer: No-till fields are less attractive to ... \n", "\n", " answer_Claude 3 Opus \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:\\nScouting for stalk borer larvae shoul... \n", " 3 Answer:\\nTo minimize feeding injury risk from ... \n", "\n", " source \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 [agllm-data/agllm-data-isu-field-insects-all-s... \n", " 3 [agllm-data/agllm-data-isu-field-insects-all-s... \n", "\n", " relevant_docs \n", "Sheet Row \n", "Researcher-IsuField-insects 2 [agllm-data/agllm-data-isu-field-insects-all-s... \n", " 3 [agllm-data/agllm-data-isu-field-insects-all-s... \n", "\n", "Processing complete.\n", " Q/GT Gemini GPT LLAMA Claude \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 3 1 1 1 3 \n", " 3 2 2 3 3 3 \n", "\n", " ground_truth_answer \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Start scouting when 10% of migrating larvae ar... \n", " 3 To minimize feeding injury risk in high-risk f... \n", "\n", " answer_Gemini-1.5 Pro \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:The provided scientific literature focu... \n", " 3 Answer:\\nDelay planting to minimize injury, an... \n", "\n", " answer_GPT-4 \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:\\nThe documents does not provide inform... \n", " 3 Answer:\\nDelay planting, use insecticide seed ... \n", "\n", " answer_GPT-3.5 \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 The documents does not provide information abo... \n", " 3 Answer: No-till fields are less attractive to ... \n", "\n", " answer_Llama-3 70B \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer: The document does not provide informat... \n", " 3 Answer: To minimize the risk of feeding injury... \n", "\n", " answer_Llama-3 8B \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer: Scouting for black cutworm larvae in c... \n", " 3 Answer: No-till fields are less attractive to ... \n", "\n", " answer_Claude 3 Opus \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 Answer:\\nScouting for stalk borer larvae shoul... \n", " 3 Answer:\\nTo minimize feeding injury risk from ... \n", "\n", " source \\\n", "Sheet Row \n", "Researcher-IsuField-insects 2 [agllm-data/agllm-data-isu-field-insects-all-s... \n", " 3 [agllm-data/agllm-data-isu-field-insects-all-s... \n", "\n", " relevant_docs \n", "Sheet Row \n", "Researcher-IsuField-insects 2 [agllm-data/agllm-data-isu-field-insects-all-s... \n", " 3 [agllm-data/agllm-data-isu-field-insects-all-s... \n", "\n", "Processing complete.\n" ] } ], "source": [ "from openpyxl import load_workbook\n", "import pandas as pd\n", "\n", "\n", "all_experts_paths={\n", "'Dr. Arti Singh': 'agllm-data/Dr. Arti Singh.xlsm',\n", "'Dr. Mathew Carroll': 'agllm-data/Dr. Matthew Carroll.xlsm',\n", "'Dr. Asheesh Singh': 'agllm-data/Dr. Asheesh Singh.xlsm'\n", "}\n", "all_experts_feedback={}\n", "\n", "for expert_name, excel_file in all_experts_paths.items():\n", "\n", " # Mappings for sheet names\n", " mappings = [\n", " {\"data_domain_identifier\": 'agllm-data-isu-field-insects-all-species', \"domain_type\": \"insects\", \"data_domain_identifier_label\": \"IsuField\", \"user_type\": \"Researcher\"},\n", " {\"data_domain_identifier\": 'agllm-data-isu-field-insects-all-species', \"domain_type\": \"insects\", \"data_domain_identifier_label\": \"IsuField\", \"user_type\": \"Farmer\"},\n", " {\"data_domain_identifier\": 'agllm-data-isu-field-weeds-all-species', \"domain_type\": \"weeds\", \"data_domain_identifier_label\": \"IsuField\", \"user_type\": \"Researcher\"},\n", " {\"data_domain_identifier\": 'agllm-data-isu-field-weeds-all-species', \"domain_type\": \"weeds\", \"data_domain_identifier_label\": \"IsuField\", \"user_type\": \"Farmer\"},\n", " {\"data_domain_identifier\": 'agllm-data-trial-all-weeds', \"domain_type\": \"weeds\", \"data_domain_identifier_label\": \"Documents\", \"user_type\": \"Researcher\"},\n", " {\"data_domain_identifier\": 'agllm-data-trial-all-weeds', \"domain_type\": \"weeds\", \"data_domain_identifier_label\": \"Documents\", \"user_type\": \"Farmer\"},\n", " {\"data_domain_identifier\": 'agllm-data-trial-all-insects', \"domain_type\": \"insects\", \"data_domain_identifier_label\": \"Documents\", \"user_type\": \"Researcher\"},\n", " {\"data_domain_identifier\": 'agllm-data-trial-all-insects', \"domain_type\": \"insects\", \"data_domain_identifier_label\": \"Documents\", \"user_type\": \"Farmer\"}\n", " ]\n", "\n", " def transform_score(score):\n", " if score is not None and isinstance(score, (int, float)):\n", " return 4 - score\n", " return score\n", "\n", " # Load the workbook\n", " wb = load_workbook(excel_file, data_only=True)\n", "\n", " # Create an empty list to store the data\n", " data_list = []\n", "\n", " # Column mappings\n", " column_mappings = {\n", " 18: \"Q/GT\",\n", " 19: \"Gemini\",\n", " 20: \"GPT\",\n", " 21: \"LLAMA\",\n", " 22: \"Claude\"\n", " }\n", "\n", " # Additional columns to fetch\n", " additional_columns = [\n", " \"ground_truth_answer\",\n", " \"answer_Gemini-1.5 Pro\",\n", " \"answer_GPT-4\",\n", " \"answer_GPT-3.5\",\n", " \"answer_Llama-3 70B\",\n", " \"answer_Llama-3 8B\",\n", " \"answer_Claude 3 Opus\",\n", " \"source\", \n", " \"relevant_docs\"\n", " ]\n", "\n", " # Iterate through the mappings\n", " for mapping in mappings:\n", " sheet_name = f\"{mapping['user_type']}-{mapping['data_domain_identifier_label']}-{mapping['domain_type']}\"\n", " \n", " try:\n", " # Get the worksheet\n", " ws = wb[sheet_name]\n", " \n", " # Get the column indices for additional columns\n", " additional_column_indices = {}\n", " for col in range(1, ws.max_column + 1):\n", " cell_value = ws.cell(row=1, column=col).value\n", " if cell_value in additional_columns:\n", " additional_column_indices[cell_value] = col\n", " \n", " # Start from row 2 (index 1) and continue until column 18 is empty\n", " row = 2\n", " while ws.cell(row=row, column=18).value is not None:\n", " row_data = [transform_score(ws.cell(row=row, column=col).value) for col in list(column_mappings.keys())]\n", " \n", " # Fetch additional column values\n", " for col_name in additional_columns:\n", " if col_name in additional_column_indices:\n", " col_index = additional_column_indices[col_name]\n", " row_data.append(ws.cell(row=row, column=col_index).value)\n", " \n", " # Create a dictionary with the data\n", " data_dict = {\n", " \"Sheet\": sheet_name,\n", " \"Row\": row,\n", " **{column_mappings[col]: value for col, value in zip(list(column_mappings.keys()), row_data[:len(column_mappings)])},\n", " **{col_name: value for col_name, value in zip(additional_columns, row_data[len(column_mappings):])}\n", " }\n", " \n", " # Append the dictionary to the data list\n", " data_list.append(data_dict)\n", " \n", " row += 1\n", " \n", " except KeyError:\n", " print(f\"Sheet '{sheet_name}' not found in the workbook.\")\n", " except Exception as e:\n", " print(f\"Error reading sheet '{sheet_name}': {str(e)}\")\n", "\n", " # Create a DataFrame from the data list\n", " df = pd.DataFrame(data_list)\n", "\n", " # Set multi-index with 'Sheet' and 'Row'\n", " df.set_index(['Sheet', 'Row'], inplace=True)\n", " df[\"source\"] = df[\"source\"].apply(safe_literal_eval)\n", " df[\"relevant_docs\"] = df[\"relevant_docs\"].apply(safe_literal_eval)\n", " # Display the DataFrame\n", " print(df.head(2))\n", " all_experts_feedback[expert_name]=df.copy(deep=True)\n", " print(\"\\nProcessing complete.\")\n", " " ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "# all_experts_feedback['Dr. Mathew Carroll']" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "# df.to_csv(\"temp_results.csv\")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n", "MultiIndex: 97 entries, ('Researcher-IsuField-insects', 2) to ('Farmer-Documents-insects', 10)\n", "Columns: 14 entries, Q/GT to relevant_docs\n", "dtypes: int64(5), object(9)\n", "DataFrame Schema:\n", "\n", "Number of rows: 88\n", "Number of columns: 14\n", "\n", "Column Information:\n", "Q/GT int64\n", "Gemini int64\n", "GPT int64\n", "LLAMA int64\n", "Claude int64\n", "ground_truth_answer object\n", "answer_Gemini-1.5 Pro object\n", "answer_GPT-4 object\n", "answer_GPT-3.5 object\n", "answer_Llama-3 70B object\n", "answer_Llama-3 8B object\n", "answer_Claude 3 Opus object\n", "source object\n", "relevant_docs object\n", "\n", "Basic Statistics:\n", " Q/GT Gemini GPT LLAMA Claude ground_truth_answer answer_Gemini-1.5 Pro answer_GPT-4 answer_GPT-3.5 answer_Llama-3 70B answer_Llama-3 8B answer_Claude 3 Opus source relevant_docs\n", "count 97.000000 97.000000 97.000000 97.000000 97.000000 97 93 97 97 97 97 97 97 97\n", "unique NaN NaN NaN NaN NaN 96 93 93 92 97 97 97 68 79\n", "top NaN NaN NaN NaN NaN Consider applying insecticides when defoliation reaches or exceeds 20% in the pod-fill stages (R4-R5) or when there is at least one corn earworm per linear foot of row. Answer:The provided scientific literature focuses on stalk borer and does not contain information about black cutworm. \\n\\nReference:Nothing. \\n Answer:\\nThe documents does not provide information about it.\\n\\nReference:\\nNothing. The documents does not provide information about it. Answer: The document does not provide information about black cutworm larvae, it provides information about Stalk borer (Papaipema nebris).\\n\\nReference: Nothing Answer: Scouting for black cutworm larvae in corn fields should begin when 10% movement of migrating larvae is predicted, or 1,300-1,400 degree days have accumulated since January 1 (base 41°F).\\n\\nReference: \"Scouting. Scouting should begin when 10% movement of migrating larvae is predicted, or 1,300-1,400 degree days have accumulated since January 1 (base 41°F).\" Answer:\\nScouting for stalk borer larvae should begin when 10% movement of migrating larvae is predicted, or 1,300-1,400 degree days have accumulated since January 1 (base 41°F).\\n\\nReference:\\n\"Scouting should begin when 10% movement \\nof migrating larvae is predicted, or 1,300-1,400 degree \\ndays have accumulated since January 1 (base 41°F).\" [agllm-data/agllm-data-isu-field-insects-all-species/papaipema nebris.pdf#page1#chunk1] []\n", "freq NaN NaN NaN NaN NaN 2 1 5 6 1 1 1 2 4\n", "mean 2.536082 2.412371 2.494845 2.463918 2.783505 NaN NaN NaN NaN NaN NaN NaN NaN NaN\n", "std 0.693149 0.703452 0.678904 0.736855 0.563227 NaN NaN NaN NaN NaN NaN NaN NaN NaN\n", "min 1.000000 1.000000 1.000000 1.000000 1.000000 NaN NaN NaN NaN NaN NaN NaN NaN NaN\n", "25% 2.000000 2.000000 2.000000 2.000000 3.000000 NaN NaN NaN NaN NaN NaN NaN NaN NaN\n", "50% 3.000000 3.000000 3.000000 3.000000 3.000000 NaN NaN NaN NaN NaN NaN NaN NaN NaN\n", "75% 3.000000 3.000000 3.000000 3.000000 3.000000 NaN NaN NaN NaN NaN NaN NaN NaN NaN\n", "max 3.000000 3.000000 3.000000 3.000000 3.000000 NaN NaN NaN NaN NaN NaN NaN NaN NaN\n", "\n", "Additional Information:\n", "None\n", "\n" ] } ], "source": [ "import pandas as pd\n", "\n", "\n", "# Get basic information about the DataFrame\n", "info_string = all_experts_feedback['Dr. Arti Singh'].info(memory_usage=False, verbose=False)\n", "\n", "# Get column names and data types\n", "columns_info = all_experts_feedback['Dr. Arti Singh'].dtypes.to_string()\n", "\n", "# Get basic statistics\n", "stats = all_experts_feedback['Dr. Arti Singh'].describe(include='all').to_string()\n", "\n", "# Combine all information\n", "schema_description = f\"\"\"\n", "DataFrame Schema:\n", "\n", "Number of rows: {len(df)}\n", "Number of columns: {len(df.columns)}\n", "\n", "Column Information:\n", "{columns_info}\n", "\n", "Basic Statistics:\n", "{stats}\n", "\n", "Additional Information:\n", "{info_string}\n", "\"\"\"\n", "\n", "print(schema_description)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "# import pandas as pd\n", "# import matplotlib.pyplot as plt\n", "# import seaborn as sns\n", "# import os\n", "\n", "# # Set up plot style and fonts at the start\n", "# plt.rcParams.update({\n", "# 'font.family': 'Arial',\n", "# 'font.size': 12,\n", "# 'axes.labelsize': 12,\n", "# 'axes.titlesize': 13,\n", "# 'xtick.labelsize': 10,\n", "# 'ytick.labelsize': 10,\n", "# 'legend.fontsize': 11,\n", "# })\n", "# plt.style.use('seaborn-v0_8-whitegrid')\n", "# plt.rcParams['font.size'] = 10\n", "\n", "# def plot_overall_performance(df, expert_name):\n", "# # Calculate average scores and perfect score percentages\n", "# model_performance = df[['Gemini', 'GPT', 'LLAMA', 'Claude']].agg(['mean', lambda x: (x == 3).mean() * 100])\n", "# model_performance.index = ['Average Score', 'Perfect Score %']\n", "# model_performance = model_performance.transpose()\n", "\n", "# # Plot: Overall Model Performance\n", "# fig, ax1 = plt.subplots(figsize=(8, 4))\n", "# ax2 = ax1.twinx()\n", "\n", "# x = range(len(model_performance.index))\n", "# ax1.bar(x, model_performance['Average Score'], width=0.4, align='center', color='skyblue', label='Average Score')\n", "# ax2.bar([i+0.4 for i in x], model_performance['Perfect Score %'], width=0.4, align='center', color='lightgreen', label='Perfect Score %')\n", "\n", "# ax1.set_ylim(1, 3)\n", "# ax2.set_ylim(0, 100)\n", "\n", "# ax1.set_xlabel('Model')\n", "# ax1.set_ylabel('Average Score')\n", "# ax2.set_ylabel('Perfect Score %')\n", "\n", "# plt.title(f'Overall Model Performance - {expert_name}', fontsize=12)\n", "# ax1.set_xticks([i+0.2 for i in x])\n", "# ax1.set_xticklabels(model_performance.index, rotation=0)\n", "\n", "# lines1, labels1 = ax1.get_legend_handles_labels()\n", "# lines2, labels2 = ax2.get_legend_handles_labels()\n", "# ax1.legend(lines1 + lines2, labels1 + labels2, loc='upper left', bbox_to_anchor=(0, 1), fontsize=8)\n", "\n", "# plt.tight_layout()\n", "# plt.savefig(f'writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/writing/65d4fadc59fceb1a54d1aae6/analysis/overall_model_performance_{expert_name}.png', dpi=300, bbox_inches='tight')\n", "# plt.show()\n", "# plt.close()\n", "\n", "# def plot_score_distribution(df, expert_name):\n", "# plt.figure(figsize=(6, 3))\n", "# sns.violinplot(data=df[['Gemini', 'GPT', 'LLAMA', 'Claude']])\n", "# plt.title(f'Distribution of Scores by Model - {expert_name}', fontsize=12)\n", "# plt.xlabel('Model', fontsize=10)\n", "# plt.ylabel('Score', fontsize=10)\n", "# plt.tight_layout()\n", "# plt.savefig(f'analysis/score_distribution_{expert_name}.png', dpi=300, bbox_inches='tight')\n", "# plt.show()\n", "# plt.close()\n", "\n", "# # Ensure the analysis directory exists\n", "# os.makedirs('analysis', exist_ok=True)\n", "\n", "# # Assume all_experts_feedback is already given\n", "# for expert_name, df in all_experts_feedback.items():\n", "# plot_overall_performance(df, expert_name)\n", "# plot_score_distribution(df, expert_name)\n", "\n", "# print(\"All plots have been generated and saved.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Interexpert agreeableness and overall scores" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "All plots have been generated and saved.\n", "Inter-expert agreement calculations and visualization have been completed.\n" ] } ], "source": [ "# THIS IS COMMENTED. MAKE SURE TO UNCOMMENT WHEN NEEDED\n", "# import pandas as pd\n", "# import matplotlib.pyplot as plt\n", "# import seaborn as sns\n", "# import os\n", "# import numpy as np\n", "# from scipy import stats\n", "# import itertools\n", "\n", "# # Set up plot style and fonts at the start\n", "# plt.rcParams.update({\n", "# 'font.family': 'Arial',\n", "# 'font.size': 12,\n", "# 'axes.labelsize': 12,\n", "# 'axes.titlesize': 13,\n", "# 'xtick.labelsize': 10,\n", "# 'ytick.labelsize': 10,\n", "# 'legend.fontsize': 11,\n", "# })\n", "# plt.style.use('seaborn-v0_8-whitegrid')\n", "# plt.rcParams['font.size'] = 10\n", "\n", "# def plot_overall_performance(df, expert_name):\n", "# # Calculate average scores and perfect score percentages\n", "# model_performance = df[['Gemini', 'GPT', 'LLAMA', 'Claude']].agg(['mean', lambda x: (x == 3).mean() * 100])\n", "# model_performance.index = ['Average Score', 'Perfect Score %']\n", "# model_performance = model_performance.transpose()\n", "\n", "# # Plot: Overall Model Performance\n", "# fig, ax1 = plt.subplots(figsize=(4, 2))\n", "# ax2 = ax1.twinx()\n", "\n", "# x = range(len(model_performance.index))\n", "# ax1.bar(x, model_performance['Average Score'], width=0.4, align='center', color='skyblue', label='Average Score')\n", "# ax2.bar([i+0.4 for i in x], model_performance['Perfect Score %'], width=0.4, align='center', color='lightgreen', label='Perfect Score %')\n", "\n", "# ax1.set_ylim(1, 3)\n", "# ax2.set_ylim(0, 100)\n", "\n", "# ax1.set_xlabel('Model')\n", "# ax1.set_ylabel('Average Score')\n", "# ax2.set_ylabel('Perfect Score %')\n", "\n", "# plt.title(f'Overall Model Performance - {expert_name}', fontsize=12)\n", "# ax1.set_xticks([i+0.2 for i in x])\n", "# ax1.set_xticklabels(model_performance.index, rotation=0)\n", "\n", "# lines1, labels1 = ax1.get_legend_handles_labels()\n", "# lines2, labels2 = ax2.get_legend_handles_labels()\n", "# ax1.legend(lines1 + lines2, labels1 + labels2, loc='upper left', bbox_to_anchor=(0, 1), fontsize=8)\n", "\n", "# plt.tight_layout()\n", "# plt.savefig(f'writing/65d4fadc59fceb1a54d1aae6/analysis/overall_model_performance_{expert_name}.png', dpi=300, bbox_inches='tight')\n", "# plt.close()\n", "\n", "# def plot_score_distribution(df, expert_name):\n", "# plt.figure(figsize=(6, 3))\n", "# sns.violinplot(data=df[['Gemini', 'GPT', 'LLAMA', 'Claude']])\n", "# plt.title(f'Distribution of Scores by Model - {expert_name}', fontsize=12)\n", "# plt.xlabel('Model', fontsize=10)\n", "# plt.ylabel('Score', fontsize=10)\n", "# plt.tight_layout()\n", "# plt.savefig(f'writing/65d4fadc59fceb1a54d1aae6/analysis/score_distribution_{expert_name}.png', dpi=300, bbox_inches='tight')\n", "# plt.close()\n", "\n", "# def calculate_expert_agreement(all_experts_feedback):\n", "# models = ['Gemini', 'GPT', 'LLAMA', 'Claude']\n", " \n", "# def pairwise_correlations(data):\n", "# correlations = []\n", "# for (expert1, ratings1), (expert2, ratings2) in itertools.combinations(data.items(), 2):\n", "# common_index = ratings1.index.intersection(ratings2.index)\n", "# if len(common_index) > 1:\n", "# corr, _ = stats.pearsonr(ratings1[common_index], ratings2[common_index])\n", "# correlations.append(corr)\n", "# return np.mean(correlations) if correlations else np.nan\n", "\n", "# # Calculate agreement for each model and overall\n", "# agreement_results = {}\n", "# for model in models + ['Overall']:\n", "# if model == 'Overall':\n", "# model_data = {expert: df[models].mean(axis=1) for expert, df in all_experts_feedback.items()}\n", "# else:\n", "# model_data = {expert: df[model] for expert, df in all_experts_feedback.items()}\n", "# agreement_results[model] = pairwise_correlations(model_data)\n", "\n", "# return pd.Series(agreement_results, name=\"Average Pairwise Correlation\")\n", "\n", "# # Ensure the analysis directory exists\n", "# os.makedirs('analysis', exist_ok=True)\n", "\n", "# # Assume all_experts_feedback is already given\n", "# for expert_name, df in all_experts_feedback.items():\n", "# plot_overall_performance(df, expert_name)\n", "# plot_score_distribution(df, expert_name)\n", "\n", "# # Calculate and save inter-expert agreement\n", "# agreement_series = calculate_expert_agreement(all_experts_feedback)\n", "# agreement_series.to_csv('writing/65d4fadc59fceb1a54d1aae6/analysis/inter_expert_agreement.csv')\n", "\n", "# print(\"All plots have been generated and saved.\")\n", "\n", "# # Visualize the agreement results\n", "# plt.figure(figsize=(10, 6))\n", "# agreement_series.plot(kind='bar')\n", "# plt.title('Inter-Expert Agreement: Average Pairwise Correlation')\n", "# plt.xlabel('Model')\n", "# plt.ylabel('Correlation')\n", "# plt.ylim(0, 1) # Correlation ranges from 0 to 1\n", "# plt.xticks(rotation=45)\n", "# plt.tight_layout()\n", "# plt.savefig('writing/65d4fadc59fceb1a54d1aae6/analysis/inter_expert_agreement_bar.png', dpi=300, bbox_inches='tight')\n", "# plt.close()\n", "\n", "# print(\"Inter-expert agreement calculations and visualization have been completed.\")" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overall performance and score distribution plots have been generated and saved.\n", "Inter-expert agreement calculations and visualization have been completed.\n" ] } ], "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "import os\n", "import numpy as np\n", "from scipy import stats\n", "import itertools\n", "\n", "# Set up plot style and fonts\n", "plt.rcParams.update({\n", " 'font.family': 'Arial',\n", " 'font.size': 12,\n", " 'axes.labelsize': 12,\n", " 'axes.titlesize': 13,\n", " 'xtick.labelsize': 10,\n", " 'ytick.labelsize': 10,\n", " 'legend.fontsize': 11,\n", "})\n", "\n", "plt.style.use('seaborn-v0_8-whitegrid')\n", "plt.rcParams['font.size'] = 10\n", "\n", "def plot_overall_performance(all_experts_feedback):\n", " # Calculate average scores and perfect score percentages across all experts\n", " combined_df = pd.concat(all_experts_feedback.values())\n", " model_performance = combined_df[['Gemini', 'GPT', 'LLAMA', 'Claude']].agg(['mean', lambda x: (x == 3).mean() * 100])\n", " model_performance.index = ['Average Score', 'Perfect Score %']\n", " model_performance = model_performance.transpose()\n", "\n", " # Plot: Overall Model Performance\n", " fig, ax1 = plt.subplots(figsize=(4, 2))\n", " ax2 = ax1.twinx()\n", "\n", " x = range(len(model_performance.index))\n", " ax1.bar(x, model_performance['Average Score'], width=0.4, align='center', color='skyblue', label='Average Score')\n", " ax2.bar([i+0.4 for i in x], model_performance['Perfect Score %'], width=0.4, align='center', color='lightgreen', label='Perfect Score %')\n", "\n", " ax1.set_ylim(1, 3)\n", " ax2.set_ylim(0, 100)\n", "\n", " ax1.set_xlabel('Model')\n", " ax1.set_ylabel('Average Score')\n", " ax2.set_ylabel('Perfect Score %')\n", "\n", " plt.title('Overall Model Performance', fontsize=12)\n", " ax1.set_xticks([i+0.2 for i in x])\n", " ax1.set_xticklabels(model_performance.index, rotation=0)\n", "\n", " lines1, labels1 = ax1.get_legend_handles_labels()\n", " lines2, labels2 = ax2.get_legend_handles_labels()\n", " ax1.legend(lines1 + lines2, labels1 + labels2, loc='upper left', bbox_to_anchor=(0, 1), fontsize=8)\n", "\n", " plt.tight_layout()\n", " plt.savefig('writing/65d4fadc59fceb1a54d1aae6/analysis/overall_model_performance.pdf', dpi=300, bbox_inches='tight')\n", " plt.close()\n", "\n", " # Save data as CSV\n", " model_performance.to_csv('writing/65d4fadc59fceb1a54d1aae6/analysis/overall_model_performance.csv')\n", "\n", "def plot_score_distribution(all_experts_feedback):\n", " combined_df = pd.concat(all_experts_feedback.values())\n", " plt.figure(figsize=(6, 3))\n", " sns.violinplot(data=combined_df[['Gemini', 'GPT', 'LLAMA', 'Claude']])\n", " plt.title('Distribution of Scores by Model', fontsize=12)\n", " plt.xlabel('Model', fontsize=10)\n", " plt.ylabel('Score', fontsize=10)\n", " plt.tight_layout()\n", " plt.savefig('writing/65d4fadc59fceb1a54d1aae6/analysis/score_distribution.png', dpi=300, bbox_inches='tight')\n", " plt.close()\n", "\n", " # Save data as CSV\n", " score_distribution = combined_df[['Gemini', 'GPT', 'LLAMA', 'Claude']].describe()\n", " score_distribution.to_csv('writing/65d4fadc59fceb1a54d1aae6/analysis/score_distribution.csv')\n", "\n", "def calculate_expert_agreement(all_experts_feedback):\n", " models = ['Gemini', 'GPT', 'LLAMA', 'Claude']\n", " \n", " def pairwise_correlations(data):\n", " correlations = []\n", " for (expert1, ratings1), (expert2, ratings2) in itertools.combinations(data.items(), 2):\n", " common_index = ratings1.index.intersection(ratings2.index)\n", " if len(common_index) > 1:\n", " corr, _ = stats.pearsonr(ratings1[common_index], ratings2[common_index])\n", " correlations.append(corr)\n", " return np.mean(correlations) if correlations else np.nan\n", "\n", " # Calculate agreement for each model and overall\n", " agreement_results = {}\n", " for model in models + ['Overall']:\n", " if model == 'Overall':\n", " model_data = {expert: df[models].mean(axis=1) for expert, df in all_experts_feedback.items()}\n", " else:\n", " model_data = {expert: df[model] for expert, df in all_experts_feedback.items()}\n", " agreement_results[model] = pairwise_correlations(model_data)\n", "\n", " return pd.Series(agreement_results, name=\"Average Pairwise Correlation\")\n", "\n", "# Ensure the analysis directory exists\n", "os.makedirs('writing/65d4fadc59fceb1a54d1aae6/analysis', exist_ok=True)\n", "\n", "# Assume all_experts_feedback is already given\n", "plot_overall_performance(all_experts_feedback)\n", "plot_score_distribution(all_experts_feedback)\n", "\n", "# Calculate and save inter-expert agreement\n", "agreement_series = calculate_expert_agreement(all_experts_feedback)\n", "agreement_series.to_csv('writing/65d4fadc59fceb1a54d1aae6/analysis/inter_expert_agreement.csv')\n", "\n", "print(\"Overall performance and score distribution plots have been generated and saved.\")\n", "\n", "# Visualize the agreement results\n", "plt.figure(figsize=(10, 6))\n", "agreement_series.plot(kind='bar')\n", "plt.title('Inter-Expert Agreement: Average Pairwise Correlation')\n", "plt.xlabel('Model')\n", "plt.ylabel('Correlation')\n", "plt.ylim(0, 1) # Correlation ranges from 0 to 1\n", "plt.xticks(rotation=45)\n", "plt.tight_layout()\n", "plt.savefig('writing/65d4fadc59fceb1a54d1aae6/analysis/inter_expert_agreement_bar.png', dpi=300, bbox_inches='tight')\n", "plt.close()\n", "\n", "# Save agreement results as CSV\n", "agreement_series.to_csv('writing/65d4fadc59fceb1a54d1aae6/analysis/inter_expert_agreement_bar.csv')\n", "\n", "print(\"Inter-expert agreement calculations and visualization have been completed.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# By Category" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Category Performance Dataframe:\n", " Gemini GPT LLAMA Claude\n", "Sheet \n", "Farmer-Documents-insects 2.250000 2.250000 2.000000 2.250000\n", "Farmer-Documents-weeds 2.666667 2.666667 2.666667 2.666667\n", "Farmer-IsuField-insects 2.090909 2.181818 2.090909 2.181818\n", "Farmer-IsuField-weeds 2.000000 2.400000 2.000000 2.600000\n", "Researcher-Documents-insects 2.000000 2.250000 2.250000 2.500000\n", "Researcher-Documents-weeds 1.666667 2.000000 2.000000 2.666667\n", "Researcher-IsuField-insects 2.090909 2.181818 2.363636 2.636364\n", "Researcher-IsuField-weeds 2.200000 2.600000 2.800000 2.600000\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAABHcAAAKQCAYAAAAVPpYbAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB6CklEQVR4nOzdeXhM5///8VdEIkjsoZaKNfYQCaGlFKWU2teKWmr5Kqkuamm1/RSNStGi0iq1VVF7iaW1VHctrdpDEkvQEmoPQnJ+f/iZmiZINJPJHc/HdfWqOeee+7znvOdM5OWcMy6WZVkCAAAAAACAkbI5uwAAAAAAAADcP8IdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgsOzOLgAAYLbhw4dr+fLldsuyZcsmNzc35c+fX3Xq1NGgQYP08MMPO2T758+fV2hoqDZv3qzLly/roYce0pgxY1SnTh2HbO9Bd+zYMTVu3FiS1LZtW40bNy7Dtn3+/HnNnTtXmzZt0tGjR5WQkKAiRYqoTp066tWrl8qWLfuft7F//35VrFgxHaoFAADIOIQ7AIB0l5SUpGvXrumvv/7SihUr9OOPP+qLL75Q0aJF031bYWFhduFSbGyscufOne7bgXPt2LFDAwcO1JkzZ+yWx8bGKjY2VitWrNC4cePUsmXL+5o/Ojpa7733niIjI7Vp06b0KBkAACDDcFkWACDdLFq0SFu2bNGGDRs0b948+fv7S5JOnTqladOmOWSbO3bskCS5ublp8eLFWrlyJWdeZDHHjh1Tv379dObMGeXIkUMvv/yyIiIitHz5cj333HNycXHR9evXNWLECMXExNzXNvr27UuoAwAAjMWZOwCAdFOoUCE99NBDkqSHH35YkydPVsOGDZWYmKgffvjBIduMj4+XJHl7e8vPz88h24BzTZo0SefPn5ckTZgwQU888YRtXeXKleXm5qbw8HAlJCRo9erVCgkJcVapAAAATkG4AwBwmMKFCyt//vw6ffq04uLi7Nb98ccf+vDDD/Xbb78pISFBpUuXVocOHfTMM88oW7Z/TiytUKGCJKlXr16Sbp4d5OLioldffVVvvvmmbdyJEydUoUIF1a5dW/PmzZMkXbp0STNnztRXX32lY8eOyc3NTVWqVFH37t3tAoJ7bcfd3V0jRoyQJK1cuVJfffWVli9frjNnzqhixYoaNmyYqlWrpo8++khLlizR+fPnVaFCBb388ssKCgqybcOyLM2bN0/Lly/XsWPHdOXKFeXPn1+1atXS888/b3fPmFv19O7dW08++aQmTpyonTt3ys3NTY899pheffVVFS5c2O41fP/995o1a5Z27typhIQEPfTQQ2rcuLH69u2r/Pnz28Zdu3ZNn3zyiVatWqXjx48rT548tnsjlSlTJtX9laT169frww8/1KFDh+Tt7a3WrVvr//7v/+Tu7q6DBw/aLpN66qmnNHHiRLt98dhjj+nUqVMqU6aM1q5dm+L8V65c0fr16yVJ5cuXT9Y3SerRo4fKlSunWrVqqUiRImna37ffQ0iSjh8/rgoVKtjdT+jkyZOaPHmytmzZonPnzqlw4cJq2rSpBg4cqDx58tjVsn//fk2aNEm//vqrXFxc1KBBAw0fPlxdunTR8ePHk92nKD3eoy+88ILeeecdSVK/fv308ssv281ft25dJSQkqF69epo5c2aK+xkAAJiNcAcA4DAnT57U2bNnJUkFCxa0Ld+4caNeeOEFXb9+3bZs//79GjNmjHbs2KEJEyYkm2vp0qW6cOGC7fGNGzfuue1u3brp2LFjtmVXr17Vzz//rJ9//lnPPvusRo4cec/t1KxZU7t377Y9Hjp0qA4cOGB7/Mcff+i5555TtWrVtHXrVrvl/fv315o1a1SsWDFJ0vjx4/Xpp5/abe/UqVOKiIjQjz/+qPXr1ytv3rx267dv36558+bZ7atVq1bpzz//1Pz5823L5s2bpzFjxtg99/Dhw5o5c6a+++47ff755/Ly8lJCQoJ69+6tbdu22cadOXNGERER+uabbzR37lxVrVr1DnvV3vfff293v6Pjx49r2rRp2rdvn8LDw1W+fHn5+flp586d2rx5s65cuaKcOXPaXtepU6ck3bwx853s2bPH9tpvXeb3bwUKFEjxXjup2d/3Ehsbq65du9qFk8ePH9esWbP07bffatGiRfLy8pIk7dq1S8HBwbpy5YptbEREhHbt2mU7w+x26fUeDQoKUuHChXXq1CmtXbvWLtzZuHGjEhISJEnt2rW75+sFAABm4p47AIB0c/r0af3111+KjY3Vjz/+qIEDByoxMVGS1KhRI0k3z8R4/fXXdf36dZUsWVIzZ87U2rVrNXDgQEnS6tWr9fXXXyeb+8KFC+revbvWrVun999/X+3bt9eWLVtsl4E99NBD2rJliz744ANJ0siRI22/NPfo0UMrV67U7NmzVaVKFUnSnDlzFBERcc/t+Pr62q0/duyYJkyYoNWrV6tBgwaSbl4atm3bNr355ptas2aNmjRpYnut3377raSbZ1AsW7ZMkvT4448rIiJCa9asUZs2bSRJZ8+e1e+//56snj/++ENNmzbVl19+qVmzZtluFr1t2zYdPXpU0s2zlt59911JUtGiRfXRRx9p3bp16tOnjyTpwIEDmjNnjiRp7ty5tmDnueee05o1azRnzhyVKVNGly9f1muvvZZCZ1MWFxendu3a6csvv9TMmTNVsmRJSdLmzZtt969p3769bR9t3rzZ9tx169ZJuvnNak8//fRdt3FLgQIFUl1bavd30aJFU3wf3TpTa/To0YqLi1OOHDk0btw4rV+/Xu+9955y5syp6Oho2/tNksaOHWsLdnr37q3Vq1frww8/VEJCgi3kvF16vUcrVqxoe12xsbHauXOnbeyt/ezl5WV7XwIAgKyHM3cAAOmmc+fOKS4vWbKkBg0aJEn64Ycf9Pfff0uSunfvrnLlytmeGxERoSNHjmj58uXJLknx8PDQ0KFD5eHhodKlS0uScubMKVdXV0mSq6ur7Rf0I0eO6Pvvv5ckPfbYY3aBRXh4uJ544gldu3ZN8+bN01NPPXXP7dyuffv2trNEOnXqpC1btkiS6tevr27dutmWb9iwQZJsv9R7enrq559/1rFjx5Q3b17lyZNH586dU6lSpWxznzt3Ltn28uXLp3Hjxsnd3V2S1KZNG9sZO3FxcSpZsqS++uor29ktL730kh5//HFJN88y8vT0VKlSpWxn46xevVqSVKxYMQUHB0uScufOreDgYP3vf//T/v37tW/fPlWqVClZLf9WokQJjRkzRq6urqpQoYLeeust9e7dW5L0zTffqHHjxmrZsqVCQ0N19epVrV27Vi1atJBlWfrqq68kSXXr1rX1LSVJSUkp/vleUru/b71vUnofnT9/Xt99950kqXHjxqpbt64kqVatWmratKlWrlypL7/8Uq+99ppdOFejRg0NGzZM0s1LyVxcXGzh5S3p/R5t3769pk+fLklas2aN/Pz8dOnSJds2mjdvrhw5cqR6/wEAALMQ7gAA0p2rq6s8PDxUpEgR1atXT//3f/9nO+vi8OHDtnHvvPOO7V4ht9uzZ0+yZQ8//LA8PDxStf39+/fb/vzII4/YrStSpIjKli2rvXv3KjIyMs3buf2eNLd/5frt98u5ffntl49dv35dv/76q7777jvt3LnT7nIcKeXwomTJkrZgR7I/e+XW3Lfv09tDmZRChVtjT5w4YTvz6N92796dqnCnSpUqtlBEkqpXr27784kTJyTdDFmaNWumlStXasuWLbp06ZIiIyN18uRJSbKdcXInt7/ef38N+u2SkpLs7tUk3d/+vt2RI0dsY9asWaM1a9YkG3P+/HnFxsbanZlTo0YNuzGBgYHJnpfe79FSpUopMDBQ27Zt09q1azVs2DBt2rTJdknWvfYzAAAwG+EOACDdbNy4USVKlLjrmOzZ7/2j59aZPbfz9PRMdR23Bw4psSzrjuvutZ3bf6m+PUy4dS+ZO4mPj9czzzyjvXv3Knfu3GratKn69++vpKQkuxtD3217/97mLbe/nnvdi+he+0ZSipcQpeTf27p97tv/3L59e61cuVLXrl3Txo0bbfcw8vT0VNOmTe+6japVqypbtmxKSkqyfe39v+3bt0+9e/dWw4YN1aZNGwUFBd33/r5dat6r0s336+0B3N3eX7c44j3avn17bdu2TX/99Ze2b99uuySrVKlSCggIuGdNAADAXIQ7AIAMdeu+LJL03nvvqVWrVrbHO3fuVMmSJZUvX75kz3Nzc0v1Nm4/i+ann36yfbuQdPMmtjExMZKU4tkpadlOWqxdu1Z79+6VJL399tu2S7vu9C1RaXH7pUb/Puumb9++cnd3V61atdSzZ0/5+Phoz549KlWqlN0Nhf/8809duXJFJUuWTHWo8ccffyghIcEWbNx+r5fb+1y7dm2VLFlSR48eVUREhO2slSeffPKeZ2N5eXmpUaNG2rBhg6Kjo7Vhw4Zk946ZMWOG/v77by1btkw+Pj4KCgq67/19e6hy+2vo1KmTRo8ebXt88OBBeXp6qmjRopKkixcv2tb9O4S6/UbbtzjiPfrkk09qzJgxunz5shYvXmy7JKt169YpjgcAAFkHN1QGAGSounXrqlChQpKkSZMmafPmzTp69Kg+/fRTdezYUUFBQXZfFX0/SpcubTtTYcuWLQoNDVVkZKS2bt2qgQMH6tq1a5Ju3vMno1y+fNn256+++koxMTHasmWL3deD37r5dFo1bdrUFsh88MEH2rBhgw4dOqT3339f3377rTZs2GA7G+dWmHb48GGNGTNGBw8e1K5du9S/f381b95ctWrVsn2L1b2cPn1agwcP1s6dO7V161a7M2Juv2eSi4uL7RuxtmzZYrsk627fknW7l19+2Xap2yuvvKJPP/1UMTEx2rlzp0aOHGm7j1DBggX1zDPPSEr7/r51P5pz585p//79ioqKkqenp+3+RcuXL9fChQt1+PBhbd68WV26dFHDhg3Vrl07WZYlLy8vPfbYY5Juhl7vvvuuoqKitH79er399tvJXpMj3qO5cuVS8+bNJUkrVqzQtWvX5OLiQrgDAMADgDN3AAAZKmfOnBoxYoSGDh2q48ePa8CAAXbrixcvbncWw/1699131b17d/3111+aPXu2Zs+ebbf+2WefVYsWLf7zdlLrscce04QJE3T16lWtX78+xa/hvv2bodKiePHiGjp0qEJDQxUXF6fnn3/ebr2vr6/tm7O6du2qVatWac+ePZo3b57mzZtnN7Z///4qXLhwqrZbo0YNffPNN/rmm2/slrdu3VpBQUF2y9q1a6cpU6bY7mHz8MMPp/pSoTJlyig8PFyDBw/W+fPn9e6779q+HeyWPHnyaOrUqbavJU/r/q5YsaJiYmIUHx+v1q1bq0GDBpo+fbqGDh2q33//XefOnUt2OdetGxu7uLhIuhlC/frrr7py5Yo+/fRT29ewly1b1ratW2Mlx7xHO3TooCVLltge165dW8WLF0/THAAAwDycuQMAyHAtW7bUnDlz1LBhQ+XLl09ubm4qUaKEgoODtWjRIhUpUuQ/b+Phhx/W6tWrNXDgQJUvX14eHh7KnTu3goKCNHXqVI0cOTIdXknqlSpVSjNmzFBAQIBy586tfPnyKTAwUJ988ol8fHwkyfb14fejZ8+emj59uh555BF5eXnJw8NDZcqUUb9+/TRv3jzlyZNH0s1AYu7cuRo4cKDKli2rHDlyKG/evAoMDNTkyZOThW1306pVK02ePFmVK1eWu7u7SpQooZdfflmhoaHJxj700EN69NFHbY/btGljF3Tcy61Lrfr27aty5copZ86ccnd3V+nSpfXss89q9erVqlmzpm18Wvf34MGDFRQUpFy5cilPnjy292DZsmW1ZMkStWvXTg899JDc3Nzk7e2tZs2a6fPPP7d9g5Z0MyCaP3++HnnkEeXKlUt58+ZVhw4dNGPGDNuY2y+pcsR71N/f3+6Sr9SeHQUAAMzmYqXmrn8AAAD/Ub9+/bRlyxa5uLjo66+/1sMPP+zsktLVxo0blS9fPhUtWlQPPfSQ7ebXp06dUv369SXd3Acvv/yyw2qwLEtPP/20Dhw4oFy5cun777+3+/Y2AACQNXFZFgAAcJiTJ0/KxcVF69ev15YtWyTdPAsnqwU7kvT+++/rwIEDkqTnn39ebdu21aVLlzRr1izbmGrVqjlk23/++aeyZcumzz77zFZD8+bNCXYAAHhAcOYOAABwmFdeeUWrVq2yPXZzc9PcuXPtLqHKKhYvXqzXX3/9jusrVaqkJUuWpPrbyNKia9eu+u2332yPc+fOrRUrVth94xcAAMi6nH7PnSNHjqhPnz7y9/dXw4YN7a5L/7e9e/eqY8eOql69utq3b6/du3dnYKUAACCtKlSooNy5cyt37tzy9/dXeHh4lgx2JKljx46aPHmy6tatq/z588vV1VU5cuRQmTJl1KdPH82bN88hwY4kVa5cWTlz5lSePHn0yCOPaPbs2QQ7AAA8QJx65k5SUpKaN2+uatWqadCgQTpy5IheeuklvfXWW7avar0lPj5eTZs2VatWrdShQwctWLBAa9eu1ddff61cuXI56RUAAAAAAAA4l1PP3Dl9+rQqVaqkt956S6VKlVKDBg1Ut25dbd++PdnYNWvWKEeOHHr11VdVtmxZvfbaa8qdO7fWrVvnhMoBAAAAAAAyB6feULlw4cJ6//33Jd38dofffvtNv/76q958881kY//44w8FBATYvjbVxcVFNWvW1I4dO9SuXbtk42/cuKHz588rR44ctm+rAAAAAADAUZKSknTt2jXlzZvXYZfiAinJNO+2Ro0a6cSJE3r88cfVrFmzZOvj4uJUrlw5u2UFCxbUwYMHU5zv/PnzOnz4sCNKBQAAAADgjkqVKqWCBQs6uww8QDJNuDN58mSdPn1ab731lkJDQ5N928SVK1fk7u5ut8zd3V0JCQkpzpcjRw5JUsmSJeXh4eGYog2XlJSkqKgolStXjrObDET/zEb/zEb/zEb/zEb/zEb/zEb/7u3q1as6evSo7fdRIKNkmnCnWrVqkqRr167plVde0auvvmoX5uTIkSNZkJOQkHDH4ObWh03u3Lm54fIdJCYmSpI8PT3l6urq5GqQVvTPbPTPbPTPbPTPbPTPbPTPbPTv3m7tF8IvZDSn31B5w4YNdsvKlSun69ev69KlS3bLixQpotOnTyd7fuHChR1eJwAAAAAAQGbl1HDn2LFjGjRokE6ePGlbtnv3bhUoUEAFChSwG1u9enX9/vvvuvXN7bduwFy9evUMrRkAAAAAACAzcWq4U61aNVWpUkUjR45UVFSUtmzZorCwMA0YMEDSzZsoX716VZL05JNP6sKFCxo7dqyioqI0duxYXblyRc2bN3fmSwAAAAAAAHAqp4Y7rq6umjZtmnLmzKnOnTvrtddeU3BwsHr06CFJqlevntasWSPp5nWdH3/8sbZv36527drpjz/+0PTp07mfDgAAAAAAeKA5/YbKRYoU0dSpU1NcFxkZaffYz89Py5cvz4iyAAAAAAAAjMAtvAEAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABnP6V6EDAAAAAADzlBoekaHbOzzuqTQ/Jz4+XtOnT9e6det04sQJ5cyZU0FBQRo8eLDKly+frvUdO3ZMjRs31saNG1WiRIm7jt26dat69OihyMjIdNk24Q4AAAAAAMhyLl++rG7duik+Pl7Dhw9XxYoVdfbsWc2fP19dunTRihUr9PDDD6fb9ooWLarvv/9eBQoUuOdYf39/ff/99+m2bcIdAAAAAACQ5Xz44Yc6c+aM1qxZozx58kiSihcvrtDQUP3555+aPXu2Ro0alW7bc3V1lbe3d6rGuru7p3psanDPHQAAAAAAkKUkJSVp+fLl6tWrly3Yud348eM1dOhQSdK2bdvUrl07+fn5qVWrVlq/fr1t3PDhwxUWFqYhQ4aoevXqatGihfbu3atJkyYpMDBQjz32mNauXSvp5mVZFSpU0LFjxyRJFSpU0MqVK9WyZUtVrVpV3bp1U2xsrKSbl2VVqFAh3V4v4Q4AAAAAAMhSjh49qr///luBgYEpri9cuLA8PDwUFxen/v37q127dlq1apWee+45DR8+XNu2bbONnTNnjmrXrq0vv/xS+fLl07PPPqszZ85o0aJFatSokd58800lJSWluJ0pU6botdde07Jly3T27Fm9//77jni5XJYFAAAAAACylrNnz0qS8ubNa1v2448/6vnnn7c9LlasmJ544gk98sgj6t69uyTJx8dH+/bt05w5c2zB0K2zbiSpZcuWeuedd/T666/Lw8NDwcHBWrBggU6fPp1iHb169VLdunUlSV27dtX8+fPT/8WKcAcAAAAAAGQxty7FunDhgm2Zv7+/VqxYIUn66quvtGDBAsXExGjz5s3y9/e3jbt+/bpKly5te3z7N195eHioUKFC8vDwkCTlyJFDkpSQkJBiHT4+PrY/e3p66vr16//xlaWMcAcAAAAAAGQpPj4+ypcvn37//Xf5+flJknLmzGkLWwoWLChJunHjhlq1aqUBAwbYPT979uwp/lmSsmVL/R1u3Nzc7qv+tOKeOwAAAAAAIEvJnj272rdvrzlz5ujSpUvJ1p88eVKSVLp0aR05ckQ+Pj62/zZu3KhVq1ZldMn/CeEOAAAAAADIcgYPHixvb2916dJF69atU2xsrHbu3KlRo0Zp8uTJCggIULdu3bR7925NmjRJhw8f1qpVqzRx4kQVK1bM2eWnCZdlAQAAAACANDs87ilnl3BXOXPm1Lx58zRnzhxNmzZNR44ckbu7u/z8/DRlyhQ1adJEkvTRRx/pvffe08yZM1WkSBENHz5cTz/9tJOrTxsXy7IsZxfhCPHx8dq3b58qVaqkXLlyObucTCkxMVE7duxQjRo15Orq6uxykEb0z2z0z2z0z2z0z2z0z2z0z2z07974PRTOwmVZAAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABsvu7AIAAAAAAICB3sqbwds7n+anxMfHa/r06Vq3bp1OnDihnDlzKigoSIMHD1b58uV17NgxNW7c2O45OXLkUJUqVTRgwAA1aNBAU6ZM0dSpU++4jdDQULVr1y7NtaUnwh0AAAAAAJDlXL58Wd26dVN8fLyGDx+uihUr6uzZs5o/f766dOmiFStWyMXFRZK0ePFiFS1aVJJ09epVzZkzR88//7zWrFmj3r17q0uXLpKk33//XYMHD9b3339v246Xl1fGv7h/4bIsAAAAAACQ5Xz44Yc6c+aMli5dqsaNG6t48eKqWrWqQkNDVa1aNc2ePds2tkCBAvL29pa3t7cefvhhDRs2TO7u7tq0aZNy585tW5c3782zlW499vb2loeHh5Ne4T84cwcAAAAAAGQpSUlJWr58uZ577jnlyZMn2frx48crT548On36dIrPz579Zlzi5ubm0DrTC+EOAAAAAADIUo4ePaq///5bgYGBKa4vXLjwHZ97+fJlffzxx7p+/brq16/vqBLTFeEOAAAAAADIUs6ePStJtsuoJOnHH3/U888/b3tcrFgxffzxx5Kkli1bysXFRZZl6cqVKypSpIhCQ0NVsmTJjC38PhHuAAAAAACALOXWpVgXLlywLfP399eKFSskSV999ZUWLFhgWzd9+nQVKVJELi4uypUrlwoVKpSh9f5XhDsAAAAAACBL8fHxUb58+fT777/Lz89PkpQzZ075+PhIkgoWLGg3vlixYipRokSG15le+LYsAAAAAACQpWTPnl3t27fXnDlzdOnSpWTrT5486YSqHIdwBwAAAAAAZDmDBw+Wt7e3unTponXr1ik2NlY7d+7UqFGjNHnyZAUEBDi7xHTDZVkAAAAAACDt3jrv7AruKmfOnJo3b57mzJmjadOm6ciRI3J3d5efn5+mTJmiJk2a6NixY84uM10Q7gAAAAAAgCzJ3d1dffv2Vd++fVNcX6JECUVGRqZ6vqCgoDSNzyhclgUAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGCw7M4uAAAAAAAAmKfanGoZur1dz+5K0/gKFSpo7ty5CgoKSrZu2bJlmjp1qjZt2nTXOWJjY9WkSRM9/fTTCgsLs1s3ZcoUTZ06Ve3atVNoaKjdOsuyVL9+fcXFxSkyMjLZtkeMGKExY8aoY8eOaXpNd+L0M3dOnjypkJAQ1a5dW/Xr11doaKiuXbuW4tivv/5azZs3l7+/v7p27ao9e/ZkcLUAAAAAAOBBsWbNGpUsWVIbNmzQ5cuXk613c3PTli1blJSUZLd8x44dOn36dIpzRkREqGTJklq5cmW61enUcMeyLIWEhOjKlSuaP3++Jk2apM2bN+v9999PNvbgwYN6+eWX1b9/f61cuVKVKlVS//79deXKlYwvHAAAAAAAZHmrV69W9+7d5ebmpvXr1ydbX7lyZV25ckU7duywW75hwwbVqFEj2fgzZ87op59+0vPPP69t27YpNjY2Xep0argTExOjHTt2KDQ0VOXLl1dgYKBCQkK0evXqZGN/+OEHlStXTm3atFHJkiX10ksvKS4uTlFRUU6oHAAAAAAAZGVRUVE6cOCAgoKCVL9+fS1fvjzZmBw5cqhevXrJLu/asGGDmjRpkmz8unXr5OXlpaefflqFCxdOt7N3nBrueHt7a8aMGSpUqJDd8kuXLiUbmy9fPkVFRWn79u1KSkrSsmXL5OnpqZIlS2ZUuQAAAAAA4AGxevVqFS9eXBUrVlTjxo3166+/6vjx48nGNW7c2C7ciYqK0tWrV1W1atVkYyMiItSwYUNly5ZNjRo10ooVK2RZ1n+u1ak3VM6TJ4/q169ve5yUlKTPPvtMderUSTa2RYsW2rRpk7p16yZXV1dly5ZNH3/8sfLmzXvXbSQmJioxMTHda88Kbu0X9o+Z6J/Z6J/Z6J/Z6J/Z6J/Z6J/Z6N+9sW+yljVr1qhRo0aSpAYNGsjd3V0rVqzQ888/bzeuQYMGGjlypI4cOSIfHx9t2LBBjRs3louLi924P//8U7/99pt69eolSWratKkWLFig7du3KzAw8D/Vmqm+LSssLEx79+7VkiVLkq07e/as4uLi9MYbb6h69epasGCBRowYoeXLl6tgwYJ3nPPAgQOOLDlL2LUrbXccR+ZC/8xG/8xG/8xG/8xG/8xG/8zVc3dPaXf6zzu76uz0nxT4D3bu3KkjR47YLq3KnTu3HnnkEa1cuTJZuJM/f34FBARo06ZN6tWrlzZs2KCXX3452ZwRERG2y7gkqXbt2sqbN6+WL1+edcKdsLAwzZkzR5MmTZKvr2+y9e+99558fX31zDPPSJJGjx6t5s2ba+nSperXr98d5/X19VWuXLkcVrfJEhMTtWvXLlWrVk2urq7OLgdpRP/MRv/MRv/MRv/MRv/MRv/MlpiY6JBgR1KKN541UXx8PCcYZBERERGSpN69e9uWJSUlybIsbd++XQEBAXbjGzdurI0bN6pFixaKjY1VrVq1tH379mRzXr161e65iYmJWrdunUaNGiUPD4/7rjdThDujR4/WggULFBYWpmbNmqU4Zs+ePQoODrY9zpYtmypWrKgTJ07cdW5XV1d+cNwD+8hs9M9s9M9s9M9s9M9s9M9s9A//llXeD1nldTzokpKStHbtWrVu3VrPPfecbXliYqK6d++uFStWpBjuhIWFafny5WrYsKGyZ7ePWw4dOqS9e/fq9ddfV1BQkG15VFSUXnzxRX399ddq1arVfdfs9HBn6tSpWrhwoSZOnKgnn3zyjuMKFy6s6Ohou2WHDh1StWrVHF0iAAAAAAAw0M6dO3Xt2jW7ZbVq1ZIkXb16Vd9++63durx58+ratWs6efKkgoODk11Z9PTTT2vVqlV6/fXX7ZY//PDDKlOmjKZPn67x48cnqyMiIkL58uVT586d5e7ublvu6+urDz/8UCtWrDA33ImOjta0adPUr18/BQQEKC4uzrbO29tbcXFx8vLykoeHhzp16qThw4eratWq8vf31+LFi3XixAm1bdvWia8AAAAAAIAH065nM//9s957771ky7766itJ0pkzZ9S3b1+7dTVr1lT58uVVoUKFFE8m6dq1qz7//HNt2LAh2bpGjRpp9uzZevTRR5Oti4iIUKtWreyCndvnHDt2rE6ePKkiRYqk+rXdzqnhzsaNG5WYmKjw8HCFh4fbrYuMjFS9evUUGhqqdu3aqUWLFrp8+bI+/vhj/fXXX6pUqZLmzJlz15spAwAAAACAB1NkZOQd1/n4+Khdu3ZpntPX1/eO87744ot68cUXbY+DgoJsY9euXXvHObt3767u3bunuZbbOTXc6dev311vhvzvHdaxY0d17NjR0WUBAAAAAAAYI5uzCwAAAAAAAMD9I9wBAAAAAAAwGOEOAAAAAACAwQh3AAAAAAAADEa4AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABiMcAcAAAAAAMBg2Z1dAAAAAAAAMM++ipUydHuV9u9L83POnz+v8PBwffXVVzpz5oyKFSumzp07q0ePHsqWLZsqVKiguXPnKigoKN3rDQ4OVu3atTV48OB0n/vfCHcAAAAAAECWc/bsWXXu3FmFCxfW2LFjVaJECe3atUujR49WbGysRo0a5ewS0w3hDgAAAAAAyHImTJggd3d3zZw5Uzly5JAkPfzww/Lw8NDAgQPVvXt3J1eYfrjnDgAAAAAAyFISEhIUERGhZ555xhbs3PL4449r9uzZKl68uN3ykydPKiQkRLVq1VLVqlXVtm1bbd++XZJ07NgxVahQQceOHbONnzJlioKDg22Pv/76azVr1kw1atTQ22+/rcTERLv5Fy5cqEaNGsnf31/BwcGKjIxMt9dLuAMAAAAAALKUo0ePKj4+XtWqVUu2zsXFRXXq1JG7u7vd8ldeeUWJiYlauHChVqxYoSJFiuitt95K1faioqI0ZMgQde3aVUuXLtWNGzdswZAkbdq0SVOnTtWoUaO0fPlyBQQEqEePHjp//vx/ep23EO4AAAAAAIAs5cKFC5IkLy+vVI23LEtNmjTRqFGjVLZsWZUrV07PPPOMoqKiUvX8pUuXKjAwUD179lTZsmU1atQoFS5c2LZ+xowZ6t+/vx5//HGVKlVKQ4YMUfHixfXll1+m/cWlgHvuAAAAAACALCVfvnySlOozY1xcXNS1a1etWbNGv/32mw4dOqTdu3crKSkpVc+Pjo5WpUr/fHuYm5ub3ePo6GiFhYVp4sSJtmXXrl3T4cOHUzX/vRDuAAAAAACALKVkyZLy8vLSnj175Ofnl2z9//3f/9ndLycpKUm9e/fWhQsX1KJFCzVq1EjXr1/XoEGDJN0Mf/7txo0bdo8ty7J77ObmZvtzYmKiRo4cqbp169qN8fT0TPuLSwGXZQEAAAAAgCwle/bsatGihebPn6+EhAS7dZs2bdKmTZvsLpuKiorSr7/+qtmzZ2vAgAFq2LChTp06JelmaHMrqLl8+bLtObffXLl8+fLatWuX7XFSUpL2799ve1y6dGn99ddf8vHxsf330UcfaceOHenyegl3AAAAAABAljN48GBdunRJffr00S+//KKjR49q8eLFGj58uHr06KFy5crZxubJk0fZsmVTRESEjh8/rnXr1mnKlCmSbn7zVqFChVS0aFHNnDlTsbGxWrZsmb755hvb8zt16qTdu3crPDxcMTExevfdd3XixAnb+l69emnOnDlasWKFjh49qrCwMK1du1Zly5ZNl9fKZVkAAAAAACDNKu3f5+wS7srb21sLFizQlClT9Morr+jcuXMqWbKkQkJC1LVrV7uxDz30kN566y19+OGHmjhxokqXLq3XX39dw4YN0969e+Xv76+xY8dq9OjRatGiherWrasBAwbo22+/lST5+PgoPDxcoaGhCg8PV5MmTdSgQQPb/C1atNDp06c1efJknT59WuXKlVN4eLhKlSqVLq+VcAcAAAAAAGRJRYsW1TvvvHPH9ZGRkbY/d+7cWZ07d7Zb37JlS9ufH330Ua1bt85ufd++fe3Wr169+o7b6tGjh3r06JHq2tOCy7IAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAbL7uwCgMyi1PAIh8x72KObQ+Z1ldSzdElpd/rP/UXojfSfVFKl/fscMi8AAOlhX8VKDpubn4EAAEfizB0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGCw7M4uAAAAAAD+q30VKzls7kr79zlsbgBID5y5AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwmNPDnZMnTyokJES1a9dW/fr1FRoaqmvXrqU4NjIyUl27dpWfn59atWqln3/+OYOrBQAAAAAAyFycGu5YlqWQkBBduXJF8+fP16RJk7R582a9//77ycZevHhRvXv3Vrly5bRq1So98cQTGjRokM6cOZPxhQMAAAAAAGQSTg13YmJitGPHDoWGhqp8+fIKDAxUSEiIVq9enWzs8uXLlStXLr311lvy8fFRSEiIfHx8tHv3bidUDgAAAAAAkDlkd+bGvb29NWPGDBUqVMhu+aVLl5KN/eWXX9S4cWO5urrali1dutThNQIAAAAAAGRmTg138uTJo/r169seJyUl6bPPPlOdOnWSjY2NjZWfn59GjRqlTZs2qXjx4ho2bJgCAgLuuo3ExEQlJiame+1Zwa39wv5BRuG99g+OP7PRP7PRP7OZ2j/T6nUU+mc2R+6HrLKPs8rrgHmcGu78W1hYmPbu3aslS5YkWxcfH6/p06erR48e+uSTTxQREaE+ffpo7dq1Klq06B3nPHDggCNLzhJ27drl7BLwgNixY4ezS8h0OP7M5oj+5Xqme7rPeUv8/M8cNreJOP7M5pDjL91n/Ac/A+3RP8cLWNUo3ed0laTSJdN9Xkk6UKWqQ+aV+PmHB0OmCXfCwsI0Z84cTZo0Sb6+vsnWu7q6qlKlSgoJCZEkVa5cWT/88INWrlypAQMG3HFeX19f5crlyI96cyUmJmrXrl2qVq2a3eVuD6zF65xdQZZXo0YNZ5eQaXD8mc2R/XPkP0lwDN7E8Wc2jj+z0b8MtMrZBWQeGdm/+Ph4TjCAU2SKcGf06NFasGCBwsLC1KxZsxTHeHt7q0yZMnbLSpUqpT///POuc7u6uvIXt3tgHyGj8D5LjuPPbKb1z6RaM4Jp/YM90/pnUq0Zgf4hI2Vk/3ivwFmc+m1ZkjR16lQtXLhQEydO1FNPPXXHcTVq1FBkZKTdspiYGBUvXtzRJQIAAAAAAGRaTg13oqOjNW3aNPXt21cBAQGKi4uz/SdJcXFxunr1qiSpS5cuioyM1JQpU3TkyBF98MEHio2NVevWrZ35EgAAAAAAAJzKqeHOxo0blZiYqPDwcNWrV8/uP0mqV6+e1qxZI0kqXry4ZsyYoc2bN6tly5bavHmzpk+friJFijjzJQAAAAAAADiVU++5069fP/Xr1++O6/99GVZAQICWLVvm6LIAAAAAAACM4fR77gAAAAAAAOD+Ee4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAyW3dkFAAAAZBX7KlZy2NyV9u9z2NwAAMBsnLkDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDB7jvcSUpK0v79+/Xtt9/q0qVLOnfuXDqWBQAAAAAAgNTIfj9PWrlypSZMmKBTp04pW7ZsWrx4saZMmSI3NzdNmDBB7u7u6V0nAAAAAAAAUpDmM3fWrFmjYcOGqU6dOpo0aZKSkpIkSU888YS2bNmiadOmpXuRAAAAAAAASFmaw52PPvpIXbp00fjx49W0aVPb8vbt22vw4MGKiIhI03wnT55USEiIateurfr16ys0NFTXrl2763OOHTsmf39/bd26Na3lAwAAAAAAZClpDncOHTqkJ554IsV11atX18mTJ1M9l2VZCgkJ0ZUrVzR//nxNmjRJmzdv1vvvv3/X57311luKj49PS9kAAAAAAABZUprDnYIFCyo6OjrFddHR0SpYsGCq54qJidGOHTsUGhqq8uXLKzAwUCEhIVq9evUdn/Pll1/q8uXLaS0bAAAAAAAgS0rzDZVbtGihyZMnq3DhwmrQoIEkycXFRbt379a0adPUsmXLVM/l7e2tGTNmqFChQnbLL126lOL4s2fPKiwsTJ9++mmqt5OYmKjExMRU1/QgubVf2D/IKLzX/sHxZzZT+2davY5C/8xG/8xG/zKOq7MLyEQysn8mvleQNaQ53BkyZIgOHDigIUOGKFu2myf+BAcHKz4+XoGBgXrhhRdSPVeePHlUv3592+OkpCR99tlnqlOnTorjx40bp7Zt26p8+fKp3saBAwdSPfZBtWvXLmeXgAfEjh07nF1CpsPx51gBqxo5ZF5XSWq1ySH9y5XuM/6DY9Ae/TMb/TMb/XO8AGcXkImY2D8grdIc7ri7u2vGjBn64Ycf9PPPP+vcuXPy8vJS7dq11aBBA7m4uNx3MWFhYdq7d6+WLFmSbN2PP/6o7du33/WSrZT4+voqVy5HftSbKzExUbt27VK1atXk6kq2r8XrnF1BllejRg1nl5BpcPxlkFWOnd4R/XPkP0lwDN7kyOOP/jke/TMb/ctADv4ZaJKM7F98fDwnGMAp0hzu9OnTR88995weffRRPfroo+lWSFhYmObMmaNJkybJ19fXbt3Vq1f1xhtv6M0335SHh0ea5nV1deUXp3tgHyGj8D5LjuPPbKb1z6RaMwL9Mxv9Mxv9Q0bKyP7xXoGzpDnc+e233/7T2TkpGT16tBYsWKCwsDA1a9Ys2fqdO3cqNjZWISEhdsv79u2rNm3a6O23307XegAAAAAAAEyR5nCnfv36+vLLLxUQECA3N7f/XMDUqVO1cOFCTZw4UU8++WSKY/z8/PTVV1/ZLWvatKnGjBmTrmcPAQAAAAAAmCbN4U6OHDn05Zdfau3atSpbtmyy+9m4uLhozpw5qZorOjpa06ZNU79+/RQQEKC4uDjbOm9vb8XFxcnLy0seHh7y8fFJ9vwiRYqk6avXAQAAAAAAspo0hzt//fWX/P39bY8ty7Jb/+/Hd7Nx40YlJiYqPDxc4eHhdusiIyNVr149hYaGql27dmktEwAAAAAA4IGQ5nBn3rx56bbxfv36qV+/fndcHxkZeV/rAAAAAAAAHhRpDnduiY6O1i+//KKLFy8qf/78CggIUJkyZdKzNgAAAAAAANxDmsMdy7L05ptvavHixXaXYLm4uKht27Z655130rVAAAAAAAAA3Fmaw50ZM2Zo6dKlCgkJ0dNPPy1vb2+dOnVKK1euVHh4uHx9fdWzZ08HlAoAAAAAAIB/S3O4s2TJEj333HP6v//7P9uyEiVK6Pnnn9f169f1xRdfEO4AAAAASFHP3T2l3ek/7xfpPyUAGCNbWp/w559/qk6dOimuCwoK0rFjx/5zUQAAAAAAAEidNIc7xYsXv+M3Ve3fv18FChT4z0UBAAAAAAAgddIc7rRs2VJTpkzR2rVrbTdUtixLa9as0dSpU9WiRYt0LxIAAAAAAAApS/M9d/r27att27bpxRdf1NChQ5U/f36dPXtWN27cUFBQkF544QVH1AkAAAAAAIAUpDnccXd316xZs/Ttt9/ql19+0fnz55U3b17VqlVLDRo0cESNAAAAAAAAuIM0hzuSdPToUZ06dUqvvPKKJCk6OlpLly5V+fLlVaxYsXQtEAAAAAAAAHeW5nvu7NixQ23atNHMmTNtyy5cuKAvv/xSbdu21YEDB9K1QAAAAAAAANxZmsOdCRMmqGbNmlq+fLltmb+/vzZu3Cg/Pz+NHz8+XQsEAAAAAADAnaU53NmzZ4/69OkjDw8Pu+U5cuTQs88+qz/++CPdigMAAAAAAMDdpTnc8fDw0MmTJ1Ncd/bsWWXLluYpAQAAAAAAcJ/SnMTUr19fkydPVmRkpN3y6OhoTZkyRY899li6FQcAAAAAAIC7S/O3Zb3yyivq0qWL2rZtqxIlSqhAgQI6e/asYmNjVaJECb366quOqBMAAAAAAAApSHO44+3trVWrVmnZsmX67bffdO7cORUpUkTdu3dXu3btlDt3bkfUCQAAAAAAgBSkOdyRpFy5cql79+7q3r17etcDAAAAAACANEhTuLN7927lyZNHJUuWlHTzBsqffPKJoqOjVaFCBfXs2VMFChRwSKEAAAAAAABILlU3VL5+/boGDRqkjh07at26dZKka9eu6ZlnntGsWbN08uRJLVmyRB07dtTff//t0IIBAAAAAADwj1SFO5999pm+++47jRgxQh06dJAkzZ8/XzExMQoJCdGKFSv09ddfy9PTUx999JFDCwYAAAAAAMA/UhXurFq1Sr1791aPHj1sl12tXbtWOXPmVO/evSVJuXPnVnBwsDZt2uS4agEAAAAAAGAnVeHO4cOHFRgYaHt86dIl7dmzR/7+/sqRI4dtealSpXTy5Mn0rxIAAAAAAAApSlW4Y1mWsmX7Z+jvv/+upKQkBQUF2Y27ePGicubMmb4VAgAAAAAA4I5SFe6ULl1au3fvtj3evHmzXFxcVK9ePbtxW7ZsUalSpdK1QAAAAAAAANxZqr4K/emnn9aHH36o/PnzKykpScuWLVOlSpVUpUoV25i1a9dq6dKlevHFFx1WLAAAAAAAAOylKtwJDg5WZGSkRo0aJcuyVLRoUY0fP962vnnz5rb78gQHBzusWAAAAAAAANhLVbjj6uqq0NBQhYSE6PTp06pYsaLc3Nxs6xs2bKgyZcqoTZs2dssBAAAAAADgWKkKd24pWrSoihYtmmz5sGHD0q0gAAAAAAAApF6qbqgMAAAAAACAzIlwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADDYfwp3Ll68qOjoaCUkJCgxMTG9agIAAAAAAEAq3Ve4s3XrVnXs2FG1a9dWq1atdPDgQb388ssaN25cetcHAAAAAACAu0hzuPPTTz+pT58+8vDw0CuvvCLLsiRJFStW1Ny5czVr1qx0LxIAAAAAAAApS3O48/7776tx48aaN2+enn32WVu4M2DAAD333HNavHhxuhcJAAAAAACAlKU53Nm3b5/at28vSXJxcbFb9+ijj+r48ePpUxkAAAAAAADuKXtan+Dl5aW4uLgU1/3555/y8vL6z0WZqtTwCIfMe9ijm0PmdZWkVpscMve+ipUcMm+l/fscMi8AIBN6K69DpnWV1LN0SWl3+s/9RfpPCQAAcE9pPnOncePGmjRpknbt2mVb5uLior/++ksfffSRGjZsmJ71AQAAAAAA4C7SfObOyy+/rD/++EOdOnVSoUKFJEkvvfSS/vrrLxUtWlQvvfRSuhcJAAAAAACAlKU53MmbN68WL16sFStW6Oeff9a5c+fk5eWl4OBgtWvXTjlz5nREnQAAAAAAAEhBmsMdSXJ3d1enTp3UqVOn9K4HAAAAAAAAaZDmcGfq1Kl3XJctWzblypVLPj4+evTRR+Xu7v6figMAAAAAAMDdpTnc+fLLL/XXX38pISFB2bNnV758+XTu3DnduHFDLi4usixLklSuXDnNnTtXBQoUSPeiAQAAAAAAcFOavy3rhRdekLu7uyZOnKidO3fq+++/165duzR16lTlz59f77//vlatWiUXFxdNnDjRETUDAAAAAADg/0tzuDNlyhQNGTJELVq0ULZsN5/u4uKiJk2aKCQkRB988IHKly+vAQMGaMuWLeleMAAAAAAAAP6R5nDnzz//lI+PT4rrihcvruPHj0uSihQpovPnz/+36gAAAAAAAHBXaQ53ypUrp8WLF6e4bsmSJSpdurQk6fDhwypcuPB/qw4AAAAAAAB3leYbKg8ePFjPP/+82rZtq6ZNm6pgwYI6ffq0NmzYoMjISE2ePFl79+5VWFiY2rdvf8/5Tp48qbFjx+rnn39Wjhw51KJFC7300kvKkSNHsrHffPONJk2apKNHj6pEiRIaMmSIGjdunNaXAAAAAAAAkGWkOdxp2LChZs6cqSlTpmjq1KlKTExU9uzZFRAQoDlz5igwMFCbNm3SU089pSFDhtx1LsuyFBISojx58mj+/Pk6f/68Ro4cqWzZsmnYsGF2Y/fv369Bgwbp1VdfVYMGDfT999/rhRde0JIlS1SxYsW0vgwAAAAAAIAsIc3hjiTVqVNHderUUUJCgs6fP6+CBQvabq4sSY0aNVKjRo3uOU9MTIx27NihH374QYUKFZIkhYSE6N13300W7qxevVp16tRRjx49JEk+Pj7atGmT1q5dS7gDAAAAAAAeWPcV7ly7dk2RkZFKSEiQZVk6fPiwkpKSdOXKFW3btk2vvPJKqubx9vbWjBkzbMHOLZcuXUo2tm3btrp+/Xqy5RcvXryflwAAAAAAAJAlpDnc2bp1q1544YU7fhNW7ty5Ux3u5MmTR/Xr17c9TkpK0meffaY6deokG1u2bFm7xwcPHtRPP/2kLl263HUbiYmJSkxMTFU9DyqT9o9JtSI5+vePW/uCfeJYrg6e37T+mVavo/tnGtP65yimfn6aVq+jmLofTKybz9B/ZGT/THyvIGtIc7gzadIk5c+fX6NHj9aXX36pbNmyqV27dvr222+1YMECffLJJ/ddTFhYmPbu3aslS5bcddzff/+twYMHq2bNmve8ofKBAwfuu54Hxa5du9J9zlzpPuNNO3bscNDMyAj0LzlHHH/4R4CD5zfp81My7xh0dP9MY1r/HI3jDxnJUf1rv/gvh8wrSYc9HDa1cTj+8CBIc7gTGRmpMWPG6IknntDFixe1cOFCNWjQQA0aNND169cVHh6u6dOnp7mQsLAwzZkzR5MmTZKvr+8dx50+fVq9evWSZVmaPHmy3b1+UuLr66tcuRz5o/o2i9dlzHbSWbVq1eTqmr7ZvqMitRo1ajhoZhnbP5M4tH+GSUxM1K5duxxy/OE2qxw7vUmfn5KBx6CD+2ca4/rnII78/OT4c7zExERpt7OrSDuH9Y+/f2aIjDz+4uPjOcEATpHmcCcpKUlFihSRdPOmxgcPHrSta9asWbIbIafG6NGjtWDBAoWFhalZs2Z3HHfy5EnbDZXnzp2rAgUK3HNuV1dXfnG6B5P2kSl1ImX0LzmTjj8kZ1r/TKoVydE/exx/yEj0z2wZ2T/eK3CWu5/2koKSJUsqMjJSklS6dGlduXJFMTExkqQbN27o8uXLaZpv6tSpWrhwoSZOnKinnnrqjuPi4+P13HPPKVu2bPrss89sARMAAAAAAMCDLM1n7rRq1UrvvfeeLMtS9+7dVbVqVY0ePVrBwcH66KOPVK5cuVTPFR0drWnTpqlfv34KCAhQXFycbZ23t7fi4uLk5eUlDw8Pffzxxzp69KjmzZsnSbaxHh4e8vLySuvLAAAAAAAAyBLSHO4899xzOnv2rP744w91795db775pvr27auBAwfK09NT4eHhqZ5r48aNSkxMVHh4eLLnRUZGql69egoNDVW7du20fv16Xb16VR07drQb17ZtW40bNy6tLwMAUuetvA6Z1lWSWm1yyNz7KlZyyLySVGn/PofNDQDIZBzwM9BVkkqXTPd5AeBBl+Zw59ChQ3b31alWrZo2bNigmJgYlSlTRp6enqmeq1+/furXr98d19+6/EuS1q3jZmMAAAAAAAD/luZ77nTr1k0rVqywW+bp6Sk/P780BTsAAAAAAAD479Ic7ri5uSl//vyOqAUAAAAAAABplObLsl544QWNHz9eFy9eVMWKFZUrV65kY4oVK5YuxQEAAAAAAODu0hzuvPXWW0pMTNTQoUPvOGbfPm64CQAAAAAAkBHSHO6MGTPGEXUAAAAAAADgPqQ53Gnbtq0j6gAAAAAAAMB9SHO4I0kJCQlasmSJfvzxR8XFxemdd97RL7/8oipVqsjPzy+9awQAAAAAAMAdpPnbsv7++2+1b99eY8eO1ZEjR7Rz505dvXpV33zzjYKDg/X77787ok4AAAAAAACkIM3hzvjx43X58mWtWbNGy5cvl2VZkqTJkyerWrVqmjx5croXCQAAAAAAgJSlOdzZvHmzXnjhBfn4+MjFxcW2PEeOHOrdu7f27NmTrgUCAAAAAADgztIc7ly7dk358uVLcZ2rq6uuX7/+X2sCAAAAAABAKqU53KlWrZo+//zzFNetWrVKVatW/c9FAQAAAAAAIHXS/G1ZL7zwgnr27KnWrVurQYMGcnFx0erVqzVlyhR9//33mjFjhiPqBAAAAAAAQArSfOZOYGCgZs2apZw5c2rGjBmyLEuzZ89WXFycPv74Y9WpU8cRdQIAAAAAACAFaT5zR5Jq1aqlhQsX6urVqzp//rw8PT2VO3fu9K4NAAAAAAAA95DmM3fatGmj2bNn6/Tp0/Lw8FCRIkUIdgAAAAAAAJwkzeFOsWLFNGHCBDVo0EB9+vTRqlWrdPXqVUfUBgAAAAAAgHtIc7gzbdo0/fjjj/rf//4ny7I0fPhwPfLIIxo2bJh+/PFHWZbliDoBAAAAAACQgvu6546Xl5c6dOigDh066MyZM1q3bp3WrVunvn37qlChQtqyZUt61wkAAAAAAIAUpPnMnX87c+aMTp8+rQsXLigxMVF58+ZNj7oAAAAAAACQCvd15k5sbKxWr16tNWvWKCoqSoUKFVLLli317rvvqmLFiuldIwAAAAAAAO4gzeFO+/bttXfvXnl4eOiJJ57Q8OHDVbduXWXLdvMkIMuy5OLiku6FAgAAAAAAILk0hzv58uXTuHHj1LRpU+XMmdO2/NSpU/riiy+0dOlSbd68OV2LBAAAAAAAQMrSHO7MnDnT7vF3332nhQsXasuWLbpx44ZKlCiRbsUBAAAAAADg7u7rnjt///23lixZoi+++ELHjx+Xp6en2rZtq9atWyswMDC9awQAAAAAAMAdpCnc+fnnn7Vo0SJt2LBBiYmJCggI0PHjx/Xhhx+qdu3ajqoRAAAAAAAAd5CqcGf27NlatGiRDh06JB8fHw0cOFBt27ZVrly5VLt2bW6gDMDpSg2PcMi8hz0cMq0kqefuntLu9J/3i/SfEgCQSTnq55/k2J+BAID0lapwZ9y4capQoYLmzp1rd4bOxYsXHVYYAAAAAAAA7i1bagY99dRTOnLkiPr376+BAwfq66+/1o0bNxxdGwAAAAAAAO4hVWfuTJgwQZcuXdKqVau0bNkyDR48WPnz51eTJk3k4uLCZVkAAAAAAABOkqozdyTJ09NTXbt21eLFi7Vq1Sq1bt1amzZtkmVZGjlypD744ANFRUU5slYAAAAAAAD8S6rDnduVL19ew4cP15YtWzRlyhSVKVNGn3zyiVq1aqWnn346vWsEAAAAAADAHaTpq9CTPTl7dj3xxBN64okndPr0aS1fvlzLly9Pr9oAAAAAAABwD/d15k5KChUqpL59+2rNmjXpNSUAAAAAAADuId3CHQAAAAAAAGQ8wh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAzm9HDn5MmTCgkJUe3atVW/fn2Fhobq2rVrKY7du3evOnbsqOrVq6t9+/bavXt3BlcLAAAAAACQuTg13LEsSyEhIbpy5Yrmz5+vSZMmafPmzXr//feTjY2Pj1e/fv0UGBioZcuWyd/fX/3791d8fHzGFw4AAAAAAJBJODXciYmJ0Y4dOxQaGqry5csrMDBQISEhWr16dbKxa9asUY4cOfTqq6+qbNmyeu2115Q7d26tW7fOCZUDAAAAAABkDtmduXFvb2/NmDFDhQoVslt+6dKlZGP/+OMPBQQEyMXFRZLk4uKimjVraseOHWrXrt0dt5GYmKjExMT0LTyLMWn/mFQrkqN/5jOth64Ont+0/WFavY7un2lM65+j3NoPpu0P0+qFPfpntozsH+8VOItTw508efKofv36tsdJSUn67LPPVKdOnWRj4+LiVK5cObtlBQsW1MGDB++6jQMHDqRPsVlUz909JQfcuuiL9J9SkrRjxw4HzYyMQP/M56getl/8l0PmPezhkGklmff5KdE/05n2GRqwqpFD5nWV1LN0SY4/ZCj6Zzb6hweBU8OdfwsLC9PevXu1ZMmSZOuuXLkid3d3u2Xu7u5KSEi465y+vr7KlStXutZ5R4u5RMzRatSo4bjJ6Z/D0T/zOayH9C9D0D+zOfQz1BFWObuAzIXjz2z0z2wZ+fkZHx/PCQZwikwT7oSFhWnOnDmaNGmSfH19k63PkSNHsiAnISFBHh53/2c9V1dXubpyYndWQS/NRv/MRw/NRv/MRv/MRv/MRv/MlpH9470CZ8kU4c7o0aO1YMEChYWFqVmzZimOKVKkiE6fPm237PTp0ypcuHBGlAgAAAAAAJApOfXbsiRp6tSpWrhwoSZOnKinnnrqjuOqV6+u33//XZZlSbr5Neq//fabqlevnlGlAgAAAAAAZDpODXeio6M1bdo09e3bVwEBAYqLi7P9J928ifLVq1clSU8++aQuXLigsWPHKioqSmPHjtWVK1fUvHlzZ74EAAAAAAAAp3JquLNx40YlJiYqPDxc9erVs/tPkurVq6c1a9ZIkjw9PfXxxx9r+/btateunf744w9Nnz49426WDAAAAAAAkAk59Z47/fr1U79+/e64PjIy0u6xn5+fli9f7uiyAAAAAAAAjOH0e+4AAAAAAADg/hHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAg2WacCchIUEtW7bU1q1b7zjm66+/VvPmzeXv76+uXbtqz549GVghAAAAAABA5pMpwp1r167ppZde0sGDB+845uDBg3r55ZfVv39/rVy5UpUqVVL//v115cqVDKwUAAAAAAAgc3F6uBMVFaVOnTrp6NGjdx33ww8/qFy5cmrTpo1Kliypl156SXFxcYqKisqgSgEAAAAAADIfp4c7v/zyi4KCgrRo0aK7jsuXL5+ioqK0fft2JSUladmyZfL09FTJkiUzqFIAAAAAAIDMJ7uzC+jWrVuqxrVo0UKbNm1St27d5OrqqmzZsunjjz9W3rx57/q8xMREJSYmpkepyATopdnon/noodnon9lM65+rswvIZEzrH+zRP7NlZP94r8BZnB7upNbZs2cVFxenN954Q9WrV9eCBQs0YsQILV++XAULFrzj8w4cOJCBVcLRduzY4ewS8B/QP/PRQ7PRP7OZ1r8AZxeQyZjWP9ijf2ajf3gQGBPuvPfee/L19dUzzzwjSRo9erSaN2+upUuXql+/fnd8nq+vr3LlypUxRS5elzHbeYDVqFHDcZPTP4ejf+ZzWA/pX4agf2Zz6GeoI6xydgGZC8ef2eif2TLy8zM+Pp4TDOAUxoQ7e/bsUXBwsO1xtmzZVLFiRZ04ceKuz3N1dZWrKycGZxX00mz0z3z00Gz0z2z0z2z0z2z0z2wZ2T/eK3AWp99QObUKFy6s6Ohou2WHDh1SiRIlnFQRAAAAAACA82XqM3fi4uLk5eUlDw8PderUScOHD1fVqlXl7++vxYsX68SJE2rbtq2zywQAAAAAAHCaTB3u1KtXT6GhoWrXrp1atGihy5cv6+OPP9Zff/2lSpUqac6cOXe9mTIAAAAAAEBWl6nCncjIyLs+7tixozp27JiRJQEAAAAAAGRqxtxzBwAAAAAAAMkR7gAAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwGOEOAAAAAACAwQh3AAAAAAAADEa4AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwGOEOAAAAAACAwQh3AAAAAAAADEa4AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwGOEOAAAAAACAwQh3AAAAAAAADEa4AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwGOEOAAAAAACAwQh3AAAAAAAADEa4AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABgsu7MLAAAAgPOUGh7hkHkPezhkWgAAkALO3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCZJtxJSEhQy5YttXXr1juOiYyMVNeuXeXn56dWrVrp559/zsAKAQAAAAAAMp9MEe5cu3ZNL730kg4ePHjHMRcvXlTv3r1Vrlw5rVq1Sk888YQGDRqkM2fOZGClAAAAAAAAmYvTw52oqCh16tRJR48eveu45cuXK1euXHrrrbfk4+OjkJAQ+fj4aPfu3RlUKQAAAAAAQObj9HDnl19+UVBQkBYtWnTPcY0bN5arq6tt2dKlS9WgQQNHlwgAAAAAAJBpZXd2Ad26dUvVuNjYWPn5+WnUqFHatGmTihcvrmHDhikgIOCuz0tMTFRiYmJ6lIpMgF6ajf6Zjx6ajf6Zjf6Zjf6Zjf6ZLSP7x3sFzuL0cCe14uPjNX36dPXo0UOffPKJIiIi1KdPH61du1ZFixa94/MOHDiQgVXC0Xbs2OHsEvAf0D/z0UOz0T+z0T+z0T+z0T+z0T88CIwJd1xdXVWpUiWFhIRIkipXrqwffvhBK1eu1IABA+74PF9fX+XKlStjily8LmO28wCrUaOG4yanfw5H/8znsB7SvwxB/8xG/8xG/8xG/8zm0L+D/kt8fDwnGMApjAl3vL29VaZMGbtlpUqV0p9//nnX57m6utrdpwdmo5dmo3/mo4dmo39mo39mo39mo39my8j+8V6Bszj9hsqpVaNGDUVGRtoti4mJUfHixZ1UEQAAAAAAgPNl6nAnLi5OV69elSR16dJFkZGRmjJlio4cOaIPPvhAsbGxat26tZOrBAAAAAAAcJ5MHe7Uq1dPa9askSQVL15cM2bM0ObNm9WyZUtt3rxZ06dPV5EiRZxcJQAAAAAAgPNkqnvu/Puyq38/DggI0LJlyzKyJAAAAAAAgEwtU5+5AwAAAAAAgLsj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAg2V3dgGOkpSUJEm6cuVKhm2zdD7H7M74HOUdMq8k+Xg85JB5k0olOmTe+Ph4h8wr0b/b0b9/0D97juoh/fsH/fsH/bNH//5B//7hqP5Jjush/fsH/fuHif1Lya3fP2/9PgpkFBfLsixnF+EIZ86c0eHDh51dBgAAAADgAVOqVCkVLFjQ2WXgAZJlw50bN27o/PnzypEjh7Jl4+ozAAAAAIBjJSUl6dq1a8qbN6+yZ8+yF8ogE8qy4Q4AAAAAAMCDgFNaAAAAAAAADEa4AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwGOEOAAAAAACAwQh3AACAQ1mW5ewSAAAAsrTszi4AAACkjwMHDig8PFy//PKLzp8/r3z58ikwMFADBgxQxYoVJUnBwcGSpHnz5mVITYsXL1Z0dLSGDx+eIdsDAAB4EHHmDgAAWcDBgwfVuXNnnTt3Tq+//ro+/fRTvfrqqzpx4oQ6deqkHTt2OKWu8PBwnTt3zinbBgAAeFBw5g4AAFnArFmzlD9/fn3yySfKnv2fH+9NmjTRk08+qWnTpmn69OlOrBAAAACOwpk7AABkAadPn5ZlWUpKSrJbnitXLo0cOVLNmze3LbMsS5988okaNmwoPz8/de7cWTt37rR73oEDB9S/f3/VrFlTNWvW1PPPP6/Y2Fi7MefOndMbb7yhRx55RNWqVVOnTp30008/2dY3atRIx48f1/Lly1WhQgUdO3bMAa8cAAAALhZ3OQQAwHiff/65/ve//6lKlSpq37696tSpozJlysjFxcVuXHBwsLZt26Zq1aqpf//+unHjhsaNG6eEhARt2bJF2bNn16FDh9S+fXuVKVPGNiY8PFx///23Vq5cqYIFC+ratWvq1KmTTp8+rSFDhqhw4cJaunSpNm7cqBkzZqhu3brau3ev+vXrp8qVK2vgwIGqXLmy3N3dnbSHAAAAsi4uywIAIAvo1q2b4uLiNHPmTL399tuSpPz586tevXrq0aOH/Pz8bGPd3d01ffp05cuXT5J04cIFvf7664qKilLFihU1depU5cyZU7Nnz5anp6ckqW7dumrSpIlmzJihYcOGaeXKldq/f7+++OILVa9eXZL02GOPKTg4WO+9956WLl1qC3MKFCigGjVqZOj+AAAAeJBwWRYAAFnECy+8oO+++04TJkxQhw4d5OnpqVWrVqlTp06aO3eubVy5cuVswY4klShRQpJ08eJFSdLPP/+s2rVry8PDQzdu3NCNGzfk6empwMBA/fjjj5Kkn376Sd7e3qpSpYptTGJioh5//HHt3r1b58+fz7gXDgAA8IDjzB0AALKQvHnzqmXLlmrZsqUkae/evRo6dKjCwsLUqlUrSTfvw3O7bNlu/lvPrfv1nDt3TmvWrNGaNWuSzV+gQAHbmLi4OFWpUiXFOuLi4pQ3b970eVEAAAC4K8IdAAAMd/LkSbVv314vvPCCOnbsaLeucuXKevHFF1O8IfKdeHl56ZFHHlGvXr2Srbv1TVxeXl4qVaqU3nvvvRTnuHU2EAAAAByPy7IAADBcoUKFlD17dn3++ee6du1asvUxMTHKkSOHfHx8UjVf7dq1FRUVpUqVKqlatWqqVq2aqlatqtmzZ+vrr7+2jfnzzz9VsGBB25hq1arphx9+0IwZM+Tq6irpn7OCAAAA4Dj8jQsAAMO5urrqrbfe0oEDB9S+fXstWLBAv/zyi7Zs2aJ33nlHH3zwgQYNGpTqy6QGDhyoo0ePqn///tqwYYO+++47DR48WBEREapYsaIkqV27dipWrJh69eql5cuX6+eff9bEiRP1wQcfqHDhwnJzc5Mk5cmTR3v37tUvv/yiq1evOmwfAAAAPMj4KnQAALKIPXv2aObMmdq+fbv+/vtvubu7q3LlygoODlbTpk0l3fwqdEmaN2+e7Xlbt25Vjx49NHfuXAUFBdnmmjRpkn777TdZliVfX1/169dPjRs3tj3vzJkzmjBhgr755htdvHhRxYsXV4cOHdS7d2/bGTurV6/WO++8o4sXL2rWrFkKDAzMqN0BAADwwCDcAQAAAAAAMBiXZQEAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwGOEOAAAAAACAwQh3AAAAAAAADEa4AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwGOEOAAAAAACAwQh3AAAAAAAADEa4AwAAAAAAYDDCHQAAAAAAAIMR7gAAAAAAABiMcAcAAAAAAMBghDsAAAAAAAAGI9wBAAAAAAAwGOEOAAAAAACAwQh3AAAAAAAADEa4AwAAAAAAYLDszi4AqdOoUSMdP3482fKaNWtqwYIFTqgIaUH/zEcPzUb/Mod/98HFxUV58uRRQECA3njjDRUtWtSJ1aXesmXLNHXqVG3atMnZpWQo+mc2+mc2+gfgXgh3DDJy5Ei1aNHCbpmbm5uTqkFa0T/z0UOz0b/M4fY+JCUlKSoqSm+++aaGDRumuXPnOrk63Av9Mxv9Mxv9A3A3hDsG8fLykre3t7PLwH2if+ajh2ajf5nDv/tQpEgRhYSEaOjQobp48aK8vLycWB3uhf6Zjf6Zjf4BuBvuuZMFXLp0SSNGjFDdunVVtWpVPfnkk9qwYYNtfYUKFfTBBx8oKChIAwYM0LJlyxQcHKzw8HDVqlVLjz76qFasWKF169bp8ccfV2BgoMLCwmzPT0hI0JgxYxQUFKSgoCC98sorOnfunCTp2LFjqlChgj788EPVqlVLb7/9dka/fOPRP/PRQ7PRP+dzd3eXJGXLlk0XLlzQ0KFDVbNmTdWrV0+jR4/W1atXbWMnTpyoevXqyc/PT8HBwTp48KBt3bZt29SuXTv5+fmpVatWWr9+vW1dQkKCQkNDVb9+fVWpUkWNGjXSokWLbOsbNWqksLAw1atXT23atJFlWdq5c6e6du2q6tWrq1mzZoqIiLCNtyxLU6ZMUVBQkAIDA/Xuu+/avaaFCxeqUaNG8vf3V3BwsCIjI++6LZPRP/on0T9noX9m9w9IVxaM8Pjjj1tLly5Ncd3w4cOtzp07W3v37rUOHTpkvfbaa1bt2rWta9euWZZlWb6+vlbr1q2t6Oho6+DBg9bSpUutKlWqWCNGjLAOHz5sjR071qpevbrVtWtXa9++fdbixYstX19fa8+ePZZlWVZoaKjVuXNn648//rD2799v9e/f3+rRo4dlWZYVGxtr+fr6Wr1797aOHDliHTp0KEP2h2non/noodnoX+aQUh+OHDlitW3b1urTp49lWZY1aNAgq3///tb+/futP/74w+rYsaM1YsQIy7Is66uvvrJq165t/frrr9aRI0esIUOGWO3bt7csy7JOnTpl1axZ05o3b551+PBha8WKFVaNGjWsX3/91bIsy5oyZYrVtGlT6/fff7eOHj1qffDBB1aVKlWsuLg4W23169e39u/fb+3bt886ffq0FRAQYP3vf/+zoqOjbX3ft2+ftXTpUsvX19d68cUXrZiYGCsiIsKqUKGCtWXLFsuyLGvjxo3Wo48+am3atMk6dOiQNWnSJKt27drWuXPnUtyWKegf/aN/zkP/zO4fkBEIdwzx+OOPW1WrVrVq1Khh99/ly5etpUuXWpGRkbax0dHRlq+vr3XixAnLsm7+YvL555/b1i9dutSqXLmydfnyZcuyLCsqKsry9fW1fvzxR9uYunXrWqtWrbLi4+OtKlWqWPv377etO3/+vFWxYkVr//79tl9Mbn0gI2X0z3z00Gz0L3P4dx+qVq1q+fv7W6+88or1999/W0eOHLEqVqxoXbhwwfac/fv325bNmjXLevTRR63jx49blmVZZ86csf3yMWnSJGvQoEF22wsNDbUt+/rrr21jLcuyrl27Zvn6+tqWPf7441ZYWJht/Zw5c6xGjRpZiYmJtmWffvqp9fvvv9t+Ubn1HrAsy2rdurX18ccfW5ZlWV27drXmzp1rV0vbtm1ty/69LVPQP/pH/5yH/pndPyAjcM8dg4SEhKhp06Z2y3LmzKk2bdpow4YN+uKLLxQTE6M9e/ZIkhITE23jihcvbve8ggULKleuXJKkHDlySJJKlChhW+/h4aGEhATFxsbq+vXr6tKli93zk5KSdPjwYVWpUiXF+ZEc/TMfPTQb/cscbvXh8uXLmjJlio4fP66XX35Z+fPn144dO5SUlKTHHnvM7jlJSUk6cuSInnrqKX322Wdq3LixatSooSZNmqhDhw6SpJiYGG3evFn+/v62512/fl2lS5eWJDVp0kQ//PCDxo0bp5iYGO3du1fSnft86NAhVa5cWdmy/XMFe69evWzbuv09IN28F0ZCQoIkKTo6WmFhYZo4caJt/bVr13T48OEUt2US+pd8Wyahf8m3ZRL6l3xbAP5BuGOQggULysfHJ9nyoUOH6vfff1fr1q3VtWtXeXt7q3PnznZjbv3ycUv27Mlb7+LikmzZrQ/tzz//3O5D+FY9t+4b8e/5kRz9Mx89NBv9yxxu78MHH3ygDh06aODAgVq0aJESExPl5eWlpUuXJntekSJF5OHhobVr1+qHH37Q5s2bNXPmTH3xxRdasWKFbty4oVatWmnAgAF2z7vVq0mTJmnx4sVq166d2rRpozfffFONGjWyG3t7H1Lq8e1cXV2TLbP+/70fEhMTNXLkSNWtW9duvaenZ4rbMgn9S74tk9C/5NsyCf1Lvi0A/+CGyoa7dOmSVq9erUmTJikkJERPPPGEzp8/L0npcoOxhx9+WK6urjp37px8fHzk4+MjT09PhYaG6syZM/95/gcd/TMfPTQb/XMud3d3jRkzRvv27dPs2bNVunRpXbx4US4uLrb9dfXqVY0fP14JCQn65ptvtHjxYjVs2FD/+9//tHLlSh0+fFgHDhxQ6dKldeTIEdvzfHx8tHHjRq1atUrSzRt0jho1Sq+88opatGihK1euSLpzn0uVKqXIyEi79UOGDNGMGTPu+bpKly6tv/76y66Wjz76SDt27PjvOy0ToX9mo39mo38A/o1wx3Du7u7KmTOnvvrqKx07dkzfffed7dtWbp3e+F94enqqY8eOeuutt7R161ZFRUXp1Vdf1ZEjR+wuQcD9oX/mo4dmo3/O5+fnpw4dOmjatGny9PRU/fr19corr2jnzp3as2ePRowYofj4eOXJk0dJSUkaP368vv76ax07dkzLli1Tzpw5VapUKXXr1k27d+/WpEmTdPjwYa1atUoTJ05UsWLFJEn58uXT5s2bFRsbq23btunVV1+VdOc+t2rVSufOndP48eN1+PBhLVu2TBs3btSjjz56z9fUq1cvzZkzRytWrNDRo0cVFhamtWvXqmzZsum34zIJ+mc2+mc2+gfgdoQ7hnN3d1dYWJjWr1+vp556SuPGjdP//d//ydvbW/v27UuXbQwfPlx169ZVSEiIOnXqpOzZs2v69OkpnlKJtKF/5qOHZqN/mcOLL74oNzc3hYWFafz48SpRooR69uypXr16qXTp0rZ7LzRq1EghISEKDQ1V8+bNtWbNGk2bNk158+ZV8eLF9dFHH+m7775Ty5Yt9f7772v48OF6+umnJUnvvPOO9u3bp6eeekojRozQk08+KT8/vzv2OU+ePPr444+1bds2tWzZUp988okmTJigSpUq3fP1tGjRQi+++KImT56sli1b6qefflJ4eLhKlSqVbvssM6F/ZqN/ZqN/AG5xsdLjvHMAAAAAAAA4BWfuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEOwAAAAAAAAYj3AEAAAAAADAY4Q4AAAAAAIDBCHcAAAAAAAAMRrgDAAAAAABgMMIdAAAAAAAAgxHuAAAAAAAAGIxwBwAAAAAAwGCEO07SqFEjVahQQRUqVFDFihXl7++vLl266LvvvnN2aekqISFBX3zxhbPLSHf078ERHBysKVOm/Kc5pkyZYnu//Pu/I0eOaNmyZWrUqFGq5woODnZovVkJ/ct8+Pw0G/17cPD5aTb6Bzx4sju7gAfZyJEj1aJFCyUlJen8+fNasWKF+vfvrxkzZuiRRx5xdnnpIiIiQh999JE6derk7FLSHf1DWvj7+6f4l5YCBQqoSJEiatiwYcYXhVSjf+mLz0+z0T+kBZ+fZqN/gDkId5zIy8tL3t7ekqQiRYro1VdfVVxcnEJDQ7Vq1SonV5c+LMtydgkOQ/+QFm5ubrb3y7+5urrKw8MjgytCWtC/9MXnp9noH9KCz0+z0T/AHFyWlcl07txZBw4c0JEjR3T+/HmNGjVKjzzyiAICAjR06FCdP3/eNnbnzp3q2rWrqlevrmbNmikiIkKSUjxF8vZTHYcPH66wsDANGTJE1atXV4sWLbR3715NmjRJgYGBeuyxx7R27Vrbc//8808NGDBA1atXV6NGjTR16lQlJibathUcHKzJkycrKChIgYGBCg0NlWVZ2rp1q0aMGKHjx4+rQoUKOnbsmPbv368uXbqoevXqql+/vqZOneroXZqh6F/Wdf36db3++usKCgqSv7+/BgwYoJMnT0pK+VTjRo0aadmyZama+989P3DggIKDg+Xn56dmzZpp/vz5d3zu119/rWbNmqlGjRp6++23bb2FPfqX+fH5aTb6l3Xx+Wk2+gc8OAh3MpmyZctKkqKiojRo0CDt27dPH330kWbNmqXo6GgNHz5cknTmzBn17t1blSpV0vLly9W/f38NGzZM+/fvT9V25syZo9q1a+vLL79Uvnz59Oyzz+rMmTNatGiRGjVqpDfffFNJSUmyLEuDBg1SwYIFtXz5ctu/yn300Ue2uX7//XcdOnRICxYs0KhRozR37lz9+OOP8vf318iRI/XQQw/p+++/V9GiRfXqq6+qUqVKWr16tcaOHasZM2Zoy5Yt6b8jnYT+ZV3z58/Xr7/+qk8//VRLlizR5cuX9c4776T7dq5evaq+ffsqICBAX375pYYNG6Zp06ZpxYoVycZGRUVpyJAh6tq1q5YuXaobN25o+/bt6V5TVkD/Mj8+P81G/7IuPj/NRv+ABweXZWUyXl5ekqR9+/bpl19+0bp161S6dGlJUlhYmFq0aKGYmBh9//33yps3r15//XVly5ZNZcqU0fnz53X16tVUbadq1arq1q2bJKlly5Z655139Prrr8vDw0PBwcFasGCBTp8+rejoaJ04cUKLFy+2bWfYsGEaMWKEnn/+eUlSYmKiRo8eLU9PT5UpU0azZ8/Wrl279Oijj8rLy0uurq620zmPHz+uxo0bq3jx4nr44Yc1a9YslShRIr13o9PQv6zr2LFjypEjh4oXL658+fJp3LhxOnfuXKqfv23bNvn7+9stGz58uDp37my3bNWqVSpYsKCGDBkiSSpVqpSOHz+uuXPnqk2bNnZjly5dqsDAQPXs2VOSNGrUKG3evDmtL+2BQP8yPz4/zUb/si4+P81G/4AHB+FOJnPp0iVJUvHixZUnTx7bX4ykm/8qljdvXsXExOjQoUOqXLmysmX75+SrXr16SZJiYmLuuZ3b/0Li4eGhQoUK2a6ZzZEjh6Sb3xQRHR2tc+fOKSAgwDY+KSlJV69e1dmzZyVJBQsWlKenp229p6enbty4keJ2+/fvr4kTJ2rRokVq2LChWrdufcfreE1E/7Kuzp07KyIiQvXq1VPt2rXVpEkTtWvXLtXPr1q1qt577z27ZQUKFEg2LiYmRvv377f7i1RiYqJcXV2TjY2OjlalSpVsj93c3Owe4x/0L/Pj89Ns9C/r4vPTbPQPeHAQ7mQykZGRkqQLFy6kuD4xMVGJiYnKnv3OrXNxcUm27N9/Wfn382//S9a/n1emTBlNmzYt2bpb/0rn7u6ebN2dbkTYr18/NW/eXBs2bNCmTZv07LPPavTo0erYsWPKL8Yw9C/rKl++vDZt2qRvvvlG33zzjSZOnKjVq1dr/vz5qeqZh4eHfHx87rmdGzduqG7dunrjjTdSVde/e+Xm5paq5z1o6F/mx+en2ehf1sXnp9noH/Dg4J47mczSpUtVpUoV1atXTxcuXLD7V6yoqChdunRJpUuXVqlSpRQZGWn3wThkyBDNmDFDbm5uunz5sm25ZVk6duzYfdVTunRpnThxQgUKFJCPj498fHx07NgxTZ48OcUfCP92+5hr165pzJgxcnd3V69evTRv3jx16tRJ69evv6/aMiP6l3WtWLFCmzdvVvPmzfXuu+9qxowZ2r59u86cOZOsZ5cvX9bff/99X9spXbq0Dh06pBIlSth6tmPHDs2bNy/Z2PLly2vXrl22x0lJSam+b8WDhv5lfnx+mo3+ZV18fpqN/gEPDsIdJ7p48aLi4uJ06tQpRUZGauzYsVqzZo2GDx+usmXL6rHHHtOwYcO0c+dO7dy5U8OGDVOtWrXk6+urVq1a6dy5cxo/frwOHz6sZcuWaePGjXr00UdVtWpVnTt3TvPmzVNsbKxCQ0PtvqUiLerVq6fixYtr6NChioyM1LZt2zRq1CjlzJkzxdMs/y1nzpw6f/68Dh8+LFdXV/32228aPXq0YmJitGvXLm3btk2VK1e+r9qcjf6Z3b+0unjxosaOHauffvpJsbGxWrVqlR566CHlz59f1apV0/79+7V27VodOnRIb7zxxh3/Nflenn76aV29elVvvPGGoqOjtWXLFo0dO1YFCxZMNrZTp07avXu3wsPDFRMTo3fffVcnTpz4ry81S6J/mQufn2Z/ftI/s/uXVnx+mo3+AQ8Owh0neuedd1SvXj099thj6tWrlw4dOqTZs2erdu3akqR3331XDz/8sHr27Kk+ffqofPny+vDDDyVJefLk0ccff6xt27apZcuW+uSTTzRhwgRVqlRJpUqV0rBhwxQeHq42bdrIsiw1a9bsvmp0dXVVeHi4kpKS1KlTJw0ePFgNGjTQ66+/nqrn16lTRz4+PmrVqpX27dunSZMm6cqVK+rQoYP69OmjwMBADRw48L5qczb6Z3b/0uqZZ55RmzZtNHToUNvX74aHh8vV1VV169ZVz5499cYbb6hLly4qX768qlevfl/b8fT01CeffKLDhw+rTZs2ev311/XMM8+of//+ycb6+PgoPDxcERERatOmjeLi4tSgQYP/+lKzJPqXufD5afbnJ/0zu39pxeen2egf8OBwse50cTEAAAAAAAAyPc7cAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwAAAAAAAAxGuAMAAAAAAGAwwh0AAAAAAACDEe4AAAAAAAAYjHAHAAAAAADAYIQ7AAAAAAAABiPcAQAAAAAAMBjhDgAAAAAAgMEIdwxy7NgxVahQQceOHXPK9i9duqQVK1Y4Zdv4bypUqKCtW7c6uwyjcfzhfnH8/Xccf7hfHH//Hccf7hfHH5CxCHcMUrRoUX3//fcqWrSoU7Y/e/ZsLV261CnbBpyN4w9wHo4/wHk4/gDADNmdXQBSz9XVVd7e3k7bvmVZTts24Gwcf4DzcPwBzsPxBwBm4Mwdg9x+WmyFChW0cuVKtWzZUlWrVlW3bt0UGxtrGztx4kTVq1dPfn5+Cg4O1sGDB23rtm3bpnbt2snPz0+tWrXS+vXr7bYza9YsNWrUSP7+/urTp49iY2O1bNkyTZ06Vb/88osqVKggSfrpp5/UunVrVatWTY0bN9bChQszZkcY7Omnn9Znn31me9yrVy91797d9njRokXq2rWr/vzzTw0YMEDVq1dXo0aNNHXqVCUmJtrG3auHU6dOVd26dRUUFKTFixfbraNv94fjz3wcf+bi+DMfx5+5OP7Mx/EHPCAsGCM2Ntby9fW1/b9x48bWjz/+aEVGRlpPPvmk9dJLL1mWZVlfffWVVbt2bevXX3+1jhw5Yg0ZMsRq3769ZVmWderUKatmzZrWvHnzrMOHD1srVqywatSoYf3666+WZVnWggULrJo1a1oRERHWoUOHrMGDB1tt27a1rly5Yo0bN87q3LmzderUKevGjRtW7dq1rWnTplmxsbHWypUrrYoVK1oHDx502v4xwbvvvmsNHjzYsizLSkhIsGrUqGH5+flZCQkJlmVZ1uDBg62pU6da7dq1s0aOHGlFR0dbP//8s9W0aVNr6tSplmXdu4cLFy60atWqZW3atMnau3ev1blzZ8vX19f6+eef6dt/wPFnPo4/c3H8mY/jz1wcf+bj+AMeDFyWZbBevXqpbt26kqSuXbtq/vz5kqTjx4/Lzc1NxYoVU7FixTRq1CjFxMRIkubPn69HHnnEltb7+Pho3759mjNnjgIDA7Vo0SL17NlTLVq0kCS98cYbmjlzpiQpV65ccnNzk7e3t86dO6dz586pUKFCKlGihEqUKKHChQs79bRdE9SrV08vv/yyLMvSnj17VLJkSf3999/au3evqlWrpq1bt6pq1ao6ceKEFi9erGzZsqlMmTIaNmyYRowYoeeff/6ePfziiy/07LPP6vHHH5ckjRkzRk899ZQk6eLFi/QtnXD8mYfjL+vg+DMPx1/WwfFnHo4/4MFAuGMwHx8f2589PT11/fp1SdJTTz2lzz77TI0bN1aNGjXUpEkTdejQQZIUExOjzZs3y9/f3/bc69evq3Tp0pKkQ4cOqcr/a+8OQqLowziO/1YrWEmMKII8mFIXFQ12kSXXbMNrhXiQzA4eCsWzdNBQLx48KHhY7CCRniyMxIOIm3lwQdxNlDyYIqmMHtZA6bId2v2/h2B4q5fX942ynfH7gWWZeZgd/vvs7/IMM1tSYtfOnTunR48e/XDuM2fO6O7du+ro6FA4HFYoFFJdXZ3y8vJ+y1rdwu/3K5lMan19XbFYTH6/X4lEQm/fvlV2draysrLk9Xp1cHAgn89nH5dOp/X582ft7+8f2sONjQ21trbatcuXLysnJ0cSffuVyJ/zkD/3IH/OQ/7cg/w5D/kDjgeGOw528uTJf9x//vx5TU5OKhqN6s2bNxoaGtLz58/16tUrffnyRbdu3VJzc/M3x5w4ceKb9/+iq6tL9+7dUyQSUSQS0ejoqMLhsKqrq39+US536tQp+f1+LSwsKB6P686dO0okEorH40qlUqqsrFQqlVJRUZHC4fAPx+fm5h7aQ+nHhw/+vUbffg3y5zzkzz3In/OQP/cgf85D/oDjgQcqu9Ds7KxevHihGzduqLu7W+Pj49rc3NTa2poKCwu1tbWlgoIC+/X69WtNTExI+no1ZnV11f6s/f19BQIBWZYlj8dj79/b21N3d7cKCgrU0tKisbExBQIBzczMHPl6nSYYDGphYUFLS0vy+Xzy+XxaXFzU3NycqqqqVFhYqN3dXZ09e9bukWVZGhgYkMfjObSHV65c0bt37+zzWZalT58+SaJvR4H8ZTby527kL7ORP3cjf5mN/AHux3DHhdLptHp7ezU9PS3LsvTy5Ut5vV5dunRJDQ0NWllZUX9/vzY3NzUxMaG+vj5dvHhRknT//n09e/ZMkUhEHz58UGdnp31vrNfrVSKRkGVZysvL0/T0tHp6erS9va1YLKbV1VUVFxf/4dVnvmAwqJmZGZ0+fVoXLlxQcXGxksmkYrGYqqqqFAwGlZ+fr7a2Nr1//17xeFyPHz+W1+tVdnb2oT1sbGzU8PCwpqamtLa2pvb2dmVlfY06ffv9yF9mI3/uRv4yG/lzN/KX2cgfcAz8uWc54//6/t8K5ufn7drY2JgJhUL29tDQkAmFQqa0tNTcvn3bRKNRuxaNRk1tba0pKSkxN2/eNCMjI3YtnU6bwcFBEwwGzdWrV83Dhw/Nzs6OMcaYra0tU1NTY8rKyszHjx/N8vKyqa+vN+Xl5ebatWumr6/PpFKpI/gmnO/69eumra3N3m5qajK1tbX29vb2tnnw4IEpKyszgUDAdHV1mWQyadf/rYfGGPP06VNTWVlpfD6fefLkiamoqLB/L/Tt55A/9yB/zkP+3IP8OQ/5cw/yB7ibx5jvbo4EAAAAAACAY3BbFgAAAAAAgIMx3AEAAAAAAHAwhjsAAAAAAAAOxnAHAAAAAADAwRjuAAAAAAAAOBjDHQAAAAAAAAdjuAMAAAAAAOBgDHcAAAAAAAAcjOEOAAAAAACAgzHcAQAAAAAAcDCGOwAAAAAAAA72F6HJ1HWl9Kp2AAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Performance by Category plot has been generated, cropped, and saved.\n" ] } ], "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt\n", "import seaborn as sns\n", "from PIL import Image\n", "import numpy as np\n", "\n", "# Set consistent theme and font\n", "plt.style.use('seaborn-v0_8-whitegrid')\n", "plt.rcParams['font.size'] = 10\n", "\n", "\n", "plt.rcParams.update({\n", " # 'font.family': 'Arial',\n", " 'font.size': 12,\n", " 'axes.labelsize': 12,\n", " 'axes.titlesize': 13,\n", " 'xtick.labelsize': 10,\n", " 'ytick.labelsize': 10,\n", " 'legend.fontsize': 11,\n", "})\n", "# Load the data\n", "df = pd.read_csv('temp_results.csv')\n", "\n", "# Calculate performance by category\n", "category_performance = df.groupby('Sheet')[['Gemini', 'GPT', 'LLAMA', 'Claude']].mean()\n", "print(\"Category Performance Dataframe:\")\n", "print(category_performance)\n", "\n", "\n", "# Create the plot\n", "fig, ax = plt.subplots(figsize=(12, 6))\n", "category_performance.plot(kind='bar', ax=ax, width=0.8)\n", "\n", "# Customize the plot\n", "plt.title('Performance by Category', fontsize=14, fontweight='bold')\n", "plt.ylabel('Average Score', fontsize=12)\n", "plt.ylim(1.5, 3)\n", "\n", "# Customize the legend\n", "plt.legend(fontsize=10, bbox_to_anchor=(1.05, 1), loc='upper left')\n", "\n", "# Customize x-axis labels\n", "ax.set_xticklabels([]) # Remove existing labels\n", "categories = category_performance.index.tolist()\n", "\n", "# Split categories and determine how many parts we have\n", "split_categories = [cat.split('-') for cat in categories]\n", "max_parts = max(len(parts) for parts in split_categories)\n", "\n", "# Create labels for each part\n", "for i, parts in enumerate(split_categories):\n", " for j, part in enumerate(parts):\n", " ax.text(i, -0.1 - (j * 0.1), part, \n", " ha='center', va='top', rotation=0, fontsize=10, \n", " transform=ax.get_xaxis_transform())\n", "\n", "# Add horizontal lines to separate label rows (optional)\n", "for j in range(1, max_parts):\n", " ax.axhline(y=-0.05 - (j * 0.1), color='lightgray', linewidth=0.5, \n", " xmin=0, xmax=1, clip_on=False)\n", "\n", "# Save the figure with extra padding\n", "plt.savefig('temp_plot.png', dpi=300, bbox_inches='tight', pad_inches=0.5)\n", "plt.show()\n", "plt.close()\n", "\n", "# Open the saved image\n", "img = Image.open('temp_plot.png')\n", "\n", "# Calculate the height to keep (60% of the original height)\n", "new_height = int(img.height * 0.65)\n", "\n", "# Crop the image\n", "cropped_img = img.crop((0, 0, img.width, new_height))\n", "\n", "# Save the cropped image\n", "cropped_img.save('writing/65d4fadc59fceb1a54d1aae6/analysis/performance_by_category.png')\n", "\n", "print(\"Performance by Category plot has been generated, cropped, and saved.\")" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
SheetRowQ/GTGeminiGPTLLAMAClaudeground_truth_answeranswer_Gemini-1.5 Proanswer_GPT-4answer_GPT-3.5answer_Llama-3 70Banswer_Llama-3 8Banswer_Claude 3 Opus
41Researcher-Documents-insects523222The fall armyworm (FAW), specifically Spodopte...Answer:\\nFall armyworm (FAW) was first reporte...Answer:\\nThe fall armyworm was first reported ...The fall armyworm (FAW) was first reported out...Answer: Fall armyworm (FAW) was first reported...Answer:\\nThe fall armyworm (FAW) was first rep...Answer:\\nFall armyworm (Spodoptera frugiperda)...
42Farmer-Documents-insects233323The fall armyworm has spread to over 70 countr...Answer:Fall armyworm has affected more than 10...Answer:\\nOver 100 countries have been affected...The documents does not provide information abo...Answer: The fall armyworm has spread to over 7...Answer: Over 70 countries.\\n\\nReference: \\n\"Th...Answer:\\nFall armyworm has spread to over 70 c...
43Farmer-Documents-insects322222Not managing pests could lead to substantial e...Answer: If you don't manage pests like Fall ar...Answer:\\nThe potential economic impact of not ...The documents does not provide information abo...Answer: The potential economic impact of not m...Answer: The potential economic impact of not m...Answer:\\nThe potential economic impact of not ...
44Farmer-Documents-insects413333Fall armyworm can cause significant crop destr...Answer:\\nFall armyworm is a serious pest and c...Answer:\\nFall armyworm can cause more than 50%...The fall armyworm can drastically impact maize...Answer: Fall armyworm can cause more than 50% ...Answer: \\nFall armyworm can cause more than 50...Answer:\\nFall armyworm can cause more than 50%...
45Farmer-Documents-insects511111CABI estimates that the fall armyworm invasion...Answer:\\nThis bugger could cost us farmers aro...Answer:\\nIt could cost up to USD 9.4 billion a...Answer:\\nThe fall armyworm invasion could cost...Answer: The fall armyworm invasion could cost ...Answer:\\nAccording to the provided information...Answer:\\nThe fall armyworm invasion could cost...
\n", "
" ], "text/plain": [ " Sheet Row Q/GT Gemini GPT LLAMA Claude \\\n", "41 Researcher-Documents-insects 5 2 3 2 2 2 \n", "42 Farmer-Documents-insects 2 3 3 3 2 3 \n", "43 Farmer-Documents-insects 3 2 2 2 2 2 \n", "44 Farmer-Documents-insects 4 1 3 3 3 3 \n", "45 Farmer-Documents-insects 5 1 1 1 1 1 \n", "\n", " ground_truth_answer \\\n", "41 The fall armyworm (FAW), specifically Spodopte... \n", "42 The fall armyworm has spread to over 70 countr... \n", "43 Not managing pests could lead to substantial e... \n", "44 Fall armyworm can cause significant crop destr... \n", "45 CABI estimates that the fall armyworm invasion... \n", "\n", " answer_Gemini-1.5 Pro \\\n", "41 Answer:\\nFall armyworm (FAW) was first reporte... \n", "42 Answer:Fall armyworm has affected more than 10... \n", "43 Answer: If you don't manage pests like Fall ar... \n", "44 Answer:\\nFall armyworm is a serious pest and c... \n", "45 Answer:\\nThis bugger could cost us farmers aro... \n", "\n", " answer_GPT-4 \\\n", "41 Answer:\\nThe fall armyworm was first reported ... \n", "42 Answer:\\nOver 100 countries have been affected... \n", "43 Answer:\\nThe potential economic impact of not ... \n", "44 Answer:\\nFall armyworm can cause more than 50%... \n", "45 Answer:\\nIt could cost up to USD 9.4 billion a... \n", "\n", " answer_GPT-3.5 \\\n", "41 The fall armyworm (FAW) was first reported out... \n", "42 The documents does not provide information abo... \n", "43 The documents does not provide information abo... \n", "44 The fall armyworm can drastically impact maize... \n", "45 Answer:\\nThe fall armyworm invasion could cost... \n", "\n", " answer_Llama-3 70B \\\n", "41 Answer: Fall armyworm (FAW) was first reported... \n", "42 Answer: The fall armyworm has spread to over 7... \n", "43 Answer: The potential economic impact of not m... \n", "44 Answer: Fall armyworm can cause more than 50% ... \n", "45 Answer: The fall armyworm invasion could cost ... \n", "\n", " answer_Llama-3 8B \\\n", "41 Answer:\\nThe fall armyworm (FAW) was first rep... \n", "42 Answer: Over 70 countries.\\n\\nReference: \\n\"Th... \n", "43 Answer: The potential economic impact of not m... \n", "44 Answer: \\nFall armyworm can cause more than 50... \n", "45 Answer:\\nAccording to the provided information... \n", "\n", " answer_Claude 3 Opus \n", "41 Answer:\\nFall armyworm (Spodoptera frugiperda)... \n", "42 Answer:\\nFall armyworm has spread to over 70 c... \n", "43 Answer:\\nThe potential economic impact of not ... \n", "44 Answer:\\nFall armyworm can cause more than 50%... \n", "45 Answer:\\nThe fall armyworm invasion could cost... " ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df.tail()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Type 1 metrics" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Average scores by model:\n", " bleu_score mer_score ter_score\n", "Model \n", "answer_Claude 3 Opus 0.225 0.713 0.961\n", "answer_GPT-3.5 0.192 0.747 1.148\n", "answer_GPT-4 0.185 0.726 0.877\n", "answer_Gemini-1.5 Pro 0.111 0.810 0.844\n", "answer_Llama-3 70B 0.194 0.729 0.913\n", "answer_Llama-3 8B 0.175 0.758 0.914\n" ] } ], "source": [ "import pandas as pd\n", "from torchmetrics.text import BLEUScore, MatchErrorRate, TranslationEditRate\n", "from torchmetrics.text.bert import BERTScore\n", "\n", "def split_response(ret):\n", " if not isinstance(ret, str):\n", " return \"None\", \"None\"\n", " try:\n", " answer_start = ret.find(\"Answer:\")\n", " reference_start = ret.find(\"Reference:\")\n", " if answer_start != -1 and reference_start != -1:\n", " model_answer = ret[answer_start + len(\"Answer:\"):reference_start].strip()\n", " reference = ret[reference_start + len(\"Reference:\"):].strip()\n", " return model_answer, reference\n", " else:\n", " return ret, \"None\" # Return the full text as answer if markers are not found\n", " except:\n", " return \"None\", \"None\"\n", "\n", "\n", "\n", "# Initialize metrics\n", "\n", "bleu = BLEUScore(n_gram=4)\n", "mer = MatchErrorRate()\n", "# bertscore = BERTScore()\n", "ter = TranslationEditRate()\n", "\n", "# Function to calculate scores\n", "def calculate_scores(pred, target):\n", " if not isinstance(pred, str) or not isinstance(target, str):\n", " print(\"Incompatible row found for evaluation \")\n", " return {\n", " 'bleu_score': 0.0,\n", " 'mer_score': 1.0,\n", " 'bertscore_f1': 0.0,\n", " 'ter_score': 1.0\n", " }\n", " \n", " bleu_score = bleu([pred], [[target]]).item()\n", " return {\n", " 'bleu_score': bleu_score,\n", " 'mer_score': mer([pred], [target]).item(),\n", " # 'bertscore_f1': bertscore([pred], [target])['f1'].item(),\n", " 'ter_score': ter([pred], [[target]]).item()\n", " }\n", "\n", "# Create a list to store the results\n", "results = []\n", "\n", "# Iterate through the DataFrame\n", "for index, row in df.iterrows():\n", " ground_truth = row['ground_truth_answer']\n", " \n", " for model in [\"answer_Gemini-1.5 Pro\", \"answer_GPT-4\", \"answer_GPT-3.5\", \"answer_Llama-3 70B\", \"answer_Llama-3 8B\", \"answer_Claude 3 Opus\"]:\n", " prediction = row[model]\n", " just_answer, just_reference= split_response(prediction)\n", " scores = calculate_scores(just_answer, ground_truth)\n", " \n", " result = {\n", " 'Sheet': row.name[0] if isinstance(row.name, tuple) else row['Sheet'],\n", " 'Row': row.name[1] if isinstance(row.name, tuple) else index,\n", " 'Model': model,\n", " **scores\n", " }\n", " results.append(result)\n", "\n", "# Create a new DataFrame with the results\n", "results_df = pd.DataFrame(results)\n", "\n", "# Calculate average scores\n", "avg_scores = results_df.groupby('Model')[['bleu_score', 'mer_score',\n", "# 'bertscore_f1',\n", " 'ter_score']].mean()\n", "\n", "print(\"Average scores by model:\")\n", "print(avg_scores.round(3))\n", "\n", "# Save results to CSV\n", "results_df.to_csv('writing/65d4fadc59fceb1a54d1aae6/analysis/Type-1-All.csv', index=False)\n", "avg_scores.round(3).rename(index={\n", " \"answer_Gemini-1.5 Pro\": \"answer Gemini 1.5 Pro\",\n", " \"answer_GPT-4\": \"answer GPT 4\",\n", " \"answer_GPT-3.5\": \"answer GPT 3.5\",\n", " \"answer_Llama-3 70B\": \"answer Llama 3 70B\",\n", " \"answer_Llama-3 8B\": \"answer Llama 3 8B\",\n", " \"answer_Claude 3 Opus\": \"answer Claude 3 Opus\"\n", "}).to_csv(\"writing/65d4fadc59fceb1a54d1aae6/analysis/Type-1-Averaged.csv\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# RAG metrics" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "ename": "ModuleNotFoundError", "evalue": "No module named 'mlflow'", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", "Cell \u001b[0;32mIn[21], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m get_ipython()\u001b[38;5;241m.\u001b[39mrun_line_magic(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmatplotlib\u001b[39m\u001b[38;5;124m'\u001b[39m, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124minline\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m----> 2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mmlflow\u001b[39;00m\n\u001b[1;32m 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpprint\u001b[39;00m\n\u001b[1;32m 4\u001b[0m data_to_eval\u001b[38;5;241m=\u001b[39mall_experts_feedback[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mDr. Arti Singh\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n", "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'mlflow'" ] } ], "source": [ "%matplotlib inline\n", "import mlflow\n", "import pprint\n", "data_to_eval=all_experts_feedback[\"Dr. Arti Singh\"]\n", "with mlflow.start_run() as run:\n", " evaluate_results = mlflow.evaluate(\n", " data=data_to_eval,\n", " targets=\"source\",\n", " predictions=\"relevant_docs\",\n", " evaluators=\"default\",\n", " extra_metrics=[\n", " mlflow.metrics.precision_at_k(1),\n", " mlflow.metrics.precision_at_k(2),\n", " mlflow.metrics.precision_at_k(3),\n", " mlflow.metrics.recall_at_k(1),\n", " mlflow.metrics.recall_at_k(2),\n", " mlflow.metrics.recall_at_k(3),\n", " mlflow.metrics.ndcg_at_k(1),\n", " mlflow.metrics.ndcg_at_k(2),\n", " mlflow.metrics.ndcg_at_k(3),\n", " ],\n", " )\n", "\n", "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "\n", "# Prepare data\n", "metrics = [\"precision\", \"recall\", \"ndcg\"]\n", "k_values = [1, 2, 3]\n", "bar_width = 0.15\n", "opacity = 0.8\n", "\n", "# Create subplots\n", "fig, ax = plt.subplots(figsize=(5, 3))\n", "\n", "# Plotting each metric\n", "for i, metric_name in enumerate(metrics):\n", " y = [evaluate_results.metrics[f\"{metric_name}_at_{k}/mean\"] for k in k_values]\n", " x = np.arange(len(k_values)) + i * bar_width\n", " ax.bar(x, y, width=bar_width, alpha=opacity, label=f\"{metric_name}@k\")\n", "\n", "# Adding labels and title\n", "ax.set_xlabel(\"k\", fontsize=12)\n", "ax.set_ylabel(\"Metric Value\", fontsize=12)\n", "ax.set_title(\"Metrics Comparison at Different Ks\", fontsize=14)\n", "\n", "# Setting x-axis ticks\n", "ax.set_xticks(np.arange(len(k_values)) + bar_width)\n", "ax.set_xticklabels(k_values)\n", "\n", "# Add legend and adjust layout\n", "ax.legend(fontsize=8)\n", "fig.tight_layout()\n", "\n", "# Display the plot\n", "plt.show()\n", "pp = pprint.PrettyPrinter(indent=4)\n", "pp.pprint(evaluate_results.metrics)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# RAG Metrics without metadata" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Data from agllm-data/Responses V3 _no_metadata_filter.xlsx:\n", " Unnamed: 0 category About \\\n", "0 0 Identification and Control papaipema nebris \n", "1 1 Integrated Pest Management delia platura \n", "2 2 Identification and Control spodoptera frugiperda \n", "3 3 Identification and Control vanessa cardui \n", "4 4 Identification and Control helicoverpa zea \n", "\n", " question \\\n", "0 When should I begin scouting for black cutworm... \n", "1 What management practices can I use to minimiz... \n", "2 When should I start scouting for armyworm larv... \n", "3 What growth stage of soybean is most vulnerabl... \n", "4 At what level of defoliation or infestation sh... \n", "\n", " ground_truth_answer knowledge_vs_management \\\n", "0 Start scouting when 10% of migrating larvae ar... Management \n", "1 To minimize feeding injury risk in high-risk f... Management \n", "2 Begin scouting for armyworm larvae in mid-to-l... Management \n", "3 Soybean is most vulnerable to thistle caterpil... Management \n", "4 Consider applying insecticides when defoliatio... Management \n", "\n", " LLM Confidence chunk \\\n", "0 85 58\\nScouting. Scouting should begin when 10% m... \n", "1 85 up seed. The potential for injury is minimal i... \n", "2 85 35\\nScouting. In soybean and corn, grassy area... \n", "3 90 reach 1 1/2 inches long. Adult butterflies are... \n", "4 90 applications for corn earworm in Iowa soybean ... \n", "\n", " source \\\n", "0 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "1 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "2 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "3 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "4 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "\n", " relevant_docs \\\n", "0 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "1 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "2 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "3 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "4 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "\n", " relavent_context \\\n", "0 ['51°F) after eggs are laid. The best time to ... \n", "1 ['up seed. The potential for injury is minimal... \n", "2 ['35\\nScouting. In soybean and corn, grassy ar... \n", "3 ['Scouting. The best time to look for thistle ... \n", "4 ['applications for corn earworm in Iowa soybea... \n", "\n", " answer_Claude 3 Opus \n", "0 Answer:\\nScouting for black cutworm larvae in ... \n", "1 Answer:\\nTo minimize feeding injury risk in hi... \n", "2 Answer:\\nThe best time to scout for fall armyw... \n", "3 Answer:\\nSoybean is most vulnerable to thistle... \n", "4 Answer:\\nConsider applying insecticides for co... \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/Users/muhammadarbabarshad/miniconda3/envs/agllm-env1-updates-1/lib/python3.9/site-packages/mlflow/types/utils.py:407: UserWarning: Hint: Inferred schema contains integer column(s). Integer columns in Python cannot represent missing values. If your input data contains missing values at inference time, it will be encoded as floats and will cause a schema enforcement error. The best way to avoid this problem is to infer the model schema based on a realistic data sample (training dataset) that includes missing values. Alternatively, you can declare integer columns as doubles (float64) whenever these columns may have missing values. See `Handling Integers With Missing Values `_ for more details.\n", " warnings.warn(\n", "2024/09/02 18:44:52 INFO mlflow.models.evaluation.default_evaluator: Testing metrics on first row...\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAEcCAYAAAAFuId5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGIUlEQVR4nO3deVxN+f8H8NftokWLiBCSoSRFiyXlm4mMJcbWWCL7TjP2ClPWkLHVWAaNJcYwsjUlOzPMZJvsiezrhArdFnXv749+nXG1uDdd19Xr+Xh46J5z7ue8z7mfc973fM7nno9IJpPJQERERJ80LXUHQERERO/HhE1ERKQBmLCJiIg0ABM2ERGRBmDCJiIi0gBM2ERERBqACZuIiEgDMGETERFpACZsIiIiDaDxCdvd3R1WVlb4+eefC53//fffw8rKCqGhoQqXmZKSgh07dhS7jJ+fHwYMGKBUrMo6e/Ysxo4dCxcXFzRt2hSenp5Yt24dsrOzVbpeVYmMjISVlZW6wyiWTCbDrl278Pz58yKXsbKykvtna2sLDw8PLF68GBKJRFjuwYMHsLKyQlxcHAAgNTUVgwYNgq2tLXr16lXgtTq9r84X9dndvHkTrVq1Qq9evZCWlqay+NR1nL9PXFwcrKys8ODBA4Xfc+7cOZw9e7bE68yvV0X9S0xMLFD3BgwYAD8/v1KLAQBCQ0Ph7u7+QWW8z4ABA+S2zcbGBq6urpg8eXKBfe7u7q7U518cRT7Xd/dpSShbRrkPWtsnonz58oiNjcXgwYPlpufk5ODAgQMQiURKlbdo0SI8ePAAXl5eRS4zffp05ObmliheRWzevBkLFiyAj48PxowZA0NDQ5w/fx4LFy7EmTNnsGrVKmhpadb3rU6dOqF169bqDqNYZ86cgZ+fHw4fPlzscgEBAejUqRMAQCKR4OLFi1i4cCEuXLiA8PBwlC9fHjVq1MCff/4JIyMjAMDevXtx9uxZbN26FaampgVeq5Midf5dSUlJGDhwIOrUqYN169ZBX19fhRGq5zhXhX79+iE4OBhOTk4fVE5oaCjs7e0LTDc2NoZIJJKre6qIYciQIfD29i7x+xXVsWNHTJ8+HQCQlZWF+/fvY+nSpejTpw+2b9+OmjVrAgB+++03aGtrqzwedfosErazszP++OMPPHnyBNWrVxem//3339DT04Ourq5S5SnyeHUDAwOl41RUQkICFixYgKlTp2LgwIHC9Nq1a6NmzZro378/oqOj4enpqbIYVEFHRwc6OjrqDqNYij5a38DAAFWrVhVem5ubw8LCAr169cLu3bvh5eUFsVgst8zLly9RtWpV2NnZFfpanZQdUiApKQk+Pj6wsLDAmjVrULFiRRVF9h91HOefMiMjI7n69a7i5pWGihUrfpTPXUdHR25batWqhcaNG8PT0xNLlizB4sWLAQCVK1dWeSzqplmXaEWws7NDzZo1sX//frnp0dHR6NixY4Fv3ufPn4e3tzfs7OzQpk0bzJo1C69fvwaQ19S9a9cunD59WmgCHDBgAGbOnAkvLy84OTlh7969BZrE7969i9GjR8PR0REtWrTAxIkThWbV58+fw9fXFy1atICdnR369OmD06dPF7k9O3bsgIGBQaHfXps1a4YNGzbgf//7nzBt9+7d6Nq1K+zs7ODu7o6VK1cKV//5TWO///47unXrBltbW/To0QNJSUn48ccf0apVKzRv3hyzZs0STmChoaHo27cvfvzxR7Ro0QJOTk7w9/cX9hEAJCYmYuTIkWjWrBkaN26Mtm3bIjw8XJgfGhqK/v37Y8KECXBwcMCcOXMKNKseP34cPXr0QJMmTeDs7Aw/Pz+5ZtWkpCSMGjUKLVq0gKOjI3x9ffHw4UNh/oABA7B48WIEBATAyckJDg4OmDRpklyc7you7ri4OPj4+AAA2rZti8jIyCLLKUzjxo3h6OiIqKgouX0fFxcHPz8/hIaG4tGjR7CyshKa7/Jf56/r6NGj6NGjB+zs7ODh4YFly5bJ3QKxsrLCihUr8OWXX8LV1RV37txBdnY2QkJC0Lp1a9jb2+Obb77Bn3/+KbwnMjISHh4ewv+NGzdGjx49cO7cOQCF1/ni3Lp1CwMHDkSDBg2wdu1auZN2bm4uQkJC4ObmhsaNG6NDhw745ZdflNqPRVH1cZ6WloYZM2agdevWsLGxgbOzM2bMmIGMjAyhzLNnz8LLywt2dnbo2rUrEhIS5Nb5vjLy1+Xv7y80hZ49exY+Pj5wcHBA48aN0bFjR+zZs+eD9tW7TeJvKyyGp0+fYsKECXByckKLFi0watQo3LlzR3iPn58ffH19MWTIEDg4OGDt2rVyTeL564uNjYWXlxcaN24Md3d3/Prrr3Lr3rBhA9zd3WFnZ4fBgwcjLCysRM3qBgYG6NGjBw4ePCgcH283iWdkZGD69OlwcXGBra0tunXrhgMHDgjvV+SzBoAjR46gXbt2sLW1xYABAwp83m9LSkrC8OHDYW9vD1dXV0yaNAnJycnC/OzsbMyfPx/Ozs5wdHRESEgIpFKpUtv9WSRsIK/Z5O0DOTs7G4cOHULnzp3llktISMDgwYPRunVr7N27F4sXL8aVK1cwZMgQyGQyTJ8+HR07doS9vb3cSW/Hjh3w8fHB1q1bCzTrvnz5Et7e3sjOzsbGjRvx888/4969e/juu+8AAEFBQcjKykJERAT27dsHCwsLjBkzRu5+59suX74MOzs7lCtXeAOIs7MzDA0NAeQdADNnzkTv3r2xd+9efPvtt1i/fj0WLFgg956lS5ciICAAO3bswMuXL9G3b1/cuXMHmzdvxoQJE7B161YcPXpUWP7SpUv4888/ER4ejh9//BFnzpwRticjIwNDhgxBpUqVsG3bNkRFRaFDhw5YuHAhrl27JpRx5swZmJiYYM+ePQXu97948QLjxo1Dz549ER0djbCwMJw5cwaLFi0CADx8+BC9e/dGhQoVsHHjRoSHhyM5ORn9+/eXS8gbNmyAiYkJfvvtN4SEhODw4cPYsGFDofvtfXHb29sLB/yOHTuEJm9lWFpaFnpQT58+HUOGDEH16tXx559/Ys+ePXKvO3XqhBMnTuC7777DN998g6ioKAQGBiImJgZTpkyRK2vr1q1YsWIFwsLCULduXfj7++PkyZNYvHgxdu3ahY4dO2LUqFE4duyY8J7Hjx9j27ZtCAkJwa5du6Crqws/P79i63xhbt26BR8fH6SnpyM0NLTAVe3WrVuxf/9+LF26FLGxsejfvz+CgoI++H5pPlUe535+frh69SrCwsIQGxsLf39/7N69W0g69+/fx5AhQ2BtbY1du3Zh7NixWLlypdx631dG/roCAgIwffp0PH36FEOHDoWtrS127dqF3bt3w87ODtOnT8ezZ89KZZ+9690YJBKJcHxGRERg8+bNMDY2xjfffIOnT58K74uNjUWrVq2wc+fOIlv3goODMWrUKMTExKBNmzYICgrC/fv3AQBbtmzB0qVLMWbMGOzZswfNmzfHjz/+WOLtsLS0RGZmptwXi3zLly/H9evX8dNPPyE6Ohr/+9//MGHCBOGe9Ps+p3zh4eEIDAzEzp07UbFiRQwbNqxAUgfyvvD069cP5ubm+O2337B69Wq8fv0avXv3Fs7zc+fORXR0NBYsWIBt27bhyZMnSh8Xn0WTOJB3IK9fvx5Pnz6FqakpTp48icqVK6NRo0Zyy61fvx4uLi4YNWoUAKBu3br44Ycf0K5dO5w+fRotWrSAjo4OypcvL9cMY21tjS5duhS67ujoaKSnp2PJkiXCPaO5c+fi999/R3Z2Nu7duwdLS0vUrl0bOjo6mD59Orp06QKxWFxoeampqahdu/Z7t1kmk2Ht2rXo37+/cDVet25dpKamIiQkBL6+vsKyQ4YMQfPmzQEAHh4e2Lx5M2bPng1dXV188cUXCA0NxY0bN4RvuyKRCMuWLRPurX7//fcYPnw4bt26hUqVKsHHxwfe3t7C1ZWvry/WrVuH69evw9raWlivr6+vcPvg/PnzwvSnT58iOzsbNWvWhJmZGczMzLB69WqhZWDr1q3Q09PD4sWLUaFCBQDAihUr0LZtW+zZs0fY3vr162PixInCtru4uOCff/4pdH9lZGS8N+78z69y5colar43NDQs9ArfwMAAenp6cs3k775evXo1vvnmG/Tp0wcAUKdOHcyaNQsDBw7EgwcPUKtWLQDA119/DVtbWwB5LTtRUVHYvXu3sN8HDx6MhIQErF+/Hm3atAEAvHnzBrNmzZJbZuzYsUhOTka1atUKrfOFGThwICwsLPDPP/8gNDQUAQEBcvPv3bsHPT091KpVC9WqVUP//v1Rr149WFhYKL0vC6PK49zFxQXNmjUTrkBr1aqFiIgIJCYmAgC2b98OExMTBAYGQiwW44svvsDjx48RHBwsrPd9ZeSvy8DAAAYGBkhJScH48eMxdOhQoYVgxIgR2L17N+7cuQMTE5Mi98Xw4cMLnENmzZqFrl27FrsP340h/0t8SEiIcJEwb948xMXFYfv27Rg/fjyAvCb4YcOGFVv2oEGD0LZtWwDAhAkTsGXLFly4cAG1a9fG+vXr4ePjI3SwHD16NK5cuYKrV68WW2ZR8i9aXr16VWDevXv3ULFiRdSuXRuGhob49ttv0axZM+H4ft/nlG/mzJnCBdqiRYvg5uaGqKioAv0efvnlF1SvXh0zZswQpi1btgwtW7bE/v370b59e0RGRiIwMBBubm4AgPnz5+Pvv/9Waps/m4TduHFj1K5dG7GxsfDx8UF0dHSBb90AcPXqVdy9e7fQzhpJSUlo0aJFoeWbm5sXue7ExETUrVtXroNHw4YN0bBhQwDAuHHjMGXKFMTGxsLR0RGurq7w9PQssoNE5cqVkZqaWtzmAsi7Sn327BkcHR3lpjdv3hxv3rzBrVu3UKVKlQLx6+npwcTERO7qSEdHR67ptW7dunIdoRwcHIRt7dChA/r164eoqChcvXoV9+7dE64q327iqVKlSpH3+q2treHp6YlRo0ahatWqcHFxQZs2beDh4SGsp3HjxkKyBvJONBYWFnIHVb169eTKNTAwwMuXLwtdZ+XKlRWK+0O8evWqxP0brl69iosXL+K3334TpuXfpkhKShIS9tufZf7Jrl+/fnJlvXnzRjih5fviiy+Ev/NjfPPmjVIxNmnSBEuWLMGGDRuwZMkStGrVSvhSAADe3t44dOgQ3NzcYG1tDRcXF3Tu3Fmoh+/q3LkzHj16JLxeu3ZtsR2hVHmc9+vXD0eOHMGuXbtw584d3Lx5Ew8ePBDqWGJiIho1aiSXJPOPC0XLeFedOnXQo0cPbNq0CYmJiXJ18n2dWufOnYsmTZrITStqPxfn6tWrSEtLQ7NmzeSmZ2VlISkpSXhd3DkwX1F1LCUlBQ8fPkTTpk3llndycipxws5P1O/WcyDvy8yoUaPg7OwMOzs7uLi4oEuXLkJMin5Ob59bDQ0NUbdu3QJJHcjbhzdu3ChQ3/L34e3bt/HmzRvhizYAaGtrF/ii+T6fTcIG/msu6927Nw4fPlzoTzakUim6dOkifPN+W3GdFoq72iqq6Tqfh4cH/vjjD/zxxx84deoUfv75Z4SFhWH79u1o0KBBgeXt7e3x22+/ITc3t9Cr8MmTJ8PBwQHt27cvdH35yeftuN6N8X09zMuXLy/3Ov/kIRaLkZycjN69e6Ny5cpwd3eHq6srbG1thW+O+d53hfrDDz9g7NixOHHiBE6dOoUpU6bA0dERGzduLLJDkFQqlYvt7YT+PorG/SGuXLmi9EGYTyqVYtiwYejevXuBeW9f+b69X/P305YtWwp0AHr3My5sXynb8WrZsmUoV64chg0bhuPHj8PPzw979uwRvtzVrVsXBw4cwOnTp3Hy5EkcO3YMa9euRXBwcKHb9dNPPyEnJ0d4rUhveVUc51KpFCNHjsSNGzfg6emJTp06wcbGBjNnzhSWEYlEBb7YvX1cKVLGu27evIl+/frBxsYGrVq1Qvv27WFsbKxQz3VTU1OFkuj7SKVSWFhYYNWqVQXm6enpCX8r0uJUVB3L30+l2dHvypUr0NPTQ926dQvMs7e3x/Hjx3Hy5En89ddf2L17N1atWoV169ahRYsWCn9O755/c3NzC91GqVSKli1bIjAwsMA8AwMD4Uvpu9v/vtzxrs/mHjaQdyCfP38eO3fuRO3ateW+7eVr0KABbt68CXNzc+FfTk4OgoOD8fjxYwBQ+uch9evXx507d+SaZq5cuQJnZ2c8fPgQwcHBuH//Pjp16oS5c+fi0KFD0NLSkrvH+LaePXsiPT0dERERBebFxcVh37590NfXh4mJCUxMTITOQ/nOnj2L8uXLo06dOkptx9tu374ttz35zcyNGjVCVFQUUlNT8csvv2DMmDHw8PAQOospekBeuHAB8+fPR7169TBo0CD89NNPQhPR8+fPYWVlhUuXLsld9T979gx3794t9HNVhCJxK/vZv+3y5cuIj48v8tbJ+zRo0AC3b9+Wq5tPnjzBokWLkJ6eXuR7gLwvI2+/LzIyUqlOc4pud/4JRktLCwsXLsSbN28wZcoUIZFt2rQJBw4cgIuLC6ZOnYp9+/bB2dkZ0dHRhZZnZmYmF7ciSUEVx/m1a9dw4sQJLF++HJMnT0bXrl1Rp04d3Lt3T6gbDRs2xOXLl+Xq5OXLl5Uq413btm1DlSpV8PPPP2P48OFwc3MT7l1/rF7slpaWePToEQwMDIR9VbNmTfzwww84c+ZMqazDwMAAZmZmiI+Pl5v+7mtFvX79Grt370aHDh0KXFwAebfPzp07h7Zt22LGjBmIjY0VWmaU+Zze/nxfvHiBO3fuFHqR1aBBAyQlJaFGjRrCPjQyMsL8+fORmJgICwsLaGtry90WzMnJKbYTW2E+q4RtbW0Nc3Nz/PDDD4U2kwF593KvXr2KWbNmISkpCf/88w8mTZqEO3fuCN/U9PT08O+//wqdJd6nS5cuMDIywpQpU5CQkIDLly8jMDAQlpaWMDMzw6VLlzBz5kzEx8fjwYMHiIyMhEQiKbS5DshrVvr222+xYMECLFq0CAkJCbh9+za2bt2K8ePHw8PDQ9i+oUOHIiIiAlu3bsXdu3exb98+hIWFoXfv3h/00zOJRIKpU6ciMTERp06dwuzZs9GpUyeYmZmhevXqyMjIwP79+/Ho0SP8+eefwn1kRR/qoq+vj61btyIkJAR3795FYmIioqOjUbduXRgbG6Nv375IT08X9unFixfx7bffwtjYuMjP9n0UiTv/iiIhIaHIJAnkNcclJycjOTlZ2O/jxo1DixYt3nsPsSjDhw9HbGwswsLCcPv2bfz111/w9/fHq1eviry33KBBA3z55ZcIDAzEkSNHcP/+faxduxZr1qxR6gubsnUeyLvvN336dMTFxWHNmjUA8k5qs2fPxuHDh/Hw4UP88ccfQoe+0qKK49zExATlypVDTEwM7t+/j0uXLuG7775DcnKyUDf69u2LjIwMBAQEICkpCUePHpV7UIciZeSvNykpCSkpKahevTqePHmC48eP4+HDhzhw4ACCgoIAKH4slcTbMXTt2hVGRkbw9fXFhQsXkJSUBD8/P5w4caJUH3Q0fPhwREREIDIyEnfv3sX69esRGxv73vdlZmYKx1r+cTtixAjIZDKhI+y77t+/j8DAQPz11194+PAhYmNj8ejRI9jb2yv8OQF5fXf++usvXLt2DRMmTECNGjUK7Yzar18/vHr1CpMnT0ZCQgISEhIwYcIEXLp0CZaWlqhYsSL69++PFStW4MCBA0hKSkJgYKBcpz5FfFZN4kDet+9Vq1YV2cO3adOmWLduHZYvX47u3btDT08Pzs7OmDZtmtDU0a1bNxw8eBCenp5yPwUoiq6uLtavX4/g4GD06dMHOjo6aNOmDaZNmwYgr4d2cHAwRo8ejVevXqFevXpYvHhxsffqRowYgXr16mHz5s2IjIxEZmYmateujTFjxqBfv35CU82QIUOEntTz589H9erVMXz4cAwdOlTZXSenRo0asLa2hre3N8RiMbp06YLJkycDADp06IArV65gwYIFeP36NczMzODl5YXDhw/j0qVL6Nu373vLz+/oFhYWhq1bt0JLSwstW7bE2rVroaWlJXQCCQkJEXqLu7i4ICQkpNB7VopQJG5LS0u4ubnhu+++w8SJEzFkyJBCy5o/fz7mz58PIK8Z0NzcHN7e3vDx8SmyM6Ei8S1duhRr1qzB6tWrUalSJbi7uwv7vShLly7F0qVL8f333yMtLQ116tTBvHnzCm2CLsq7dV7RB7n06NEDR48eRVhYGFq0aIFx48bhzZs3mDt3LpKTk1G1alX07dsXI0eOVDgWRajiOF+wYAFCQ0OxZcsWVK1aFW3atMGgQYNw5MgRAHlN0PnHWffu3VGjRg2MHj0as2bNEua/rwwg75hdt24dkpKSsGLFCty6dQtTp05FdnY26tati4kTJ2LFihW4dOmS3M83S9PbMaxevRoRERFYtGgRhg4ditzcXNjY2CA8PLzErVmF6du3L9LS0rBs2TKkpKSgefPm6N69e4EWwnfFxMQgJiYGQF4LT9WqVdGuXTssWbKkyHoaGBiIhQsXYsqUKUhNTYWZmRkmT56Mr7/+GgAU+pwAYMyYMfD398eLFy/QokULrFu3rtAm8dq1ayMiIgI//PAD+vbtC7FYDAcHB2zatEm4BTNp0iRoa2tj9uzZSE9PR8eOHZX+SZtIpulPD6BSFxoail27dhWovEREJXXixAnUr19feDIZkNcL+969e9i4caMaI9Mcn1WTOBERfZr27NmDMWPGID4+Hg8fPsTu3buxd+9e4aqX3u+zaxInIqJPz8yZM7FgwQKMHTsWL1++hLm5OQICAtCjRw91h6Yx2CRORESkAdgkTkREpAGYsImIiDQAEzYREZEG+Gw7neXk5CAtLQ3a2trvfQwnERGRKkilUmRlZcHIyEjpR5G+67NN2GlpaYUOu0ZERPSx1a1bt0SDs7zts03Y+SNh1a1bt8CYvZosNzcXiYmJsLS0LPETtYgUwbpGH9PnWt8yMjJw586dIkdnVMZnm7Dzm8F1dXXlRpzRdPmjZuWPpUykKqxr9DF97vWtNG7N8uYuERGRBmDCJiIi0gBM2ERERBpA7Qk7KysLAQEBcHJygqurK8LDw4tc9uDBg+jYsSPs7e3Rt29fXLly5SNGSkREpD5qT9iLFi3C5cuXsXHjRgQGBiIsLAz79+8vsNyNGzcwadIkjBw5Env27IG1tTVGjhyJjIwMNURNRET0cak1YUskEuzYsQPTp0+HjY0NPDw8MGzYMGzZsqXAsidPnkT9+vXRrVs31KlTBxMnTkRycjJu3ryphsiJiIg+LrUm7ISEBOTk5MDe3l6Y5ujoiAsXLkAqlcotW6lSJdy8eRPnzp2DVCpFZGQk9PX1UadOnY8dNhER0Uen1t9hJycnw9jYGBUqVBCmmZiYICsrC6mpqahcubIwvVOnTjhy5Aj69esHsVgMLS0trFmzBkZGRuoI/ZO3fPlyNG7cGG3bti3RfEVcvXoV69evR1JSEmQyGaRSKezt7TF27FiYmpoCAEJDQwEA48ePL/F6iEhDrXFTeFEtAA0lEmidVeK5GSOPKx+TBlNrws7IyJBL1gCE19nZ2XLTU1JSkJycjO+//x5NmjTBL7/8An9/f+zatavYx73l5uYKP8gvytc/nirhFhRvz9hWpV5m/ra8b5vGjRtX7HLvm/8+MTExiIiIwOTJk4UWktzcXBw9ehTDhw/HsmXLYG5uLrSUlHQ9pD6K1jWioijThCuTyf77XyRS6D1SDaibpXn8qDVha2trF0jM+a91dHTkpi9evBiWlpbw9vYGAMyZMwcdO3bEzp07MWLEiCLXkZiY+N44MjIkyoaukPj4eIWWu3r1Knbs2IEKFSrg2bNnsLCwwIgRIzBu3Dh88cUXePHiBebMmYMDBw7g5MmTkMlkaNiwIQYMGACxWIz9+/fj4MGD0NLSQuPGjeHj44M1a9bA2toazs7OWLlyJZ4+fQqRSIS2bduibdu2WL16NaytreHm5oZjx44hOjoaIpEIFhYWGDRoEHR0dDBq1Ci4uLgIty5GjhyJevXqITk5GWvXroW/vz/i4+Ph5+cHLS0tNGvWDNeuXcPQoUPh5+eHadOm4cmTJwCA8+fPY9WqVdDV1cXgwYMhUvCAJPW7dOmSukMgDdVQovy5VZmOxAkKnmM/F2pN2KampkhJSUFOTo4wiklycjJ0dHRgaGgot+yVK1cwYMAA4bWWlhYaNmyIR48eFbsOS0vL9z6aVPekaq6wmzZtqtBy2dnZuH37Nnbu3Il69ephwoQJuHTpEl6/fo3x48ejVatWOHnyJJKTk7F7925cvXoVkZGRSEhIQNOmTXH06FFERkbCwMAA48aNw5s3b1C5cmXUqVMHr169gpGRETZs2IDU1FQEBwejadOmwnw9PT3ExMTg119/hbGxMebMmYM//vgDU6ZMwcuXL/HVV1/hhx9+wMaNG3H8+HH06NEDixcvhp+fH0xMTLBt2zZERETAyMgIvXr1gpubGzw9PbF7925YWVmhevXqAIA9e/agdu3amDlzJpO1hsjNzcWlS5dga2v7WT4qklRPmeZtmUyGjIwM6OrqKnyOUPQcq04SiUShC0dFqDVhW1tbo1y5coiPj4eTkxMA4Ny5c7C1tS3w3NVq1aohKSlJbtrt27dha2tb7DrEYvF7TzaqSiCKnuS0tLTg4OAAS0tLAEC3bt2wfft2AICTkxPEYjH++usvXLp0CX369EFmZia0tLRQoUIF5OTkwN3dXbjfv2bNGgB5CVJLSwtNmjRBcHAwRowYATc3N/j5+UEsFkMkEkFLSwvnzp3Dl19+CRMTEwBAnz594O/vL8Tu7u4OsViMRo0a4ejRoxCLxbh8+TKmTp2KefPmoU+fPkJStrS0FE7uBgYGQpzbtm1DWloaDh48+MHDy9HHp8gxRPTB/v88LBKJoOgZWRPqZWnGqNZe4rq6uujWrRuCgoJw8eJFHDp0COHh4fDx8QGQd7WdmZkJAPjmm2+wfft27N69G3fv3sXixYvx6NEjdO/eXZ2bUGreTmQymUxu8BIg72pn0KBB2LVrF4KDg7Ft2zZMnjxZSL75kpOTkZqaKrw2NTVFTEwM+vXrh1u3bqF79+54+fKlMP/d3vgymQw5OTnC6/wRZt5eh1QqhUgkwu3bt9GgQQMhvkuXLqFx48bIyMjAo0ePhC8BTZo0wXfffYdZs2Z90D4iIirL1P7gFH9/f9jY2GDgwIGYNWsWxo8fj/bt2wMAXF1dER0dDSCvl/jMmTOxZs0adOvWDefPn8fGjRs/eHzRT8X58+fx+PFjSKVS7N69G66urnLzW7ZsiT179iA9PR1SqRSTJk3Czp070axZM5w4cQKvX7+GVCpFQEAA4uLihPft27cPQUFBaNu2LWbMmAE9PT08fvxYmN+8eXMcPnwYL168AABs374dzZo1KzZWPT09PH/+HBYWFvj777+Rm5uLFStWIDk5GSYmJggICMCQIUOE5a2trTF48GA8efJE+DyJiEg5am+f1NXVxcKFC7Fw4cIC865fvy732svLC15eXh8rtI+qevXq8Pf3x+PHj+Hs7IzevXtj9uzZwnx3d3dcv34dffr0gUQiQevWreHt7Y1y5crBx8cHffv2hVQqxf/+9z989dVXOHr0KACgffv2OHz4MDp37ozy5cvjq6++gpWVlVBuw4YNMXr0aAwcOBBv3rxBo0aN3nsl3KVLF6xcuRIjR47EqFGj4ObmhuHDh8PQ0BCenp4YM2YMnJ2d5TqPiMVizJ49W5hnbGxcynuQiOjzJpLl96X/zEgkEly7dg3W1taf/HjYcXFxCAsLw+bNm9+7bG5uLuLj49G0aVO13b+RyWQICAgAAIwYMQIWFhbCvLS0NOzatQuHDx/GqlWroK+vr5YY6cN9CnWNNJwSv8OWIe+8raenp/A9bE34HXZp5iK1X2GT5hGJRAgODsbBgwcxf/58PHnyBFpaWpBKpdDX10enTp0QHh6O8uXLqztUIqLPBhP2J6BFixZo0aKFusNQmoeHBzw8PNQdBhFRmaD2TmdERET0fkzYREREGoAJm4iISAMwYRMREWkAJmwiIiINwF7igFK/FVSKGn8jOGDAAGEIzXd/481xrImINA8TdhkTHR2NTZs2YerUqXBwcACQ94CMI0eOYPjw4QgNDYW5ubmaoyQioncxYX8C4uLisGjRIshkMtSoUQOGhoZISEhAbm4uBgwYAC8vL2RnZ2POnDmIi4tDTk4OxowZg169eiEmJgY///wzMjMzkZmZidmzZ6Nly5aFrufhw4fYunUrNm3ahFOnTqFLly4Qi8Vo164dzpw5gx9++AFz5szBunXrhPdIpVJMnToV+vr6CAwM5NCYRERqwoT9ibh9+zaOHj2K9evXw9jYGMHBwZBIJPD29oatrS3++usvpKWlISoqCqdOncLChQvx1Vdf4ZdffsGqVatQpUoV7Ny5Exs2bCgyYW/ZsgXjx4/HgwcPMG/ePGzduhVGRkbo0aMH3Nzc0KBBA0ilUrlngAcGBjJZExF9ApiwPxH16tWDkZER/vzzT2RkZGD37t0AgNevX+P69euIi4tDr169hLGmo6KiIBaLsXLlShw5cgS3b9/G6dOnC4wj/rZLly5hypQpmDdvHnr37o2qVasCABo0aIDGjRsDAPT19ZGeng4AcuNYM1kTEakXE/YnIn/ca6lUisWLF8PGxgYA8Pz5cxgYGODAgQNySfPevXuoVq0aevbsia5du6JZs2awsrLCli1bilzH2+NYt27dGsB/41hPnDix0HGsHRwcMGvWLKxevVpVm04fm4KdLLUANJRIoHVWiQELNGAwBiJNxYT9iWnZsiV++eUXzJkzBykpKejevTvWrFmD5s2bIyYmBm5ubnj9+jUmTpyIpUuXQiQSYfTo0QCAGTNmIDc3t8iy3x3H2tXVVaFxrKOiohAdHY1OnTqpfPuJiBTVO6q3ysr+1fNXlZVdUvwd9idm3LhxyMrKQpcuXdCvXz+MHj0a1tbW6Nu3LypWrIju3btj9uzZmDBhApo0aYJGjRqhY8eO6N69O/T09PDw4UMUNWJq/jjWI0aMQFxcHNzc3FC5cmVhHGtXV9cix7GeP38+UlJSPtZuICKid3A8bA3zIWMUcxxrAqBwk/jnOj4xfUQqHg+7t1n1EoWliNK6wuZ42FQiHMeaiEhzMWGXQRzHmohI8/AeNhERkQZgwiYiItIATNhEREQagAmbiIhIA7DTmYaIjIzE6dOnMW/evBK9XyaTITIyErt27UJ6ejqkUikqVKiArl27wtvbW3ikqbu7OzZt2oRatWqVZvhERPSBmLChuqflfCpPysnNzcXEiRNRvXp1LFmyBNWqVQMAvHjxAj///DPGjx+PsLAwPi+cPlhZe/IU0cfEhP0JiIuLw6pVq2BgYICkpCQhsR47dkx4iImZmZnwo/srV65g1qxZkMlkqFatGhYvXgx9fX1h+E1jY2OYmJjA3d0dPXr0wMaNG2Fra4tBgwZh0aJF+OOPP1CzZk0YGhrCzS3vwQb79u1D165dhZju37+PYcOGISgoCM7OzmrZL0RE9B/ew/5EXLhwAf7+/oiOjoa2tjYiIiIQEhKCiIgI/Prrr8IIWtnZ2QgLC8PcuXOxb98+ODk5Ydu2bfj111+RkpKC6OhorFy5EleuXAGQN+DHgQMHMHToUKxatQoSiQQxMTEYM2YMoqOjYWtri/79++PIkSNCLE+ePMHo0aOZrImIPiG8wv5ENGjQADVr1gSQN+gGANjb2wtDYHbp0gV///03bty4gUqVKgmjeeUP/DFq1Ch4eXlBLBajcuXKwoNRbt++DSsrK0ilUkRERCA6OhoA0LhxY1SsWBEWFhbIzMzE69evhVi+/fZbNGzYkMmaiOgTwivsT4S2trbwt0gkQmxsrNwgHvmPCxWLxXL3mtPT0/Hw4UOIxeJCB/149eoVKlasiBcvXkAsFqNKlSoAgPj4eDRq1AhaWlo4fvw4mjZtKrxn+vTpeP78OQ4cOFDam0lERCXEhP2JcnBwQHx8PB4/fgypVIqYmBgAgIWFBV69eoUbN24AACIiIhAeHg4XFxfs3bsXUqkUL1++xJEjRyASiWBmZoakpCQYGxtDJpMhISEBz549Q3BwMOrXr4/r169j/fr18Pb2FtZtZ2eH2bNnY+7cuXj58qVatp+IiOSxSfwTZWJigpkzZ2Lw4MHQ09ND/fr1AeRdiY8dOxYzZsxAdnY2atSogUWLFkFPTw83btxA165dYWhoiOrVq0NbWxtVq1ZFTk4Orl69ipkzZ2LQoEGoXbs2PDw8sH79eiQlJWHRokXIyMiAsbGxsH47Ozu0a9cOCxYswPz589W1G4iI6P9xeE0NU9TwmsePH0d2djY8PDzw5s0beHt7Y968eWjQoAEePHgAX19feHl5wdPTEwYGBsL7bty4gdDQUDRu3BgjRoxQxybRx6bC4TU1YbhD+og4vCaH16SCvvjiC0ybNg1hYWGQSqXo1asXGjRoAACoVasWNm3ahA0bNmDIkCHIzMyESCSCTCaDhYUFfHx84OTkpOYtICKi4jBhfyZq1aqFLVu2FDlfX18f48aNw7hx4z5iVEREVFrY6YyIiEgDMGETERFpACZsIiIiDaD2hJ2VlYWAgAA4OTnB1dUV4eHhRS57/fp19O3bF3Z2dsKTv4iIiMoCtSfsRYsW4fLly9i4cSMCAwMRFhaG/fv3F1ju1atXGDJkCOrXr499+/bBw8MD48aNw/Pnz9UQNRER0cel1oQtkUiwY8cOTJ8+HTY2NvDw8MCwYcMK7e28a9cu6OnpISgoCObm5vD19YW5uTkuX76shsiJiIg+LrX+rCshIQE5OTmwt7cXpjk6OmL16tWQSqXQ0vrv+8Tp06fRtm1buYeF7Ny586PGS0REpC4lTthJSUk4efIk/v33XwwYMAD3799Hw4YNoa+vr3AZycnJMDY2RoUKFYRpJiYmyMrKQmpqKipXrixMv3//Puzs7DBz5kwcOXIEZmZmmDZtGhwdHYtdR25uLnJzc5XfwE9U/rZ8TttEH5eizWr5D0GUyWSASMFnT6nwuYms85pHmSbcz7W+lWa9VTphS6VSfP/999i5cydkMhlEIhE6duyIlStX4t69e4iIiED16oo9Li4jI0MuWQMQXmdnZ8tNl0gk+Omnn+Dj44O1a9fi999/x9ChQxETE4MaNWoUuY7ExEQlt1AzXLp0Sd0hkIZqKJEotXxGRobCy0qULFsZ8fHxKiubVEPZugawvhVH6YS9cuVK7Nu3D3PnzkWbNm3g4uICAJgyZQrGjh2LpUuXYuHChQqVpa2tXSAx57/W0dGRmy4Wi2FtbQ1fX18AQKNGjXDy5Ens2bMHo0aNKnIdlpaWpfIs8a9/PPXBZRRln/YMhZeVyWTIyMiArq6u3DCbxelTU3XP293aaavKyibV0Dqr2PFQkrqmyuf2vz0ELGkGResa8PnWN4lEUmoXjkon7J07d8LX1xc9e/aUu9TPT6aLFy9WuCxTU1OkpKQgJycH5crlhZKcnAwdHR0YGhrKLVu1alXUq1dPblrdunXx+PHjYtchFovl7nuXlKIVqERlK7Ww6P//Eyn+PtWFXir7lj5RrGv0MX2m9a00663SvcSfPXsGa2vrQueZmpoqNX6ytbU1ypUrJ9f0cO7cOdja2sp1OAPyvu1cv35dbtqtW7dgZmamePBEREQaSumEbW5ujuPHjxc67/Tp0zA3N1e4LF1dXXTr1g1BQUG4ePEiDh06hPDwcPj4+ADIu9rOzMwEAPTp0wfXr19HaGgo7t69i+XLl+P+/fv4+uuvld0EIiIijaN0wh44cCA2bdqE2bNn49SpUxCJRLh79y7Cw8MRHh6Ofv36KVWev78/bGxsMHDgQMyaNQvjx49H+/btAQCurq6Ijo4GAJiZmWHdunU4evQoPD09cfToUfz0008wNTVVdhOIiIg0jtL3sL28vPDixQusWrUKv/zyC2QyGSZOnIjy5ctj2LBh6Nu3r1Ll6erqYuHChYV2VHu3CdzR0RGRkZHKhkxERKTxSvQ77JEjR8Lb2xvnz59HWloaDA0N0aRJE1SqVKmUwyMiIiLgAx6coq+vj//973+lGQsREREVQemEnd8hrDibNm0qUTBERERUOKUTdv7j494mkUiQlJQEPT09ocMYERERlR6lE/bmzZsLnZ6Wlobhw4cXeLgJERERfbhSG17TyMgII0aMwIYNG0qrSCIiIvp/pT4e9vPnz0u7SCIiojJP6SbxM2fOFJiWm5uLJ0+eYOXKlbCxsSmVwIiIiOg/SifsAQMGFDoQhkwmQ40aNRAQEFAqgREREdF/lE7Yhf1kSyQSQV9fH1ZWVgUG7SAiIqIPp3TCbt68uSriICIiomIolLD9/f0VLlAkEmH+/PklDoiIiIgKUihhx8XFKVxgYfe3iYiI6MMolLCPHDmi6jiIiIioGKXaQ0wikeDEiROlWSQRERGhBJ3OHj58iKCgIJw+fRrZ2dmFLnPt2rUPDoyIiIj+o3TCDg4Oxvnz5+Hl5YXz589DV1cXTZs2xcmTJ5GYmIjQ0FBVxElERFSmKd0kfubMGUyYMAEzZsxAjx49oK2tjSlTpmDnzp1o1qwZDh8+rIo4iYiIyjSlE3Z6ejqsrKwAAPXq1cPVq1cBAGKxGP369cPff/9duhESERGR8gm7WrVqePbsGQDA3NwcaWlpSE5OBgBUqlSJg38QERGpgNIJ283NDcuWLcM///wDMzMzVK9eHeHh4Xj9+jV27twJU1NTVcRJRERUpimdsH19fWFoaIjly5cDACZMmICNGzeiWbNm2LdvHwYPHlzqQRIREZV1CvUSHzBgALy8vPDVV1/B2NgYO3bswL///gsA6Nq1K2rWrIn4+HjY2dnxWeNEREQqoFDCTk1NxdSpUzFnzhx4enrCy8sLjRo1EuY7OTnByclJZUESERGVdQo1ie/btw87d+7E119/jdjYWPTs2RPdunXDli1b8PLlS1XHSEREVOYpfA/bxsYGM2bMwIkTJxAWFobatWtjwYIFaN26NSZNmsSfcxEREamQ0k86K1euHNq2bYu2bdsiLS0NUVFR2Lt3LwYNGoTatWujZ8+eGDVqlCpiJSIiKrM+aPAPIyMjeHt749dff8XmzZshFouF3uNERERUepS+wn5bcnIyfv/9d0RFReHKlSuoUaMGxowZU1qxERER0f9TOmGnp6fjwIED2LdvH+Li4iAWi9GuXTtMmDABrVq1gkgkUkWcREREZZpCCTsnJwfHjx/Hvn37cOzYMWRmZsLa2hr+/v7o0qULjIyMVB0nERFRmaZQwnZxccHLly9haGiInj17omfPnnK/wyYiIiLVUihh29jYoGfPnvDw8ECFChVUHRMRERG9Q6GEHR4eruo4iIiIqBgf9LMuIiIi+jiYsImIiDSA2hN2VlYWAgIC4OTkBFdXV4Wa3x88eAB7e3vExcV9hAiJiIjUr8QPTsnNzYVYLAYAZGZm4s2bNzAwMFC6nEWLFuHy5cvYuHEjHj16hGnTpqFmzZro0KFDke8JCgqCRCIpaehEREQaR+kr7Ddv3iAwMBDffPONMO38+fNwdnbGwoULIZVKFS5LIpFgx44dmD59OmxsbODh4YFhw4Zhy5YtRb5n7969SE9PVzZsIiIijaZ0wg4NDcXevXvh6ekpTGvUqBEmT56M7du3Y926dQqXlZCQgJycHNjb2wvTHB0dceHChUITf0pKCkJCQjB79mxlwyYiItJoSifsffv2Ydq0aRg8eLAwrVKlShg0aBAmTJiA3377TeGykpOTYWxsLPfbbhMTE2RlZSE1NbXA8gsWLED37t3RoEEDZcMmIiLSaErfw05JSUHt2rULnVevXj08efJE4bIyMjIKPIgl/3V2drbc9FOnTuHcuXOIiopSKt7c3Fzk5uYq9Z7CyGSyDy6jyLJLEIdMJgMUfW676kIvlX1LH5ei39JZ1+hDKXNF+LnWt9Kst0on7Hr16iE2NhYuLi4F5h05cgTm5uYKl6WtrV0gMee/1tHREaZlZmbi+++/R2BgoNx0RSQmJiq1fFEyMlTXyU2Sq3zZGRkZipevwg568fHxKiubVKOhkvWBdY1KStm6BrC+FUfphO3j4wM/Pz+kpqaiXbt2qFKlCl68eIGjR48iJiYGwcHBCpdlamqKlJQU5OTkoFy5vFCSk5Oho6MDQ0NDYbmLFy/i/v378PX1lXv/8OHD0a1bt2LvaVtaWkJPT0/JrSxI9+SpDy6jKHraiscnk8mQkZEBXV1dhUdGK43tL0rTpk1VVjaphtZZxeoD6xp9KEXrGvD51jeJRFJqF45KJ+xu3bohPT0dK1euxIEDB4TpxsbGmDlzJrp166ZwWdbW1ihXrhzi4+Ph5OQEADh37hxsbW2hpfVfY4qdnZ3cugCgffv2mDt3bqFX+m8Ti8XCz88+hCqHDVWq5P+PQyQSKf4+FY54Whr7lj5RrGv0MX2m9a00622Jfoft7e2Nfv364fbt20hNTYWhoSHq1asnl2QVoauri27duiEoKAjz58/Hv//+i/DwcOEqPTk5GQYGBtDR0Sm0qd3U1BRVqlQpySYQERFplBI/6UwkEqFevXpwcHBA/fr1lU7W+fz9/WFjY4OBAwdi1qxZGD9+PNq3bw8AcHV1RXR0dElDJCIi+mwodIVtbW2NX3/9FXZ2dmjYsGGxzcMikQhXr15VOABdXV0sXLgQCxcuLDDv+vXrRb6vuHlERESfG4US9tixY2FqagoAGDdunEoDIiIiooIUSthvJ2kzMzO0atVKSOBERESkekrfeJ49ezYuXryoiliIiIioCEr3Eq9evTpev36tiliICECX0D9VWv6+Cu9fhog+PUon7N69e2PevHn4559/YGVlhYoVKxZYRpnfYhMREdH7KZ2wFyxYAADYvn17ofNFIhETNhERUSlTOmEfPnxYFXEQERFRMZTudHbmzBno6enBzMyswL8KFSrwQSdEREQqoHTC9vf3x/379wudd+3aNaxYseKDgyIiIiJ5CjWJjxgxAklJSQDyRlQZO3ZsgXGsAeD58+eoU6dO6UZIREREiiXsUaNGYceOHQCAXbt2oVGjRqhcubLcMlpaWjA0NESPHj1KP0oiIqIyTqGE7eDgAAcHB+H1mDFjULt2bZUFRURERPKU7iWeP/RlWloazp49i3///RdfffUVUlNTYWFhodJxo4mIiMqqEo2HvWrVKqxZswaZmZkQiUSws7PDsmXLkJKSgvDwcBgaGpZ2nERERGWa0r3EIyIiEBoaisGDB2P79u2QyWQAgP79++P+/ftYvnx5qQdJRERU1imdsDdv3owRI0bg22+/hY2NjTDdzc0N3333HY4cOVKqARIREVEJEvajR4/QvHnzQufVq1cPz549++CgiIiISJ7SCbtGjRr4559/Cp13+fJl1KhR44ODIiIiInlKdzrr1asXQkNDoaOjgzZt2gAAJBIJYmNjsWbNGgwePLi0YyQiIirzlE7Yw4cPx4MHD7B48WIsXrwYAODj4wMA6NKlC0aOHFm6ERIREZHyCVskEmH27NkYPHgw/v77b6SlpcHAwADNmjWDpaWlKmIkIiIq80r0O2wAsLCwgIWFRWnGQkREREVQKGH7+/srXKBIJML8+fNLHBAREREVpFDC3rVrF0QiEUxNTaGlVXzHcj6alIiIqPQplLA7duyIY8eOITs7Gx06dEDnzp3h6Oio6tiIiIjo/ymUsJcuXYqMjAwcPXoU0dHRGDx4MExMTNCpUyd07twZ1tbWqo6TiIhUoEvonyore18FlRVdJinc6UxXVxedOnVCp06d8Pr1axw8eBDR0dHYsGEDatWqBU9PT3Tu3Jkd0YiIiFSgRL3E9fX10b17d3Tv3h2pqak4ePAgYmJisHr1alhaWiIyMrK04yQiIirTlH406buysrKQkZGBzMxM5Obm4uHDh6URFxEREb2lRFfYT58+xf79+7F//35cuHABenp6aNeuHUaOHAkXF5fSjpGIiKjMUzhhv52k4+Pjoauriy+//BLDhg1D69atUaECexcQERGpikIJu2/fvrhw4QK0tbXh5uaG5cuXw83NDdra2qqOj4iIiKBgwv7nn38gFotRv359vHjxAhEREYiIiCh0WZFIhI0bN5ZqkERERGWdQgm7WbNmwt8ymazYZd83n4iIiJSnUMLevHmzquMgIiKiYnzwz7qIiIhI9ZiwiYiINIDaE3ZWVhYCAgLg5OQEV1dXhIeHF7nssWPH8PXXX8Pe3h5dunTB4cOHP2KkRERE6qP2hL1o0SJcvnwZGzduRGBgIMLCwrB///4CyyUkJGDcuHHo2bMndu/ejT59+uDbb79FQkKCGqImIiL6uEr0pLPSIpFIsGPHDqxduxY2NjawsbHBjRs3sGXLFnTo0EFu2aioKLRs2RI+Pj4AAHNzcxw5cgQxMTFo2LChOsInIiL6aNSasBMSEpCTkwN7e3thmqOjI1avXg2pVAotrf8aALp37443b94UKOPVq1cfJVYiIiJ1UmvCTk5OhrGxsdxjTU1MTJCVlYXU1FRUrlxZmP7FF1/IvffGjRv466+/0KdPn2LXkZubi9zc3A+OVZW/L1em5Pw4ZDIZIBKV/gqUVBr7luSp+lkGipbOulY28NxWuNKqb6VZb9WasDMyMgo8gzz/dXZ2dpHve/HiBcaPHw8HBwe0bdu22HUkJiZ+eKAAMjIkpVJOYSS5ypedkZGhePkS1cUeHx+vsrLLKlXWNUD5+sa69nnjua1wn2J9U2vC1tbWLpCY81/r6OgU+p5nz55h8ODBkMlkWLFihVyzeWEsLS2hp6f3wbHqnjz1wWUURU9b8fhkMhkyMjKgq6sLkYLfQktj+4vStGlTlZVdVqmyrgGK1zfWtbKB57bClVZ9k0gkpXbhqNaEbWpqipSUFOTk5KBcubxQkpOToaOjA0NDwwLLP336VOh0tmnTJrkm86KIxWKIxeIPjlXRClSispVaWPT//4kUf5/qQi+VfUvyVFnXACWqA+tamcBzW+FKq76VZr1V68+6rK2tUa5cObmmh3PnzsHW1rbAlbNEIsGwYcOgpaWFiIgImJqafuRoiYiI1EetCVtXVxfdunVDUFAQLl68iEOHDiE8PFy4ik5OTkZmZiYAYM2aNbh37x4WLlwozEtOTmYvcSIiKhPU2iQOAP7+/ggKCsLAgQOhr6+P8ePHo3379gAAV1dXBAcHo0ePHoiNjUVmZia8vLzk3t+9e3csWLBAHaETERF9NGpP2Lq6uli4cKFw5fy269evC38X9vQzIiKiskLtjyYlIiKi92PCJiIi0gBM2ERERBqACZuIiEgDMGETERFpACZsIiIiDcCETUREpAGYsImIiDQAEzYREZEGYMImIiLSAEzYREREGoAJm4iISAMwYRMREWkAJmwiIiINwIRNRESkAZiwiYiINAATNhERkQZgwiYiItIATNhEREQagAmbiIhIAzBhExERaQAmbCIiIg3AhE1ERKQBmLCJiIg0ABM2ERGRBmDCJiIi0gBM2ERERBqACZuIiEgDMGETERFpACZsIiIiDcCETUREpAGYsImIiDQAEzYREZEGYMImIiLSAEzYREREGoAJm4iISAOoPWFnZWUhICAATk5OcHV1RXh4eJHLXr16FV5eXmjSpAl69uyJy5cvf8RIiYiI1EftCXvRokW4fPkyNm7ciMDAQISFhWH//v0FlpNIJBgxYgScnJwQGRkJe3t7jBw5EhKJRA1RExERfVxqTdgSiQQ7duzA9OnTYWNjAw8PDwwbNgxbtmwpsGx0dDS0tbUxdepUfPHFF5g+fToqVqxYaHInIiL63Kg1YSckJCAnJwf29vbCNEdHR1y4cAFSqVRu2QsXLsDR0REikQgAIBKJ4ODggPj4+I8ZMhERkVqoNWEnJyfD2NgYFSpUEKaZmJggKysLqampBZatVq2a3LQqVargyZMnHyNUIiIitSqnzpVnZGTIJWsAwuvs7GyFln13uXz5V+jp6enIzc394Fir631wEUVKr2Cm8LIyGZCllQmZtg7+v7HhvUzLmZQwsvd79eqVysouq1RZ1wDF6xvrWtnAc1vhSqu+ZWZmAkCBVuOSUGvC1tbWLpBw81/r6OgotOy7y+XLysoCANy7d69UYh3VVLdUyilMAiaprGwAGKjCshMTE1VYetmkyroGqLa+sa5pHp7bClfa9S0rKwv6+vofVIZaE7apqSlSUlKQk5ODcuXyQklOToaOjg4MDQ0LLPvs2TO5ac+ePSvQTJ7PyMgIdevWhba2NrS01N4ZnoiIyiCpVIqsrCwYGRl9cFlqTdjW1tYoV64c4uPj4eTkBAA4d+4cbG1tCyTZJk2aYO3atZDJZBCJRJDJZDh//jxGjRpVaNnlypVDlSpVVL4NRERExfnQK+t8ar301NXVRbdu3RAUFISLFy/i0KFDCA8Ph4+PD4C8q+389v8OHTrg5cuXmDdvHm7evIl58+YhIyMDHTt2VOcmEBERfRRqbyv29/eHjY0NBg4ciFmzZmH8+PFo3749AMDV1RXR0dEA8r6hrFmzBufOnUOPHj1w4cIF/PTTT9DTU3EPnU9QdnY2PD09ERcXp+5Q6DP19OlT+Pr6onnz5mjdujWCg4OFfiFEpe3u3bsYOnQo7O3t0aZNG6xbt07dIX2SRDKZTKbuIEhxWVlZmDRpEg4ePIhNmzahRYsW6g6JPjMymQx9+vSBoaEhpk6dirS0NAQEBKBt27aYNm2ausOjz4xUKkXHjh1ha2uLcePG4e7du5g4cSKCgoLQpUsXdYf3SVH7FTYp7ubNm/jmm29Krec7UWFu3bqF+Ph4BAcHo0GDBnBycoKvry+ioqLUHRp9hp49ewZra2sEBQWhbt26cHNzg7OzM86dO6fu0D45TNga5PTp02jRogV+/fVXdYdCn7GqVati3bp1MDGR/43r69ev1RQRfc6qVauGZcuWQV9fHzKZDOfOncOZM2fQvHlzdYf2yVFrL3FSTr9+/dQdApUBhoaGaN26tfBaKpUiIiICLVu2VGNUVBa4u7vj0aNH+PLLL/HVV1+pO5xPDq+wiahYISEhuHr1KiZMmKDuUOgzt2LFCqxevRrXrl1DcHCwusP55PAKm4iKFBISgo0bN2Lp0qWwtLRUdzj0mbO1tQWQ17l28uTJmDp1aoFHUpdlvMImokLNmTMHP//8M0JCQtg8SSrz7NkzHDp0SG5a/fr18ebNG/abeAcTNhEVEBYWhm3btmHJkiXo3LmzusOhz9iDBw8wbtw4PH36VJh2+fJlVK5cGZUrV1ZjZJ8eJmwikpOUlISVK1di+PDhcHR0RHJysvCPqLTZ2trCxsYGAQEBuHnzJo4fP46QkJAiHztdlvEeNhHJOXz4MHJzc7Fq1SqsWrVKbt7169fVFBV9rsRiMVauXIk5c+agd+/e0NXVxYABA4RHVNN/+KQzIiIiDcAmcSIiIg3AhE1ERKQBmLCJiIg0ABM2ERGRBmDCJiIi0gBM2ERERBqACZuIiEgDMGETERFpACZsIpLj7u4OPz8/dYdBRO9gwiYiItIATNhEREQagAmbiIr122+/oWHDhvjxxx/VHQpRmcaETURFio6OxsyZMzFmzBiMHTtW3eEQlWlM2ERUqKNHj2Lq1KkYMWIEfH191R0OUZnH4TWJSI67uzsqVqyIu3fvwsTEBIcOHYKWFr/bE6kbj0IiKiAxMRHOzs54+PAhtmzZou5wiAhM2ERUiNatW2PNmjXo1KkTlixZgsePH6s7JKIyjwmbiAowMTEBAPj7+0MsFiMoKEi9AREREzYRFa1atWqYMGECjh07hqioKHWHQ1SmMWETUbH69u0LOzs7zJs3DykpKeoOh6jMYi9xIiIiDcArbCIiIg3AhE1ERKQBmLCJiIg0ABM2ERGRBmDCJiIi0gBM2ERERBqACZuIiEgDMGETERFpACZsIiIiDcCETUREpAGYsImIiDQAEzYREZEG+D+1pfzCPyDgvAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Metrics for agllm-data/Responses V3 _no_metadata_filter.xlsx:\n", "{ 'ndcg_at_1/mean': 0.5961538461538461,\n", " 'ndcg_at_1/p90': 1.0,\n", " 'ndcg_at_1/variance': 0.24075443786982248,\n", " 'ndcg_at_2/mean': 0.6750200653502785,\n", " 'ndcg_at_2/p90': 1.0,\n", " 'ndcg_at_2/variance': 0.1902608017710696,\n", " 'ndcg_at_3/mean': 0.7182892961195092,\n", " 'ndcg_at_3/p90': 1.0,\n", " 'ndcg_at_3/variance': 0.15160799286131924,\n", " 'precision_at_1/mean': 0.5961538461538461,\n", " 'precision_at_1/p90': 1.0,\n", " 'precision_at_1/variance': 0.24075443786982248,\n", " 'precision_at_2/mean': 0.3605769230769231,\n", " 'precision_at_2/p90': 0.5,\n", " 'precision_at_2/variance': 0.05027274408284023,\n", " 'precision_at_3/mean': 0.2692307692307692,\n", " 'precision_at_3/p90': 0.3333333333333333,\n", " 'precision_at_3/variance': 0.017258382642998026,\n", " 'recall_at_1/mean': 0.5961538461538461,\n", " 'recall_at_1/p90': 1.0,\n", " 'recall_at_1/variance': 0.24075443786982248,\n", " 'recall_at_2/mean': 0.7211538461538461,\n", " 'recall_at_2/p90': 1.0,\n", " 'recall_at_2/variance': 0.20109097633136092,\n", " 'recall_at_3/mean': 0.8076923076923077,\n", " 'recall_at_3/p90': 1.0,\n", " 'recall_at_3/variance': 0.15532544378698224}\n", "Metrics saved to writing/65d4fadc59fceb1a54d1aae6/analysis/rag_metrics_without_metadata.csv\n", "Data from agllm-data/Responses V3.xlsx:\n", " Unnamed: 0 category About \\\n", "0 0 Identification and Control papaipema nebris \n", "1 1 Integrated Pest Management delia platura \n", "2 2 Identification and Control spodoptera frugiperda \n", "3 3 Identification and Control vanessa cardui \n", "4 4 Identification and Control helicoverpa zea \n", "\n", " question \\\n", "0 When should I begin scouting for black cutworm... \n", "1 What management practices can I use to minimiz... \n", "2 When should I start scouting for armyworm larv... \n", "3 What growth stage of soybean is most vulnerabl... \n", "4 At what level of defoliation or infestation sh... \n", "\n", " ground_truth_answer knowledge_vs_management \\\n", "0 Start scouting when 10% of migrating larvae ar... Management \n", "1 To minimize feeding injury risk in high-risk f... Management \n", "2 Begin scouting for armyworm larvae in mid-to-l... Management \n", "3 Soybean is most vulnerable to thistle caterpil... Management \n", "4 Consider applying insecticides when defoliatio... Management \n", "\n", " LLM Confidence chunk \\\n", "0 85 58\\nScouting. Scouting should begin when 10% m... \n", "1 85 up seed. The potential for injury is minimal i... \n", "2 85 35\\nScouting. In soybean and corn, grassy area... \n", "3 90 reach 1 1/2 inches long. Adult butterflies are... \n", "4 90 applications for corn earworm in Iowa soybean ... \n", "\n", " source \\\n", "0 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "1 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "2 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "3 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "4 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "\n", " relevant_docs \\\n", "0 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "1 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "2 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "3 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "4 ['agllm-data/agllm-data-isu-field-insects-all-... \n", "\n", " relavent_context \\\n", "0 ['58\\nScouting. Scouting should begin when 10%... \n", "1 ['up seed. The potential for injury is minimal... \n", "2 ['35\\nScouting. In soybean and corn, grassy ar... \n", "3 ['Scouting. The best time to look for thistle ... \n", "4 ['applications for corn earworm in Iowa soybea... \n", "\n", " answer_Gemini-1.5 Pro \\\n", "0 Answer:The provided scientific literature focu... \n", "1 Answer:\\nDelay planting to minimize injury, an... \n", "2 Answer:\\nStart scouting for armyworm larvae in... \n", "3 Answer:Soybean growth stages V3 to V4 are most... \n", "4 Answer: Insecticide application for corn earwo... \n", "\n", " answer_GPT-4 \\\n", "0 Answer:\\nThe documents does not provide inform... \n", "1 Answer:\\nDelay planting, use insecticide seed ... \n", "2 Answer:\\nBegin scouting for armyworm larvae in... \n", "3 Answer:\\nThe V3 to V4 growth stage of soybean ... \n", "4 Answer:\\nApplications for corn earworm in Iowa... \n", "\n", " answer_GPT-3.5 \\\n", "0 The documents does not provide information abo... \n", "1 Answer: No-till fields are less attractive to ... \n", "2 Answer: \\nBegin scouting for armyworm larvae i... \n", "3 Answer: Thistle caterpillars cause the most da... \n", "4 Answer: In Iowa soybean, applications for corn... \n", "\n", " answer_Llama-3 70B \\\n", "0 Answer: The document does not provide informat... \n", "1 Answer: To minimize the risk of feeding injury... \n", "2 Answer: Begin scouting for fall armyworm in mi... \n", "3 Answer: The growth stage of soybean most vulne... \n", "4 Answer: Applications for corn earworm in Iowa ... \n", "\n", " answer_Llama-3 8B \\\n", "0 Answer: Scouting for black cutworm larvae in c... \n", "1 Answer: No-till fields are less attractive to ... \n", "2 Answer: Begin scouting for armyworm larvae in ... \n", "3 Answer:\\nThistle caterpillars damage soybean b... \n", "4 Answer: You should consider applying insectici... \n", "\n", " answer_Claude 3 Opus \n", "0 Answer:\\nScouting for stalk borer larvae shoul... \n", "1 Answer:\\nTo minimize feeding injury risk from ... \n", "2 Answer:\\nBegin scouting for fall armyworm larv... \n", "3 Answer:\\nSoybean is most vulnerable to thistle... \n", "4 Answer:\\nConsider applying insecticides to con... \n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/Users/muhammadarbabarshad/miniconda3/envs/agllm-env1-updates-1/lib/python3.9/site-packages/mlflow/types/utils.py:407: UserWarning: Hint: Inferred schema contains integer column(s). Integer columns in Python cannot represent missing values. If your input data contains missing values at inference time, it will be encoded as floats and will cause a schema enforcement error. The best way to avoid this problem is to infer the model schema based on a realistic data sample (training dataset) that includes missing values. Alternatively, you can declare integer columns as doubles (float64) whenever these columns may have missing values. See `Handling Integers With Missing Values `_ for more details.\n", " warnings.warn(\n", "2024/09/02 18:44:53 INFO mlflow.models.evaluation.default_evaluator: Testing metrics on first row...\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeoAAAEcCAYAAAAIpvc+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABGFElEQVR4nO3dd1gUV9sG8HuBSBFREHtvCFKUomjAoCh27MSOvStGjQWMESuKJlHhtYsNTKKxYrBr1JhPrNgRxYYaFRVQqcKe7w9e9nUFdBd23RXu33Vx6c6cPfPM7Nl5dmbOzJEIIQSIiIhIK+loOgAiIiLKHxM1ERGRFmOiJiIi0mJM1ERERFqMiZqIiEiLMVETERFpMSZqIiIiLcZETUREpMWYqImIiLRYkUjU7u7uqF+/PjZs2JDn/B9//BH169dHUFCQwnUmJCRg+/btHy0zffp0DBgwQKlYlXX+/HmMHTsWLi4uaNSoETp16oR169YhIyNDrctVl507d6J+/fqaDuOjhBDYtWsXXr58mW+Z+vXry/3Z2trCw8MDS5YsQUpKiqzco0ePUL9+fURGRgIAEhMTMWjQINja2qJnz565XmvSp9p8fp/dnTt38PXXX6Nnz55ISkpSW3ya+p5/SmRkJOrXr49Hjx4p/J4LFy7g/PnzBV5mTrvK7y8mJiZX2xswYACmT5+ushgAICgoCO7u7oWq41OmT5/+0XW9evWqypb14Tb6UEE+a1XUoVfgpWmZr776CgcPHsTgwYPlpmdmZuLQoUOQSCRK1RcYGIhHjx7By8sr3zIzZsxAVlZWgeJVxJYtW7Bw4UJ4e3tjzJgxMDExwcWLF7Fo0SKcO3cOK1euhI7Ol/Vbq0OHDmjevLmmw/ioc+fOYfr06Th69OhHy/n5+aFDhw4AgJSUFFy5cgWLFi3C5cuXERISgq+++gqVKlXC33//jdKlSwMA9u7di/Pnz2Pr1q2oUKFCrteapEib/1BsbCwGDhyI6tWrY926dTA2NlZjhJr5nqtD3759ERAQACcnp0LVExQUBHt7+1zTTU1NIZFI5NqeOmIYMmQI+vXrV+D3K8re3j7fH2CmpqZqX76mFZlE3axZM5w6dQpPnz5FxYoVZdPPnDkDIyMjGBoaKlWfIo9AL1WqlNJxKio6OhoLFy7E1KlTMXDgQNn0atWqoXLlyujfvz8iIiLQqVMntcWgDgYGBjAwMNB0GB+l6OPvS5UqhXLlysle16hRA7Vq1ULPnj2xe/dueHl5QVdXV67M69evUa5cOdjZ2eX5WpOUfex/bGwsvL29UatWLaxevRolS5ZUU2T/o4nvuTYrXbq0XPv60MfmqULJkiU/y+f+1VdfqX1dtNmXdTj2EXZ2dqhcuTIOHDggNz0iIgLt27fP9Uv74sWL6NevH+zs7NCiRQvMnj0bb9++BZB9qmXXrl04e/as7FTfgAEDMHPmTHh5ecHJyQl79+7Nder7wYMHGD16NBwdHeHs7IxJkybJTp++fPkSPj4+cHZ2hp2dHXr37o2zZ8/muz7bt29HqVKl8vy12rhxY2zcuBHffPONbNru3bvRuXNn2NnZwd3dHStWrJAd7eecAvvzzz/RtWtX2Nraonv37oiNjcV//vMffP3112jSpAlmz54t23EFBQWhT58++M9//gNnZ2c4OTnB19dXto0AICYmBiNHjkTjxo1hY2ODVq1aISQkRDY/KCgI/fv3x8SJE+Hg4IC5c+fmOn164sQJdO/eHQ0bNkSzZs0wffp0udOnsbGxGDVqFJydneHo6AgfHx88fvxYNn/AgAFYsmQJ/Pz84OTkBAcHB0yePFkuzg99LO7IyEh4e3sDAFq1aoWdO3fmW09ebGxs4OjoiH379slt+8jISEyfPh1BQUF48uQJ6tevD3d3d7nXOcs6fvw4unfvDjs7O3h4eGDp0qVylzrq16+P5cuXo2XLlnB1dcX9+/eRkZGBxYsXo3nz5rC3t8e3336Lv//+W/aenTt3wsPDQ/avjY0NunfvjgsXLgDIu81/zN27dzFw4EDUq1cPa9euldtZZ2VlYfHixXBzc4ONjQ3atWuHX3/9VantmB91f8+TkpLwww8/oHnz5rC2tkazZs3www8/IDU1VVbn+fPn4eXlBTs7O3Tu3BnR0dFyy/xUHTnL8vX1lZ1mPX/+PLy9veHg4AAbGxu0b98ee/bsKdS2+vDU9/vyiuHZs2eYOHEinJyc4OzsjFGjRuH+/fuy90yfPh0+Pj4YMmQIHBwcsHbtWrlT3znLO3jwILy8vGBjYwN3d3f8/vvvcsveuHEj3N3dYWdnh8GDByM4OFglp88V2RccOXIEXl5eaNSokWw/eOrUKbl6kpOTMXnyZDRs2BCurq5Yvnw5pFJpnssUQmDt2rVo1aoVGjZsiC5dumDv3r1yZT7VXhQiioCWLVuK5cuXi8DAQNGrVy/Z9PT0dNG4cWNx/fp1WRkhhLh586aws7MTK1euFPfu3RPnzp0TXl5ewsvLS0ilUvH69WsxYcIE0atXL/H8+XMhhBD9+/cX9evXF3v37hW3bt0Sr169EtOmTRP9+/cXQgiRlJQkXFxcxJAhQ8TVq1fF9evXRY8ePWTzx40bJ0aMGCFiYmLE/fv3xfTp04Wjo6NITk7Oc52+/fZbMXz4cIXWf8OGDcLGxkaEhoaKe/fuid27dwsHBwcxb948IYQQcXFxwsLCQrRq1UpERkaKmzdvilatWonGjRuL77//Xty5c0ds3bpVWFhYiKNHjwohhFi+fLmwtrYWvXv3FteuXRNnzpwRrVq1EkOHDhVCCJGSkiJcXFzE1KlTxZ07d8S9e/dEYGCgsLCwEDdu3JDVYWFhIebNmycePnwo7t27J3bs2CEsLCyEEEK8fPlSFvejR4/E+fPnhbu7u/Dz8xNCCPHo0SPh6Ogoxo8fL27evCmioqJE7969RYsWLcSbN29kn4u1tbX46aefxL1798SRI0dEw4YNRVBQUJ7b6lNxp6eni4MHDwoLCwtx+fJlkZqammc9FhYWYseOHXnO8/f3F02aNJHb9mfOnBGvX78WCxcuFN988414/vx5rtepqanixIkTws7OTvz666/iwYMH4tSpU6JNmzbCx8dHbtnOzs7iypUr4tKlS0IIISZNmiS6dOkizpw5I+7duydCQkKEtbW1OH78uBBCiB07dghra2vh5eUlLl26JGJiYkTfvn1F69at823zH8r57GJjY4WLi4to1KiReP36da5ymzdvFu7u7uLChQvi0aNHYsuWLcLCwkKcO3cuz3oV9Tm+56NGjRLdunUTUVFRIi4uTuzZs0dYW1uLDRs2CCGEePjwobC1tRUzZ84Ud+7cEQcOHBBNmjQRFhYWIi4uTqE6nj9/LiwsLMTGjRvF69evxdOnT4WdnZ0IDAwU9+/fF7dv3xZTp04V1tbWIj4+Ps9t8X67ys+HZfr37y+mTZuWZwzJycnCw8NDfPfdd+LmzZvi1q1bYvr06aJx48bi6dOnQgghpk2bJiwsLMTatWvF3bt3xZMnT8Ty5ctFy5Yt5Zbn5uYmjhw5Ih4+fChmz54tLC0txcOHD4UQQoSGhgo7Ozuxfft2cffuXbFixQphaWkpqyMv7+9nP+ZT+4KrV68KS0tLsWHDBvHw4UNx48YNMXToUNG0aVORnp4uq8PCwkLMnTtX3LlzR+zZs0c0atRI9tmdOXNG7rP+6aefRMuWLcXx48fFgwcPxB9//CHs7e1FaGioEEKx9qKIInPqGwDat2+P9evX49mzZ6hQoQJOnz4NMzMzNGjQQK7c+vXr4eLiglGjRgEAatasiZ9++gmtW7fG2bNn4ezsDAMDg1ynW6ysrODp6ZnnsiMiIpCcnIyff/5Zdk1o3rx5+PPPP5GRkYGHDx/CwsIC1apVg4GBAWbMmAFPT0/o6urmWV9iYiKqVav2yXUW//1F179/f9nRd82aNZGYmIjFixfDx8dHVnbIkCFo0qQJAMDDwwNbtmzBnDlzYGhoiDp16iAoKAi3b9+W/bqVSCRYunSp7Nrpjz/+iOHDh+Pu3bsoU6YMvL290a9fP9nRlI+PD9atW4dbt27ByspKtlwfHx/ZZYKLFy/Kpj979gwZGRmoXLkyqlSpgipVqmDVqlWyMwFbt26FkZERlixZghIlSgAAli9fjlatWmHPnj2y9a1bty4mTZokW3cXFxdcunQpz+2Vmpr6ybhzPj8zM7MCnaY3MTHJ84i+VKlSMDIykjsd/uHrVatW4dtvv0Xv3r0BANWrV8fs2bMxcOBAPHr0CFWrVgUAdOnSBba2tgCyz+Ts27cPu3fvlm33wYMHIzo6GuvXr0eLFi0AAO/evcPs2bPlyowdOxbx8fEoX758nm0+LwMHDkStWrVw6dIlBAUFwc/PT27+w4cPYWRkhKpVq6J8+fLo378/ateujVq1aim9LfOizu+5i4sLGjduLDvirFq1KkJDQxETEwMA2LZtG8zNzTFr1izo6uqiTp06+PfffxEQECBb7qfqyFlWqVKlUKpUKSQkJGD8+PEYOnSo7IzAiBEjsHv3bty/fx/m5ub5bovhw4fn2ofMnj0bnTt3/ug2/DCG7du34/Xr11i8eDH09LLTwvz58xEZGYlt27Zh/PjxALJPtQ8bNuyjdQ8aNAitWrUCAEycOBFhYWG4fPkyqlWrhvXr18Pb21vWcXL06NG4fv06bty48dE6z58/n+e1+AYNGiAsLEz2+mP7Al1dXcycORN9+/aVlff29sbw4cPx8uVLVKpUSVbnDz/8AACoU6cOYmNjERISgkGDBsktOyUlBRs3bsTPP/8s+45Vr14djx8/xvr169GvXz+F2osiilSitrGxQbVq1XDw4EF4e3sjIiICHTt2zFXuxo0bePDgQZ4ffGxsLJydnfOsv0aNGvkuOyYmBjVr1pTruGFpaQlLS0sAwLhx4zBlyhQcPHgQjo6OcHV1RadOnaCvr59nfWZmZkhMTPzY6gIAXr16hRcvXsDR0VFuepMmTfDu3TvcvXsXZcuWzRW/kZERzM3N5a7pGRgYyJ1irVmzplwHJwcHB9m6tmvXDn379sW+fftw48YNPHz4UHZK5/3TRGXLls33Wr6VlRU6deqEUaNGoVy5cnBxcUGLFi3g4eEhW46NjY0sSQPZO5hatWrJdnoAULt2bbl6S5UqhdevX+e5TDMzM4XiLow3b94UuP/CjRs3cOXKFfzxxx+yaeK/lyNiY2Nlifr9zzJnJ/f+DgjITswmJiZy0+rUqSP7f06M7969UyrGhg0b4ueff5btpL7++mvZjgoA+vXrhyNHjsDNzQ1WVlZwcXFBx44dZe3wQx07dsSTJ09kr9euXfvRDk7q/J737dsXx44dw65du3D//n3cuXMHjx49krWxmJgYNGjQQC455nwvFK3jQ9WrV0f37t2xefNmxMTEyLXJT3VWnTdvHho2bCg3Lb/t/DE3btxAUlISGjduLDc9PT0dsbGxstcf2wfmyK+NJSQk4PHjx2jUqJFceScnp08mahsbGyxZsiTX9A/3nx/bF+T8CF+zZg3u3r2LBw8e5LmdP9yX2tnZYdWqVbn2KXfu3EF6ejomT54s16k3MzMTGRkZSEtLU6i9KKJIJWog+9f2gQMH0KtXLxw9ejTPWy+kUik8PT1lv7TfZ2Zmlm/dHzu6yvkVmh8PDw+cOnUKp06dwj///IMNGzYgODgY27ZtQ7169XKVt7e3xx9//IGsrKw8j7q///57ODg4oE2bNnkuLyfpvB/XhzF+qsf4V199Jfc6pzHr6uoiPj4evXr1gpmZGdzd3eHq6gpbW1u4ubnJvedTR6Q//fQTxo4di5MnT+Kff/7BlClT4OjoiE2bNuXb0UcqlcrF9n4i/xRF4y6M69ev5zq6U5RUKsWwYcPQrVu3XPPeP9J9f7vmbKewsLBcHXs+/Izz2lb5bef8LF26FHp6ehg2bBhOnDiB6dOnY8+ePbIfdTVr1sShQ4dw9uxZnD59Gn/99RfWrl2LgICAPNdrzZo1yMzMlL1WpPe7Or7nUqkUI0eOxO3bt9GpUyd06NAB1tbWmDlzpqyMRCLJ9YPu/e+VInV86M6dO+jbty+sra3x9ddfo02bNjA1NVWoJ3qFChUUSp6fIpVKUatWLaxcuTLXPCMjI9n/FTnDlF8by9lOyra3nOUqsp4f2xecPXsWQ4cORYsWLeDo6AhPT0+kpqZi7NixcuU+/M5IpVJIJJJc+8Oc9Vi6dGmeP8JKlCjxyfaiqCLTmSxH+/btcfHiRezYsQPVqlWT+3WXo169erhz5w5q1Kgh+8vMzERAQAD+/fdfAFD6No+6devi/v37ePPmjWza9evX0axZMzx+/BgBAQGIi4tDhw4dMG/ePBw5cgQ6Ojr466+/8qyvR48eSE5ORmhoaK55kZGRCA8Ph7GxMczNzWFubi7rFJTj/Pnz+Oqrr1C9enWl1uN99+7dk1ufnFNIDRo0wL59+5CYmIhff/0VY8aMgYeHh6wTmKJfxMuXL2PBggWoXbs2Bg0ahDVr1mDBggU4c+YMXr58KbtH8v2j/BcvXuDBgwd5fq6KUCRuZT/79127dg1RUVH5XiL5lHr16uHevXtybfPp06cIDAxEcnJyvu8Bsn+EvP++nTt3KtUZTtH1ztnR6OjoYNGiRXj37h2mTJki2yFt3rwZhw4dgouLC6ZOnYrw8HA0a9YMERERedZXpUoVubgVSQbq+J7fvHkTJ0+exLJly/D999+jc+fOqF69Oh4+fChrG5aWlrh27Zpcm7x27ZpSdXzot99+Q9myZbFhwwYMHz4cbm5uePHiBYDP1yvdwsICT548QalSpWTbqnLlyvjpp59w7tw5lSyjVKlSqFKlCqKiouSmf/haXUJCQuDs7IygoCAMGjQILi4usnbw/na+fv263PsuXLiAqlWr5rqjoHbt2tDT08OTJ0/k2tiJEyewfv166OjofLK9KKrIJWorKyvUqFEDP/30U56nw4Dsa7U3btzA7NmzERsbi0uXLmHy5Mm4f/8+atasCSD7V+Tz588RFxen0HI9PT1RunRpTJkyBdHR0bh27RpmzZoFCwsLVKlSBVevXsXMmTMRFRWFR48eYefOnUhJScnztByQffpowoQJWLhwIQIDAxEdHY179+5h69atGD9+PDw8PGTrN3ToUISGhmLr1q148OABwsPDERwcjF69ehXqFrKUlBRMnToVMTEx+OeffzBnzhx06NABVapUQcWKFZGamooDBw7gyZMn+Pvvv2XXhhR9GIuxsTG2bt2KxYsX48GDB4iJiUFERARq1qwJU1NT9OnTB8nJybJteuXKFUyYMAGmpqb5frafokjcOUcQ0dHR+SZHIPsUd3x8POLj42Xbfdy4cXB2dv7kNcL8DB8+HAcPHkRwcDDu3buH//u//4Ovry/evHmT77XjevXqoWXLlpg1axaOHTuGuLg4rF27FqtXr1bqh5qybR7Ivv46Y8YMREZGYvXq1QCyL8fMmTMHR48exePHj3Hq1CncvHkz37ZeEOr4npubm0NPTw/79+9HXFwcrl69iu+++w7x8fGyttGnTx+kpqbCz88PsbGxOH78uNz9vYrUkbPc2NhYJCQkoGLFinj69ClOnDiBx48f49ChQ/D39weg+HepIN6PoXPnzihdujR8fHxw+fJlxMbGYvr06Th58qRKH1A0fPhwhIaGYufOnXjw4AHWr1+PgwcPfvJ97969k33XPvx7v0f+x1SqVAm3bt3C+fPn8ejRI+zYsQPLli0DIL+dL168iMWLFyM2Nhbbt2/H1q1bMWbMmFz1lSpVCr1798ayZcuwZ88exMXF4Y8//sDixYtRvnx5AJ9uL4oqcqe+gexf2ytXrpQ9jOJDjRo1wrp167Bs2TJ069YNRkZGaNasGaZNmyY7ddK1a1ccPnwYnTp1wqFDhz65TENDQ6xfvx4BAQHo3bs3DAwM0KJFC0ybNg0A8MsvvyAgIACjR4/GmzdvULt2bSxZsuSj1+JGjBiB2rVrY8uWLdi5cyfS0tJQrVo1jBkzBn379pWdEh8yZAhKlCiBTZs2YcGCBahYsSKGDx+OoUOHKrvp5FSqVAlWVlbo168fdHV14enpie+//x4A0K5dO1y/fh0LFy7E27dvUaVKFXh5eeHo0aO4evUq+vTp88n6czqwBQcHY+vWrdDR0UHTpk2xdu1a6OjoyDrhLF68GL169UKJEiXg4uKCxYsX57r2qihF4rawsICbmxu+++47TJo0CUOGDMmzrgULFmDBggUAsk9z1ahRA/369YO3t3e+nQQVie+XX37B6tWrsWrVKpQpUwbu7u6y7Z6fX375Bb/88gt+/PFHJCUloXr16pg/f36ep5rz82GbV/QBLN27d8fx48cRHBwMZ2dnjBs3Du/evcO8efMQHx+PcuXKoU+fPhg5cqTCsShCHd/zhQsXIigoCGFhYShXrhxatGiBQYMG4dixYwCyTzXnfM+6deuGSpUqYfTo0Zg9e7Zs/qfqALK/s+vWrUNsbCyWL1+Ou3fvYurUqcjIyEDNmjUxadIkLF++HFevXpW7DVOV3o9h1apVCA0NRWBgIIYOHYqsrCxYW1sjJCSkwGev8tKnTx8kJSVh6dKlSEhIQJMmTdCtW7dcZwQ/dOnSJbi6uuY5b+rUqQrt63x8fPDixQvZpZC6detiwYIFmDJlCq5evSpbTy8vL9y/fx/dunWDmZkZJk+ejO7du+dZp6+vL0xNTbFs2TI8f/4clSpVgo+Pj6zD3afai6Ik4nOdW6EvSlBQEHbt2iW3cyEiKoyTJ0+ibt26qFy5smzazJkz8fDhQ2zatEmDkWm3Infqm4iItNOePXswZswYREVF4fHjx9i9ezf27t2LLl26aDo0rVYkT30TEZH2mTlzJhYuXIixY8fi9evXqFGjBvz8/PI9tUzZeOqbiIhIi/HUNxERkRZjoiYiItJiTNRERERarMh2JsvMzERSUhL09fU/+ahMIiIidZBKpUhPT0fp0qUL9PhQoAgn6qSkJLmxVImIiDSlZs2aBRowBSjCiTpnVJWaNWvmekbrlywrKwsxMTGwsLAo8NOviBTBtkafU1Ftb6mpqbh//36+IyUqosgm6pzT3YaGhnKjv3zpckawyhnHmEhd2Nbocyrq7a0wl2B58ZaIiEiLMVETERFpMSZqIiIiLcZETUREpMWYqImIiLQYEzUREZEWY6ImIiLSYkX2PuribtmyZbCxsUGrVq0KNF8RN27cwPr16xEbGwshBKRSKezt7TF27FhUqFABABAUFAQAGD9+fIGXQ0RfqNVuChfVAWCZkgKd80o892LkCeVj+gIxUQPwDPpbLfWGj3dVS72KmDBhQqHmf0pERAQ2b96MqVOnwsHBAUD2AwuOHTuG4cOHIygoCDVq1CjUMoiIiIlaK0RGRmLZsmUwMDDAkydPYG1tjQULFuCbb76BnZ0dnj59ij/++AOhoaHYu3cvUlJS4OrqihkzZkBPTw+bN2/G1q1boaOjg6+//hozZsyAr68vmjRpgg4dOmDq1Kl4+PAhJBIJevXqhd69e2P69Olo0qQJunfvjh07dmDDhg2QSCSwtrbGzJkzUbJkSTRr1gyenp44f/483r17h4CAANjY2ODx48fYunUrNm/ejH/++Qeenp7Q1dVF69atce7cOfz000+YO3cu1q1bJ1tHqVSKqVOnwtjYGLNmzYJEItHgFici+nLwGrWWuHr1KmbMmIH9+/fj3bt32LJlCxITEzF48GCEh4fj3LlziIqKwrZt2xAQEIB3797ht99+w7Vr17B582b8/vvv2LdvHx49eoSzZ8/K6v37778hhMDu3bsREhKCCxcuyC331q1bWLlyJTZv3ozw8HAYGhoiODgYAPDq1Ss4Ojpi586d6NGjB1avXg0ACAsLw/jx4/Ho0SPMnz8fISEh2LZtGw4cOAAbGxvUq1cPUqkUqampsuXMmjWLSZqIqAB4RK0lHBwcUKdOHQBAly5dsG3bNgCAvb09AOD06dO4cuUKvLy8kJaWBh0dHejp6SEjIwPu7u4oXbo0AGDVqlUAgF27dgEAbG1tMX/+fAwdOhRubm6YNm2a3HLPnTuHli1bwszMDADQq1cv+Pr6yua3aNECAGBpaYmjR48CyP5RMWXKFMyfPx+9evVCuXLlAAD16tWDjY0NAMDY2BjJyckAgN9++w1JSUk4fPgwkzQRkZJ4RK0l3h+nVAghN6gIkH39d9CgQdi1axcCAgLw22+/4fvvv4eurq5c8ouPj0diYqLsdYUKFbB//3707dsXd+/eRbdu3fD69WvZfKlUKheHEAKZmZmy1zkjvry/DKlUColEgnv37qFevXqy+K5evQobGxukpqbiyZMnMDc3BwA0bNgQ3333HWbPnl2obUREVBwxUWuJixcv4t9//4VUKsXu3bvh6irfEa1p06bYs2cPkpOTIZVKMXnyZOzYsQONGzfGyZMn8fbtW0ilUvj5+SEyMlL2vvDwcPj7+6NVq1b44YcfYGRkhH///Vc2v0mTJjh69ChevXoFANi2bRsaN2780ViNjIzw8uVL1KpVC2fOnEFWVhaWL1+O+Ph4mJubw8/PD0OGDJGVt7KywuDBg/H06VNERESoYnMRERUbTNRaomLFivD19UX79u1hbm6OXr16yc13d3dH27Zt0bt3b0yZMgWVK1dGv3790KBBA3h7e6NPnz7w9PRE3bp10bZtW9n72rRpg7S0NHTs2BE9e/ZE27ZtUb9+fdl8S0tLjB49GgMHDkS7du2QlJSEiRMnfjRWT09PrFixAiNGjEBkZCTc3NxgZmYGExMTdOrUCa6urmjWrJncNWpdXV3MmTMHCxYsQEJCgoq2GhFR0ScRQghNB6EOKSkpuHnzJqysrLR+POrIyEgEBwdjy5YtnyyblZWFqKgoNGrUSGNjtgoh4OfnBwAYMWIEatWqJZuXlJSEXbt24ejRo1i5ciWMjY01EiMVnja0NfrCKXEftUD2ftvIyAgK92T5Au6jVkUuYmcyUppEIkFAQAAOHz6MBQsW4OnTp9DR0YFUKoWxsTE6dOiAkJAQfPXVV5oOlYjoi8dErQWcnZ3h7Oys6TCU5uHhAQ8PD02HQURUpPEaNRERkRZjoiYiItJiTNRERERajImaiIhIizFRExERaTH2+gaUutdPKRq8x2/AgAEYN24cAOS6R5vjSBMRfTmYqIsZjiNNiv4w1QFgmZICnfNKPKThC3gABdGXholaC0RGRiIwMBBCCFSqVAkmJiaIjo5GVlYWBgwYAC8vL2RkZGDu3LmIjIxEZmYmxowZg549e2L//v3YsGED0tLSkJaWhjlz5qBp06Z5LofjSBMRfXk0fo06PT0dfn5+cHJygqurK0JCQvIte/jwYbRv3x729vbo06cPrl+//hkjVa979+5hw4YNqFOnDiwsLLBr1y789ttv2Lp1K6KjoxEWFoakpCTs27cPP/74I9avX483b97g119/xcqVK7F3716MHDkSGzduzHcZHEeaiOjLo/Ej6sDAQFy7dg2bNm3CkydPMG3aNFSuXBnt2rWTK3f79m1MnjwZc+bMgYODAzZu3IiRI0fi8OHDsqEgv2S1a9dG6dKl8ffffyM1NRW7d+8GALx9+xa3bt1CZGQkevbsCV1dXZQqVQr79u2Drq4uVqxYgWPHjuHevXs4e/asbHjMvHAcaSKiL49GE3VKSgq2b9+OtWvXwtraGtbW1rh9+zbCwsJyJerTp0+jbt266Nq1KwBg0qRJCAsLw507d2Bra6uB6FUr58eGVCrFkiVLYG1tDQB4+fIlSpUqhUOHDskly4cPH6J8+fLo0aMHOnfujMaNG6N+/foICwvLdxnvjyPdvHlzAP8bR3rSpEl5jiPt4OCA2bNnY9WqVepadSKiAum1r9enCxXQ751+V1vdytLoqe/o6GhkZmbC3t5eNs3R0RGXL1+GVCqVK1umTBncuXMHFy5cgFQqxc6dO2FsbIzq1at/7rDVqmnTpvj1118hhMCrV6/QrVs3xMbGokmTJti/fz+EEHj79i0GDRqEW7duQSKRYPTo0WjatClOnjyJrKysfOvmONJERF8ejR5Rx8fHw9TUFCVKlJBNMzc3R3p6OhITE2FmZiab3qFDBxw7dgx9+/aFrq4udHR0sHr1apQuXfqjy8jKyvpo8gLU92tF+onlyspJpRBCICsrC6NHj8bcuXPRqVMnZGVlYeTIkbCwsEDNmjURGxuLrl27IjU1FRMmTICNjQ2srKzQrl07GBgYoHHjxnj8+DEyMzNlt10BkNXdsWNHBAcHY+TIkRgzZgy++eYbDB8+XDaO9KhRo+Ds7Iy3b9/K/VCaNWsWxo0bB2dnZ5QpU0Ydm4o+I0Xbe84IuEIIQMFLH4q2eSoelNm3FqS9QY2DNH8qb3zOejQ6HvXu3buxbNkyHD9+XDYtLi4OrVu3xokTJ1CxYkXZ9GfPnuG7775Dp06d0LBhQ/z66684deoUdu3ahbJly+aqO2cMUPofIQTWrFkDIQS6dOmCSpUqyea9ffsWJ0+exMWLFzFp0iStH8ObCs7y5Ci11R39DS+R0P+os60BQO/KFT9dqID86/qrtL4vdjxqfX19ZGRkyE3LeW1gYCA3fcmSJbCwsEC/fv0AAHPnzkX79u2xY8cOjBgxIt9lWFhYFKmkk3NN2dbWFrq6ukq/f+XKlThy5Ai2b98uG0daCAFjY2O0a9cOU6ZM4TjSRZyi90ULIZCamgpDQ0OFOxM2atSoEJFRUaPMPfgFaW/q3Lerqi2npKQgJiamUHVoNFFXqFABCQkJyMzMhJ5edijx8fEwMDCAiYmJXNnr169jwIABstc6OjqwtLTEkydPProMXV3dAiU0bVeY9Wrbti3atm2r4oioyPnvzlIikUDRPv9F8btGn0kB2pviBZWnqrasino02pnMysoKenp6iIqKkk27cOECbG1tc91mVL58ecTGxspNu3fvHqpWrfo5QiUiItIIjSZqQ0NDdO3aFf7+/rhy5QqOHDmCkJAQeHt7A8g+uk5LSwMAfPvtt9i2bRt2796NBw8eYMmSJXjy5Am6deumyVUgIiJSK40/8MTX1xf+/v4YOHAgjI2NMX78eLRp0wYA4OrqioCAAHTv3h0dOnRAcnIyVq9ejadPn8LKygqbNm3KsyMZEWlGcbmvlehz0niiNjQ0xKJFi7Bo0aJc827duiX32svLC15eXp8rNCIiIo3TeKImxezcuRNnz57F/PnzC/R+IQR27tyJXbt2ITk5GVKpFCVKlEDnzp3Rr18/WZ8Ad3d3bN68mdf+iYi0BBM11He6TltO1WVlZWHSpEmoWLEifv75Z5QvXx4A8OrVK2zYsAHjx49HcHAwn+dNRKSFmKi1QGRkJFauXIlSpUohNjZWllD/+usvrFy5EsbGxqhSpYrsnsHr169j9uzZEEKgfPnyWLJkCYyNjWXDYJqamsLc3Bzu7u7o3r07Nm3aBFtbWwwaNAiBgYE4deoUKleuDBMTE7i5ZY9NHB4ejs6dO8tiiouLw7Bhw+Dv749mzZppZLsQEZEWDHNJ2S5fvgxfX19ERERAX18foaGhWLx4MUJDQ/H777/LRrTKyMhAcHAw5s2bh/DwcDg5OeG3337D77//joSEBERERGDFihWyIUClUikOHTqEoUOHYuXKlUhJScH+/fsxZswYREREwNbWFv3798exY8dksTx9+hSjR49mkiYi0gI8otYS9erVQ+XKlQFk318OAPb29rKhKD09PXHmzBncvn0bZcqUkY2uNXr0aADAqFGj4OXlBV1dXZiZmcHDwwNA9r3m9evXh1QqRWhoqGxwDRsbG5QsWRK1atVCWloa3r59K4tlwoQJsLS0ZJImItICPKLWEvr6+rL/SyQSHDx4EO8/hj3nsZ66urpy15KTk5Px+PFj6OrqIq/Htr958wYlS5bEq1evoKurK7udLSoqCg0aNICOjg5OnDgh97i8GTNm4OXLlzh06JCqV5OIiJTERK2lHBwcEBUVhX///RdSqRT79+8HANSqVQtv3rzB7du3AQChoaEICQmBi4sL9u7dC6lUitevX+PYsWOQSCSoUqUKYmNjYWpqCiEEoqOj8eLFCwQEBKBu3bq4desW1q9fL3uGOgDY2dlhzpw5mDdvHl6/fq2R9Sciomw89a2lzM3NMXPmTAwePBhGRkaoW7cugOwj77Fjx+KHH35ARkYGKlWqhMDAQBgZGeH27dvo3LkzTExMULFiRejr66NcuXLIzMzEjRs3MHPmTAwaNAjVqlWDh4cH1q9fj9jYWAQGBiI1NRWmpqay5dvZ2aF169ZYuHAhFixYoKnNQERU7Gl0mEt1yhnmsjBDi2mjrKwsREVFoVGjRnIPez9x4gQyMjLg4eGBd+/eoV+/fpg/fz7q1auHR48ewcfHB15eXujUqRNKlSole9/t27cRFBQEGxubj45CRkXIajeFiglkf4+MjIwUHvugVxX1DTuoLbc7khIUbGtA0W1vqshFPKIuIurUqYNp06YhODgYUqkUPXv2RL169QAAVatWxebNm7Fx40YMGTIEaWlpkEgkEEKgVq1a8Pb2hpOTk4bXgIiI8sJEXURUrVoVYWFh+c43NjbGuHHjMG7cuM8YFRERFRY7kxEREWkxJmoiIiItxkRNRESkxZioiYiItBgTNRERkRZjoiYiItJiTNRERERarMD3UcfGxuL06dN4/vw5BgwYgLi4OFhaWsLY2FiV8RERERVrSidqqVSKH3/8ETt27IAQAhKJBO3bt8eKFSvw8OFDhIaGomJF9T3WjYiIqDhR+tT3ihUrEB4ejnnz5uH06dOyoRWnTJkCqVSKX375ReVBEhERFVdKH1Hv2LEDPj4+6NGjB7KysmTTrays4OPjgyVLlqg0QG3hGfS32uoOLzFD4bI6ACxTUqBzXvGHu38JD64nIqK8KX1E/eLFC1hZWeU5r0KFChy/mIiISIWUTtQ1atTAiRMn8px39uxZ1KhRo9BBERERUTalT30PHDgQP/74I969e4eWLVtCIpHgwYMHiIyMREhICKZPn66OOImIiIolpRO1l5cXXr16hZUrV+LXX3+FEAKTJk3CV199hWHDhqFPnz7qiJOIiKhYKtB91CNHjkS/fv1w8eJFJCUlwcTEBA0bNkSZMmVUHB4REVHxVuAHnhgbG+Obb75RZSxERET0AaUTtbe39yfLbN68uUDBEBERkTylE3XOA07el5KSgtjYWBgZGaFNmzYqCYyIiIgKkKi3bNmS5/SkpCQMHz4ctWvXLnRQRERElE1lo2eVLl0aI0aMwMaNG5V6X3p6Ovz8/ODk5ARXV1eEhITkW/bWrVvo06cP7Ozs4OnpiTNnzhQyaiIiIu2m8mEuX758qVT5wMBAXLt2DZs2bcKsWbMQHByMAwcO5Cr35s0bDBkyBHXr1kV4eDg8PDwwbtw4pZdHRET0JVH61Pe5c+dyTcvKysLTp0+xYsUKWFtbK1xXSkoKtm/fjrVr18La2hrW1ta4ffs2wsLC0K5dO7myu3btgpGREfz9/aGrqwsfHx+cOHEC165dg5ubm7KrQURE9EVQOlEPGDAAEokk13QhBCpVqgQ/Pz+F64qOjkZmZibs7e1l0xwdHbFq1SpIpVLo6PzvgP/s2bNo1aoVdHV1ZdN27NihbPhERERfFKUTdV63XkkkEhgbG6N+/fpyyfVT4uPjYWpqihIlSsimmZubIz09HYmJiTAzM5NNj4uLg52dHWbOnIljx46hSpUqmDZtGhwdHT+6jKysLLlRvgoqr97uqqJMzTlxCCGAPH4wFXoBSlLFtqXPS9FvKNsaFZYy11aLantTRT1KJ+omTZoUeqE5UlNT5ZI0ANnrjIwMuekpKSlYs2YNvL29sXbtWvz5558YOnQo9u/fj0qVKuW7jJiYGBXFmqKSevKSkqV83ampqYrXn6K+2KOiotRWN6mHpZLtgW2NCkrZtgawveVFoUTt6+urcIUSiQQLFixQqKy+vn6uhJzz2sDAQG66rq6ubMxrAGjQoAFOnz6NPXv2YNSoUfkuw8LCAkZGio/dnB/D0/8Uuo78GOkrHp8QAqmpqTA0NMzzEkSe9atg/fPTqFEjtdVN6qHoWOZsa1RYirY1oOi2t5SUlEIfMCqUqCMjIxWuUNENDGSPX52QkIDMzEzo6WWHEh8fDwMDA5iYmMiVLVeuXK57tGvWrIl///33o8vQ1dWVu65dUMqsl9J1K1VY8t9/JIq/T32hq2TbkpZiW6PPqYi2N1XUo1CiPnbsWKEXlBcrKyvo6ekhKioKTk5OAIALFy7A1tY217XuRo0a5epxfvfuXXTq1EktsREREWkDld5HnZKSgpMnTypc3tDQEF27doW/vz+uXLmCI0eOICQkRPY88fj4eKSlpQEAevfujVu3biEoKAgPHjzAsmXLEBcXhy5duqhyFYiIiLSK0p3JHj9+DH9/f5w9ezbX9eUcN2/eVLg+X19f+Pv7Y+DAgTA2Nsb48eNlzwt3dXVFQEAAunfvjipVqmDdunWYP38+1qxZgzp16mDNmjWoUKGCsqtARET0xVA6UQcEBODixYvw8vLCxYsXYWhoiEaNGuH06dOIiYlBUFCQUvUZGhpi0aJFWLRoUa55t27dknvt6OiInTt3KhsyERHRF0vpU9/nzp3DxIkT8cMPP6B79+7Q19fHlClTsGPHDjRu3BhHjx5VR5xERETFktKJOjk5GfXr1wcA1K5dGzdu3ACQ3bOtb9++HCiDiIhIhZRO1OXLl8eLFy8AADVq1EBSUhLi4+MBAGXKlOEgGURERCqkdKJ2c3PD0qVLcenSJVSpUgUVK1ZESEgI3r59ix07drBzFxERkQopnah9fHxgYmKCZcuWAQAmTpyITZs2oXHjxggPD8fgwYNVHiQREVFxpVCv7wEDBsDLywtt27aFqakptm/fjufPnwMAOnfujMqVKyMqKgp2dnYqfRY4ERFRcadQok5MTMTUqVMxd+5cdOrUCV5eXmjQoIFsvpOTk+zJYkRERKQ6Cp36Dg8Px44dO9ClSxccPHgQPXr0QNeuXREWFobXr1+rO0YiIqJiS+Fr1NbW1vjhhx9w8uRJBAcHo1q1ali4cCGaN2+OyZMn87YsIiIiNVD6yWR6enpo1aoVWrVqhaSkJOzbtw979+7FoEGDUK1aNfTo0eOjw04SERGR4go1KEfp0qXRr18//P7779iyZQt0dXVlvcGJiIio8JQ+on5ffHw8/vzzT+zbtw/Xr19HpUqVMGbMGFXFRkREVOwpnaiTk5Nx6NAhhIeHIzIyErq6umjdujUmTpyIr7/+GhKJGkfyJiIiKmYUStSZmZk4ceIEwsPD8ddffyEtLQ1WVlbw9fWFp6cnSpcure44iYiIiiWFErWLiwtev34NExMT9OjRAz169JC7j5qIiIjUQ6FEbW1tjR49esDDwwMlSpRQd0xERET0Xwol6pCQEHXHQURERHko1O1ZREREpF5M1ERERFqMiZqIiEiLFThRZ2Vlyf6flpaGN2/eqCQgIiIi+h+lE/W7d+8wa9YsfPvtt7JpFy9eRLNmzbBo0SJIpVKVBkhERFScKZ2og4KCsHfvXnTq1Ek2rUGDBvj++++xbds2rFu3TqUBEhERFWdKP0I0PDwc06ZNQ+/evWXTypQpg0GDBkFPTw+bN2/GiBEjVBokERFRcaX0EXVCQgKqVauW57zatWvj6dOnhQ6KiIiIsimdqGvXro2DBw/mOe/YsWOoUaNGoYMiIiKibEqf+vb29sb06dORmJiI1q1bo2zZsnj16hWOHz+O/fv3IyAgQB1xEhERFUtKJ+quXbsiOTkZK1aswKFDh2TTTU1NMXPmTHTt2lWV8RERERVrSidqAOjXrx/69u2Le/fuITExESYmJqhduzZ0dPj8FCIiIlUqUKIGAIlEgtq1a6syFiIiIvqAQofAVlZWuHLlCgDA0tISVlZW+f4pO051eno6/Pz84OTkBFdXV4VG6nr06BHs7e0RGRmp1LKIiIi+NAodUY8dOxYVKlQAAIwbN06lAQQGBuLatWvYtGkTnjx5gmnTpqFy5cpo165dvu/x9/dHSkqKSuMgIiLSRgol6veTc5UqVfD111/LEndhpKSkYPv27Vi7di2sra1hbW2N27dvIywsLN9EvXfvXiQnJxd62UTayjPob7XWH15CrdUTkYop3ftrzpw5stPghRUdHY3MzEzY29vLpjk6OuLy5ct5PjM8ISEBixcvxpw5c1SyfCIiIm2ndKKuWLEi3r59q5KFx8fHw9TUFCVK/O8nvrm5OdLT05GYmJir/MKFC9GtWzfUq1dPJcsnIiLSdkr3+u7Vqxfmz5+PS5cuoX79+ihZsmSuMoreS52amiqXpAHIXmdkZMhN/+eff3DhwgXs27dPqXizsrLkhuQsKCFEoevIt+4CxCGEACQS1S9ASarYtiRPnW0NULw5sK1RYSlzJFhU25sq6lE6US9cuBAAsG3btjznSyQShRO1vr5+roSc89rAwEA2LS0tDT/++CNmzZolN10RMTExSpXPT2qq+jqvpWQpX3dqaqri9aux411UVJTa6i6u1NnWAOXbG9saFZRlAdoD21tuSifqo0ePqmzhFSpUQEJCAjIzM6Gnlx1KfHw8DAwMYGJiIit35coVxMXFwcfHR+79w4cPR9euXT96zdrCwgJGRkaFjtXw9D+FriM/RvqKxyeEQGpqKgwNDSFR8FenKtY/P40aNVJb3cWVOtsaoHh7Y1ujwtI5z31bSkpKoQ8YlU7U586dg5ubG0xNTXPNi4+Px+7duzF8+HCF6rKysoKenh6ioqLg5OQEALhw4QJsbW3lnnJmZ2cn97hSAGjTpg3mzZsHFxeXjy5DV1cXurq6CsXzMYo2nALVrVRhyX//kSj+PvWFrpJtS/LU2dYAJZoD2xp9TkW0vamiHqU7k/n6+iIuLi7PeTdv3sTy5csVrsvQ0BBdu3aFv78/rly5giNHjiAkJATe3t4AshN/WloaDAwMUKNGDbk/IPuIvGzZssquAhER0RdDoSPqESNGIDY2FkD26YmxY8fm6gQGAC9fvkT16tWVCsDX1xf+/v4YOHAgjI2NMX78eLRp0wYA4OrqioCAAHTv3l2pOomIiIoKhRL1qFGjsH37dgDArl270KBBA5iZmcmV0dHRgYmJidJJ1dDQEIsWLcKiRYtyzbt161a+7/vYPCIioqJCoUTt4OAABwcH2esxY8agWrVqaguKiIiIsindmSwgIAAAkJSUhPPnz+P58+do27YtEhMTUatWLbV3hCEiIipOCjTM5cqVK7F69WqkpaVBIpHAzs4OS5cuRUJCAkJCQuRurSIiIqKCU7rXd2hoKIKCgjB48GBs27ZN9jSZ/v37Iy4uDsuWLVN5kERERMWV0ol6y5YtGDFiBCZMmABra2vZdDc3N3z33Xc4duyYSgMkIiIqzpRO1E+ePEGTJk3ynFe7dm28ePGi0EERERFRNqUTdaVKlXDp0qU85127dg2VKlUqdFBERESUTenOZD179kRQUBAMDAzQokULANnPMj148CBWr16NwYMHqzpGIiKiYkvpRD18+HA8evQIS5YswZIlSwBA9shPT09PjBw5UrUREhERFWNKJ2qJRII5c+Zg8ODBOHPmDJKSklCqVCk0btwYFhYW6oiRiIio2CrQfdQAUKtWLdSqVUuVsRAREdEHFErUvr6+ClcokUiwYMGCAgdERERE/6NQot61axckEgkqVKggN050XvgIUSIiItVRKFG3b98ef/31FzIyMtCuXTt07NgRjo6O6o6NiIio2FMoUf/yyy9ITU3F8ePHERERgcGDB8Pc3BwdOnRAx44dYWVlpe44iYhIDTyD/lZb3eEl1FZ1saJwZzJDQ0N06NABHTp0wNu3b3H48GFERERg48aNqFq1Kjp16oSOHTuygxkREZEKFajXt7GxMbp164Zu3bohMTERhw8fxv79+7Fq1SpYWFhg586dqo6TiIioWFL6EaIfSk9PR2pqKtLS0pCVlYXHjx+rIi4iIiJCAY+onz17hgMHDuDAgQO4fPkyjIyM0Lp1a4wcORIuLi6qjpGIiKjYUjhRv5+co6KiYGhoiJYtW2LYsGFo3rw5SpRgrwEiIiJVUyhR9+nTB5cvX4a+vj7c3NywbNkyuLm5QV9fX93xERERFWsKJepLly5BV1cXdevWxatXrxAaGorQ0NA8y0okEmzatEmlQRIRERVXCiXqxo0by/4vhPho2U/NJyIiIsUplKi3bNmi7jiIiIgoD4W+PYuIiIjUh4maiIhIizFRExERaTEmaiIiIi3GRE1ERKTFmKiJiIi0GBM1ERGRFtN4ok5PT4efnx+cnJzg6uqKkJCQfMv+9ddf6NKlC+zt7eHp6YmjR49+xkiJiIg+P40n6sDAQFy7dg2bNm3CrFmzEBwcjAMHDuQqFx0djXHjxqFHjx7YvXs3evfujQkTJiA6OloDURMREX0eBRrmUlVSUlKwfft2rF27FtbW1rC2tsbt27cRFhaGdu3ayZXdt28fmjZtCm9vbwBAjRo1cOzYMezfvx+WlpaaCJ+IiEjtNJqoo6OjkZmZCXt7e9k0R0dHrFq1ClKpFDo6/zvg79atG969e5erjjdv3nyWWImIiDRBo6e+4+PjYWpqKjeWtbm5OdLT05GYmChXtk6dOnJHzrdv38b//d//oVmzZp8rXCIios9Oo0fUqampckkagOx1RkZGvu979eoVxo8fDwcHB7Rq1eqjy8jKykJWVlahY1XnqGDK1JwThxACkEhUvwAlqWLbkjx1j0CnaO1sa8UD9215U1V7U0U9Gk3U+vr6uRJyzmsDA4M83/PixQsMHjwYQggsX75c7vR4XmJiYlQSa2pqikrqyUtKlvJ1p6amKl5/ivpij4qKUlvdxZU62xqgfHtjWyvauG/Lmza1N40m6goVKiAhIQGZmZnQ08sOJT4+HgYGBjAxMclV/tmzZ7LOZJs3b4aZmdknl2FhYQEjI6NCx2p4+p9C15EfI33F4xNCIDU1FYaGhpAo+KtTFeufn0aNGqmt7uJKnW0NULy9sa0VD9y35U1V7S0lJaXQB4waTdRWVlbQ09NDVFQUnJycAAAXLlyAra1triPllJQUDBs2DDo6Oti8eTPKlSun0DJ0dXWhq6tb6FgVbTgFqlupwpL//iNR/H3qC10l25bkqbOtAUo0B7a1YoH7trypqr2poh6NdiYzNDRE165d4e/vjytXruDIkSMICQmRHTXHx8cjLS0NALB69Wo8fPgQixYtks2Lj49nr28iIirSNHpEDQC+vr7w9/fHwIEDYWxsjPHjx6NNmzYAAFdXVwQEBKB79+44ePAg0tLS4OXlJff+bt26YeHChZoInYiISO00nqgNDQ2xaNEi2ZHy+27duiX7f15PKyMiIirqNP4IUSIiIsofEzUREZEWY6ImIiLSYkzUREREWoyJmoiISIsxURMREWkxJmoiIiItxkRNRESkxZioiYiItBgTNRERkRZjoiYiItJiTNRERERajImaiIhIizFRExERaTEmaiIiIi3GRE1ERKTFmKiJiIi0GBM1ERGRFmOiJiIi0mJM1ERERFqMiZqIiEiLMVETERFpMSZqIiIiLcZETUREpMWYqImIiLQYEzUREZEWY6ImIiLSYkzUREREWoyJmoiISIsxURMREWkxJmoiIiItpvFEnZ6eDj8/Pzg5OcHV1RUhISH5lr1x4wa8vLzQsGFD9OjRA9euXfuMkRIREX1+Gk/UgYGBuHbtGjZt2oRZs2YhODgYBw4cyFUuJSUFI0aMgJOTE3bu3Al7e3uMHDkSKSkpGoiaiIjo89Book5JScH27dsxY8YMWFtbw8PDA8OGDUNYWFiushEREdDX18fUqVNRp04dzJgxAyVLlswzqRMRERUVGk3U0dHRyMzMhL29vWyao6MjLl++DKlUKlf28uXLcHR0hEQiAQBIJBI4ODggKirqc4ZMRET0WWk0UcfHx8PU1BQlSpSQTTM3N0d6ejoSExNzlS1fvrzctLJly+Lp06efI1QiIiKN0NPkwlNTU+WSNADZ64yMDIXKflguR84ReXJyMrKysgoda0WjQleRr+QSVRQuKwSQrpMGoW+A/55c+KQKeuYFjOzT3rx5o7a6iyt1tjVA8fbGtlY8cN+WN1W1t7S0NADIdZZYGRpN1Pr6+rkSbc5rAwMDhcp+WC5Heno6AODhw4cqiXVUI0OV1JOXaExWW90AMFCNdcfExKix9uJJnW0NUG97Y1v78nDfljdVt7f09HQYGxsX6L0aTdQVKlRAQkICMjMzoaeXHUp8fDwMDAxgYmKSq+yLFy/kpr148SLX6fAcpUuXRs2aNaGvrw8dHY13biciomJIKpUiPT0dpUuXLnAdGk3UVlZW0NPTQ1RUFJycnAAAFy5cgK2tba7k2rBhQ6xduxZCCEgkEgghcPHiRYwaNSrPuvX09FC2bFm1rwMREdHHFPRIOodGDzUNDQ3RtWtX+Pv748qVKzhy5AhCQkLg7e0NIPvoOuf8frt27fD69WvMnz8fd+7cwfz585Gamor27dtrchWIiIjUSuPnhH19fWFtbY2BAwdi9uzZGD9+PNq0aQMAcHV1RUREBIDsXySrV6/GhQsX0L17d1y+fBlr1qyBkZGae95ooYyMDHTq1AmRkZGaDoWKqGfPnsHHxwdNmjRB8+bNERAQIOv3QaRqDx48wNChQ2Fvb48WLVpg3bp1mg5Jq0iEEELTQZDi0tPTMXnyZBw+fBibN2+Gs7OzpkOiIkYIgd69e8PExARTp05FUlIS/Pz80KpVK0ybNk3T4VERI5VK0b59e9ja2mLcuHF48OABJk2aBH9/f3h6emo6PK2g8SNqUtydO3fw7bffqqwnO1Fe7t69i6ioKAQEBKBevXpwcnKCj48P9u3bp+nQqAh68eIFrKys4O/vj5o1a8LNzQ3NmjXDhQsXNB2a1mCi/oKcPXsWzs7O+P333zUdChVh5cqVw7p162BuLn+P6tu3bzUUERVl5cuXx9KlS2FsbAwhBC5cuIBz586hSZMmmg5Na2i01zcpp2/fvpoOgYoBExMTNG/eXPZaKpUiNDQUTZs21WBUVBy4u7vjyZMnaNmyJdq2bavpcLQGj6iJ6KMWL16MGzduYOLEiZoOhYq45cuXY9WqVbh58yYCAgI0HY7W4BE1EeVr8eLF2LRpE3755RdYWFhoOhwq4mxtbQFkd5r9/vvvMXXq1FyPji6OeERNRHmaO3cuNmzYgMWLF/M0JKnNixcvcOTIEblpdevWxbt379gv4r+YqIkol+DgYPz222/4+eef0bFjR02HQ0XYo0ePMG7cODx79kw27dq1azAzM4OZmZkGI9MeTNREJCc2NhYrVqzA8OHD4ejoiPj4eNkfkarZ2trC2toafn5+uHPnDk6cOIHFixfn+3jo4ojXqIlIztGjR5GVlYWVK1di5cqVcvNu3bqloaioqNLV1cWKFSswd+5c9OrVC4aGhhgwYIDsUdLEJ5MRERFpNZ76JiIi0mJM1ERERFqMiZqIiEiLMVETERFpMSZqIiIiLcZETUREpMWYqImIiLQYEzUREZEWY6ImIjnu7u6YPn26psMgov9ioiYiItJiTNRERERajImaiD7qjz/+gKWlJf7zn/9oOhSiYomJmojyFRERgZkzZ2LMmDEYO3aspsMhKpaYqIkoT8ePH8fUqVMxYsQI+Pj4aDocomKLw1wSkRx3d3eULFkSDx48gLm5OY4cOQIdHf6mJ9IUfvuIKJeYmBg0a9YMjx8/RlhYmKbDISrWmKiJKJfmzZtj9erV6NChA37++Wf8+++/mg6JqNhioiaiXMzNzQEAvr6+0NXVhb+/v2YDIirGmKiJKF/ly5fHxIkT8ddff2Hfvn2aDoeoWGKiJqKP6tOnD+zs7DB//nwkJCRoOhyiYoe9vomIiLQYj6iJiIi0GBM1ERGRFmOiJiIi0mJM1ERERFqMiZqIiEiLMVETERFpMSZqIiIiLcZETUREpMWYqImIiLQYEzUREZEWY6ImIiLSYkzUREREWuz/AavvHLybJXhoAAAAAElFTkSuQmCC", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stdout", "output_type": "stream", "text": [ "Metrics for agllm-data/Responses V3.xlsx:\n", "{ 'ndcg_at_1/mean': 0.6730769230769231,\n", " 'ndcg_at_1/p90': 1.0,\n", " 'ndcg_at_1/variance': 0.22004437869822488,\n", " 'ndcg_at_2/mean': 0.7762096712568728,\n", " 'ndcg_at_2/p90': 1.0,\n", " 'ndcg_at_2/variance': 0.1356449887185433,\n", " 'ndcg_at_3/mean': 0.8146712097184113,\n", " 'ndcg_at_3/p90': 1.0,\n", " 'ndcg_at_3/variance': 0.09368803175795551,\n", " 'precision_at_1/mean': 0.6730769230769231,\n", " 'precision_at_1/p90': 1.0,\n", " 'precision_at_1/variance': 0.22004437869822488,\n", " 'precision_at_2/mean': 0.4182692307692308,\n", " 'precision_at_2/p90': 0.5,\n", " 'precision_at_2/variance': 0.03418546597633137,\n", " 'precision_at_3/mean': 0.30448717948717946,\n", " 'precision_at_3/p90': 0.3333333333333333,\n", " 'precision_at_3/variance': 0.008783284023668635,\n", " 'recall_at_1/mean': 0.6730769230769231,\n", " 'recall_at_1/p90': 1.0,\n", " 'recall_at_1/variance': 0.22004437869822488,\n", " 'recall_at_2/mean': 0.8365384615384616,\n", " 'recall_at_2/p90': 1.0,\n", " 'recall_at_2/variance': 0.13674186390532547,\n", " 'recall_at_3/mean': 0.9134615384615384,\n", " 'recall_at_3/p90': 1.0,\n", " 'recall_at_3/variance': 0.07904955621301776}\n", "Metrics saved to writing/65d4fadc59fceb1a54d1aae6/analysis/rag_metrics_with_metadata.csv\n" ] } ], "source": [ "%matplotlib inline\n", "\n", "import pandas as pd\n", "import mlflow\n", "import pprint\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "import csv\n", "\n", "# Define file paths\n", "file_paths = [\n", " 'agllm-data/Responses V3 _no_metadata_filter.xlsx',\n", " 'agllm-data/Responses V3.xlsx'\n", "]\n", "\n", "sheet_name = 'Researcher-IsuField-insects'\n", "\n", "for i, file_path in enumerate(file_paths):\n", " # Read the data and store it in the variable data_to_eval\n", " data_to_eval = pd.read_excel(file_path, sheet_name=sheet_name)\n", "\n", " # Print the first few rows to verify the data (optional)\n", " print(f\"Data from {file_path}:\")\n", " print(data_to_eval.head())\n", "\n", " data_to_eval[\"source\"] = data_to_eval[\"source\"].apply(safe_literal_eval)\n", " data_to_eval[\"relevant_docs\"] = data_to_eval[\"relevant_docs\"].apply(safe_literal_eval)\n", "\n", " with mlflow.start_run() as run:\n", " evaluate_results = mlflow.evaluate(\n", " data=data_to_eval,\n", " targets=\"source\",\n", " predictions=\"relevant_docs\",\n", " evaluators=\"default\",\n", " extra_metrics=[\n", " mlflow.metrics.precision_at_k(1),\n", " mlflow.metrics.precision_at_k(2),\n", " mlflow.metrics.precision_at_k(3),\n", " mlflow.metrics.recall_at_k(1),\n", " mlflow.metrics.recall_at_k(2),\n", " mlflow.metrics.recall_at_k(3),\n", " mlflow.metrics.ndcg_at_k(1),\n", " mlflow.metrics.ndcg_at_k(2),\n", " mlflow.metrics.ndcg_at_k(3),\n", " ],\n", " )\n", "\n", " # Prepare data for plotting\n", " metrics = [\"precision\", \"recall\", \"ndcg\"]\n", " k_values = [1, 2, 3]\n", " bar_width = 0.15\n", " opacity = 0.8\n", "\n", " # Create subplots\n", " fig, ax = plt.subplots(figsize=(5, 3))\n", "\n", " # Plotting each metric\n", " for j, metric_name in enumerate(metrics):\n", " y = [evaluate_results.metrics[f\"{metric_name}_at_{k}/mean\"] for k in k_values]\n", " x = np.arange(len(k_values)) + j * bar_width\n", " ax.bar(x, y, width=bar_width, alpha=opacity, label=f\"{metric_name}@k\")\n", "\n", " # Adding labels and title\n", " ax.set_xlabel(\"k\", fontsize=12)\n", " ax.set_ylabel(\"Metric Value\", fontsize=12)\n", " title = \"Metrics Comparison at Different Ks - Metadata Filtering \" + (\"Disabled\" if i == 0 else \"Enabled\")\n", " ax.set_title(title, fontsize=11)\n", "\n", " # Setting x-axis ticks\n", " ax.set_xticks(np.arange(len(k_values)) + bar_width)\n", " ax.set_xticklabels(k_values)\n", "\n", " # Add legend and adjust layout\n", " ax.legend(fontsize=8)\n", " fig.tight_layout()\n", "\n", " # Save the plot as PDF\n", " plt.savefig(f'writing/65d4fadc59fceb1a54d1aae6/analysis/rag_{\"without\" if i == 0 else \"with\"}.pdf')\n", "\n", " # Display the plot\n", " plt.show()\n", "\n", " pp = pprint.PrettyPrinter(indent=4)\n", " print(f\"Metrics for {file_path}:\")\n", " pp.pprint(evaluate_results.metrics)\n", "\n", " # Save metrics to CSV\n", " csv_filename = f'writing/65d4fadc59fceb1a54d1aae6/analysis/rag_metrics_{\"without\" if i == 0 else \"with\"}_metadata.csv'\n", " with open(csv_filename, 'w', newline='') as csvfile:\n", " writer = csv.writer(csvfile)\n", " writer.writerow(['Metric', 'k=1', 'k=2', 'k=3'])\n", " for metric in metrics:\n", " row = [metric]\n", " for k in k_values:\n", " value = evaluate_results.metrics[f\"{metric}_at_{k}/mean\"]\n", " row.append(round(value, 3))\n", " writer.writerow(row)\n", "\n", " print(f\"Metrics saved to {csv_filename}\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Pie chart" ] }, { "cell_type": "markdown", "metadata": {}, "source": [] }, { "cell_type": "code", "execution_count": 49, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Plots have been saved in the 'analysis/pie' folder.\n" ] } ], "source": [ "import pandas as pd\n", "from openpyxl import load_workbook\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", "from matplotlib.patches import Patch\n", "import os\n", "\n", "# Set up the plot parameters\n", "plt.rcParams.update({\n", " 'font.family': 'Times New Roman',\n", " 'font.size': 8,\n", " 'axes.labelsize': 8,\n", " 'axes.titlesize': 9,\n", " 'xtick.labelsize': 7,\n", " 'ytick.labelsize': 7,\n", "})\n", "\n", "# Path to Dr. Arti Singh's excel file\n", "file_path = 'agllm-data/Dr. Arti Singh.xlsm'\n", "\n", "def initialize_categories(category_type):\n", " if category_type == 'insect':\n", " return {\n", " \"Identification and Control\": 0,\n", " \"Pesticide Use\": 0,\n", " \"Integrated Pest Management\": 0,\n", " \"Damage Assessment\": 0,\n", " \"Other\": 0\n", " }\n", " elif category_type == 'weed':\n", " return {\n", " \"Identification and Control\": 0,\n", " \"Herbicide Use\": 0,\n", " \"Cultural and Mechanical Control\": 0,\n", " \"Integrated Weed Management\": 0,\n", " \"Other\": 0\n", " }\n", "\n", "def match_category(category, category_type):\n", " categories = initialize_categories(category_type)\n", " for key in categories.keys():\n", " if key.lower() in str(category).lower():\n", " return key\n", " return \"Other\"\n", "\n", "def process_excel_data(file_path, category_type):\n", " wb = load_workbook(file_path, data_only=True)\n", " category_counts = initialize_categories(category_type)\n", " knowledge_management_split = {cat: {'Knowledge': 0, 'Management': 0} for cat in category_counts}\n", "\n", " for sheet_name in wb.sheetnames:\n", " if category_type in sheet_name.lower():\n", " df = pd.read_excel(file_path, sheet_name=sheet_name, engine='openpyxl')\n", " \n", " if 'category' in df.columns and 'knowledge_vs_management' in df.columns:\n", " for _, row in df.iterrows():\n", " category = match_category(row['category'], category_type)\n", " km_type = row['knowledge_vs_management']\n", " \n", " category_counts[category] += 1\n", " if pd.notna(km_type):\n", " knowledge_management_split[category][km_type] += 1\n", "\n", " return category_counts, knowledge_management_split\n", "\n", "def create_pie_chart(category_counts, knowledge_management_split, category_type):\n", " categories = list(category_counts.keys())\n", " counts = list(category_counts.values())\n", " knowledge_percentages = [knowledge_management_split[cat]['Knowledge'] / count if count > 0 else 0 for cat, count in zip(categories, counts)]\n", "\n", " base_colors = plt.cm.Pastel1(np.linspace(0, 1, len(categories)))\n", "\n", " fig, ax = plt.subplots(figsize=(6, 3), dpi=300)\n", "\n", " inner_wedges, _ = ax.pie(counts, colors=base_colors, \n", " wedgeprops=dict(width=0.5, edgecolor='white'),\n", " startangle=90, radius=1)\n", "\n", " outer_colors = []\n", " outer_data = []\n", " for i, count in enumerate(counts):\n", " knowledge = count * knowledge_percentages[i]\n", " management = count - knowledge\n", " outer_colors.extend([base_colors[i], base_colors[i]])\n", " outer_data.extend([knowledge, management])\n", "\n", " outer_wedges, _ = ax.pie(outer_data, colors=outer_colors, radius=1.3, \n", " wedgeprops=dict(width=0.3, edgecolor='white'),\n", " startangle=90)\n", "\n", " for i, wedge in enumerate(outer_wedges):\n", " if i % 2 == 0: # Knowledge wedges\n", " wedge.set_hatch('/////')\n", " else: # Management wedges\n", " wedge.set_hatch('.....')\n", "\n", " total = sum(counts)\n", " for i, p in enumerate(inner_wedges):\n", " ang = (p.theta2 + p.theta1)/2\n", " y = np.sin(np.deg2rad(ang))\n", " x = np.cos(np.deg2rad(ang))\n", " horizontalalignment = {-1: \"right\", 1: \"left\"}[int(np.sign(x))]\n", " connectionstyle = f\"angle,angleA=0,angleB={ang}\"\n", " kw = dict(xycoords='data', textcoords='data',\n", " arrowprops=dict(arrowstyle=\"-\", color='gray', connectionstyle=connectionstyle),\n", " bbox=dict(boxstyle=\"round\", fc=base_colors[i], ec=\"gray\", alpha=0.8),\n", " zorder=0, va=\"center\")\n", " percentage = counts[i] / total * 100\n", " label = f\"{categories[i]} ({counts[i]})\"\n", " ax.annotate(label, xy=(x, y), xytext=(1.35*np.sign(x), 1.4*y),\n", " horizontalalignment=horizontalalignment, **kw)\n", "\n", " plt.axis('equal')\n", " plt.tight_layout()\n", "\n", " # Create the 'analysis/pie' folder if it doesn't exist\n", " os.makedirs('writing/65d4fadc59fceb1a54d1aae6/analysis/pie', exist_ok=True)\n", " \n", " # Save the plot\n", " plt.savefig(f'writing/65d4fadc59fceb1a54d1aae6/analysis/pie/{category_type}_categories.pdf', bbox_inches='tight')\n", " plt.close()\n", "\n", "\n", "# Process data and create plots for both insect and weed categories\n", "for category_type in ['insect', 'weed']:\n", " category_counts, knowledge_management_split = process_excel_data(file_path, category_type)\n", " create_pie_chart(category_counts, knowledge_management_split, category_type)\n", "\n", "print(\"Plots have been saved in the 'analysis/pie' folder.\")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "agllm-env1-updates-1", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.19" } }, "nbformat": 4, "nbformat_minor": 2 }