File size: 3,770 Bytes
64c992d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 | {
"cells": [
{
"cell_type": "markdown",
"id": "8a468c14",
"metadata": {},
"source": [
"# Table S5: Performance statistics for predicting QCD corrections\n",
"\n",
"Foundation MagNET's accuracy at predicting the rovibrational (QCD) correction (stationary vs\n",
"trajectory-averaged shielding) over qcdtraj2500 (2500 molecules), ¹H and ¹³C, all shieldings at\n",
"PBE0/pcSseg-1."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "aca3e24b",
"metadata": {},
"outputs": [],
"source": [
"import os, sys\n",
"\n",
"# make the in-repo modules importable (not pip-installed)\n",
"REPO = os.path.abspath(\"../..\")\n",
"for _p in (\"data/magnet_test_predictions\", \"analysis/code\", \"analysis/code/shared\"):\n",
" sys.path.insert(0, os.path.join(REPO, _p))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bbdcbe1d",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"import magnet_test_predictions_reader\n",
"import magnet_benchmark\n",
"import paths"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c98484b1",
"metadata": {},
"outputs": [],
"source": [
"PREDICTIONS = paths.dataset_file(\"magnet_test_predictions\", root=REPO)\n",
"\n",
"def document_path(name):\n",
" # table/spreadsheet outputs go under this notebook's documents/ folder, created on first save\n",
" os.makedirs(\"documents\", exist_ok=True)\n",
" return os.path.join(\"documents\", name)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "da4e04e1",
"metadata": {},
"outputs": [],
"source": [
"rows = magnet_benchmark.qcd_stats_table(PREDICTIONS, magnet_test_predictions_reader)\n",
"table_rows = []\n",
"for r in rows:\n",
" nucleus = r[\"model\"].split(\"(\")[1].rstrip(\")\") # \"1H\" / \"13C\"\n",
" pmed, pmae, prmse = magnet_benchmark.PUBLISHED_S5[nucleus]\n",
" table_rows.append(dict(model=r[\"model\"], n=int(r[\"n\"]),\n",
" median_repro=round(r[\"median_ae\"], 8), median_SI=round(pmed, 8),\n",
" mae_repro=round(r[\"mae\"], 8), mae_SI=round(pmae, 8),\n",
" rmse_repro=round(r[\"rmse\"], 8), rmse_SI=round(prmse, 8)))\n",
"table_s5 = pd.DataFrame(table_rows)\n",
"print(\"Table S5 (QCD corrections):\"); display(table_s5)\n",
"\n",
"# write the reproduced table to this notebook's documents/ folder\n",
"out = document_path(\"si_table_s05_qcd.xlsx\")\n",
"with pd.ExcelWriter(out) as writer:\n",
" table_s5.to_excel(writer, sheet_name=\"Table S5\", index=False)\n",
"print(\"wrote\", os.path.relpath(out, REPO))"
]
},
{
"cell_type": "markdown",
"id": "9971c179",
"metadata": {},
"source": [
"## Exact-reproduction check"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b2c792c0",
"metadata": {},
"outputs": [],
"source": [
"# every reproduced median/MAE/RMSE should match the published SI value to a few parts per million\n",
"dev = max((table_s5[[\"median_repro\", \"median_SI\"]].diff(axis=1).iloc[:, -1].abs().max(),\n",
" table_s5[[\"mae_repro\", \"mae_SI\"]].diff(axis=1).iloc[:, -1].abs().max(),\n",
" table_s5[[\"rmse_repro\", \"rmse_SI\"]].diff(axis=1).iloc[:, -1].abs().max()))\n",
"print(\"largest reproduced-vs-published deviation:\", dev)\n",
"assert dev < 1e-3, \"a row diverged from the SI by more than float rounding\""
]
}
],
"metadata": {
"language_info": {
"name": "python"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
|