{ "cells": [ { "cell_type": "code", "id": "initial_id", "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2026-05-19T12:42:02.280702Z", "start_time": "2026-05-19T12:42:02.276769Z" } }, "source": [ "from pathlib import Path\n", "import re\n", "import pandas as pd" ], "outputs": [], "execution_count": 19 }, { "metadata": {}, "cell_type": "markdown", "source": "Intra-dataset evaluation: Train on and test on the same ", "id": "145ddc0eb7f59e45" }, { "metadata": { "ExecuteTime": { "end_time": "2026-05-19T12:42:02.291070Z", "start_time": "2026-05-19T12:42:02.288978Z" } }, "cell_type": "code", "source": "LOG_ROOT = Path(\"./checkpoints\")", "id": "eba6249c8ad34da8", "outputs": [], "execution_count": 20 }, { "metadata": { "ExecuteTime": { "end_time": "2026-05-19T12:42:02.315625Z", "start_time": "2026-05-19T12:42:02.302579Z" } }, "cell_type": "code", "source": [ "filename_pattern = re.compile(r\"^train_(?P.+)\\.log$\")\n", "result_pattern = re.compile(\n", " r\"TEST\\s*\\|\\s*\"\n", " r\"loss=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"plcc=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"srcc=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"rmse=(?P[-+]?\\d*\\.?\\d+)\"\n", ")\n", "records = []\n", "for log_path in LOG_ROOT.rglob(\"*.log\"):\n", " filename_match = filename_pattern.match(log_path.name)\n", " dataset = filename_match.group(\"dataset\")\n", "\n", " text = log_path.read_text(encoding=\"utf-8\", errors=\"ignore\")\n", " matches = list(result_pattern.finditer(text))\n", " if not matches:\n", " continue\n", " records.append({\n", " \"dataset\": dataset,\n", " \"srcc\": float(matches[-1].group(\"srcc\")),\n", " \"plcc\": float(matches[-1].group(\"plcc\")),\n", " })\n", "train_df = (pd.DataFrame(records).sort_values(\"dataset\", ignore_index=True)[[\"dataset\", \"srcc\", \"plcc\"]])\n", "train_df" ], "id": "4f2532e62a94f758", "outputs": [ { "data": { "text/plain": [ " dataset srcc plcc\n", "0 finevd 0.8596 0.8579\n", "1 konvid_1k 0.7970 0.8171\n", "2 kvq 0.8768 0.8781\n", "3 live_vqc 0.6386 0.7192\n", "4 live_yt_gaming 0.8554 0.8755\n", "5 lsvq 0.8753 0.8760\n", "6 shorts-hdr-dataset_hdr2sdr 0.5431 0.6657\n", "7 shorts-hdr-dataset_sdr 0.7884 0.8182\n", "8 youtube_ugc 0.8325 0.8548" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
datasetsrccplcc
0finevd0.85960.8579
1konvid_1k0.79700.8171
2kvq0.87680.8781
3live_vqc0.63860.7192
4live_yt_gaming0.85540.8755
5lsvq0.87530.8760
6shorts-hdr-dataset_hdr2sdr0.54310.6657
7shorts-hdr-dataset_sdr0.78840.8182
8youtube_ugc0.83250.8548
\n", "
" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 21 }, { "metadata": {}, "cell_type": "markdown", "source": "Cross-dataset evaluation: Train on dataset and test on \"testOn\" dataset", "id": "68dbaf20d93a49ed" }, { "metadata": { "ExecuteTime": { "end_time": "2026-05-19T12:42:02.318434Z", "start_time": "2026-05-19T12:42:02.316713Z" } }, "cell_type": "code", "source": "LOG_ROOT = Path(\"./checkpoints_transfer\")", "id": "e8583a08cc496a7c", "outputs": [], "execution_count": 22 }, { "metadata": { "ExecuteTime": { "end_time": "2026-05-19T12:42:02.328167Z", "start_time": "2026-05-19T12:42:02.319595Z" } }, "cell_type": "code", "source": [ "filename_pattern = re.compile(r\"^transfer_(?P.+?)_test_on_(?P.+?)\\.log$\")\n", "result_pattern = re.compile(\n", " r\"TEST_ONLY\\s*\\|\\s*\"\n", " r\"loss=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"plcc=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"srcc=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"rmse=(?P[-+]?\\d*\\.?\\d+)\"\n", ")\n", "records = []\n", "for log_path in LOG_ROOT.rglob(\"transfer_*_test_on_*.log\"):\n", " filename_match = filename_pattern.match(log_path.name)\n", " trainOn = filename_match.group(\"trainOn\")\n", " testOn = filename_match.group(\"testOn\")\n", "\n", " text = log_path.read_text(encoding=\"utf-8\", errors=\"ignore\")\n", " matches = list(result_pattern.finditer(text))\n", " if not matches:\n", " continue\n", " records.append({\n", " \"trainOn\": trainOn,\n", " \"testOn\": testOn,\n", " \"srcc\": float(matches[-1].group(\"srcc\")),\n", " \"plcc\": float(matches[-1].group(\"plcc\")),\n", " })\n", "transfer_df = (pd.DataFrame(records).sort_values([\"trainOn\", \"testOn\"], ignore_index=True))\n", "transfer_df" ], "id": "b196c863f89e0e0d", "outputs": [ { "data": { "text/plain": [ " trainOn testOn srcc plcc\n", "0 kvq shorts-hdr-dataset_hdr2sdr 0.4761 0.5885\n", "1 kvq shorts-hdr-dataset_sdr 0.7548 0.8065\n", "2 shorts-hdr-dataset_hdr2sdr kvq 0.5976 0.5693\n", "3 shorts-hdr-dataset_hdr2sdr shorts-hdr-dataset_sdr 0.6166 0.6802\n", "4 shorts-hdr-dataset_sdr kvq 0.7342 0.7451\n", "5 shorts-hdr-dataset_sdr shorts-hdr-dataset_hdr2sdr 0.5447 0.6958" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
trainOntestOnsrccplcc
0kvqshorts-hdr-dataset_hdr2sdr0.47610.5885
1kvqshorts-hdr-dataset_sdr0.75480.8065
2shorts-hdr-dataset_hdr2sdrkvq0.59760.5693
3shorts-hdr-dataset_hdr2sdrshorts-hdr-dataset_sdr0.61660.6802
4shorts-hdr-dataset_sdrkvq0.73420.7451
5shorts-hdr-dataset_sdrshorts-hdr-dataset_hdr2sdr0.54470.6958
\n", "
" ] }, "execution_count": 23, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 23 }, { "metadata": {}, "cell_type": "markdown", "source": "Cross-dataset evaluation: Train on \"trainOn\" dataset and finetune on \"fintuneOn\" dataset", "id": "e08409c56d2eaea3" }, { "metadata": { "ExecuteTime": { "end_time": "2026-05-19T12:42:02.337383Z", "start_time": "2026-05-19T12:42:02.329308Z" } }, "cell_type": "code", "source": [ "filename_pattern = re.compile(r\"^transfer_(?P.+?)_finetune_on_(?P.+?)\\.log$\")\n", "finetune_result_pattern = re.compile(\n", " r\"FINETUNE TEST\\s*\\|\\s*\"\n", " r\"loss=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"plcc=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"srcc=(?P[-+]?\\d*\\.?\\d+)\\s+\"\n", " r\"rmse=(?P[-+]?\\d*\\.?\\d+)\"\n", ")\n", "records = []\n", "for log_path in LOG_ROOT.rglob(\"transfer_*_finetune_on_*.log\"):\n", " filename_match = filename_pattern.match(log_path.name)\n", " trainOn = filename_match.group(\"trainOn\")\n", " finetuneOn = filename_match.group(\"finetuneOn\")\n", "\n", " text = log_path.read_text(encoding=\"utf-8\", errors=\"ignore\")\n", " matches = list(finetune_result_pattern.finditer(text))\n", " if not matches:\n", " continue\n", " records.append({\n", " \"trainOn\": trainOn,\n", " \"finetuneOn\": finetuneOn,\n", " \"srcc\": float(matches[-1].group(\"srcc\")),\n", " \"plcc\": float(matches[-1].group(\"plcc\")),\n", " })\n", "finetune_df = (pd.DataFrame(records).sort_values([\"trainOn\", \"finetuneOn\"], ignore_index=True))\n", "finetune_df" ], "id": "bfe4a5008377348d", "outputs": [ { "data": { "text/plain": [ " trainOn finetuneOn srcc plcc\n", "0 kvq shorts-hdr-dataset_hdr2sdr 0.6412 0.7225\n", "1 kvq shorts-hdr-dataset_sdr 0.8291 0.8683\n", "2 shorts-hdr-dataset_hdr2sdr kvq 0.8743 0.8792\n", "3 shorts-hdr-dataset_hdr2sdr shorts-hdr-dataset_sdr 0.8183 0.8561\n", "4 shorts-hdr-dataset_sdr kvq 0.8862 0.8883\n", "5 shorts-hdr-dataset_sdr shorts-hdr-dataset_hdr2sdr 0.6589 0.8013" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
trainOnfinetuneOnsrccplcc
0kvqshorts-hdr-dataset_hdr2sdr0.64120.7225
1kvqshorts-hdr-dataset_sdr0.82910.8683
2shorts-hdr-dataset_hdr2sdrkvq0.87430.8792
3shorts-hdr-dataset_hdr2sdrshorts-hdr-dataset_sdr0.81830.8561
4shorts-hdr-dataset_sdrkvq0.88620.8883
5shorts-hdr-dataset_sdrshorts-hdr-dataset_hdr2sdr0.65890.8013
\n", "
" ] }, "execution_count": 24, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 24 }, { "metadata": {}, "cell_type": "markdown", "source": "Complexity_test", "id": "544f3bed13c8f3d9" }, { "metadata": { "ExecuteTime": { "end_time": "2026-05-19T12:42:02.339580Z", "start_time": "2026-05-19T12:42:02.337969Z" } }, "cell_type": "code", "source": "complexity_csv_path = \"../test_videos/complexity_test/\"", "id": "9d718a191ee6bdf2", "outputs": [], "execution_count": 25 }, { "metadata": { "ExecuteTime": { "end_time": "2026-05-19T12:42:02.347488Z", "start_time": "2026-05-19T12:42:02.340191Z" } }, "cell_type": "code", "source": [ "resolution_df1 = pd.read_csv(f\"{complexity_csv_path}complexity_resolution_KVQ_0546.csv\").assign(video=\"KVQ_0546\")\n", "resolution_df2 = pd.read_csv(f\"{complexity_csv_path}complexity_resolution_SDR_Animal_5ngj.csv\").assign(video=\"SDR_Animal_5ngj\")\n", "resolution_df = pd.concat([resolution_df1, resolution_df2], ignore_index=True)\n", "resolution_df" ], "id": "5c99ce5680f78c34", "outputs": [ { "data": { "text/plain": [ " model 540P 720P 1080P 2160P MOS video\n", "0 FAST-VQA 0.638 0.745 1.020 2.680 4.017 KVQ_0546\n", "1 FasterVQA 0.428 0.480 0.573 1.077 4.356 KVQ_0546\n", "2 DOVER 1.018 1.163 1.511 3.662 4.223 KVQ_0546\n", "3 Our 0.364 0.485 0.753 2.437 3.650 KVQ_0546\n", "4 FAST-VQA 0.599 0.673 0.909 2.217 3.319 SDR_Animal_5ngj\n", "5 FasterVQA 0.489 0.547 0.696 1.343 3.556 SDR_Animal_5ngj\n", "6 DOVER 0.920 1.022 1.293 2.783 3.814 SDR_Animal_5ngj\n", "7 Our 0.313 0.405 0.697 2.137 3.878 SDR_Animal_5ngj" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
model540P720P1080P2160PMOSvideo
0FAST-VQA0.6380.7451.0202.6804.017KVQ_0546
1FasterVQA0.4280.4800.5731.0774.356KVQ_0546
2DOVER1.0181.1631.5113.6624.223KVQ_0546
3Our0.3640.4850.7532.4373.650KVQ_0546
4FAST-VQA0.5990.6730.9092.2173.319SDR_Animal_5ngj
5FasterVQA0.4890.5470.6961.3433.556SDR_Animal_5ngj
6DOVER0.9201.0221.2932.7833.814SDR_Animal_5ngj
7Our0.3130.4050.6972.1373.878SDR_Animal_5ngj
\n", "
" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 26 }, { "metadata": { "ExecuteTime": { "end_time": "2026-05-19T12:42:02.353757Z", "start_time": "2026-05-19T12:42:02.348243Z" } }, "cell_type": "code", "source": [ "complexity_df = pd.read_csv(f\"{complexity_csv_path}complexity_test.csv\")\n", "complexity_df" ], "id": "f639ed0f66a50c57", "outputs": [ { "data": { "text/plain": [ " model video Time Predicted_MOS Scaled_MOS MOS\n", "0 FAST-VQA SDR_Gameplay_wcq2 0.822 0.248790 1.995 4.156\n", "1 FAST-VQA SDR_Gameplay_s0pc 0.820 0.550190 3.201 4.264\n", "2 FAST-VQA SDR_Gameplay_z5fz 0.823 0.538800 3.155 4.221\n", "3 FAST-VQA SDR_Animal_5ngj 0.846 0.579850 3.319 4.308\n", "4 FAST-VQA 0546 0.895 0.754290 4.017 2.389\n", "5 FasterVQA SDR_Gameplay_wcq2 0.502 0.299890 2.200 4.156\n", "6 FasterVQA SDR_Gameplay_s0pc 0.524 0.706330 3.825 4.264\n", "7 FasterVQA SDR_Gameplay_z5fz 0.499 0.400900 2.604 4.221\n", "8 FasterVQA SDR_Animal_5ngj 0.500 0.638990 3.556 4.308\n", "9 FasterVQA 0546 0.484 0.839070 4.356 2.389\n", "10 DOVER SDR_Gameplay_wcq2 1.407 0.347305 2.389 4.156\n", "11 DOVER SDR_Gameplay_s0pc 1.345 0.582339 3.329 4.264\n", "12 DOVER SDR_Gameplay_z5fz 1.341 0.524504 3.098 4.221\n", "13 DOVER SDR_Animal_5ngj 1.326 0.703516 3.814 4.308\n", "14 DOVER 0546 1.489 0.805786 4.223 2.389\n", "15 Our SDR_Gameplay_wcq2 0.594 59.864200 3.395 4.156\n", "16 Our SDR_Gameplay_s0pc 0.605 61.772900 3.471 4.264\n", "17 Our SDR_Gameplay_z5fz 0.602 61.077000 3.443 4.221\n", "18 Our SDR_Animal_5ngj 0.598 71.957800 3.878 4.308\n", "19 Our 0546 0.734 66.257200 3.650 2.389" ], "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
modelvideoTimePredicted_MOSScaled_MOSMOS
0FAST-VQASDR_Gameplay_wcq20.8220.2487901.9954.156
1FAST-VQASDR_Gameplay_s0pc0.8200.5501903.2014.264
2FAST-VQASDR_Gameplay_z5fz0.8230.5388003.1554.221
3FAST-VQASDR_Animal_5ngj0.8460.5798503.3194.308
4FAST-VQA05460.8950.7542904.0172.389
5FasterVQASDR_Gameplay_wcq20.5020.2998902.2004.156
6FasterVQASDR_Gameplay_s0pc0.5240.7063303.8254.264
7FasterVQASDR_Gameplay_z5fz0.4990.4009002.6044.221
8FasterVQASDR_Animal_5ngj0.5000.6389903.5564.308
9FasterVQA05460.4840.8390704.3562.389
10DOVERSDR_Gameplay_wcq21.4070.3473052.3894.156
11DOVERSDR_Gameplay_s0pc1.3450.5823393.3294.264
12DOVERSDR_Gameplay_z5fz1.3410.5245043.0984.221
13DOVERSDR_Animal_5ngj1.3260.7035163.8144.308
14DOVER05461.4890.8057864.2232.389
15OurSDR_Gameplay_wcq20.59459.8642003.3954.156
16OurSDR_Gameplay_s0pc0.60561.7729003.4714.264
17OurSDR_Gameplay_z5fz0.60261.0770003.4434.221
18OurSDR_Animal_5ngj0.59871.9578003.8784.308
19Our05460.73466.2572003.6502.389
\n", "
" ] }, "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 27 } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 5 }