Sync from GitHub Actions

.gitignore

@@ -4,5 +4,4 @@ __pycache__/
 .ipynb_checkpoints/
 .env
 .venv/
-venv/
-artifacts/
+venv/

artifacts/evaluation/model_comparison/README_model_comparison.md

@@ -0,0 +1,7 @@
+| Model | Avg Accuracy | Avg Macro F1 | Top-3 Accuracy | Exact Match | Latency |
+|---|---:|---:|---:|---:|---:|
+| Majority Baseline | 42.63% | 7.93% | 73.44% | 0.95% | 0.001 ms |
+| Frozen CLIP + Heads (V1) | 83.35% | 65.68% | 97.11% | 27.94% | 6.227 ms |
+| AutoCatalogAI V2 | 87.48% | 67.41% | 98.15% | 40.46% | 7.474 ms |
+
+> All models were evaluated on the same 6,611-image held-out test split. Metrics use raw model predictions. Latency is batch-size-1 model-forward time with preprocessing excluded.

artifacts/evaluation/model_comparison/benchmark_metadata.json

@@ -0,0 +1,37 @@
+{
+  "created_at_utc": "2026-07-06T14:54:13.900213+00:00",
+  "dataset_name": "ashraq/fashion-product-images-small",
+  "dataset_fingerprint": "5cacd020bfdb9ce5",
+  "v1_repo_id": "mohsin416/autocatalogai-clip-multitask",
+  "v2_repo_id": "mohsin416/autocatalogai-clip-multitask-v2",
+  "seed": 42,
+  "train_samples": 30850,
+  "validation_samples": 6611,
+  "test_samples": 6611,
+  "test_split_sha256": "106737acc60a436248d35e8a375a014d8b6e2f26f74b147b6444540b7781c0ec",
+  "tasks": [
+    "gender",
+    "masterCategory",
+    "subCategory",
+    "articleType",
+    "baseColour",
+    "season",
+    "usage"
+  ],
+  "metric_policy": "Raw model predictions for all rows",
+  "latency_policy": {
+    "batch_size": 1,
+    "model_forward_only": true,
+    "preprocessing_excluded": true,
+    "warmup_runs": 20,
+    "measured_runs": 100
+  },
+  "environment": {
+    "python": "3.12.13",
+    "torch": "2.5.1+cu121",
+    "transformers": "4.46.3",
+    "device": "cuda",
+    "gpu": "Tesla P100-PCIE-16GB",
+    "cuda_runtime": "12.1"
+  }
+}

artifacts/evaluation/model_comparison/model_comparison.csv

@@ -0,0 +1,4 @@
+Model,Avg Accuracy,Avg Macro F1,Top-3 Accuracy,Exact Match,Latency (ms)
+Majority Baseline,0.42628087386822827,0.07927692465823664,0.7344469174752037,0.009529571925578581,0.0005647709000186296
+Frozen CLIP + Heads (V1),0.8335026038852995,0.6568447666058456,0.9711087581303888,0.27938284677053393,6.226557110003341
+AutoCatalogAI V2,0.8747758065561727,0.6740687827687263,0.981502690321326,0.4046286492209953,7.4737231199719645

artifacts/evaluation/model_comparison/model_comparison.json

@@ -0,0 +1,23 @@
+{
+  "Majority Baseline": {
+    "average_accuracy": 0.42628087386822827,
+    "average_macro_f1": 0.07927692465823664,
+    "average_top3_accuracy": 0.7344469174752037,
+    "exact_match_accuracy": 0.009529571925578581,
+    "latency_ms": 0.0005647709000186296
+  },
+  "Frozen CLIP + Heads (V1)": {
+    "average_accuracy": 0.8335026038852995,
+    "average_macro_f1": 0.6568447666058456,
+    "average_top3_accuracy": 0.9711087581303888,
+    "exact_match_accuracy": 0.27938284677053393,
+    "latency_ms": 6.226557110003341
+  },
+  "AutoCatalogAI V2": {
+    "average_accuracy": 0.8747758065561727,
+    "average_macro_f1": 0.6740687827687263,
+    "average_top3_accuracy": 0.981502690321326,
+    "exact_match_accuracy": 0.4046286492209953,
+    "latency_ms": 7.4737231199719645
+  }
+}

artifacts/evaluation/model_comparison/model_comparison_per_task.csv

@@ -0,0 +1,22 @@
+Model,Task,Accuracy,Macro F1,Top-3 Accuracy
+Majority Baseline,gender,0.49916805324459235,0.13318534961154274,0.9698986537588867
+Majority Baseline,masterCategory,0.4844955377401301,0.10879016371170437,0.9485705642111633
+Majority Baseline,subCategory,0.34911511117833915,0.012938670254512837,0.5843291483890486
+Majority Baseline,articleType,0.16033882922402057,0.0021761065553157046,0.2978369384359401
+Majority Baseline,baseColour,0.2187263651489941,0.008157784873684092,0.45620934805626984
+Majority Baseline,season,0.48903342913326275,0.16421170255993497,0.9399485705642112
+Majority Baseline,usage,0.7830887914082589,0.12547869504096174,0.9443351989109061
+Frozen CLIP + Heads (V1),gender,0.8886703978218121,0.7766001480008942,0.9956133716533051
+Frozen CLIP + Heads (V1),masterCategory,0.9933444259567388,0.8629937982338353,0.999394947814249
+Frozen CLIP + Heads (V1),subCategory,0.9429738314929663,0.7108939726417577,0.9947057933746786
+Frozen CLIP + Heads (V1),articleType,0.8410225381939193,0.6639695724602702,0.9711087581303888
+Frozen CLIP + Heads (V1),baseColour,0.6011193465436394,0.3409578550909414,0.8538798971411284
+Frozen CLIP + Heads (V1),season,0.7070034790500681,0.7280134716349071,0.9848736953562245
+Frozen CLIP + Heads (V1),usage,0.8603842081379519,0.5144845481783126,0.9981848434427469
+AutoCatalogAI V2,gender,0.9192255332022387,0.8120023642434658,0.9974285282105582
+AutoCatalogAI V2,masterCategory,0.9944032672818031,0.8464411154349717,0.999394947814249
+AutoCatalogAI V2,subCategory,0.963545605808501,0.7593135523586715,0.9972772651641204
+AutoCatalogAI V2,articleType,0.876418091060354,0.6636649938278787,0.9815459083345939
+AutoCatalogAI V2,baseColour,0.697171381031614,0.3649654467037918,0.9069732264407805
+AutoCatalogAI V2,season,0.7548026017243987,0.767844259585529,0.9897141128422327
+AutoCatalogAI V2,usage,0.9178641657842989,0.5042497472267755,0.9981848434427469