selector_metrics [results]

.gitattributes

@@ -35,3 +35,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 data/raw/selector_dataset.csv filter=lfs diff=lfs merge=lfs -text
 data/raw/priority_dataset.csv filter=lfs diff=lfs merge=lfs -text
+results/plots/decision_tree.png filter=lfs diff=lfs merge=lfs -text
+results/plots/feature_importance.png filter=lfs diff=lfs merge=lfs -text
+results/plots/shap_selector_xgb.png filter=lfs diff=lfs merge=lfs -text

results/selector_metrics.json

@@ -0,0 +1,426 @@
+{
+  "_meta": {
+    "run_hash": "a5df24198f89b9e9",
+    "label_names": [
+      "FIFO",
+      "Priority-EDD",
+      "Critical-Ratio",
+      "ATC",
+      "WSPT",
+      "Slack"
+    ]
+  },
+  "models": {
+    "dt": {
+      "model": "dt",
+      "n_train": 156000,
+      "n_test": 39000,
+      "n_features": 32,
+      "n_classes": 6,
+      "accuracy": 0.5709487179487179,
+      "balanced_accuracy": 0.5290909413678629,
+      "mcc": 0.42112667363058276,
+      "cohens_kappa": 0.3993694507162956,
+      "f1_macro": 0.4308264689331996,
+      "f1_micro": 0.5709487179487179,
+      "f1_weighted": 0.6178696598280847,
+      "cv_accuracy_mean": 0.5634935897435899,
+      "cv_accuracy_std": 0.0032924639510805882,
+      "cv_accuracy_folds": [
+        0.5686858974358975,
+        0.564775641025641,
+        0.5625641025641026,
+        0.5585576923076923,
+        0.5628846153846154
+      ],
+      "per_class": [
+        {
+          "class": "FIFO",
+          "class_idx": 0,
+          "precision": 0.5602836879432624,
+          "recall": 0.6346880907372401,
+          "f1": 0.5951695103035675,
+          "support": 4232
+        },
+        {
+          "class": "Priority-EDD",
+          "class_idx": 1,
+          "precision": 0.4261963952765693,
+          "recall": 0.5718157181571816,
+          "f1": 0.4883824445829253,
+          "support": 4797
+        },
+        {
+          "class": "Critical-Ratio",
+          "class_idx": 2,
+          "precision": 0.14338235294117646,
+          "recall": 0.5712383488681758,
+          "f1": 0.22922789206518834,
+          "support": 751
+        },
+        {
+          "class": "ATC",
+          "class_idx": 3,
+          "precision": 0.346766330841729,
+          "recall": 0.46340933767643866,
+          "f1": 0.3966911422994702,
+          "support": 4605
+        },
+        {
+          "class": "WSPT",
+          "class_idx": 4,
+          "precision": 0.9391586197582551,
+          "recall": 0.59072375127421,
+          "f1": 0.7252626913148906,
+          "support": 23544
+        },
+        {
+          "class": "Slack",
+          "class_idx": 5,
+          "precision": 0.09619921363040629,
+          "recall": 0.3426704014939309,
+          "f1": 0.15022513303315596,
+          "support": 1071
+        }
+      ],
+      "confusion_matrix": [
+        [
+          2686,
+          502,
+          65,
+          382,
+          319,
+          278
+        ],
+        [
+          484,
+          2743,
+          316,
+          486,
+          87,
+          681
+        ],
+        [
+          17,
+          102,
+          429,
+          20,
+          14,
+          169
+        ],
+        [
+          243,
+          972,
+          264,
+          2134,
+          426,
+          566
+        ],
+        [
+          1257,
+          1908,
+          1718,
+          2999,
+          13908,
+          1754
+        ],
+        [
+          107,
+          209,
+          200,
+          133,
+          55,
+          367
+        ]
+      ],
+      "confusion_matrix_labels": [
+        "FIFO",
+        "Priority-EDD",
+        "Critical-Ratio",
+        "ATC",
+        "WSPT",
+        "Slack"
+      ],
+      "log_loss": 1.4256790366866996,
+      "roc_auc_ovr_macro": 0.8395237903638563,
+      "roc_auc_ovr_weighted": 0.846621746931995,
+      "pr_auc_macro": 0.4092273256735423,
+      "pr_auc_weighted": 0.7021611881862597,
+      "brier_mean": 0.09428976942335138
+    },
+    "rf": {
+      "model": "rf",
+      "n_train": 156000,
+      "n_test": 39000,
+      "n_features": 32,
+      "n_classes": 6,
+      "accuracy": 0.6291794871794871,
+      "balanced_accuracy": 0.5511175272026881,
+      "mcc": 0.47639913992536354,
+      "cohens_kappa": 0.4594432116719369,
+      "f1_macro": 0.4677547590963984,
+      "f1_micro": 0.6291794871794871,
+      "f1_weighted": 0.6583182850706908,
+      "cv_accuracy_mean": 0.630076923076923,
+      "cv_accuracy_std": 0.0025719111868781016,
+      "cv_accuracy_folds": [
+        0.6303525641025641,
+        0.6254807692307692,
+        0.6309935897435898,
+        0.6301602564102564,
+        0.6333974358974359
+      ],
+      "per_class": [
+        {
+          "class": "FIFO",
+          "class_idx": 0,
+          "precision": 0.5811373092926491,
+          "recall": 0.6930529300567108,
+          "f1": 0.6321801918310163,
+          "support": 4232
+        },
+        {
+          "class": "Priority-EDD",
+          "class_idx": 1,
+          "precision": 0.44614147909967844,
+          "recall": 0.6941838649155723,
+          "f1": 0.5431857107903107,
+          "support": 4797
+        },
+        {
+          "class": "Critical-Ratio",
+          "class_idx": 2,
+          "precision": 0.17252396166134185,
+          "recall": 0.5752330226364847,
+          "f1": 0.2654377880184332,
+          "support": 751
+        },
+        {
+          "class": "ATC",
+          "class_idx": 3,
+          "precision": 0.37192118226600984,
+          "recall": 0.49185667752442996,
+          "f1": 0.423562412342216,
+          "support": 4605
+        },
+        {
+          "class": "WSPT",
+          "class_idx": 4,
+          "precision": 0.9297990801258775,
+          "recall": 0.6525654094461434,
+          "f1": 0.7668962763302386,
+          "support": 23544
+        },
+        {
+          "class": "Slack",
+          "class_idx": 5,
+          "precision": 0.1560904449307075,
+          "recall": 0.19981325863678806,
+          "f1": 0.17526617526617527,
+          "support": 1071
+        }
+      ],
+      "confusion_matrix": [
+        [
+          2933,
+          413,
+          35,
+          423,
+          349,
+          79
+        ],
+        [
+          506,
+          3330,
+          199,
+          435,
+          115,
+          212
+        ],
+        [
+          19,
+          133,
+          432,
+          15,
+          39,
+          113
+        ],
+        [
+          230,
+          1100,
+          232,
+          2265,
+          575,
+          203
+        ],
+        [
+          1243,
+          2198,
+          1396,
+          2793,
+          15364,
+          550
+        ],
+        [
+          116,
+          290,
+          210,
+          159,
+          82,
+          214
+        ]
+      ],
+      "confusion_matrix_labels": [
+        "FIFO",
+        "Priority-EDD",
+        "Critical-Ratio",
+        "ATC",
+        "WSPT",
+        "Slack"
+      ],
+      "log_loss": 0.9268822996154131,
+      "roc_auc_ovr_macro": 0.8874953998881535,
+      "roc_auc_ovr_weighted": 0.8838300040841169,
+      "pr_auc_macro": 0.4734487538434295,
+      "pr_auc_weighted": 0.7516964651745404,
+      "brier_mean": 0.0791191637167047
+    },
+    "xgb": {
+      "model": "xgb",
+      "n_train": 156000,
+      "n_test": 39000,
+      "n_features": 32,
+      "n_classes": 6,
+      "accuracy": 0.6185641025641025,
+      "balanced_accuracy": 0.5593364090484011,
+      "mcc": 0.46905461959242584,
+      "cohens_kappa": 0.450831708063013,
+      "f1_macro": 0.464199081171982,
+      "f1_micro": 0.6185641025641025,
+      "f1_weighted": 0.6536438489561579,
+      "cv_accuracy_mean": 0.7154102564102565,
+      "cv_accuracy_std": 0.002060216919188168,
+      "cv_accuracy_folds": [
+        0.7143910256410256,
+        0.7163461538461539,
+        0.7119871794871795,
+        0.7180128205128206,
+        0.7163141025641026
+      ],
+      "per_class": [
+        {
+          "class": "FIFO",
+          "class_idx": 0,
+          "precision": 0.5687185443283004,
+          "recall": 0.694234404536862,
+          "f1": 0.6252394126409875,
+          "support": 4232
+        },
+        {
+          "class": "Priority-EDD",
+          "class_idx": 1,
+          "precision": 0.4744243548137846,
+          "recall": 0.6399833229101521,
+          "f1": 0.5449059282925097,
+          "support": 4797
+        },
+        {
+          "class": "Critical-Ratio",
+          "class_idx": 2,
+          "precision": 0.1625,
+          "recall": 0.6231691078561917,
+          "f1": 0.25778022583310384,
+          "support": 751
+        },
+        {
+          "class": "ATC",
+          "class_idx": 3,
+          "precision": 0.3714376691609616,
+          "recall": 0.5066232356134637,
+          "f1": 0.4286239206320044,
+          "support": 4605
+        },
+        {
+          "class": "WSPT",
+          "class_idx": 4,
+          "precision": 0.9363291218024522,
+          "recall": 0.6389738362215427,
+          "f1": 0.7595869834136982,
+          "support": 23544
+        },
+        {
+          "class": "Slack",
+          "class_idx": 5,
+          "precision": 0.1269320843091335,
+          "recall": 0.2530345471521942,
+          "f1": 0.16905801621958827,
+          "support": 1071
+        }
+      ],
+      "confusion_matrix": [
+        [
+          2938,
+          381,
+          48,
+          392,
+          323,
+          150
+        ],
+        [
+          476,
+          3070,
+          275,
+          493,
+          101,
+          382
+        ],
+        [
+          19,
+          100,
+          468,
+          15,
+          18,
+          131
+        ],
+        [
+          256,
+          924,
+          263,
+          2333,
+          512,
+          317
+        ],
+        [
+          1367,
+          1751,
+          1586,
+          2912,
+          15044,
+          884
+        ],
+        [
+          110,
+          245,
+          240,
+          136,
+          69,
+          271
+        ]
+      ],
+      "confusion_matrix_labels": [
+        "FIFO",
+        "Priority-EDD",
+        "Critical-Ratio",
+        "ATC",
+        "WSPT",
+        "Slack"
+      ],
+      "log_loss": 0.9664218852784615,
+      "roc_auc_ovr_macro": 0.8884795139901561,
+      "roc_auc_ovr_weighted": 0.8861972617734345,
+      "pr_auc_macro": 0.47825064337426243,
+      "pr_auc_weighted": 0.757008292981398,
+      "brier_mean": 0.08144319678346316
+    }
+  }
+}

results/selector_metrics_table.csv

@@ -0,0 +1,4 @@
+model,accuracy,balanced_accuracy,f1_macro,f1_weighted,mcc,cohens_kappa,roc_auc_ovr_macro,pr_auc_macro,log_loss,brier_mean,cv_acc_mean,cv_acc_std
+dt,0.5709487179487179,0.5290909413678629,0.4308264689331996,0.6178696598280847,0.42112667363058276,0.3993694507162956,0.8395237903638563,0.4092273256735423,1.4256790366866996,0.09428976942335138,0.5634935897435899,0.0032924639510805882
+rf,0.6291794871794871,0.5511175272026881,0.4677547590963984,0.6583182850706908,0.47639913992536354,0.4594432116719369,0.8874953998881535,0.4734487538434295,0.9268822996154131,0.0791191637167047,0.630076923076923,0.0025719111868781016
+xgb,0.6185641025641025,0.5593364090484011,0.464199081171982,0.6536438489561579,0.46905461959242584,0.450831708063013,0.8884795139901561,0.47825064337426243,0.9664218852784615,0.08144319678346316,0.7154102564102565,0.002060216919188168