Upload folder using huggingface_hub

README.md

@@ -0,0 +1,49 @@
+---
+license: mit
+tags:
+  - vulnerability-detection
+  - linux-kernel
+  - security
+  - pytorch
+datasets:
+  - pebblebed/kernel-vuln-dataset
+metrics:
+  - auc
+  - recall
+  - precision
+pipeline_tag: text-classification
+---
+
+# VulnBERT v8
+
+Vulnerability detection model for Linux kernel commits.
+
+## Results
+
+| Metric | Value |
+|--------|-------|
+| AUC | 0.987 |
+| Recall | 91.4% |
+| Precision | 88.4% |
+| F1 | 0.899 |
+| FPR | 5.9% |
+
+## Usage
+```python
+import torch
+
+checkpoint = torch.load("pytorch_model.pt", map_location="cpu")
+model.load_state_dict(checkpoint["model_state_dict"])
+```
+
+Full code: [github.com/quguanni/vulnbert](https://github.com/quguanni/vulnbert)
+
+## Training
+
+- Dataset: [pebblebed/kernel-vuln-dataset](https://huggingface.co/datasets/pebblebed/kernel-vuln-dataset) (650K commits)
+- Architecture: CodeBERT + 118 handcrafted features
+- Time: ~7 hours on NVIDIA GH200
+
+## License
+
+MIT

pytorch_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4733cc4836e5949051e5d4900a961382d4c0a90bdb4dd62711db9bab33917182
+size 517265547

test_results.json

@@ -0,0 +1,15 @@
+{
+  "precision": 0.8840327467285503,
+  "recall": 0.9142666666666667,
+  "f1": 0.8988955527152361,
+  "auc": 0.9869358197843104,
+  "ap": 0.9764627774456178,
+  "fpr": 0.05909016258827394,
+  "tp": 13714,
+  "fp": 1799,
+  "tn": 28646,
+  "fn": 1286,
+  "recall_at_1_fpr": 0.8097333333333333,
+  "recall_at_5_fpr": 0.9033333333333333,
+  "recall_at_10_fpr": 0.9540666666666666
+}

training_history.json

@@ -0,0 +1,431 @@
+[
+  {
+    "epoch": 1,
+    "train": {
+      "precision": 0.41892358882712455,
+      "recall": 0.44379013869422423,
+      "f1": 0.43099849152051684,
+      "auc": 0.7680436984019521,
+      "ap": 0.46493909710249876,
+      "fpr": 0.127397138694727,
+      "tp": 42429,
+      "fp": 58852,
+      "tn": 403105,
+      "fn": 53177,
+      "recall_at_1_fpr": 0.14793004623140807,
+      "recall_at_5_fpr": 0.28439637679643537,
+      "recall_at_10_fpr": 0.39409660481559733,
+      "loss": 0.442193871078037,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.8442053864347295,
+      "recall": 0.7590725114907045,
+      "f1": 0.7993787024996388,
+      "auc": 0.9481094739921635,
+      "ap": 0.9114432578319719,
+      "fpr": 0.06192945743487095,
+      "tp": 11065,
+      "fp": 2042,
+      "tn": 30931,
+      "fn": 3512,
+      "recall_at_1_fpr": 0.6148041435137546,
+      "recall_at_5_fpr": 0.7389037524867943,
+      "recall_at_10_fpr": 0.8147767030253138,
+      "loss": 0.20687348265100577
+    }
+  },
+  {
+    "epoch": 2,
+    "train": {
+      "precision": 0.8307668515122727,
+      "recall": 0.7866242704432777,
+      "f1": 0.8080931811832462,
+      "auc": 0.9692768001107926,
+      "ap": 0.9092969502291814,
+      "fpr": 0.033163259783919284,
+      "tp": 75206,
+      "fp": 15320,
+      "tn": 446637,
+      "fn": 20400,
+      "recall_at_1_fpr": 0.6966612974081124,
+      "recall_at_5_fpr": 0.8268518712214714,
+      "recall_at_10_fpr": 0.9016065937284271,
+      "loss": 0.1825303359336363,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.9014335639398046,
+      "recall": 0.8670508335048364,
+      "f1": 0.8839079655919995,
+      "auc": 0.9825398959126007,
+      "ap": 0.9665405246343721,
+      "fpr": 0.04191308039911443,
+      "tp": 12639,
+      "fp": 1382,
+      "tn": 31591,
+      "fn": 1938,
+      "recall_at_1_fpr": 0.7705289154146944,
+      "recall_at_5_fpr": 0.8828977155793373,
+      "recall_at_10_fpr": 0.9390821156616588,
+      "loss": 0.14055028911934397
+    }
+  },
+  {
+    "epoch": 3,
+    "train": {
+      "precision": 0.8819841739549769,
+      "recall": 0.8417254147229254,
+      "f1": 0.861384654906662,
+      "auc": 0.985384220872594,
+      "ap": 0.9498956506035895,
+      "fpr": 0.02330952880895841,
+      "tp": 80474,
+      "fp": 10768,
+      "tn": 451189,
+      "fn": 15132,
+      "recall_at_1_fpr": 0.7859757755789386,
+      "recall_at_5_fpr": 0.9042319519695416,
+      "recall_at_10_fpr": 0.9581407024663724,
+      "loss": 0.1255587323948855,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.887695445275323,
+      "recall": 0.8957947451464636,
+      "f1": 0.8917267046812579,
+      "auc": 0.9852800333656634,
+      "ap": 0.9714275151122855,
+      "fpr": 0.050101598277378465,
+      "tp": 13058,
+      "fp": 1652,
+      "tn": 31321,
+      "fn": 1519,
+      "recall_at_1_fpr": 0.7969403855388626,
+      "recall_at_5_fpr": 0.8957261439253619,
+      "recall_at_10_fpr": 0.9481374768470878,
+      "loss": 0.13353063743442475
+    }
+  },
+  {
+    "epoch": 4,
+    "train": {
+      "precision": 0.8926473129428029,
+      "recall": 0.8645064117314812,
+      "f1": 0.8783515233955728,
+      "auc": 0.9892002520538117,
+      "ap": 0.9607525382115066,
+      "fpr": 0.02151715419400507,
+      "tp": 82652,
+      "fp": 9940,
+      "tn": 452017,
+      "fn": 12954,
+      "recall_at_1_fpr": 0.8136623224483819,
+      "recall_at_5_fpr": 0.9279124741125034,
+      "recall_at_10_fpr": 0.9739660690751627,
+      "loss": 0.10802158596400661,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.9209117221418235,
+      "recall": 0.8730877409617891,
+      "f1": 0.8963622917913864,
+      "auc": 0.9866947658083867,
+      "ap": 0.973685911231753,
+      "fpr": 0.03314833348497255,
+      "tp": 12727,
+      "fp": 1093,
+      "tn": 31880,
+      "fn": 1850,
+      "recall_at_1_fpr": 0.8020168759003911,
+      "recall_at_5_fpr": 0.9010770391712972,
+      "recall_at_10_fpr": 0.9547231940728544,
+      "loss": 0.12385919165582603
+    }
+  },
+  {
+    "epoch": 5,
+    "train": {
+      "precision": 0.8966450101182235,
+      "recall": 0.88055142982658,
+      "f1": 0.888525351458606,
+      "auc": 0.9912534168074455,
+      "ap": 0.9672059599340288,
+      "fpr": 0.021006284134670542,
+      "tp": 84186,
+      "fp": 9704,
+      "tn": 452253,
+      "fn": 11420,
+      "recall_at_1_fpr": 0.8321653452712173,
+      "recall_at_5_fpr": 0.9446792042340438,
+      "recall_at_10_fpr": 0.9820408760956425,
+      "loss": 0.0977837755253274,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.9377220164764568,
+      "recall": 0.8511353502092337,
+      "f1": 0.8923331415420023,
+      "auc": 0.986774076542897,
+      "ap": 0.9739167475982135,
+      "fpr": 0.02499014345070209,
+      "tp": 12407,
+      "fp": 824,
+      "tn": 32149,
+      "fn": 2170,
+      "recall_at_1_fpr": 0.80592714550319,
+      "recall_at_5_fpr": 0.9027920696988406,
+      "recall_at_10_fpr": 0.9549975989572614,
+      "loss": 0.12919959488739488
+    }
+  },
+  {
+    "epoch": 6,
+    "train": {
+      "precision": 0.9038794926004229,
+      "recall": 0.8943685542748363,
+      "f1": 0.8990988717495768,
+      "auc": 0.9927986150339585,
+      "ap": 0.9721630461464356,
+      "fpr": 0.01968365020986802,
+      "tp": 85507,
+      "fp": 9093,
+      "tn": 452864,
+      "fn": 10099,
+      "recall_at_1_fpr": 0.8480011714745936,
+      "recall_at_5_fpr": 0.9573771520615861,
+      "recall_at_10_fpr": 0.9866535573081188,
+      "loss": 0.08971706647469802,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": 0.1413140798166237,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.8790910295481649,
+      "recall": 0.9102696027989298,
+      "f1": 0.894408681877928,
+      "auc": 0.9869048064651947,
+      "ap": 0.9740420754054223,
+      "fpr": 0.055348315288266156,
+      "tp": 13269,
+      "fp": 1825,
+      "tn": 31148,
+      "fn": 1308,
+      "recall_at_1_fpr": 0.8023598820058997,
+      "recall_at_5_fpr": 0.9039582904575701,
+      "recall_at_10_fpr": 0.9556150099471771,
+      "loss": 0.12849023392180398
+    }
+  },
+  {
+    "epoch": 7,
+    "train": {
+      "precision": 0.9111462997724389,
+      "recall": 0.9087923352090873,
+      "f1": 0.9099677951457074,
+      "auc": 0.9941540713009294,
+      "ap": 0.9769532634680379,
+      "fpr": 0.01834153395229426,
+      "tp": 86886,
+      "fp": 8473,
+      "tn": 453484,
+      "fn": 8720,
+      "recall_at_1_fpr": 0.8675919921343849,
+      "recall_at_5_fpr": 0.9682760496203168,
+      "recall_at_10_fpr": 0.9903771729807752,
+      "loss": 0.08156081970673255,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.9283284023668639,
+      "recall": 0.8610139260478836,
+      "f1": 0.8934049898565684,
+      "auc": 0.9863749242919583,
+      "ap": 0.9732129729985973,
+      "fpr": 0.02938768082976981,
+      "tp": 12551,
+      "fp": 969,
+      "tn": 32004,
+      "fn": 2026,
+      "recall_at_1_fpr": 0.8036633052068327,
+      "recall_at_5_fpr": 0.8994992110859573,
+      "recall_at_10_fpr": 0.9515675379021746,
+      "loss": 0.13835142146230034
+    }
+  },
+  {
+    "epoch": 8,
+    "train": {
+      "precision": 0.9220899220899221,
+      "recall": 0.9185406773633454,
+      "f1": 0.9203118777640377,
+      "auc": 0.9952344204548305,
+      "ap": 0.980939889918141,
+      "fpr": 0.01606210101806012,
+      "tp": 87818,
+      "fp": 7420,
+      "tn": 454537,
+      "fn": 7788,
+      "recall_at_1_fpr": 0.8882287722527875,
+      "recall_at_5_fpr": 0.9762253415057632,
+      "recall_at_10_fpr": 0.9930339100056482,
+      "loss": 0.07410608262540465,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.9039670345020254,
+      "recall": 0.887905604719764,
+      "f1": 0.895864336390379,
+      "auc": 0.9868423812888824,
+      "ap": 0.973946457446816,
+      "fpr": 0.04170078549115944,
+      "tp": 12943,
+      "fp": 1375,
+      "tn": 31598,
+      "fn": 1634,
+      "recall_at_1_fpr": 0.8015366673526789,
+      "recall_at_5_fpr": 0.9025176648144337,
+      "recall_at_10_fpr": 0.9567126294848048,
+      "loss": 0.13774085793055554
+    }
+  },
+  {
+    "epoch": 9,
+    "train": {
+      "precision": 0.9304854287569924,
+      "recall": 0.9290839487061482,
+      "f1": 0.9297841606129755,
+      "auc": 0.9962078429373322,
+      "ap": 0.9846454862326747,
+      "fpr": 0.014364973363321695,
+      "tp": 88826,
+      "fp": 6636,
+      "tn": 455321,
+      "fn": 6780,
+      "recall_at_1_fpr": 0.9071710980482397,
+      "recall_at_5_fpr": 0.9825952346087066,
+      "recall_at_10_fpr": 0.9955442127063155,
+      "loss": 0.06646002379976858,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.8970751316883837,
+      "recall": 0.887905604719764,
+      "f1": 0.8924668160661955,
+      "auc": 0.9861394086206904,
+      "ap": 0.9726516277893582,
+      "fpr": 0.04503684833045219,
+      "tp": 12943,
+      "fp": 1485,
+      "tn": 31488,
+      "fn": 1634,
+      "recall_at_1_fpr": 0.7995472319407285,
+      "recall_at_5_fpr": 0.8962749536941758,
+      "recall_at_10_fpr": 0.9534197708719215,
+      "loss": 0.15375348549817783
+    }
+  },
+  {
+    "epoch": 10,
+    "train": {
+      "precision": 0.9386959937156324,
+      "recall": 0.9374097859966948,
+      "f1": 0.9380524489614352,
+      "auc": 0.996945361357788,
+      "ap": 0.9875233253666762,
+      "fpr": 0.012670010412224514,
+      "tp": 89622,
+      "fp": 5853,
+      "tn": 456104,
+      "fn": 5984,
+      "recall_at_1_fpr": 0.9258310147898667,
+      "recall_at_5_fpr": 0.9870091835240467,
+      "recall_at_10_fpr": 0.996422818651549,
+      "loss": 0.05996785134017926,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.9055374592833876,
+      "recall": 0.877272415448995,
+      "f1": 0.8911808773824872,
+      "auc": 0.9858778145820947,
+      "ap": 0.9721916153483744,
+      "fpr": 0.040457343887423045,
+      "tp": 12788,
+      "fp": 1334,
+      "tn": 31639,
+      "fn": 1789,
+      "recall_at_1_fpr": 0.7960485696645401,
+      "recall_at_5_fpr": 0.896480757357481,
+      "recall_at_10_fpr": 0.950950126912259,
+      "loss": 0.16451230935118763
+    }
+  },
+  {
+    "epoch": 11,
+    "train": {
+      "precision": 0.9472834067547724,
+      "recall": 0.9446478254502856,
+      "f1": 0.9459637803358018,
+      "auc": 0.9975255730858837,
+      "ap": 0.9897969104682971,
+      "fpr": 0.010879800500912423,
+      "tp": 90314,
+      "fp": 5026,
+      "tn": 456931,
+      "fn": 5292,
+      "recall_at_1_fpr": 0.9403907704537372,
+      "recall_at_5_fpr": 0.9903980921699475,
+      "recall_at_10_fpr": 0.9972595862184381,
+      "loss": 0.05451454288937023,
+      "focal_loss": NaN,
+      "dice_loss": NaN,
+      "contrastive_loss": NaN,
+      "nan_batches": 0
+    },
+    "val": {
+      "precision": 0.8955048409405256,
+      "recall": 0.8883172120463744,
+      "f1": 0.8918965457864104,
+      "auc": 0.9859243882221933,
+      "ap": 0.9721878200096978,
+      "fpr": 0.04582537227428502,
+      "tp": 12949,
+      "fp": 1511,
+      "tn": 31462,
+      "fn": 1628,
+      "recall_at_1_fpr": 0.7936475269259793,
+      "recall_at_5_fpr": 0.8964121561363793,
+      "recall_at_10_fpr": 0.9514989366810729,
+      "loss": 0.17257934353854557
+    }
+  }
+]