Upload EMBER2024 ember-ml models
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +8 -0
- README.md +359 -0
- dnn/dnn_APK.onnx +3 -0
- dnn/dnn_APK.pt +3 -0
- dnn/dnn_APK_benchmark.json +43 -0
- dnn/dnn_APK_metrics.json +19 -0
- dnn/dnn_ELF.onnx +3 -0
- dnn/dnn_ELF.pt +3 -0
- dnn/dnn_ELF_benchmark.json +43 -0
- dnn/dnn_ELF_metrics.json +19 -0
- dnn/dnn_PDF.onnx +3 -0
- dnn/dnn_PDF.pt +3 -0
- dnn/dnn_PDF_benchmark.json +43 -0
- dnn/dnn_PDF_metrics.json +19 -0
- dnn/dnn_PE.onnx +3 -0
- dnn/dnn_PE.pt +3 -0
- dnn/dnn_PE_benchmark.json +43 -0
- dnn/dnn_PE_fp32.onnx +3 -0
- dnn/dnn_PE_metrics.json +19 -0
- dnn/dnn_Win32.onnx +3 -0
- dnn/dnn_Win32.pt +3 -0
- dnn/dnn_Win32_benchmark.json +43 -0
- dnn/dnn_Win32_fp32.onnx +3 -0
- dnn/dnn_Win32_metrics.json +19 -0
- dnn/dnn_Win64.onnx +3 -0
- dnn/dnn_Win64.pt +3 -0
- dnn/dnn_Win64_benchmark.json +43 -0
- dnn/dnn_Win64_fp32.onnx +3 -0
- dnn/dnn_Win64_metrics.json +19 -0
- dnn/dnn_all.onnx +3 -0
- dnn/dnn_all.pt +3 -0
- dnn/dnn_all_benchmark.json +43 -0
- dnn/dnn_all_fp32.onnx +3 -0
- dnn/dnn_all_metrics.json +19 -0
- dnn/dnn_dotnet.onnx +3 -0
- dnn/dnn_dotnet.pt +3 -0
- dnn/dnn_dotnet_benchmark.json +43 -0
- dnn/dnn_dotnet_fp32.onnx +3 -0
- dnn/dnn_dotnet_metrics.json +19 -0
- hybrid/hybrid_APK.pt +3 -0
- hybrid/hybrid_APK_benchmark.json +43 -0
- hybrid/hybrid_APK_lgbm.model +3 -0
- hybrid/hybrid_APK_metrics.json +19 -0
- hybrid/hybrid_APK_nnpart.onnx +3 -0
- hybrid/hybrid_ELF.pt +3 -0
- hybrid/hybrid_ELF_benchmark.json +43 -0
- hybrid/hybrid_ELF_lgbm.model +3 -0
- hybrid/hybrid_ELF_metrics.json +19 -0
- hybrid/hybrid_ELF_nnpart.onnx +3 -0
- hybrid/hybrid_PDF.pt +3 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,11 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
lightgbm/lightgbm_APK.tl filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
lightgbm/lightgbm_ELF.tl filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
lightgbm/lightgbm_PDF.tl filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
lightgbm/lightgbm_PE.tl filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
lightgbm/lightgbm_Win32.tl filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
lightgbm/lightgbm_Win64.tl filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
lightgbm/lightgbm_all.tl filter=lfs diff=lfs merge=lfs -text
|
| 43 |
+
lightgbm/lightgbm_dotnet.tl filter=lfs diff=lfs merge=lfs -text
|
README.md
ADDED
|
@@ -0,0 +1,359 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
datasets:
|
| 4 |
+
- joyce8/EMBER2024
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
tags:
|
| 8 |
+
- malware-detection
|
| 9 |
+
- cybersecurity
|
| 10 |
+
- onnxruntime
|
| 11 |
+
- lightgbm
|
| 12 |
+
- pytorch
|
| 13 |
+
- tabnet
|
| 14 |
+
- binary-classification
|
| 15 |
+
pipeline_tag: text-classification
|
| 16 |
+
library_name: onnxruntime
|
| 17 |
+
---
|
| 18 |
+
|
| 19 |
+
# EMBER2024 Malware Detection Models
|
| 20 |
+
|
| 21 |
+
4종 모델 아키텍처(DNN, TabNet, Hybrid GBDT2NN, LightGBM)를 [EMBER2024](https://huggingface.co/datasets/joyce8/EMBER2024) 데이터셋의 8개 파일 타입 subset 전체에 대해 학습·평가하고 배포 가능한 포맷으로 변환한 모델 컬렉션입니다.
|
| 22 |
+
|
| 23 |
+
> **학습 환경**: NVIDIA DGX Spark (GB10 Grace Blackwell, 128 GB 통합 메모리, CUDA 13)
|
| 24 |
+
> **코드**: [github.com/evan0416/ember-ml](https://github.com/evan0416/ember-ml)
|
| 25 |
+
> **데이터셋 논문**: [Joyce et al., KDD 2025 (arXiv:2506.05074)](https://arxiv.org/abs/2506.05074)
|
| 26 |
+
|
| 27 |
+
---
|
| 28 |
+
|
| 29 |
+
## 모델 구성
|
| 30 |
+
|
| 31 |
+
| 디렉토리 | 아키텍처 | 배포 포맷 | 파라미터 |
|
| 32 |
+
|---------|---------|---------|---------|
|
| 33 |
+
| `dnn/` | Feed-Forward DNN (PReLU + Dropout) | ONNX (INT8 Static / FP32) | 13.2 M (PE) / 0.73 M (non-PE) |
|
| 34 |
+
| `tabnet/` | TabNet ([Arik & Pfister, 2021](https://arxiv.org/abs/1908.07442)) | ONNX FP32 | ~3 M |
|
| 35 |
+
| `hybrid/` | GBDT2NN ([DeepGBM, KDD 2019](https://www.microsoft.com/en-us/research/publication/deepgbm-a-deep-learning-framework-distilled-by-gbdt-for-online-prediction-tasks/)) | ONNX (nn_part) + LightGBM booster | ~1 M NN |
|
| 36 |
+
| `lightgbm/` | LightGBM (사전학습, [joyce8/EMBER2024-benchmark-models](https://huggingface.co/joyce8/EMBER2024-benchmark-models)) | Treelite `.tl` | — |
|
| 37 |
+
|
| 38 |
+
### Subset 목록
|
| 39 |
+
|
| 40 |
+
| Subset | 대상 파일 타입 | 입력 차원 |
|
| 41 |
+
|--------|-------------|---------|
|
| 42 |
+
| `PE` | PE 바이너리 전체 (Win32 + Win64 + .NET) | 2,568 |
|
| 43 |
+
| `Win32` | Windows 32-bit PE | 2,568 |
|
| 44 |
+
| `Win64` | Windows 64-bit PE | 2,568 |
|
| 45 |
+
| `.NET` | .NET 어셈블리 | 2,568 |
|
| 46 |
+
| `APK` | Android APK | 696 |
|
| 47 |
+
| `ELF` | Linux ELF | 696 |
|
| 48 |
+
| `PDF` | PDF 문서 | 696 |
|
| 49 |
+
| `all` | 전체 파일 타입 혼합 | 2,568 |
|
| 50 |
+
|
| 51 |
+
---
|
| 52 |
+
|
| 53 |
+
## 디렉토리 구조
|
| 54 |
+
|
| 55 |
+
파일명 규칙: `{모델}_{subset}[_suffix].{ext}`
|
| 56 |
+
`.NET` subset은 파일명에서 `dotnet`으로 변환됩니다.
|
| 57 |
+
|
| 58 |
+
```
|
| 59 |
+
dnn/
|
| 60 |
+
├── dnn_PE.onnx # INT8 Static (배포용, PE/Win32/Win64/dotnet/all)
|
| 61 |
+
├── dnn_PE_fp32.onnx # FP32 ONNX (참조용, INT8 계열만 추가 포함)
|
| 62 |
+
├── dnn_PE.pt # PyTorch 체크포인트
|
| 63 |
+
├── dnn_PE_metrics.json # 평가 결과 (AUC, TPR@1%FPR)
|
| 64 |
+
├── dnn_PE_benchmark.json # 크기·레이턴시
|
| 65 |
+
├── dnn_APK.onnx # FP32 (non-PE — INT8 AUC 손실 과다)
|
| 66 |
+
├── dnn_APK.pt
|
| 67 |
+
└── ...
|
| 68 |
+
|
| 69 |
+
tabnet/
|
| 70 |
+
├── tabnet_PE.onnx # FP32 ONNX (134 MB — sparsemax 언폴딩)
|
| 71 |
+
├── tabnet_PE.zip # pytorch-tabnet 네이티브 (7 MB, 경량)
|
| 72 |
+
└── ...
|
| 73 |
+
|
| 74 |
+
hybrid/
|
| 75 |
+
├── hybrid_PE_nnpart.onnx # GBDT2NN nn_part ONNX (5.1 MB)
|
| 76 |
+
├── hybrid_PE_lgbm.model # LightGBM booster (3.6 MB)
|
| 77 |
+
├── hybrid_PE.pt # PyTorch 체크포인트
|
| 78 |
+
└── ...
|
| 79 |
+
|
| 80 |
+
lightgbm/
|
| 81 |
+
├── lightgbm_PE.tl # Treelite 직렬화 (플랫폼 독립, 재컴파일 필요)
|
| 82 |
+
└── ...
|
| 83 |
+
```
|
| 84 |
+
|
| 85 |
+
---
|
| 86 |
+
|
| 87 |
+
## 성능 결과 (EMBER2024 test set)
|
| 88 |
+
|
| 89 |
+
> 평가 기준: ROC-AUC, TPR @ 1% FPR (논문 §4.1), challenge set detection rate @ FPR=1% 임계값
|
| 90 |
+
> Challenge set: 6,315 evasive malware (양성 only, Win32 3,225 / .NET 829 / Win64 814 / PDF 805 / ELF 386 / APK 256)
|
| 91 |
+
|
| 92 |
+
### DNN
|
| 93 |
+
|
| 94 |
+
| Subset | ROC-AUC | TPR@1%FPR | 배포 포맷 | 크기 |
|
| 95 |
+
|--------|---------|----------|---------|------|
|
| 96 |
+
| PE | 0.9969 | 0.9539 | INT8 Static ONNX | 13.3 MB |
|
| 97 |
+
| Win32 | 0.9966 | 0.9468 | INT8 Static ONNX | 13.3 MB |
|
| 98 |
+
| Win64 | 0.9969 | 0.9656 | INT8 Static ONNX | 13.3 MB |
|
| 99 |
+
| .NET | 0.9939 | 0.8976 | INT8 Static ONNX | 13.3 MB |
|
| 100 |
+
| all | 0.9942 | 0.9103 | INT8 Static ONNX | 13.3 MB |
|
| 101 |
+
| APK | 0.9761 | 0.7682 | FP32 ONNX | 3.9 MB |
|
| 102 |
+
| ELF | 0.9840 | 0.8103 | FP32 ONNX | 3.9 MB |
|
| 103 |
+
| PDF | 0.9795 | 0.8902 | FP32 ONNX | 3.9 MB |
|
| 104 |
+
|
| 105 |
+
> non-PE(APK/ELF/PDF)는 입력 696-dim, 파라미터 수 부족으로 INT8 AUC 손실이 크므로 FP32 유지.
|
| 106 |
+
|
| 107 |
+
### TabNet
|
| 108 |
+
|
| 109 |
+
| Subset | ROC-AUC | TPR@1%FPR | 배포 포맷 | 크기 |
|
| 110 |
+
|--------|---------|----------|---------|------|
|
| 111 |
+
| PE | 0.9951 | 0.9212 | FP32 ONNX | 134 MB |
|
| 112 |
+
| Win32 | 0.9949 | 0.9322 | FP32 ONNX | 134 MB |
|
| 113 |
+
| Win64 | 0.9946 | 0.9323 | FP32 ONNX | 134 MB |
|
| 114 |
+
| .NET | 0.9925 | 0.8702 | FP32 ONNX | 134 MB |
|
| 115 |
+
| all | 0.9922 | 0.8912 | FP32 ONNX | 134 MB |
|
| 116 |
+
| APK | 0.9741 | 0.7028 | FP32 ONNX | 13.5 MB |
|
| 117 |
+
| ELF | 0.9793 | 0.5460 | FP32 ONNX | 13.5 MB |
|
| 118 |
+
| PDF | 0.9810 | 0.8597 | FP32 ONNX | 13.5 MB |
|
| 119 |
+
|
| 120 |
+
> PE 계열 ONNX 크기 134 MB: sparsemax attention loop이 ONNX 그래프로 언폴딩되는 구조적 특성. 크기 우선이면 `tabnet_model.zip`(7 MB) 직접 사용 권장.
|
| 121 |
+
|
| 122 |
+
### Hybrid (GBDT2NN)
|
| 123 |
+
|
| 124 |
+
| Subset | ROC-AUC | TPR@1%FPR | 배포 포맷 | 크기 |
|
| 125 |
+
|--------|---------|----------|---------|------|
|
| 126 |
+
| PE | 0.9982 | 0.9752 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB |
|
| 127 |
+
| Win32 | 0.9981 | 0.9747 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB |
|
| 128 |
+
| Win64 | 0.9983 | 0.9812 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB |
|
| 129 |
+
| .NET | 0.9961 | 0.9466 | nn_part ONNX + LightGBM booster | 5.1 + 3.5 MB |
|
| 130 |
+
| all | 0.9970 | 0.9528 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB |
|
| 131 |
+
| APK | 0.9821 | 0.8003 | nn_part ONNX + LightGBM booster | 5.1 + 3.5 MB |
|
| 132 |
+
| ELF | 0.9899 | 0.8827 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB |
|
| 133 |
+
| PDF | 0.9879 | 0.9283 | nn_part ONNX + LightGBM booster | 5.1 + 3.6 MB |
|
| 134 |
+
|
| 135 |
+
### LightGBM (Treelite 컴파일)
|
| 136 |
+
|
| 137 |
+
| Subset | ROC-AUC | TPR@1%FPR | 크기 (.tl) | 크기 (원본 .model) |
|
| 138 |
+
|--------|---------|----------|-----------|-----------------|
|
| 139 |
+
| PE | 0.9983 | 0.9707 | 5.3 MB | 3.6 MB |
|
| 140 |
+
| Win32 | 0.9984 | 0.9736 | 5.3 MB | 3.6 MB |
|
| 141 |
+
| Win64 | 0.9989 | 0.9831 | 5.3 MB | 3.6 MB |
|
| 142 |
+
| .NET | 0.9980 | 0.9566 | 5.3 MB | 3.5 MB |
|
| 143 |
+
| all | 0.9968 | 0.9440 | 5.3 MB | 3.6 MB |
|
| 144 |
+
| APK | 0.9861 | 0.8157 | 5.3 MB | 3.5 MB |
|
| 145 |
+
| ELF | 0.9929 | 0.9140 | 5.3 MB | 3.6 MB |
|
| 146 |
+
| PDF | 0.9913 | 0.9275 | 5.3 MB | 3.6 MB |
|
| 147 |
+
|
| 148 |
+
> 원본 LightGBM 모델: [joyce8/EMBER2024-benchmark-models](https://huggingface.co/joyce8/EMBER2024-benchmark-models). `.tl`은 Treelite 3.9.1로 직렬화된 플랫폼 독립 파일 — 각 플랫폼에서 재컴파일 필요.
|
| 149 |
+
|
| 150 |
+
### Challenge Set Detection Rate
|
| 151 |
+
|
| 152 |
+
> Challenge set: 6,315 evasive malware (전부 양성). test set FPR=1% 임계값 적용.
|
| 153 |
+
|
| 154 |
+
| Subset | DNN | TabNet | Hybrid | LightGBM |
|
| 155 |
+
|--------|-----|--------|--------|----------|
|
| 156 |
+
| `.NET` | 58.6% | 70.0% | 80.6% | 79.6% |
|
| 157 |
+
| `APK` | 27.3% | 29.3% | 34.4% | 33.6% |
|
| 158 |
+
| `ELF` | 11.7% | 4.4% | 23.8% | 30.3% |
|
| 159 |
+
| `PDF` | 41.5% | 40.1% | 56.9% | 57.1% |
|
| 160 |
+
| `PE` | 38.5% | 36.9% | 58.2% | 58.8% |
|
| 161 |
+
| `Win32`| 36.6% | 45.3% | 58.4% | 69.9% |
|
| 162 |
+
| `Win64`| 46.3% | 44.1% | 59.5% | 59.7% |
|
| 163 |
+
| `all` | 35.3% | 42.3% | 54.1% | 48.4% |
|
| 164 |
+
|
| 165 |
+
---
|
| 166 |
+
|
| 167 |
+
## 추론 성능 (Apple M1, darwin-arm64)
|
| 168 |
+
|
| 169 |
+
> `warm_batch1` 레이턴시: 배치 크기=1, 캐시 웜업 후 측정. 배포 환경(x86_64 Linux)과 다를 수 있음.
|
| 170 |
+
|
| 171 |
+
### 레이턴시 (ms, warm batch=1)
|
| 172 |
+
|
| 173 |
+
| Subset | DNN | TabNet | Hybrid | LightGBM |
|
| 174 |
+
|--------|-----|--------|--------|----------|
|
| 175 |
+
| `.NET` | 0.248 | 5.465 | 0.151 | 0.050 |
|
| 176 |
+
| `APK` | 0.035 | 0.846 | 0.145 | 0.031 |
|
| 177 |
+
| `ELF` | 0.039 | 0.505 | 0.160 | 0.036 |
|
| 178 |
+
| `PDF` | 0.036 | 2.230 | 0.172 | 0.048 |
|
| 179 |
+
| `PE` | 0.290 | 4.402 | 0.138 | 0.028 |
|
| 180 |
+
| `Win32`| 0.288 | 4.693 | 0.141 | 0.044 |
|
| 181 |
+
| `Win64`| 0.220 | 5.621 | 0.422 | 0.039 |
|
| 182 |
+
| `all` | 0.254 | 4.788 | 0.147 | 0.068 |
|
| 183 |
+
|
| 184 |
+
> TabNet 레이턴시 높음: sparsemax attention이 ONNX 그래프로 언폴딩되는 구조적 특성.
|
| 185 |
+
> Hybrid = nn_part ONNX 추론만 측정 (LightGBM leaf extraction 제외).
|
| 186 |
+
> LightGBM 레이턴시 = 컴파일 `.dylib` 기준; 업로드 파일은 `.tl` (재컴파일 필요).
|
| 187 |
+
|
| 188 |
+
### 모델 파일 크기 (배포 포맷)
|
| 189 |
+
|
| 190 |
+
| Subset | DNN | TabNet `.onnx` | TabNet `.zip` | Hybrid (nn+lgbm) | LightGBM `.tl` |
|
| 191 |
+
|--------|-----|----------------|---------------|-------------------|----------------|
|
| 192 |
+
| PE 계열 | 13.3 MB (INT8) | 140.2 MB | 7.4 MB | 5.3 + 3.8 MB | 5.3 MB |
|
| 193 |
+
| non-PE | 3.9 MB (FP32) | 13.5 MB | 3.2 MB | 5.3 + 3.7 MB | 5.3 MB |
|
| 194 |
+
|
| 195 |
+
---
|
| 196 |
+
|
| 197 |
+
## 사용 방법
|
| 198 |
+
|
| 199 |
+
### 의존성 설치
|
| 200 |
+
|
| 201 |
+
```bash
|
| 202 |
+
pip install onnxruntime>=1.20 numpy
|
| 203 |
+
# LightGBM / Hybrid 추론 시
|
| 204 |
+
pip install "treelite==3.9.1" "treelite_runtime==3.9.1" lightgbm>=4.6
|
| 205 |
+
# TabNet 체크포인트 직접 사용 시
|
| 206 |
+
pip install pytorch-tabnet>=4.1
|
| 207 |
+
```
|
| 208 |
+
|
| 209 |
+
### DNN 추론 (ONNX Runtime)
|
| 210 |
+
|
| 211 |
+
```python
|
| 212 |
+
import numpy as np
|
| 213 |
+
import onnxruntime as ort
|
| 214 |
+
from huggingface_hub import hf_hub_download
|
| 215 |
+
|
| 216 |
+
# PE subset — INT8 Static
|
| 217 |
+
model_path = hf_hub_download(
|
| 218 |
+
repo_id="cycloevan/ember-model",
|
| 219 |
+
filename="dnn/dnn_PE.onnx",
|
| 220 |
+
)
|
| 221 |
+
sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
|
| 222 |
+
|
| 223 |
+
# X: np.ndarray shape (N, 2568), dtype float32
|
| 224 |
+
X = np.random.randn(1, 2568).astype(np.float32)
|
| 225 |
+
logit = sess.run(["logit"], {"features": X})[0] # shape (N, 1)
|
| 226 |
+
prob = 1 / (1 + np.exp(-logit.ravel())) # sigmoid → [0, 1]
|
| 227 |
+
print(f"malware probability: {prob[0]:.4f}")
|
| 228 |
+
```
|
| 229 |
+
|
| 230 |
+
```python
|
| 231 |
+
# APK subset — FP32
|
| 232 |
+
model_path = hf_hub_download(
|
| 233 |
+
repo_id="cycloevan/ember-model",
|
| 234 |
+
filename="dnn/dnn_APK.onnx",
|
| 235 |
+
)
|
| 236 |
+
sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
|
| 237 |
+
X = np.random.randn(1, 696).astype(np.float32) # non-PE: dim=696
|
| 238 |
+
prob = 1 / (1 + np.exp(-sess.run(["logit"], {"features": X})[0].ravel()))
|
| 239 |
+
```
|
| 240 |
+
|
| 241 |
+
### TabNet 추론 (ONNX Runtime)
|
| 242 |
+
|
| 243 |
+
```python
|
| 244 |
+
import numpy as np
|
| 245 |
+
import onnxruntime as ort
|
| 246 |
+
from huggingface_hub import hf_hub_download
|
| 247 |
+
|
| 248 |
+
model_path = hf_hub_download(
|
| 249 |
+
repo_id="cycloevan/ember-model",
|
| 250 |
+
filename="tabnet/tabnet_PE.onnx",
|
| 251 |
+
)
|
| 252 |
+
sess = ort.InferenceSession(model_path, providers=["CPUExecutionProvider"])
|
| 253 |
+
X = np.random.randn(1, 2568).astype(np.float32)
|
| 254 |
+
# 출력: logit (sigmoid 전)
|
| 255 |
+
logit = sess.run(["logit"], {"features": X})[0]
|
| 256 |
+
prob = 1 / (1 + np.exp(-logit.ravel()))
|
| 257 |
+
```
|
| 258 |
+
|
| 259 |
+
### Hybrid 추론 (ONNX + LightGBM)
|
| 260 |
+
|
| 261 |
+
```python
|
| 262 |
+
import numpy as np
|
| 263 |
+
import lightgbm as lgb
|
| 264 |
+
import onnxruntime as ort
|
| 265 |
+
from huggingface_hub import hf_hub_download
|
| 266 |
+
|
| 267 |
+
# 1. LightGBM booster로 leaf indices 추출
|
| 268 |
+
booster = lgb.Booster(model_file=hf_hub_download(
|
| 269 |
+
repo_id="cycloevan/ember-model",
|
| 270 |
+
filename="hybrid/hybrid_PE_lgbm.model",
|
| 271 |
+
))
|
| 272 |
+
X_raw = np.random.randn(1, 2568).astype(np.float64)
|
| 273 |
+
leaf_indices = booster.predict(X_raw, pred_leaf=True).astype(np.int64) # (N, n_trees)
|
| 274 |
+
|
| 275 |
+
# 2. GBDT2NN ONNX로 최종 분류
|
| 276 |
+
nn_sess = ort.InferenceSession(hf_hub_download(
|
| 277 |
+
repo_id="cycloevan/ember-model",
|
| 278 |
+
filename="hybrid/hybrid_PE_nnpart.onnx",
|
| 279 |
+
), providers=["CPUExecutionProvider"])
|
| 280 |
+
logit = nn_sess.run(["logit"], {"leaf_indices": leaf_indices})[0]
|
| 281 |
+
prob = 1 / (1 + np.exp(-logit.ravel()))
|
| 282 |
+
print(f"malware probability: {prob[0]:.4f}")
|
| 283 |
+
```
|
| 284 |
+
|
| 285 |
+
### LightGBM 추론 (Treelite 컴파일 — 빠른 추론)
|
| 286 |
+
|
| 287 |
+
```python
|
| 288 |
+
# 1. Treelite .tl → 플랫폼별 공유 라이브러리 컴파일 (최초 1회)
|
| 289 |
+
import treelite, treelite_runtime, sys, numpy as np
|
| 290 |
+
from huggingface_hub import hf_hub_download
|
| 291 |
+
|
| 292 |
+
tl_path = hf_hub_download(
|
| 293 |
+
repo_id="cycloevan/ember-model",
|
| 294 |
+
filename="lightgbm/lightgbm_PE.tl",
|
| 295 |
+
)
|
| 296 |
+
tl_model = treelite.Model.deserialize(tl_path)
|
| 297 |
+
lib_ext = ".dylib" if sys.platform == "darwin" else ".so"
|
| 298 |
+
lib_path = tl_path.replace(".tl", lib_ext)
|
| 299 |
+
tl_model.export_lib(
|
| 300 |
+
toolchain="clang" if sys.platform == "darwin" else "gcc",
|
| 301 |
+
libpath=lib_path,
|
| 302 |
+
verbose=False,
|
| 303 |
+
)
|
| 304 |
+
|
| 305 |
+
# 2. 추론
|
| 306 |
+
predictor = treelite_runtime.Predictor(lib_path, verbose=False)
|
| 307 |
+
X = np.random.randn(1, 2568).astype(np.float32)
|
| 308 |
+
prob = predictor.predict(treelite_runtime.DMatrix(X))
|
| 309 |
+
print(f"malware probability: {prob[0]:.4f}")
|
| 310 |
+
```
|
| 311 |
+
|
| 312 |
+
> **주의**: `treelite==3.9.1` + `treelite_runtime==3.9.1` 필요. 4.x는 `export_lib()` 미지원.
|
| 313 |
+
|
| 314 |
+
---
|
| 315 |
+
|
| 316 |
+
## 학습 및 평가 환경
|
| 317 |
+
|
| 318 |
+
| 항목 | 내용 |
|
| 319 |
+
|------|------|
|
| 320 |
+
| 데이터셋 | [EMBER2024](https://huggingface.co/datasets/joyce8/EMBER2024) — train 52주(2.6 M), test 12주(606 K), challenge 6,315 |
|
| 321 |
+
| Feature 차원 | PE 2,568 (v3) / non-PE 696 (유효 prefix) |
|
| 322 |
+
| Split 정책 | 시간적 순서 고정 (temporal split), 임의 셔플 없음 |
|
| 323 |
+
| 학습 환경 | DGX Spark (GB10 Grace Blackwell, 128 GB, CUDA 13) |
|
| 324 |
+
| 프레임워크 | PyTorch 2.11.0, pytorch-tabnet 4.1, LightGBM 4.6 |
|
| 325 |
+
| 재현 시드 | 42 |
|
| 326 |
+
| DNN 아키텍처 | Linear(2568→2568→1024→512→1) + PReLU + Dropout(0.5) |
|
| 327 |
+
| Hybrid | LightGBM leaf extraction → Linear(n_trees→512→256→1) + PReLU |
|
| 328 |
+
| 평가 지표 | ROC-AUC, PR-AUC, **TPR @ 1% FPR** (논문 §4.1) |
|
| 329 |
+
|
| 330 |
+
---
|
| 331 |
+
|
| 332 |
+
## 알려진 한계
|
| 333 |
+
|
| 334 |
+
- **TabNet ONNX 크기**: sparsemax attention loop 언폴딩으로 PE 계열 ONNX가 134 MB로 팽창. 원본 `tabnet_model.zip`(7 MB)이 경량.
|
| 335 |
+
- **Treelite `.dylib`**: Mac ARM64 전용 사전 컴파일 파일. 다른 플랫폼은 `.tl`에서 재컴파일 필요.
|
| 336 |
+
- **DNN non-PE INT8**: 696-dim 모델은 양자화 AUC 손실이 크므로 FP32 유지.
|
| 337 |
+
- **Hybrid 추론**: 단일 ONNX 파일이 아님 — LightGBM leaf extraction + nn_part ONNX 2단계.
|
| 338 |
+
- **challenge detection rate**: test set에서 FPR=1% 임계값으로 측정. subset별 분포 차이로 값이 상이할 수 있음.
|
| 339 |
+
|
| 340 |
+
---
|
| 341 |
+
|
| 342 |
+
## 인용
|
| 343 |
+
|
| 344 |
+
```bibtex
|
| 345 |
+
@inproceedings{joyce2025ember2024,
|
| 346 |
+
title = {EMBER2024: An Open Dataset for Training Behavioral Malware Detection Models},
|
| 347 |
+
author = {Joyce, Ruby and Rudd, Ethan M. and others},
|
| 348 |
+
booktitle = {Proceedings of the 31st ACM SIGKDD Conference on Knowledge Discovery and Data Mining},
|
| 349 |
+
year = {2025},
|
| 350 |
+
url = {https://arxiv.org/abs/2506.05074}
|
| 351 |
+
}
|
| 352 |
+
```
|
| 353 |
+
|
| 354 |
+
---
|
| 355 |
+
|
| 356 |
+
## 라이선스
|
| 357 |
+
|
| 358 |
+
코드 및 모델 가중치: Apache 2.0
|
| 359 |
+
LightGBM 원본 모델(`hybrid/hybrid_*_lgbm.model`): [joyce8/EMBER2024-benchmark-models](https://huggingface.co/joyce8/EMBER2024-benchmark-models) 라이선스 준수
|
dnn/dnn_APK.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f7fc471fef4a3ab3b1deaeccba5473b81a62a8b4d0c1f435c4238150765cba64
|
| 3 |
+
size 3902529
|
dnn/dnn_APK.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:82128b485920e8532d9ccc8148b52254e61ab091bacb99978f377aa64e54af3d
|
| 3 |
+
size 3929735
|
dnn/dnn_APK_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "APK",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"model_size_mb": 3.903,
|
| 6 |
+
"original_size_mb": 3.93,
|
| 7 |
+
"size_reduction_pct": 0.7,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.061,
|
| 10 |
+
"warm_batch1": 0.035,
|
| 11 |
+
"warm_batch16": 0.215,
|
| 12 |
+
"warm_batch256": 2.326,
|
| 13 |
+
"throughput_batch1": 28571.4,
|
| 14 |
+
"ms_per_sample_batch1": 0.035,
|
| 15 |
+
"throughput_batch16": 74418.6,
|
| 16 |
+
"ms_per_sample_batch16": 0.0134,
|
| 17 |
+
"throughput_batch256": 110060.2,
|
| 18 |
+
"ms_per_sample_batch256": 0.0091
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 0.196,
|
| 22 |
+
"warm_batch1": 0.094,
|
| 23 |
+
"warm_batch16": 0.528,
|
| 24 |
+
"warm_batch256": 1.149,
|
| 25 |
+
"throughput_batch1": 10638.3,
|
| 26 |
+
"ms_per_sample_batch1": 0.094,
|
| 27 |
+
"throughput_batch16": 30303.0,
|
| 28 |
+
"ms_per_sample_batch16": 0.033,
|
| 29 |
+
"throughput_batch256": 222802.4,
|
| 30 |
+
"ms_per_sample_batch256": 0.0045
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9760728567708332,
|
| 34 |
+
"pr_auc": 0.9782617133148918,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.7681666666666667,
|
| 36 |
+
"n_samples": 96000,
|
| 37 |
+
"n_positive": 48000,
|
| 38 |
+
"auc_delta_vs_original": -0.0
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 0.0,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
dnn/dnn_APK_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "APK",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9760728567708332,
|
| 7 |
+
"pr_auc": 0.9782617133148918,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.7681666666666667,
|
| 9 |
+
"n_samples": 96000,
|
| 10 |
+
"n_positive": 48000,
|
| 11 |
+
"auc_delta_vs_original": -0.0
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.2734375,
|
| 15 |
+
"threshold": 0.893254017132744
|
| 16 |
+
},
|
| 17 |
+
"auc_gate_failed": false,
|
| 18 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit"
|
| 19 |
+
}
|
dnn/dnn_ELF.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:2a3ee26c129cfcef77ab60de38f4d5296fd0cff1d7f99d3ca4bbd3bdf2d5e568
|
| 3 |
+
size 3902529
|
dnn/dnn_ELF.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f875a91e09738ee9371ccde5693f61367f41a7a2cb623096d3e3854da5c32e9c
|
| 3 |
+
size 3929735
|
dnn/dnn_ELF_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "ELF",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"model_size_mb": 3.903,
|
| 6 |
+
"original_size_mb": 3.93,
|
| 7 |
+
"size_reduction_pct": 0.7,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.055,
|
| 10 |
+
"warm_batch1": 0.039,
|
| 11 |
+
"ms_per_sample_batch1": 0.0387,
|
| 12 |
+
"throughput_batch1": 25864.6,
|
| 13 |
+
"warm_batch16": 0.183,
|
| 14 |
+
"ms_per_sample_batch16": 0.0114,
|
| 15 |
+
"throughput_batch16": 87489.4,
|
| 16 |
+
"warm_batch256": 2.135,
|
| 17 |
+
"ms_per_sample_batch256": 0.0083,
|
| 18 |
+
"throughput_batch256": 119926.0
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 0.135,
|
| 22 |
+
"warm_batch1": 0.094,
|
| 23 |
+
"ms_per_sample_batch1": 0.0942,
|
| 24 |
+
"throughput_batch1": 10612.8,
|
| 25 |
+
"warm_batch16": 0.522,
|
| 26 |
+
"ms_per_sample_batch16": 0.0326,
|
| 27 |
+
"throughput_batch16": 30663.9,
|
| 28 |
+
"warm_batch256": 1.07,
|
| 29 |
+
"ms_per_sample_batch256": 0.0042,
|
| 30 |
+
"throughput_batch256": 239209.3
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9839878888888888,
|
| 34 |
+
"pr_auc": 0.9845206683196877,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.8103333333333333,
|
| 36 |
+
"n_samples": 12000,
|
| 37 |
+
"n_positive": 6000,
|
| 38 |
+
"auc_delta_vs_original": 6e-06
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 2.2,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
dnn/dnn_ELF_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "ELF",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9839878888888888,
|
| 7 |
+
"pr_auc": 0.9845206683196877,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.8103333333333333,
|
| 9 |
+
"n_samples": 12000,
|
| 10 |
+
"n_positive": 6000,
|
| 11 |
+
"auc_delta_vs_original": 6e-06
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.11658031088082901,
|
| 15 |
+
"threshold": 0.9921941479742924
|
| 16 |
+
},
|
| 17 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 18 |
+
"auc_gate_failed": false
|
| 19 |
+
}
|
dnn/dnn_PDF.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a9817bd278a5614376a7c1e1267e4e098cab64a9684936fc847df8b3bae0beac
|
| 3 |
+
size 3902529
|
dnn/dnn_PDF.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:d742551ed4f103eabda97a63ce9fba3be032039b97dd5f8fb3c66a328da7bfa9
|
| 3 |
+
size 3929735
|
dnn/dnn_PDF_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "PDF",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"model_size_mb": 3.903,
|
| 6 |
+
"original_size_mb": 3.93,
|
| 7 |
+
"size_reduction_pct": 0.7,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.05,
|
| 10 |
+
"warm_batch1": 0.036,
|
| 11 |
+
"warm_batch16": 0.201,
|
| 12 |
+
"warm_batch256": 2.391,
|
| 13 |
+
"throughput_batch1": 27777.8,
|
| 14 |
+
"ms_per_sample_batch1": 0.036,
|
| 15 |
+
"throughput_batch16": 79602.0,
|
| 16 |
+
"ms_per_sample_batch16": 0.0126,
|
| 17 |
+
"throughput_batch256": 107068.2,
|
| 18 |
+
"ms_per_sample_batch256": 0.0093
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 0.177,
|
| 22 |
+
"warm_batch1": 0.102,
|
| 23 |
+
"warm_batch16": 0.502,
|
| 24 |
+
"warm_batch256": 1.122,
|
| 25 |
+
"throughput_batch1": 9803.9,
|
| 26 |
+
"ms_per_sample_batch1": 0.102,
|
| 27 |
+
"throughput_batch16": 31872.5,
|
| 28 |
+
"ms_per_sample_batch16": 0.0314,
|
| 29 |
+
"throughput_batch256": 228164.0,
|
| 30 |
+
"ms_per_sample_batch256": 0.0044
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9795445,
|
| 34 |
+
"pr_auc": 0.9848222884024884,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.8901666666666667,
|
| 36 |
+
"n_samples": 24000,
|
| 37 |
+
"n_positive": 12000,
|
| 38 |
+
"auc_delta_vs_original": 0.0
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 2.1,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
dnn/dnn_PDF_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "PDF",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9795445,
|
| 7 |
+
"pr_auc": 0.9848222884024884,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.8901666666666667,
|
| 9 |
+
"n_samples": 24000,
|
| 10 |
+
"n_positive": 12000,
|
| 11 |
+
"auc_delta_vs_original": 0.0
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.41490683229813663,
|
| 15 |
+
"threshold": 0.7553592072806331
|
| 16 |
+
},
|
| 17 |
+
"auc_gate_failed": false,
|
| 18 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit"
|
| 19 |
+
}
|
dnn/dnn_PE.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b9fbe6a00e369366c42b64aeec8e61bf70d96060fd78575a01354e0027459be7
|
| 3 |
+
size 13303487
|
dnn/dnn_PE.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:382d174985ddd146df8a9e0d1bae262339734aed7841b35a5000b6b7c4fcdf43
|
| 3 |
+
size 52938695
|
dnn/dnn_PE_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "PE",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"model_size_mb": 13.303,
|
| 6 |
+
"original_size_mb": 52.939,
|
| 7 |
+
"size_reduction_pct": 74.9,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.778,
|
| 10 |
+
"warm_batch1": 0.29,
|
| 11 |
+
"warm_batch16": 1.036,
|
| 12 |
+
"warm_batch256": 12.935,
|
| 13 |
+
"throughput_batch1": 3448.3,
|
| 14 |
+
"ms_per_sample_batch1": 0.29,
|
| 15 |
+
"throughput_batch16": 15444.0,
|
| 16 |
+
"ms_per_sample_batch16": 0.0648,
|
| 17 |
+
"throughput_batch256": 19791.3,
|
| 18 |
+
"ms_per_sample_batch256": 0.0505
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 2.916,
|
| 22 |
+
"warm_batch1": 2.466,
|
| 23 |
+
"warm_batch16": 4.939,
|
| 24 |
+
"warm_batch256": 11.88,
|
| 25 |
+
"throughput_batch1": 405.5,
|
| 26 |
+
"ms_per_sample_batch1": 2.466,
|
| 27 |
+
"throughput_batch16": 3239.5,
|
| 28 |
+
"ms_per_sample_batch16": 0.3087,
|
| 29 |
+
"throughput_batch256": 21548.8,
|
| 30 |
+
"ms_per_sample_batch256": 0.0464
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9968856578492659,
|
| 34 |
+
"pr_auc": 0.9969432521647822,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.9471575053488233,
|
| 36 |
+
"n_samples": 100000,
|
| 37 |
+
"n_positive": 50011,
|
| 38 |
+
"auc_delta_vs_original": -6.2e-05
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 0.0,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
dnn/dnn_PE_fp32.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:df67ff72e0ee7cfc6381dbec0f8f5945c3db77dadb46af930b3f6c4670a5e150
|
| 3 |
+
size 52851589
|
dnn/dnn_PE_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "PE",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9968856578492659,
|
| 7 |
+
"pr_auc": 0.9969432521647822,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.9471575053488233,
|
| 9 |
+
"n_samples": 100000,
|
| 10 |
+
"n_positive": 50011,
|
| 11 |
+
"auc_delta_vs_original": -6.2e-05
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.3849962862094578,
|
| 15 |
+
"threshold": 0.9285938677767953
|
| 16 |
+
},
|
| 17 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 18 |
+
"auc_gate_failed": false
|
| 19 |
+
}
|
dnn/dnn_Win32.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:500f4ed1d1317a20b95cc1147f68eecd10abcf43527ad8234d1cba6aa4fa7a28
|
| 3 |
+
size 13303486
|
dnn/dnn_Win32.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:4f0f67ab9635cdc54f253a3bc591181c0c40092ddd7613ae48cdad9cf5b633be
|
| 3 |
+
size 52938695
|
dnn/dnn_Win32_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "Win32",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"model_size_mb": 13.303,
|
| 6 |
+
"original_size_mb": 52.939,
|
| 7 |
+
"size_reduction_pct": 74.9,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.487,
|
| 10 |
+
"warm_batch1": 0.288,
|
| 11 |
+
"warm_batch16": 0.99,
|
| 12 |
+
"warm_batch256": 8.912,
|
| 13 |
+
"throughput_batch1": 3472.2,
|
| 14 |
+
"ms_per_sample_batch1": 0.288,
|
| 15 |
+
"throughput_batch16": 16161.6,
|
| 16 |
+
"ms_per_sample_batch16": 0.0619,
|
| 17 |
+
"throughput_batch256": 28725.3,
|
| 18 |
+
"ms_per_sample_batch256": 0.0348
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 2.393,
|
| 22 |
+
"warm_batch1": 2.343,
|
| 23 |
+
"warm_batch16": 4.234,
|
| 24 |
+
"warm_batch256": 11.15,
|
| 25 |
+
"throughput_batch1": 426.8,
|
| 26 |
+
"ms_per_sample_batch1": 2.343,
|
| 27 |
+
"throughput_batch16": 3778.9,
|
| 28 |
+
"ms_per_sample_batch16": 0.2646,
|
| 29 |
+
"throughput_batch256": 22959.6,
|
| 30 |
+
"ms_per_sample_batch256": 0.0436
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9965419976367095,
|
| 34 |
+
"pr_auc": 0.9966226288532287,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.9478530944531651,
|
| 36 |
+
"n_samples": 100000,
|
| 37 |
+
"n_positive": 50204,
|
| 38 |
+
"auc_delta_vs_original": -8e-05
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 5.3,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
dnn/dnn_Win32_fp32.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:359d7e4e4cc4222bc8756d179415e93affb84f2602b0587e4f063dbb8047945e
|
| 3 |
+
size 52851589
|
dnn/dnn_Win32_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "Win32",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9965419976367095,
|
| 7 |
+
"pr_auc": 0.9966226288532287,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.9478530944531651,
|
| 9 |
+
"n_samples": 100000,
|
| 10 |
+
"n_positive": 50204,
|
| 11 |
+
"auc_delta_vs_original": -8e-05
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.3655813953488372,
|
| 15 |
+
"threshold": 0.7420705254159128
|
| 16 |
+
},
|
| 17 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 18 |
+
"auc_gate_failed": false
|
| 19 |
+
}
|
dnn/dnn_Win64.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:24bac8430ecb1b15cf0e28a097eff6fe525025d115b9f1bca9fcf75a31882167
|
| 3 |
+
size 13303486
|
dnn/dnn_Win64.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:39d22c9b0123afb6c0a6c11d17a147905c35b35214bbf318a54acdfff1d892a5
|
| 3 |
+
size 52938695
|
dnn/dnn_Win64_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "Win64",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"model_size_mb": 13.303,
|
| 6 |
+
"original_size_mb": 52.939,
|
| 7 |
+
"size_reduction_pct": 74.9,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.426,
|
| 10 |
+
"warm_batch1": 0.22,
|
| 11 |
+
"warm_batch16": 0.941,
|
| 12 |
+
"warm_batch256": 8.919,
|
| 13 |
+
"throughput_batch1": 4545.5,
|
| 14 |
+
"ms_per_sample_batch1": 0.22,
|
| 15 |
+
"throughput_batch16": 17003.2,
|
| 16 |
+
"ms_per_sample_batch16": 0.0588,
|
| 17 |
+
"throughput_batch256": 28702.8,
|
| 18 |
+
"ms_per_sample_batch256": 0.0348
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 2.789,
|
| 22 |
+
"warm_batch1": 2.389,
|
| 23 |
+
"warm_batch16": 4.246,
|
| 24 |
+
"warm_batch256": 11.135,
|
| 25 |
+
"throughput_batch1": 418.6,
|
| 26 |
+
"ms_per_sample_batch1": 2.389,
|
| 27 |
+
"throughput_batch16": 3768.3,
|
| 28 |
+
"ms_per_sample_batch16": 0.2654,
|
| 29 |
+
"throughput_batch256": 22990.6,
|
| 30 |
+
"ms_per_sample_batch256": 0.0435
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9968797740122215,
|
| 34 |
+
"pr_auc": 0.9970818991078805,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.9616965868794326,
|
| 36 |
+
"n_samples": 100000,
|
| 37 |
+
"n_positive": 49920,
|
| 38 |
+
"auc_delta_vs_original": -3.3e-05
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 0.0,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
dnn/dnn_Win64_fp32.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dc072a860523d3e1557fcb98837758d5f064d5edb57cec2e98931b5d51928bb6
|
| 3 |
+
size 52851589
|
dnn/dnn_Win64_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "Win64",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9968797740122215,
|
| 7 |
+
"pr_auc": 0.9970818991078805,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.9616965868794326,
|
| 9 |
+
"n_samples": 100000,
|
| 10 |
+
"n_positive": 49920,
|
| 11 |
+
"auc_delta_vs_original": -3.3e-05
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.4631449631449631,
|
| 15 |
+
"threshold": 0.5563582119497066
|
| 16 |
+
},
|
| 17 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 18 |
+
"auc_gate_failed": false
|
| 19 |
+
}
|
dnn/dnn_all.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:55ffa1127b72aa5f593b4963f769697d6a696beb60995ac771d1a6ebbb34a1db
|
| 3 |
+
size 13303487
|
dnn/dnn_all.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b5563545cd2037339668863a623d0b6f84e90cec5053d85abbc9a4e3faa8ad3e
|
| 3 |
+
size 52938695
|
dnn/dnn_all_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "all",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"model_size_mb": 13.303,
|
| 6 |
+
"original_size_mb": 52.939,
|
| 7 |
+
"size_reduction_pct": 74.9,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.429,
|
| 10 |
+
"warm_batch1": 0.254,
|
| 11 |
+
"warm_batch16": 0.93,
|
| 12 |
+
"warm_batch256": 8.955,
|
| 13 |
+
"throughput_batch1": 3937.0,
|
| 14 |
+
"ms_per_sample_batch1": 0.254,
|
| 15 |
+
"throughput_batch16": 17204.3,
|
| 16 |
+
"ms_per_sample_batch16": 0.0581,
|
| 17 |
+
"throughput_batch256": 28587.4,
|
| 18 |
+
"ms_per_sample_batch256": 0.035
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 2.658,
|
| 22 |
+
"warm_batch1": 2.394,
|
| 23 |
+
"warm_batch16": 4.271,
|
| 24 |
+
"warm_batch256": 11.371,
|
| 25 |
+
"throughput_batch1": 417.7,
|
| 26 |
+
"ms_per_sample_batch1": 2.394,
|
| 27 |
+
"throughput_batch16": 3746.2,
|
| 28 |
+
"ms_per_sample_batch16": 0.2669,
|
| 29 |
+
"throughput_batch256": 22513.4,
|
| 30 |
+
"ms_per_sample_batch256": 0.0444
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9938331270533947,
|
| 34 |
+
"pr_auc": 0.9937970701852039,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.8870437241462001,
|
| 36 |
+
"n_samples": 100000,
|
| 37 |
+
"n_positive": 50041,
|
| 38 |
+
"auc_delta_vs_original": -0.000375
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 0.0,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
dnn/dnn_all_fp32.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:c97fe488d5dedac6e88793a271b47018301dd7c7fb59c1cdbd23dad489e86f5b
|
| 3 |
+
size 52851589
|
dnn/dnn_all_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": "all",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9938331270533947,
|
| 7 |
+
"pr_auc": 0.9937970701852039,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.8870437241462001,
|
| 9 |
+
"n_samples": 100000,
|
| 10 |
+
"n_positive": 50041,
|
| 11 |
+
"auc_delta_vs_original": -0.000375
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.3534441805225653,
|
| 15 |
+
"threshold": 0.9838834411565777
|
| 16 |
+
},
|
| 17 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 18 |
+
"auc_gate_failed": false
|
| 19 |
+
}
|
dnn/dnn_dotnet.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:dff033cb3a75fc3af64c11c6557207974480d7286fea7d50225d5f41c0f871cb
|
| 3 |
+
size 13303487
|
dnn/dnn_dotnet.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:05c84608a67b7a3de08102f0d8b75c2fbd9ae3b5a97224d854717f258dab8abf
|
| 3 |
+
size 52938695
|
dnn/dnn_dotnet_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": ".NET",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"model_size_mb": 13.303,
|
| 6 |
+
"original_size_mb": 52.939,
|
| 7 |
+
"size_reduction_pct": 74.9,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.429,
|
| 10 |
+
"warm_batch1": 0.248,
|
| 11 |
+
"warm_batch16": 0.909,
|
| 12 |
+
"warm_batch256": 8.858,
|
| 13 |
+
"throughput_batch1": 4032.3,
|
| 14 |
+
"ms_per_sample_batch1": 0.248,
|
| 15 |
+
"throughput_batch16": 17601.8,
|
| 16 |
+
"ms_per_sample_batch16": 0.0568,
|
| 17 |
+
"throughput_batch256": 28900.4,
|
| 18 |
+
"ms_per_sample_batch256": 0.0346
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 2.44,
|
| 22 |
+
"warm_batch1": 2.368,
|
| 23 |
+
"warm_batch16": 4.208,
|
| 24 |
+
"warm_batch256": 11.103,
|
| 25 |
+
"throughput_batch1": 422.3,
|
| 26 |
+
"ms_per_sample_batch1": 2.368,
|
| 27 |
+
"throughput_batch16": 3802.3,
|
| 28 |
+
"ms_per_sample_batch16": 0.263,
|
| 29 |
+
"throughput_batch256": 23056.8,
|
| 30 |
+
"ms_per_sample_batch256": 0.0434
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9919714812355357,
|
| 34 |
+
"pr_auc": 0.9900265259892996,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.8444236421489054,
|
| 36 |
+
"n_samples": 100000,
|
| 37 |
+
"n_positive": 49946,
|
| 38 |
+
"auc_delta_vs_original": -0.001909
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 0.0,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
dnn/dnn_dotnet_fp32.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:e182e7c89766c2625d318c55e854a428b7a1bccdf5e71b52d805eb9fc59aa056
|
| 3 |
+
size 52851589
|
dnn/dnn_dotnet_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "dnn",
|
| 3 |
+
"subset": ".NET",
|
| 4 |
+
"quant": "int8-static",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9919714812355357,
|
| 7 |
+
"pr_auc": 0.9900265259892996,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.8444236421489054,
|
| 9 |
+
"n_samples": 100000,
|
| 10 |
+
"n_positive": 49946,
|
| 11 |
+
"auc_delta_vs_original": -0.001909
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.586248492159228,
|
| 15 |
+
"threshold": 0.9723500243107374
|
| 16 |
+
},
|
| 17 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 18 |
+
"auc_gate_failed": false
|
| 19 |
+
}
|
hybrid/hybrid_APK.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b65ff8cb5da0cf77b0b542785967c4bf63d580a1ff8be1513b3517cf834b6653
|
| 3 |
+
size 5331649
|
hybrid/hybrid_APK_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "hybrid",
|
| 3 |
+
"subset": "APK",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"model_size_mb": 5.326,
|
| 6 |
+
"original_size_mb": 5.332,
|
| 7 |
+
"size_reduction_pct": 0.1,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.455,
|
| 10 |
+
"warm_batch1": 0.145,
|
| 11 |
+
"warm_batch16": 0.805,
|
| 12 |
+
"warm_batch256": 11.495,
|
| 13 |
+
"throughput_batch1": 6896.6,
|
| 14 |
+
"ms_per_sample_batch1": 0.145,
|
| 15 |
+
"throughput_batch16": 19875.8,
|
| 16 |
+
"ms_per_sample_batch16": 0.0503,
|
| 17 |
+
"throughput_batch256": 22270.6,
|
| 18 |
+
"ms_per_sample_batch256": 0.0449
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 0.466,
|
| 22 |
+
"warm_batch1": 0.211,
|
| 23 |
+
"warm_batch16": 0.908,
|
| 24 |
+
"warm_batch256": 6.452,
|
| 25 |
+
"throughput_batch1": 4739.3,
|
| 26 |
+
"ms_per_sample_batch1": 0.211,
|
| 27 |
+
"throughput_batch16": 17621.1,
|
| 28 |
+
"ms_per_sample_batch16": 0.0568,
|
| 29 |
+
"throughput_batch256": 39677.6,
|
| 30 |
+
"ms_per_sample_batch256": 0.0252
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9827782760416667,
|
| 34 |
+
"pr_auc": 0.9845011360409881,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.8002916666666666,
|
| 36 |
+
"n_samples": 96000,
|
| 37 |
+
"n_positive": 48000,
|
| 38 |
+
"auc_delta_vs_original": 0.000631
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 0.0,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
hybrid/hybrid_APK_lgbm.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a13274463201e7d3ddec74b098350051266fcf0812aff302617397f1765cbc90
|
| 3 |
+
size 3679987
|
hybrid/hybrid_APK_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "hybrid",
|
| 3 |
+
"subset": "APK",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9827782760416667,
|
| 7 |
+
"pr_auc": 0.9845011360409881,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.8002916666666666,
|
| 9 |
+
"n_samples": 96000,
|
| 10 |
+
"n_positive": 48000,
|
| 11 |
+
"auc_delta_vs_original": 0.000631
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.34375,
|
| 15 |
+
"threshold": 0.9999923489401177
|
| 16 |
+
},
|
| 17 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 18 |
+
"auc_gate_failed": false
|
| 19 |
+
}
|
hybrid/hybrid_APK_nnpart.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:7052c06fe40e9b2ab700f004f5d5e95f80e78d644073b13caa64e393ec2ae8e9
|
| 3 |
+
size 5325521
|
hybrid/hybrid_ELF.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:9a1e2daf9a192da640e68210bd3a6ecf9031f938e32508cd23365481b1cdc560
|
| 3 |
+
size 5331649
|
hybrid/hybrid_ELF_benchmark.json
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "hybrid",
|
| 3 |
+
"subset": "ELF",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"model_size_mb": 5.326,
|
| 6 |
+
"original_size_mb": 5.332,
|
| 7 |
+
"size_reduction_pct": 0.1,
|
| 8 |
+
"latency_ms": {
|
| 9 |
+
"cold_batch1": 0.483,
|
| 10 |
+
"warm_batch1": 0.16,
|
| 11 |
+
"warm_batch16": 1.058,
|
| 12 |
+
"warm_batch256": 12.195,
|
| 13 |
+
"throughput_batch1": 6250.0,
|
| 14 |
+
"ms_per_sample_batch1": 0.16,
|
| 15 |
+
"throughput_batch16": 15122.9,
|
| 16 |
+
"ms_per_sample_batch16": 0.0661,
|
| 17 |
+
"throughput_batch256": 20992.2,
|
| 18 |
+
"ms_per_sample_batch256": 0.0476
|
| 19 |
+
},
|
| 20 |
+
"original_latency_ms": {
|
| 21 |
+
"cold_batch1": 0.487,
|
| 22 |
+
"warm_batch1": 0.211,
|
| 23 |
+
"warm_batch16": 0.912,
|
| 24 |
+
"warm_batch256": 6.459,
|
| 25 |
+
"throughput_batch1": 4739.3,
|
| 26 |
+
"ms_per_sample_batch1": 0.211,
|
| 27 |
+
"throughput_batch16": 17543.9,
|
| 28 |
+
"ms_per_sample_batch16": 0.057,
|
| 29 |
+
"throughput_batch256": 39634.6,
|
| 30 |
+
"ms_per_sample_batch256": 0.0252
|
| 31 |
+
},
|
| 32 |
+
"test_metrics": {
|
| 33 |
+
"roc_auc": 0.9899468888888888,
|
| 34 |
+
"pr_auc": 0.9863592706579489,
|
| 35 |
+
"tpr_at_1pct_fpr": 0.8826666666666667,
|
| 36 |
+
"n_samples": 12000,
|
| 37 |
+
"n_positive": 6000,
|
| 38 |
+
"auc_delta_vs_original": 0.0
|
| 39 |
+
},
|
| 40 |
+
"peak_rss_delta_mb": 0.0,
|
| 41 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 42 |
+
"platform_short": "darwin-arm64"
|
| 43 |
+
}
|
hybrid/hybrid_ELF_lgbm.model
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:8e4072938955ac6020f80bca6cff7c12e97865102d578b3099e2011048eb9c43
|
| 3 |
+
size 3764582
|
hybrid/hybrid_ELF_metrics.json
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"model": "hybrid",
|
| 3 |
+
"subset": "ELF",
|
| 4 |
+
"quant": "fp32",
|
| 5 |
+
"test_metrics": {
|
| 6 |
+
"roc_auc": 0.9899468888888888,
|
| 7 |
+
"pr_auc": 0.9863592706579489,
|
| 8 |
+
"tpr_at_1pct_fpr": 0.8826666666666667,
|
| 9 |
+
"n_samples": 12000,
|
| 10 |
+
"n_positive": 6000,
|
| 11 |
+
"auc_delta_vs_original": 0.0
|
| 12 |
+
},
|
| 13 |
+
"challenge_metrics": {
|
| 14 |
+
"detection_rate": 0.23834196891191708,
|
| 15 |
+
"threshold": 0.7018789316233345
|
| 16 |
+
},
|
| 17 |
+
"platform": "macOS-15.7.7-arm64-arm-64bit",
|
| 18 |
+
"auc_gate_failed": false
|
| 19 |
+
}
|
hybrid/hybrid_ELF_nnpart.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:182798a23e2a8f7c478d561abd8103c32dcb9e8feb6decf0728261ab9d0893c6
|
| 3 |
+
size 5325521
|
hybrid/hybrid_PDF.pt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:5785b87d151d9f4fefd2184df92444655697fe77fc9db74fa52e787dd8139ff7
|
| 3 |
+
size 5331649
|