File size: 3,005 Bytes
902df3d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
{
 "ckpt": "tinyfacenet_v2.pt",
 "tag": "v2",
 "names": [
  "PERSON_A",
  "PERSON_B",
  "PERSON_C",
  "PERSON_D",
  "PERSON_E",
  "other",
  "PERSON_F",
  "PERSON_G",
  "PERSON_H",
  "PERSON_I"
 ],
 "n_test": 59,
 "n_test_family": 49,
 "n_agree": 1500,
 "thr": 0.45,
 "results": {
  "fp32": {
   "fam_acc": 0.9591836734693877,
   "openset_acc": 0.8983050847457628,
   "per_class": {
    "PERSON_A": 1.0,
    "PERSON_C": 0.857,
    "PERSON_D": 0.909,
    "PERSON_E": 1.0,
    "other": 0.6,
    "PERSON_G": 1.0,
    "PERSON_I": 1.0
   },
   "cos_vs_fp32": 1.0,
   "argmax_agree_vs_fp32": 1.0,
   "size_kb": 3981,
   "lat_ms": 11.95,
   "note": "PyTorch fp32 (deploy params only)"
  },
  "fp32_onnx": {
   "fam_acc": 0.9591836734693877,
   "openset_acc": 0.8983050847457628,
   "per_class": {
    "PERSON_A": 1.0,
    "PERSON_C": 0.857,
    "PERSON_D": 0.909,
    "PERSON_E": 1.0,
    "other": 0.6,
    "PERSON_G": 1.0,
    "PERSON_I": 1.0
   },
   "cos_vs_fp32": 1.0,
   "argmax_agree_vs_fp32": 1.0,
   "size_kb": 3968,
   "lat_ms": 63.22,
   "note": "ONNX Runtime fp32"
  },
  "int8_ort": {
   "fam_acc": 0.9591836734693877,
   "openset_acc": 0.9152542372881356,
   "per_class": {
    "PERSON_A": 1.0,
    "PERSON_C": 0.857,
    "PERSON_D": 0.909,
    "PERSON_E": 1.0,
    "other": 0.5,
    "PERSON_G": 1.0,
    "PERSON_I": 1.0
   },
   "cos_vs_fp32": 0.9980310797691345,
   "argmax_agree_vs_fp32": 0.9973333333333333,
   "size_kb": 1119,
   "lat_ms": 91.33,
   "note": "ONNX Runtime static int8, per-channel, 200 calib imgs (deployable)"
  },
  "int8_w": {
   "fam_acc": 0.9591836734693877,
   "openset_acc": 0.8983050847457628,
   "per_class": {
    "PERSON_A": 1.0,
    "PERSON_C": 0.857,
    "PERSON_D": 0.909,
    "PERSON_E": 1.0,
    "other": 0.6,
    "PERSON_G": 1.0,
    "PERSON_I": 1.0
   },
   "cos_vs_fp32": 0.9999483823776245,
   "argmax_agree_vs_fp32": 0.9993333333333333,
   "size_kb": 997,
   "lat_ms": null,
   "note": "simulated: int8 weights per-channel, fp32 activations"
  },
  "int4_w": {
   "fam_acc": 0.9387755102040817,
   "openset_acc": 0.864406779661017,
   "per_class": {
    "PERSON_A": 1.0,
    "PERSON_C": 0.857,
    "PERSON_D": 0.909,
    "PERSON_E": 1.0,
    "other": 0.5,
    "PERSON_G": 1.0,
    "PERSON_I": 0.923
   },
   "cos_vs_fp32": 0.9782859683036804,
   "argmax_agree_vs_fp32": 0.992,
   "size_kb": 499,
   "lat_ms": null,
   "note": "simulated: int4 weights per-channel (coarse)"
  },
  "int4_g32": {
   "fam_acc": 0.9387755102040817,
   "openset_acc": 0.864406779661017,
   "per_class": {
    "PERSON_A": 1.0,
    "PERSON_C": 0.857,
    "PERSON_D": 0.909,
    "PERSON_E": 1.0,
    "other": 0.5,
    "PERSON_G": 1.0,
    "PERSON_I": 0.923
   },
   "cos_vs_fp32": 0.9782859683036804,
   "argmax_agree_vs_fp32": 0.992,
   "size_kb": 529,
   "lat_ms": null,
   "note": "simulated: int4 weights, group=32 scales (llama.cpp-style)"
  }
 }
}