Update app.py
Browse files
app.py
CHANGED
|
@@ -13,8 +13,8 @@ with center:
|
|
| 13 |
# --------------------------------------------------
|
| 14 |
|
| 15 |
model_owner_actions = {
|
| 16 |
-
"Overfit to Audit Dataset": "overfit",
|
| 17 |
-
"Overfit to Audit Dataset-like
|
| 18 |
}
|
| 19 |
|
| 20 |
auditor_actions = {
|
|
@@ -150,7 +150,7 @@ auditor_cost = "π’ Low"
|
|
| 150 |
if "auditmany" in auditor_active_list:
|
| 151 |
auditor_cost = "π‘ Medium"
|
| 152 |
if "internals" in auditor_active_list:
|
| 153 |
-
auditor_cost = "π΄ High"
|
| 154 |
|
| 155 |
owner_cost = "None"
|
| 156 |
if "overfit" in owner_active_list:
|
|
@@ -158,47 +158,25 @@ if "overfit" in owner_active_list:
|
|
| 158 |
if "overfitdist" in owner_active_list:
|
| 159 |
owner_cost = "π‘ Medium"
|
| 160 |
|
| 161 |
-
reliability
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
for action in st.session_state.auditor_selected:
|
| 178 |
-
auditor_cost += auditor_actions[action]["cost"]
|
| 179 |
-
reliability -= auditor_actions[action]["cheating"]
|
| 180 |
-
openness += auditor_actions[action]["open"]
|
| 181 |
-
|
| 182 |
-
reliability = max(0, min(100, reliability))
|
| 183 |
-
openness = max(0, openness)
|
| 184 |
-
|
| 185 |
-
# --------------------------------------------------
|
| 186 |
-
# Convert to Levels
|
| 187 |
-
# --------------------------------------------------
|
| 188 |
-
|
| 189 |
-
def level(score):
|
| 190 |
-
if score < 30:
|
| 191 |
-
return "Low"
|
| 192 |
-
elif score < 60:
|
| 193 |
-
return "Medium"
|
| 194 |
-
else:
|
| 195 |
-
return "High"
|
| 196 |
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
"High": "π΄ High"
|
| 201 |
-
}
|
| 202 |
|
| 203 |
# --------------------------------------------------
|
| 204 |
# Dashboard
|
|
@@ -209,12 +187,12 @@ st.divider()
|
|
| 209 |
c1, c2, c3, c4 = st.columns(4)
|
| 210 |
|
| 211 |
c1.metric(
|
| 212 |
-
"
|
| 213 |
indicator_map[level(auditor_cost)]
|
| 214 |
)
|
| 215 |
|
| 216 |
c2.metric(
|
| 217 |
-
"Cost for Model Owner",
|
| 218 |
indicator_map[level(owner_cost)]
|
| 219 |
)
|
| 220 |
|
|
|
|
| 13 |
# --------------------------------------------------
|
| 14 |
|
| 15 |
model_owner_actions = {
|
| 16 |
+
"Overfit to One Audit Dataset": "overfit",
|
| 17 |
+
"Overfit to Audit Dataset-like Distributions": "overfitdist",
|
| 18 |
}
|
| 19 |
|
| 20 |
auditor_actions = {
|
|
|
|
| 150 |
if "auditmany" in auditor_active_list:
|
| 151 |
auditor_cost = "π‘ Medium"
|
| 152 |
if "internals" in auditor_active_list:
|
| 153 |
+
auditor_cost = "π΄ Very High"
|
| 154 |
|
| 155 |
owner_cost = "None"
|
| 156 |
if "overfit" in owner_active_list:
|
|
|
|
| 158 |
if "overfitdist" in owner_active_list:
|
| 159 |
owner_cost = "π‘ Medium"
|
| 160 |
|
| 161 |
+
reliability = "π΄ Low"
|
| 162 |
+
if "auditone" in auditor_active_list and len(owner_active_list)==0:
|
| 163 |
+
reliability = "π‘ Medium"
|
| 164 |
+
if "auditmany" in auditor_active_list and len(owner_active_list)==0:
|
| 165 |
+
reliability = "π’ High"
|
| 166 |
+
if "auditmany" in auditor_active_list and "overfit" in owner_active_list:
|
| 167 |
+
reliability = "π‘ Medium"
|
| 168 |
+
if "private" in auditor_active_list and "overfitdist" not in owner_active_list:
|
| 169 |
+
reliability = "π‘ Medium"
|
| 170 |
+
if "private" in auditor_active_list and "auditmany" in auditor_active_list and "overfitdist" not in owner_active_list:
|
| 171 |
+
reliability = "π’ High"
|
| 172 |
+
if "private" in auditor_active_list and "auditmany" in auditor_active_list and "overfitdist" in owner_active_list:
|
| 173 |
+
reliability = "π‘ Medium"
|
| 174 |
+
if "internals" in auditor_active_list:
|
| 175 |
+
reliability = "π’ High"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 176 |
|
| 177 |
+
openness = "None"
|
| 178 |
+
if "internals" in auditor_active_list:
|
| 179 |
+
openness = "π’ High (Model Weight Sharing)"
|
|
|
|
|
|
|
| 180 |
|
| 181 |
# --------------------------------------------------
|
| 182 |
# Dashboard
|
|
|
|
| 187 |
c1, c2, c3, c4 = st.columns(4)
|
| 188 |
|
| 189 |
c1.metric(
|
| 190 |
+
"Auditign Cost",
|
| 191 |
indicator_map[level(auditor_cost)]
|
| 192 |
)
|
| 193 |
|
| 194 |
c2.metric(
|
| 195 |
+
"Cost to Cheat for Model Owner",
|
| 196 |
indicator_map[level(owner_cost)]
|
| 197 |
)
|
| 198 |
|