results/mia_results_all.json

{
  "baseline": {
    "auc": 0.622962,
    "attack_accuracy": 0.6055,
    "precision": 0.6779089376053963,
    "recall": 0.402,
    "f1": 0.504708097928437,
    "tpr_at_5fpr": 0.185,
    "tpr_at_1fpr": 0.093,
    "loss_gap": 0.010672920569777478,
    "member_loss_mean": 0.24938265652954578,
    "member_loss_std": 0.03375208091414265,
    "non_member_loss_mean": 0.26005557709932325,
    "non_member_loss_std": 0.03391039509516577,
    "best_threshold_loss": 0.22699543833732605,
    "sample_size": 2000,
    "test_duration_minutes": 2.4
  },
  "smooth_eps_0.02": {
    "auc": 0.6040805,
    "attack_accuracy": 0.5925,
    "precision": 0.6403641881638846,
    "recall": 0.422,
    "f1": 0.5087402049427366,
    "tpr_at_5fpr": 0.146,
    "tpr_at_1fpr": 0.062,
    "loss_gap": 0.009059646993875459,
    "member_loss_mean": 0.27049152280390265,
    "member_loss_std": 0.03371441539414184,
    "non_member_loss_mean": 0.2795511697977781,
    "non_member_loss_std": 0.03366667410078979,
    "best_threshold_loss": 0.24904540181159973,
    "sample_size": 2000,
    "test_duration_minutes": 2.4
  },
  "smooth_eps_0.05": {
    "auc": 0.5934269999999999,
    "attack_accuracy": 0.5795,
    "precision": 0.6819221967963387,
    "recall": 0.298,
    "f1": 0.41475295755045233,
    "tpr_at_5fpr": 0.145,
    "tpr_at_1fpr": 0.063,
    "loss_gap": 0.008093021050095572,
    "member_loss_mean": 0.30312477512657643,
    "member_loss_std": 0.033732365377909905,
    "non_member_loss_mean": 0.311217796176672,
    "non_member_loss_std": 0.033528631313525376,
    "best_threshold_loss": 0.27569371461868286,
    "sample_size": 2000,
    "test_duration_minutes": 2.4
  },
  "smooth_eps_0.1": {
    "auc": 0.579729,
    "attack_accuracy": 0.5685,
    "precision": 0.6319845857418112,
    "recall": 0.328,
    "f1": 0.4318630678077683,
    "tpr_at_5fpr": 0.122,
    "tpr_at_1fpr": 0.038,
    "loss_gap": 0.007210358262062122,
    "member_loss_mean": 0.3609955322742462,
    "member_loss_std": 0.033379551506700585,
    "non_member_loss_mean": 0.3682058905363083,
    "non_member_loss_std": 0.03337051131271436,
    "best_threshold_loss": 0.33504441380500793,
    "sample_size": 2000,
    "test_duration_minutes": 2.4
  },
  "smooth_eps_0.2": {
    "auc": 0.570492,
    "attack_accuracy": 0.556,
    "precision": 0.6076923076923076,
    "recall": 0.316,
    "f1": 0.41578947368421054,
    "tpr_at_5fpr": 0.127,
    "tpr_at_1fpr": 0.038,
    "loss_gap": 0.006796935558319062,
    "member_loss_mean": 0.4847908769249916,
    "member_loss_std": 0.03266944093536488,
    "non_member_loss_mean": 0.4915878124833107,
    "non_member_loss_std": 0.03270088491751788,
    "best_threshold_loss": 0.45949578285217285,
    "sample_size": 2000,
    "test_duration_minutes": 2.4
  }
}