data/processed/analysis_results.json

{
  "frequentist": {
    "n_control": 21306,
    "n_treatment": 42694,
    "conv_control": 122,
    "conv_treatment": 456,
    "cvr_control": 0.00573,
    "cvr_treatment": 0.01068,
    "spend_mean_control": 0.6528,
    "spend_mean_treatment": 1.2496,
    "ztest": {
      "test_name": "Two-proportion Z-test",
      "statistic": 6.2438,
      "p_value": 0.0,
      "ci_lower": 0.003548,
      "ci_upper": 0.006361,
      "observed_diff": 0.004955,
      "relative_lift": 86.53,
      "effect_size": 0.0556,
      "effect_name": "Cohen's h",
      "significant": true,
      "alpha": 0.05
    },
    "chitest": {
      "test_name": "Chi-square test",
      "statistic": 38.9846,
      "p_value": 0.0,
      "ci_lower": NaN,
      "ci_upper": NaN,
      "observed_diff": 0.004955,
      "relative_lift": 86.53,
      "effect_size": 0.0247,
      "effect_name": "Cram\u00e9r's V",
      "significant": true,
      "alpha": 0.05
    },
    "ttest_spend": {
      "test_name": "Welch's t-test",
      "statistic": -5.3022,
      "p_value": 0.0,
      "ci_lower": 0.3762,
      "ci_upper": 0.8174,
      "observed_diff": 0.5968,
      "relative_lift": 91.42,
      "effect_size": 0.0419,
      "effect_name": "Cohen's d",
      "significant": true,
      "alpha": 0.05
    },
    "power_curve": {
      "sample_sizes": [
        50,
        74,
        98,
        122,
        146,
        170,
        194,
        218,
        242,
        266,
        290,
        314,
        338,
        362,
        386,
        410,
        434,
        458,
        482,
        506,
        530,
        554,
        578,
        602,
        626,
        650,
        674,
        698,
        722,
        746,
        770,
        794,
        818,
        842,
        866,
        890,
        914,
        938,
        962,
        986,
        1010,
        1034,
        1058,
        1082,
        1106,
        1130,
        1154,
        1178,
        1202,
        1226,
        1250,
        1274,
        1298,
        1322,
        1346,
        1370,
        1394,
        1418,
        1442,
        1466,
        1490,
        1514,
        1538,
        1562,
        1586,
        1610,
        1634,
        1658,
        1682,
        1706,
        1730,
        1754,
        1778,
        1802,
        1826,
        1850,
        1874,
        1898,
        1922,
        1946,
        1970,
        1994,
        2018,
        2042,
        2066,
        2090,
        2114,
        2138,
        2162,
        2186,
        2210,
        2234,
        2258,
        2282,
        2306,
        2330,
        2354,
        2378,
        2402,
        2426,
        2450,
        2474,
        2498,
        2522,
        2546,
        2570,
        2594,
        2618,
        2642,
        2666,
        2690,
        2714,
        2738,
        2762,
        2786,
        2810,
        2834,
        2858,
        2882,
        2906,
        2930,
        2954,
        2978,
        3002,
        3026,
        3050,
        3074,
        3098,
        3122,
        3146,
        3170,
        3194,
        3218,
        3242,
        3266,
        3290,
        3314,
        3338,
        3362,
        3386,
        3410,
        3434,
        3458,
        3482,
        3506,
        3530,
        3554,
        3578,
        3602,
        3626,
        3650,
        3674,
        3698,
        3722,
        3746,
        3770,
        3794,
        3818,
        3842,
        3866,
        3890,
        3914,
        3938,
        3962,
        3986,
        4010,
        4034,
        4058,
        4082,
        4106,
        4130,
        4154,
        4178,
        4202,
        4226,
        4250,
        4274,
        4298,
        4322,
        4346,
        4370,
        4394,
        4418,
        4442,
        4466,
        4490,
        4514,
        4538,
        4562,
        4586,
        4610,
        4634,
        4658,
        4682,
        4706,
        4730,
        4754,
        4778,
        4802,
        4826,
        4850,
        4874,
        4898,
        4922,
        4946,
        4970,
        4994
      ],
      "powers": [
        0.07016359467341239,
        0.08542757443856885,
        0.10019915555509751,
        0.11472823925011488,
        0.12912675465935197,
        0.14344961163868802,
        0.15772401108397216,
        0.17196227914738482,
        0.18616823014040723,
        0.2003406111212503,
        0.21447509255068437,
        0.2285654776499641,
        0.2426044661476573,
        0.2565841513412118,
        0.270496351095622,
        0.28433283194581627,
        0.29808546242951983,
        0.31174631843369555,
        0.3253077553308209,
        0.33876245672631977,
        0.35210346648801816,
        0.3653242086767469,
        0.3784184986320982,
        0.39138054754143436,
        0.40420496218122126,
        0.416886741071703,
        0.4294212679672,
        0.4418043033745544,
        0.4540319746245751,
        0.46610076489758695,
        0.4780075015118807,
        0.4897493437143351,
        0.5013237701596133,
        0.5127285662238109,
        0.5239618112670851,
        0.5350218659353898,
        0.5459073595722768,
        0.556617177796599,
        0.5671504502899152,
        0.5775065388277701,
        0.5876850255812971,
        0.597685701709319,
        0.6075085562560554,
        0.6171537653653922,
        0.6266216818192832,
        0.6359128249050573,
        0.645027870614119,
        0.6539676421726321,
        0.6627331009032064,
        0.6713253374153239,
        0.6797455631211574,
        0.6879951020725652,
        0.6960753831143077,
        0.7039879323479434,
        0.7117343659003754,
        0.7193163829906285,
        0.7267357592881195,
        0.7339943405554568,
        0.7410940365685976,
        0.7480368153070729,
        0.7548246974068891,
        0.7614597508686624,
        0.7679440860135129,
        0.7742798506792578,
        0.7804692256494565,
        0.7865144203079224,
        0.7924176685113669,
        0.7981812246729322,
        0.8038073600494553,
        0.8092983592254133,
        0.814656516786614,
        0.8198841341768148,
        0.8249835167305837,
        0.8299569708758534,
        0.8348068014997527,
        0.8395353094714446,
        0.8441447893158516,
        0.8486375270322882,
        0.8530157980521762,
        0.8572818653301708,
        0.8614379775631683,
        0.865486367531832,
        0.8694292505594088,
        0.8732688230827741,
        0.8770072613307798,
        0.8806467201051413,
        0.8841893316592366,
        0.8876372046703389,
        0.8909924233009534,
        0.8942570463450624,
        0.8974331064552263,
        0.900522609446625,
        0.9035275336742595,
        0.9064498294796605,
        0.9092914187035881,
        0.9120541942613258,
        0.9147400197773015,
        0.9173507292758827,
        0.9198881269253207,
        0.922353986831923,
        0.9247500528816542,
        0.9270780386264705,
        0.9293396272128004,
        0.9315364713496886,
        0.9336701933142214,
        0.9357423849919462,
        0.9377546079500978,
        0.9397083935415339,
        0.9416052430373704,
        0.9434466277863967,
        0.9452339893994326,
        0.9469687399568726,
        0.9486522622377379,
        0.9502859099686382,
        0.9518710080911126,
        0.9534088530458952,
        0.954900713072718,
        0.9563478285243275,
        0.9577514121934593,
        0.9591126496515742,
        0.9604326995982188,
        0.9617126942199321,
        0.9629537395576733,
        0.964156915881799,
        0.9653232780736709,
        0.9664538560130222,
        0.9675496549702582,
        0.968611656002912,
        0.9696408163555216,
        0.9706380698622328,
        0.9716043273514758,
        0.9725404770520982,
        0.9734473850003781,
        0.9743258954473721,
        0.9751768312660896,
        0.9760009943580149,
        0.9767991660585313,
        0.97757210754083,
        0.9783205602179135,
        0.9790452461423331,
        0.9797468684033226,
        0.980426111521017,
        0.9810836418374667,
        0.9817201079041812,
        0.9823361408659569,
        0.9829323548407635,
        0.9835093472954827,
        0.9840676994173125,
        0.9846079764806646,
        0.9851307282094018,
        0.9856364891342777,
        0.9861257789454517,
        0.9865991028399735,
        0.987056951864136,
        0.9874998032506161,
        0.9879281207503289,
        0.9883423549589337,
        0.9887429436379431,
        0.9891303120303918,
        0.9895048731710359,
        0.9898670281910569,
        0.9902171666172602,
        0.990555666665757,
        0.9908828955301316,
        0.9911992096641032,
        0.9915049550586917,
        0.9918004675139114,
        0.9920860729050134,
        0.9923620874433098,
        0.992628817931613,
        0.9928865620143293,
        0.9931356084222511,
        0.9933762372120927,
        0.9936087200008223,
        0.9938333201948428,
        0.9940502932140775,
        0.9942598867110201,
        0.9944623407848115,
        0.9946578881904056,
        0.9948467545428897,
        0.9950291585170287,
        0.9952053120420993,
        0.9953754204920864,
        0.9955396828713123,
        0.9956982919955706,
        0.9958514346688386,
        0.9959992918556423,
        0.9961420388491473,
        0.9962798454350519,
        0.9964128760513562,
        0.9965412899440836,
        0.9966652413190299,
        0.9967848794896146,
        0.9969003490209106,
        0.997011789869928,
        0.9971193375222245,
        0.9972231231249193,
        0.9973232736161839,
        0.9974199118512813,
        0.9975131567252308,
        0.9976031232921657,
        0.9976899228814605,
        0.9977736632106947,
        0.9978544484955254,
        0.9979323795565384,
        0.9980075539231451,
        0.9980800659345945
      ]
    },
    "required_n_per_group": 1665
  },
  "bayesian": {
    "conversion": {
      "model": "Beta-Binomial",
      "prob_b_beats_a": 1.0,
      "expected_loss_choosing_b": 0.0,
      "expected_loss_choosing_a": 0.004932,
      "ci_lower": 0.0035,
      "ci_upper": 0.006336,
      "posterior_mean_a": 0.005772,
      "posterior_mean_b": 0.010704,
      "posterior_std_a": 0.000519,
      "posterior_std_b": 0.000498,
      "n_samples": 100000
    }
  },
  "cuped": {
    "hillstrom_cuped": {
      "theta": 0.030308,
      "corr_pre_post": 0.0331,
      "variance_reduction_pct": 0.11,
      "raw_mean_control": 0.005726,
      "raw_mean_treatment": 0.010681,
      "raw_t_stat": -6.2456,
      "raw_p_value": 0.0,
      "raw_significant": true,
      "cuped_mean_control": 0.005728,
      "cuped_mean_treatment": 0.010679,
      "cuped_t_stat": -6.2446,
      "cuped_p_value": 0.0,
      "cuped_significant": true,
      "sample_size_reduction_pct": 0.11,
      "alpha": 0.05
    },
    "power_vs_correlation": [
      {
        "n_per_group": 1000,
        "true_effect": 0.02,
        "pre_post_correlation": 0.0,
        "theoretical_variance_reduction_pct": 0.0,
        "raw_power": 0.077,
        "cuped_power": 0.079,
        "power_gain_pct": 2.6
      },
      {
        "n_per_group": 1000,
        "true_effect": 0.02,
        "pre_post_correlation": 0.2,
        "theoretical_variance_reduction_pct": 4.0,
        "raw_power": 0.079,
        "cuped_power": 0.084,
        "power_gain_pct": 6.3
      },
      {
        "n_per_group": 1000,
        "true_effect": 0.02,
        "pre_post_correlation": 0.4,
        "theoretical_variance_reduction_pct": 16.0,
        "raw_power": 0.072,
        "cuped_power": 0.085,
        "power_gain_pct": 18.1
      },
      {
        "n_per_group": 1000,
        "true_effect": 0.02,
        "pre_post_correlation": 0.6,
        "theoretical_variance_reduction_pct": 36.0,
        "raw_power": 0.075,
        "cuped_power": 0.084,
        "power_gain_pct": 12.0
      },
      {
        "n_per_group": 1000,
        "true_effect": 0.02,
        "pre_post_correlation": 0.8,
        "theoretical_variance_reduction_pct": 64.0,
        "raw_power": 0.075,
        "cuped_power": 0.093,
        "power_gain_pct": 24.0
      },
      {
        "n_per_group": 1000,
        "true_effect": 0.02,
        "pre_post_correlation": 0.95,
        "theoretical_variance_reduction_pct": 90.25,
        "raw_power": 0.069,
        "cuped_power": 0.099,
        "power_gain_pct": 43.5
      }
    ]
  }
}