diff --git "a/data.json" "b/data.json"
new file mode 100644--- /dev/null
+++ "b/data.json"
@@ -0,0 +1,21223 @@
+{
+ "meta": {
+ "source": "Upworthy Research Archive (Matias et al.), CC BY 4.0",
+ "alpha": 0.05,
+ "looks": 10,
+ "obf_bounds": [
+ 6.198,
+ 4.383,
+ 3.582,
+ 3.119,
+ 2.816,
+ 2.599,
+ 2.437,
+ 2.31,
+ 2.207,
+ 2.122
+ ],
+ "pocock_bounds": [
+ 2.655,
+ 2.623,
+ 2.59,
+ 2.562,
+ 2.54,
+ 2.521,
+ 2.506,
+ 2.493,
+ 2.482,
+ 2.472
+ ],
+ "z_fixed": 1.96
+ },
+ "dataset": {
+ "filters": {
+ "rows_raw": 150817,
+ "tests_raw": 32487,
+ "rows_kept": 150024,
+ "tests_kept": 32363,
+ "dropped_bad_counts": 0,
+ "dropped_single_arm": 0,
+ "dropped_small_arm": 409,
+ "dropped_too_many_arms": 384
+ },
+ "tests": 32363,
+ "arms": 150024,
+ "impressions": 536874152,
+ "clicks": 8160148,
+ "pooled_ctr": 0.015199368361470306,
+ "median_arm_impressions": 3120.0,
+ "median_test_impressions": 14351.0,
+ "arms_per_test": {
+ "2": 837,
+ "3": 1166,
+ "4": 16918,
+ "5": 6156,
+ "6": 5674,
+ "7": 889,
+ "8": 517,
+ "9": 112,
+ "10": 54,
+ "11": 31,
+ "12": 9
+ },
+ "variation_types": {
+ "headline": 15672,
+ "image": 9343,
+ "other": 5289,
+ "both": 2059
+ },
+ "weeks_covered": 121
+ },
+ "curse": {
+ "config": {
+ "replicates": 20,
+ "selection_fraction": 0.5,
+ "kappa": 2317.6977046341626
+ },
+ "vs_test_average": {
+ "n_decisions": 647242,
+ "median_in_sample": 0.31868131868131866,
+ "median_out_of_sample": 0.14206439625286016,
+ "mean_in_sample": 0.3783102340137593,
+ "mean_out_of_sample": 0.1826517713416575,
+ "shrinkage_median": 0.554211722103094,
+ "shrinkage_mean": 0.5171905094827163
+ },
+ "vs_test_average_per_test": {
+ "n_tests": 32354,
+ "median_in_sample": 0.32796999656388864,
+ "median_out_of_sample": 0.12094147264307453,
+ "mean_in_sample": 0.37807799590129176,
+ "mean_out_of_sample": 0.1826698304748117,
+ "shrinkage_median": 0.6312422663348258,
+ "shrinkage_mean": 0.5168461733951242
+ },
+ "vs_runner_up": {
+ "n_tests": 31915,
+ "median_in_sample": 0.1839019416355784,
+ "median_out_of_sample": 0.08402156695679681,
+ "mean_in_sample": 0.24954621195953383,
+ "mean_out_of_sample": 0.23182197507525645,
+ "shrinkage_median": 0.5431175646677258,
+ "shrinkage_mean": 0.07102587029913132
+ },
+ "significant_winners_only": {
+ "n_decisions": 254265,
+ "median_in_sample": 0.516036467571201,
+ "median_out_of_sample": 0.2724248103230493,
+ "mean_in_sample": 0.5738378899261888,
+ "mean_out_of_sample": 0.3141318330156038,
+ "shrinkage_median": 0.47208225107567414,
+ "shrinkage_mean": 0.452577394190527,
+ "share_of_decisions_calling_a_significant_winner": 0.3928344096653586
+ },
+ "replication_rate": 0.4353953588975064,
+ "by_arm_count": {
+ "2": {
+ "n_tests": 837,
+ "median_in_sample": 0.11092271666361864,
+ "median_out_of_sample": 0.010025987512021212,
+ "replication_rate": 0.6079450418160095
+ },
+ "3": {
+ "n_tests": 1166,
+ "median_in_sample": 0.21263491821073582,
+ "median_out_of_sample": 0.07485807867954185,
+ "replication_rate": 0.5227701543739279
+ },
+ "4": {
+ "n_tests": 16918,
+ "median_in_sample": 0.2900668703837036,
+ "median_out_of_sample": 0.1121344691893478,
+ "replication_rate": 0.46564605745359966
+ },
+ "5": {
+ "n_tests": 6156,
+ "median_in_sample": 0.36963398940845227,
+ "median_out_of_sample": 0.12748433326520914,
+ "replication_rate": 0.3992608836907082
+ },
+ "6": {
+ "n_tests": 5674,
+ "median_in_sample": 0.42449651059914206,
+ "median_out_of_sample": 0.15421368957962794,
+ "replication_rate": 0.36602044413112444
+ },
+ "7": {
+ "n_tests": 889,
+ "median_in_sample": 0.45016965378795915,
+ "median_out_of_sample": 0.18826688119268736,
+ "replication_rate": 0.3787401574803149
+ },
+ "8": {
+ "n_tests": 517,
+ "median_in_sample": 0.4867977961371479,
+ "median_out_of_sample": 0.1805987566388986,
+ "replication_rate": 0.325531914893617
+ },
+ "9": {
+ "n_tests": 112,
+ "median_in_sample": 0.5414530560736706,
+ "median_out_of_sample": 0.21368699748534725,
+ "replication_rate": 0.28125
+ },
+ "10": {
+ "n_tests": 54,
+ "median_in_sample": 0.5724740546312979,
+ "median_out_of_sample": 0.20635668625340123,
+ "replication_rate": 0.24259259259259258
+ }
+ },
+ "by_traffic_quartile": {
+ "0": {
+ "n_tests": 8097,
+ "median_arm_impressions": 2328.4,
+ "median_in_sample": 0.34772269165223346,
+ "median_out_of_sample": 0.1115969694241283,
+ "replication_rate": 0.41384463381499315
+ },
+ "1": {
+ "n_tests": 8086,
+ "median_arm_impressions": 3009.875,
+ "median_in_sample": 0.3459243269824711,
+ "median_out_of_sample": 0.12285629813929196,
+ "replication_rate": 0.42260697501855055
+ },
+ "2": {
+ "n_tests": 8091,
+ "median_arm_impressions": 3565.3333333333335,
+ "median_in_sample": 0.3237282969614249,
+ "median_out_of_sample": 0.12243480212000284,
+ "replication_rate": 0.43411197626992953
+ },
+ "3": {
+ "n_tests": 8089,
+ "median_arm_impressions": 5152.833333333333,
+ "median_in_sample": 0.2936309405044912,
+ "median_out_of_sample": 0.12586099934658254,
+ "replication_rate": 0.47103473853381134
+ }
+ },
+ "by_variation_type": {
+ "both": {
+ "n_tests": 2059,
+ "median_in_sample": 0.3544615687415575,
+ "median_out_of_sample": 0.15396077938951375
+ },
+ "headline": {
+ "n_tests": 15672,
+ "median_in_sample": 0.3807955229577384,
+ "median_out_of_sample": 0.1710960371768363
+ },
+ "image": {
+ "n_tests": 9343,
+ "median_in_sample": 0.3096719983063988,
+ "median_out_of_sample": 0.10780890071112145
+ },
+ "other": {
+ "n_tests": 5289,
+ "median_in_sample": 0.23199220153270478,
+ "median_out_of_sample": 0.03363531542064699
+ }
+ },
+ "estimators": {
+ "raw": {
+ "bias": 0.0024386243267708043,
+ "relative_bias": 0.12967131621106626,
+ "rmse": 0.0030924567126945223,
+ "mae": 0.002552567502497645
+ },
+ "eb": {
+ "bias": -0.0005040170360299849,
+ "relative_bias": -0.026800582499458978,
+ "rmse": 0.00292609761262605,
+ "mae": 0.001946116177398383
+ },
+ "pool": {
+ "bias": -0.0025313968087278546,
+ "relative_bias": -0.13460439660047907,
+ "rmse": 0.004708668441440616,
+ "mae": 0.002998323774879991
+ }
+ },
+ "selection_fraction_sensitivity": {
+ "0.3": {
+ "median_in_sample": 0.37186440677966104,
+ "median_out_of_sample": 0.12236164463568211,
+ "shrinkage_median": 0.6709509100498977
+ },
+ "0.5": {
+ "median_in_sample": 0.31835916404474596,
+ "median_out_of_sample": 0.14139019815782683,
+ "shrinkage_median": 0.5558783470798592
+ },
+ "0.7": {
+ "median_in_sample": 0.2917112426035502,
+ "median_out_of_sample": 0.15568690632691196,
+ "shrinkage_median": 0.46629788781059067
+ }
+ }
+ },
+ "peeking": {
+ "tau_fitted": 0.0035931873430655294,
+ "kappa": 2317.6977046341626,
+ "aa_nulls": {
+ "config": {
+ "n_arms": 150024,
+ "replicates": 20,
+ "alpha": 0.05,
+ "tau": 0.0035931873430655294,
+ "median_impressions_per_arm": 3120.0
+ },
+ "by_looks": {
+ "1": {
+ "naive": 0.049325774542739835,
+ "fixed": 0.049325774542739835,
+ "pocock": 0.04932310830267157,
+ "obrien_fleming": 0.04932310830267157,
+ "msprt": 0.0003159494480883058,
+ "stop_look_share": [
+ 1.0
+ ],
+ "median_false_lift_at_stop": 0.7058823529411764
+ },
+ "2": {
+ "naive": 0.08149262784621127,
+ "fixed": 0.04950674558737268,
+ "pocock": 0.046423238948434906,
+ "obrien_fleming": 0.04848957500133313,
+ "msprt": 0.00031861568815656164,
+ "stop_look_share": [
+ 0.589251463088456,
+ 0.410748536911544
+ ],
+ "median_false_lift_at_stop": 0.856126301598723
+ },
+ "5": {
+ "naive": 0.13586759451821043,
+ "fixed": 0.049445088785794275,
+ "pocock": 0.04313776462432677,
+ "obrien_fleming": 0.04826827707566788,
+ "msprt": 0.00045459393163760456,
+ "stop_look_share": [
+ 0.3160635615255551,
+ 0.2418242295200997,
+ 0.18213840674274165,
+ 0.14366592423246366,
+ 0.11630787797913988
+ ],
+ "median_false_lift_at_stop": 1.0000000000000002
+ },
+ "10": {
+ "naive": 0.1787110728950035,
+ "fixed": 0.04966072095131445,
+ "pocock": 0.03914107076201141,
+ "obrien_fleming": 0.04880919052951528,
+ "msprt": 0.0005572441742654508,
+ "stop_look_share": [
+ 0.18706722439898624,
+ 0.1803069268339988,
+ 0.1351089759967476,
+ 0.10821884342031894,
+ 0.0903119807392129,
+ 0.07592233770157343,
+ 0.06600288315035462,
+ 0.058297076381105484,
+ 0.05204589915687434,
+ 0.04671785222082769
+ ],
+ "median_false_lift_at_stop": 1.1428571428571428
+ },
+ "20": {
+ "naive": 0.215034261184877,
+ "fixed": 0.04950674558737268,
+ "pocock": 0.03601623740201568,
+ "obrien_fleming": 0.04730576441102757,
+ "msprt": 0.0006462299365434863,
+ "stop_look_share": [
+ 0.09008285725799202,
+ 0.12529486706571236,
+ 0.10784307647480029,
+ 0.0886445569322046,
+ 0.07246677805228098,
+ 0.06223903683474735,
+ 0.054517471939194616,
+ 0.04839694609163585,
+ 0.04383406229948265,
+ 0.039751645210986876,
+ 0.036416276352048804,
+ 0.03357842301528504,
+ 0.030591779989646718,
+ 0.028524223271327295,
+ 0.026715498615945917,
+ 0.025117559353136826,
+ 0.023260788027389703,
+ 0.022321553116369037,
+ 0.020658518364677327,
+ 0.019744081735135754
+ ],
+ "median_false_lift_at_stop": 1.2727272727272727
+ }
+ }
+ },
+ "declarations": {
+ "config": {
+ "n_pairs": 32363,
+ "looks": 10,
+ "alpha": 0.05,
+ "tau": 0.0035931873430655294
+ },
+ "rules": {
+ "fixed_horizon": {
+ "declaration_rate": 0.17096684485369093,
+ "n_declared": 5533,
+ "mean_stop_fraction": 1.0,
+ "sign_replication_rate": 0.871294287780188,
+ "median_declared_lift": 0.7454761194531032,
+ "median_holdout_lift": 0.43277883447732646,
+ "lift_shrinkage": 0.41945982817689453
+ },
+ "naive_peeking": {
+ "declaration_rate": 0.30435991718938293,
+ "n_declared": 9850,
+ "mean_stop_fraction": 0.44750253807106594,
+ "sign_replication_rate": 0.7745964057264697,
+ "median_declared_lift": 1.0513098969995522,
+ "median_holdout_lift": 0.3023096337401675,
+ "lift_shrinkage": 0.7124447942486208
+ },
+ "pocock": {
+ "declaration_rate": 0.13123010845718877,
+ "n_declared": 4247,
+ "mean_stop_fraction": 0.5864610313162233,
+ "sign_replication_rate": 0.8707018370230806,
+ "median_declared_lift": 1.0308263441408942,
+ "median_holdout_lift": 0.4457428473778206,
+ "lift_shrinkage": 0.5675868686211069
+ },
+ "obrien_fleming": {
+ "declaration_rate": 0.16416895837839507,
+ "n_declared": 5313,
+ "mean_stop_fraction": 0.7526068134763788,
+ "sign_replication_rate": 0.8697289156626506,
+ "median_declared_lift": 0.8693797760173938,
+ "median_holdout_lift": 0.43127506704294133,
+ "lift_shrinkage": 0.5039278817611779
+ },
+ "msprt": {
+ "declaration_rate": 0.01554244044124463,
+ "n_declared": 503,
+ "mean_stop_fraction": 0.8240556660039762,
+ "sign_replication_rate": 0.9681908548707754,
+ "median_declared_lift": 1.168246463701647,
+ "median_holdout_lift": 0.7381621178342287,
+ "lift_shrinkage": 0.36814521526962274
+ }
+ }
+ },
+ "tau_sweep": {
+ "looks": 10,
+ "reference_fixed_horizon": 0.17096684485369093,
+ "rows": [
+ {
+ "tau": 0.001,
+ "false_positive_rate": 0.0,
+ "declaration_rate": 6.179896795723511e-05,
+ "sign_replication_rate": 1.0
+ },
+ {
+ "tau": 0.002,
+ "false_positive_rate": 3.9993601023836184e-05,
+ "declaration_rate": 0.0026882551061397273,
+ "sign_replication_rate": 0.9770114942528736
+ },
+ {
+ "tau": 0.005,
+ "false_positive_rate": 0.00157308164027089,
+ "declaration_rate": 0.03000339894323765,
+ "sign_replication_rate": 0.9618949536560247
+ },
+ {
+ "tau": 0.01,
+ "false_positive_rate": 0.004439289713645817,
+ "declaration_rate": 0.05166393721224855,
+ "sign_replication_rate": 0.9342105263157895
+ },
+ {
+ "tau": 0.02,
+ "false_positive_rate": 0.005725750546579214,
+ "declaration_rate": 0.05323981089515805,
+ "sign_replication_rate": 0.926291352292513
+ }
+ ]
+ }
+ },
+ "multiplicity": {
+ "config": {
+ "n_source_arms": 9435,
+ "min_impressions": 6000,
+ "replicates": 20,
+ "alpha": 0.05,
+ "median_impressions": 6980.0
+ },
+ "by_arms": {
+ "2": {
+ "false_positive_rate": 0.05049814520402755,
+ "mean_z_of_best_arm": 0.7999737674187941,
+ "median_apparent_lift": 0.07704830053667266,
+ "p90_apparent_lift": 0.22580645161290325,
+ "impressions_per_arm": 3490.0
+ },
+ "3": {
+ "false_positive_rate": 0.08264440911499735,
+ "mean_z_of_best_arm": 1.0481762339852339,
+ "median_apparent_lift": 0.15416639979504243,
+ "p90_apparent_lift": 0.3770491803278688,
+ "impressions_per_arm": 2326.6666666666665
+ },
+ "4": {
+ "false_positive_rate": 0.11354531001589827,
+ "mean_z_of_best_arm": 1.2122554958989449,
+ "median_apparent_lift": 0.2248299319727891,
+ "p90_apparent_lift": 0.5140050740702207,
+ "impressions_per_arm": 1745.0
+ },
+ "5": {
+ "false_positive_rate": 0.14760466348701645,
+ "mean_z_of_best_arm": 1.3407993799508509,
+ "median_apparent_lift": 0.2908414771253136,
+ "p90_apparent_lift": 0.6659731853906612,
+ "impressions_per_arm": 1396.0
+ },
+ "6": {
+ "false_positive_rate": 0.18210386857445682,
+ "mean_z_of_best_arm": 1.441989063728598,
+ "median_apparent_lift": 0.3528476316386132,
+ "p90_apparent_lift": 0.7634700939199208,
+ "impressions_per_arm": 1163.3333333333333
+ },
+ "8": {
+ "false_positive_rate": 0.24803921568627452,
+ "mean_z_of_best_arm": 1.5983881465015841,
+ "median_apparent_lift": 0.46626016260162606,
+ "p90_apparent_lift": 0.9993447777785404,
+ "impressions_per_arm": 872.5
+ },
+ "10": {
+ "false_positive_rate": 0.31072602013778483,
+ "mean_z_of_best_arm": 1.7271592612405282,
+ "median_apparent_lift": 0.573984181404664,
+ "p90_apparent_lift": 1.1926331414677687,
+ "impressions_per_arm": 698.0
+ }
+ }
+ },
+ "power": {
+ "config": {
+ "alpha": 0.05,
+ "power_target": 0.8,
+ "n_tests": 32363
+ },
+ "median_baseline_ctr": 0.01303538175046555,
+ "median_arm_impressions": 3089.0,
+ "median_mde_relative": 0.6841496794023025,
+ "median_mde_relative_textbook_approx": 0.5920487129962937,
+ "median_true_lift": 0.12094147264307445,
+ "share_adequately_powered": 0.04554583938448228,
+ "median_realised_power": 0.08750987660410837,
+ "impressions_needed_per_arm": 86101.29373885738,
+ "traffic_shortfall_factor": 27.873516911251983,
+ "mde_percentiles": {
+ "0.1": 0.4075025264945719,
+ "0.25": 0.5184872621134372,
+ "0.5": 0.6841496794023025,
+ "0.75": 0.9158746575643475,
+ "0.9": 1.2112357920260604
+ },
+ "true_lift_percentiles": {
+ "0.1": 0.0,
+ "0.25": 0.002579204641405175,
+ "0.5": 0.12094147264307445,
+ "0.75": 0.29955706894309253,
+ "0.9": 0.5213373762361565
+ },
+ "by_traffic_decile": [
+ {
+ "bucket": 0,
+ "arm_impressions": 2025.0,
+ "mde_rel": 0.9808284054349048,
+ "true_lift": 0.1050711503936414,
+ "powered": 0.15584199568516222
+ },
+ {
+ "bucket": 1,
+ "arm_impressions": 2527.5,
+ "mde_rel": 0.7468712650440182,
+ "true_lift": 0.1211473309573299,
+ "powered": 0.18858134726721915
+ },
+ {
+ "bucket": 2,
+ "arm_impressions": 2750.5,
+ "mde_rel": 0.5995502549171345,
+ "true_lift": 0.1118253342529459,
+ "powered": 0.20996959931462145
+ },
+ {
+ "bucket": 3,
+ "arm_impressions": 2866.0,
+ "mde_rel": 0.7447778080360874,
+ "true_lift": 0.124143202607286,
+ "powered": 0.1884605661314108
+ },
+ {
+ "bucket": 4,
+ "arm_impressions": 3060.5,
+ "mde_rel": 0.7817054107527598,
+ "true_lift": 0.1232680405448523,
+ "powered": 0.1856882725721644
+ },
+ {
+ "bucket": 5,
+ "arm_impressions": 3277.5,
+ "mde_rel": 0.6736432191043422,
+ "true_lift": 0.1182253975230108,
+ "powered": 0.197932508546423
+ },
+ {
+ "bucket": 6,
+ "arm_impressions": 3597.0,
+ "mde_rel": 0.6734410197359542,
+ "true_lift": 0.1188595265068763,
+ "powered": 0.20313257032144985
+ },
+ {
+ "bucket": 7,
+ "arm_impressions": 4097.0,
+ "mde_rel": 0.6467085432336616,
+ "true_lift": 0.1302693541561495,
+ "powered": 0.21891303590705605
+ },
+ {
+ "bucket": 8,
+ "arm_impressions": 5017.0,
+ "mde_rel": 0.6035956176663714,
+ "true_lift": 0.12436662287529635,
+ "powered": 0.2222169108018201
+ },
+ {
+ "bucket": 9,
+ "arm_impressions": 6359.5,
+ "mde_rel": 0.5113104259135894,
+ "true_lift": 0.1284647786925055,
+ "powered": 0.2593972109001994
+ }
+ ]
+ },
+ "tests": [
+ {
+ "id": "53f5f61b73269e19c2000049",
+ "variation": "headline",
+ "week": 201433,
+ "impressions": 70362,
+ "pooled_ctr": 0.009365850885421108,
+ "arms": [
+ {
+ "headline": "Yo Doctors \u2014 There's Somethin' We've Really, Really Gotta Talk About (Your Patients Can Come Too)",
+ "impressions": 11641,
+ "clicks": 77,
+ "ctr": 0.0066145520144317,
+ "ctr_select": 0.006872852233676976,
+ "ctr_holdout": 0.00635629616904312,
+ "n_select": 5820,
+ "n_holdout": 5821,
+ "eb": 0.007514070900554288,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Biggest Problem With The Culture Of Medicine? What Happens After The Mess Ups.",
+ "impressions": 11747,
+ "clicks": 89,
+ "ctr": 0.007576402485741,
+ "ctr_select": 0.008341845420497106,
+ "ctr_holdout": 0.006810829218457347,
+ "n_select": 5874,
+ "n_holdout": 5873,
+ "eb": 0.008563211198603638,
+ "labelled_winner": false
+ },
+ {
+ "headline": "'Do You Remember' Can Be The Most Gut-Wrenching Words Doctors Ever Hear. Here's Why.",
+ "impressions": 11752,
+ "clicks": 150,
+ "ctr": 0.0127637848876786,
+ "ctr_select": 0.012423417290673928,
+ "ctr_holdout": 0.013104152484683457,
+ "n_select": 5876,
+ "n_holdout": 5876,
+ "eb": 0.011490201483348783,
+ "labelled_winner": false
+ },
+ {
+ "headline": "As A Young Man, He Let A Woman Die. Now He's Here To Tell You Why You Should Care.",
+ "impressions": 11661,
+ "clicks": 108,
+ "ctr": 0.0092616413686647,
+ "ctr_select": 0.008919382504288165,
+ "ctr_holdout": 0.009603841536614645,
+ "n_select": 5830,
+ "n_holdout": 5831,
+ "eb": 0.008977657268542874,
+ "labelled_winner": false
+ },
+ {
+ "headline": "3 Of The Scariest Words Doctors Ever Have To Hear",
+ "impressions": 11782,
+ "clicks": 159,
+ "ctr": 0.0134951621116958,
+ "ctr_select": 0.01222203361059243,
+ "ctr_holdout": 0.014768290612799185,
+ "n_select": 5891,
+ "n_holdout": 5891,
+ "eb": 0.011347383089441014,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Yo Doctors \u2014 There's Something This Guy Really, Really Wants To Talk To You About",
+ "impressions": 11779,
+ "clicks": 76,
+ "ctr": 0.0064521606248408,
+ "ctr_select": 0.005942275042444821,
+ "ctr_holdout": 0.0069621327899473595,
+ "n_select": 5890,
+ "n_holdout": 5889,
+ "eb": 0.0068408023224585356,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.00912424319945425,
+ "pool_holdout": 0.009607458571387966,
+ "lift_select": 0.36158331371713226,
+ "lift_holdout": 0.36395617918239265,
+ "eb_lift": 0.25930460556290824,
+ "p_value": 0.0035658625066109315,
+ "held_up": false,
+ "z_path": [
+ 0.32,
+ 0.714,
+ 1.03,
+ 0.368,
+ -0.376,
+ -0.2,
+ -0.117,
+ -0.299,
+ -0.7,
+ -0.493
+ ],
+ "first_cross": null,
+ "final_z": -0.493
+ },
+ {
+ "id": "53f548f1ba722b505e000029",
+ "variation": "both",
+ "week": 201433,
+ "impressions": 61132,
+ "pooled_ctr": 0.0062324151017470394,
+ "arms": [
+ {
+ "headline": "HIV-Positive People Are Living Longer Than Ever. And There's A Big Problem With That.",
+ "impressions": 10280,
+ "clicks": 92,
+ "ctr": 0.0089494163424124,
+ "ctr_select": 0.0071984435797665365,
+ "ctr_holdout": 0.010700389105058366,
+ "n_select": 5140,
+ "n_holdout": 5140,
+ "eb": 0.0068219658592151055,
+ "labelled_winner": true
+ },
+ {
+ "headline": "HIV-Positive People Are Living Longer Than Ever. And There's A Big Problem With That.",
+ "impressions": 10114,
+ "clicks": 87,
+ "ctr": 0.0086019379078505,
+ "ctr_select": 0.008107573660272888,
+ "ctr_holdout": 0.009096302155428119,
+ "n_select": 5057,
+ "n_holdout": 5057,
+ "eb": 0.007441140142582069,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Can Live For Years With HIV. Here's The Problem With That.",
+ "impressions": 10078,
+ "clicks": 63,
+ "ctr": 0.0062512403254614,
+ "ctr_select": 0.006350466362373487,
+ "ctr_holdout": 0.0061520142885493154,
+ "n_select": 5039,
+ "n_holdout": 5039,
+ "eb": 0.006235971759511419,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Can Live For Years With HIV. Here's The Problem With That.",
+ "impressions": 10063,
+ "clicks": 67,
+ "ctr": 0.0066580542581735,
+ "ctr_select": 0.007551669316375199,
+ "ctr_holdout": 0.005764261578215067,
+ "n_select": 5032,
+ "n_holdout": 5031,
+ "eb": 0.007058271130886368,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Living With HIV Is Better Than Dying From It. But For How Long?",
+ "impressions": 10285,
+ "clicks": 32,
+ "ctr": 0.0031113271754982,
+ "ctr_select": 0.0031116297160637884,
+ "ctr_holdout": 0.003111024693758507,
+ "n_select": 5142,
+ "n_holdout": 5143,
+ "eb": 0.004005009359937099,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Living With HIV Is Better Than Dying From It. But For How Long?",
+ "impressions": 10312,
+ "clicks": 40,
+ "ctr": 0.0038789759503491,
+ "ctr_select": 0.0036850271528316523,
+ "ctr_holdout": 0.004072924747866563,
+ "n_select": 5156,
+ "n_holdout": 5156,
+ "eb": 0.004398914757948509,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.005987044428449911,
+ "pool_holdout": 0.006477785775044167,
+ "lift_select": 0.35418631967159087,
+ "lift_holdout": 0.40423016001422174,
+ "eb_lift": 0.24287371365116694,
+ "p_value": 0.0323757607505977,
+ "held_up": false,
+ "z_path": [
+ -0.009,
+ 0.173,
+ 1.031,
+ 0.633,
+ 1.307,
+ 1.231,
+ 1.549,
+ 1.82,
+ 2.225,
+ 1.586
+ ],
+ "first_cross": 8,
+ "final_z": 1.586
+ },
+ {
+ "id": "5420dc081afbe86848000074",
+ "variation": "both",
+ "week": 201438,
+ "impressions": 48797,
+ "pooled_ctr": 0.011086747136094432,
+ "arms": [
+ {
+ "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.",
+ "impressions": 6916,
+ "clicks": 88,
+ "ctr": 0.0127241179872758,
+ "ctr_select": 0.012434933487565067,
+ "ctr_holdout": 0.013013302486986697,
+ "n_select": 3458,
+ "n_holdout": 3458,
+ "eb": 0.011671804855457222,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.",
+ "impressions": 6942,
+ "clicks": 100,
+ "ctr": 0.0144050705848458,
+ "ctr_select": 0.012100259291270527,
+ "ctr_holdout": 0.016709881878421203,
+ "n_select": 3471,
+ "n_holdout": 3471,
+ "eb": 0.011472842407962525,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.",
+ "impressions": 6867,
+ "clicks": 72,
+ "ctr": 0.0104849279161205,
+ "ctr_select": 0.011357018054746652,
+ "ctr_holdout": 0.009612583745994756,
+ "n_select": 3434,
+ "n_holdout": 3433,
+ "eb": 0.011025060733200604,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.",
+ "impressions": 7120,
+ "clicks": 88,
+ "ctr": 0.0123595505617977,
+ "ctr_select": 0.013202247191011237,
+ "ctr_holdout": 0.011516853932584269,
+ "n_select": 3560,
+ "n_holdout": 3560,
+ "eb": 0.012149794035221992,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.",
+ "impressions": 7058,
+ "clicks": 48,
+ "ctr": 0.0068007934258996,
+ "ctr_select": 0.006517427033153868,
+ "ctr_holdout": 0.007084159818645509,
+ "n_select": 3529,
+ "n_holdout": 3529,
+ "eb": 0.008109332636613346,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.",
+ "impressions": 7014,
+ "clicks": 67,
+ "ctr": 0.0095523239235814,
+ "ctr_select": 0.007698887938408896,
+ "ctr_holdout": 0.01140575990875392,
+ "n_select": 3507,
+ "n_holdout": 3507,
+ "eb": 0.008826692666247351,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16 And He's Set On Making An Impact.",
+ "impressions": 6880,
+ "clicks": 78,
+ "ctr": 0.0113372093023255,
+ "ctr_select": 0.010465116279069767,
+ "ctr_holdout": 0.012209302325581395,
+ "n_select": 3440,
+ "n_holdout": 3440,
+ "eb": 0.010492530106951961,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.010533218574531744,
+ "pool_holdout": 0.011640298385113533,
+ "lift_select": 0.2533915533598565,
+ "lift_holdout": -0.01060492166482041,
+ "eb_lift": 0.153474025935336,
+ "p_value": 0.09143196774647187,
+ "held_up": false,
+ "z_path": [
+ -0.054,
+ -0.422,
+ 0.292,
+ 0.101,
+ 0.583,
+ 0.683,
+ 0.365,
+ -0.426,
+ -0.106,
+ -0.194
+ ],
+ "first_cross": null,
+ "final_z": -0.194
+ },
+ {
+ "id": "53f65f602f4a3a4260000025",
+ "variation": "image",
+ "week": 201433,
+ "impressions": 44996,
+ "pooled_ctr": 0.004778202506889501,
+ "arms": [
+ {
+ "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.",
+ "impressions": 8962,
+ "clicks": 47,
+ "ctr": 0.0052443650970765,
+ "ctr_select": 0.005802276277616604,
+ "ctr_holdout": 0.0046864539165364875,
+ "n_select": 4481,
+ "n_holdout": 4481,
+ "eb": 0.005627421162244576,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.",
+ "impressions": 9087,
+ "clicks": 40,
+ "ctr": 0.0044018928139099,
+ "ctr_select": 0.005061619718309859,
+ "ctr_holdout": 0.0037420206911732336,
+ "n_select": 4544,
+ "n_holdout": 4543,
+ "eb": 0.005138543966305752,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.",
+ "impressions": 8990,
+ "clicks": 50,
+ "ctr": 0.0055617352614015,
+ "ctr_select": 0.006674082313681869,
+ "ctr_holdout": 0.004449388209121246,
+ "n_select": 4495,
+ "n_holdout": 4495,
+ "eb": 0.0062029958132468165,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.",
+ "impressions": 9004,
+ "clicks": 35,
+ "ctr": 0.0038871612616614,
+ "ctr_select": 0.0046645935139937806,
+ "ctr_holdout": 0.003109729009329187,
+ "n_select": 4502,
+ "n_holdout": 4502,
+ "eb": 0.004876922230168862,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.",
+ "impressions": 8953,
+ "clicks": 43,
+ "ctr": 0.0048028593767452,
+ "ctr_select": 0.0042448614834673815,
+ "ctr_holdout": 0.005360732633459906,
+ "n_select": 4476,
+ "n_holdout": 4477,
+ "eb": 0.004601196093467512,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.005289359054138146,
+ "pool_holdout": 0.004267045959640857,
+ "lift_select": 0.2617941503631487,
+ "lift_holdout": 0.04273266592510194,
+ "eb_lift": 0.1727310908103099,
+ "p_value": 0.15248894768407653,
+ "held_up": false,
+ "z_path": [
+ -0.338,
+ -0.825,
+ -1.04,
+ -1.355,
+ -0.801,
+ -0.391,
+ -0.37,
+ 0.096,
+ 0.297,
+ 0.29
+ ],
+ "first_cross": null,
+ "final_z": 0.29
+ },
+ {
+ "id": "527d0b432a1baa15cf011f0c",
+ "variation": "headline",
+ "week": 201344,
+ "impressions": 38694,
+ "pooled_ctr": 0.015583811443634672,
+ "arms": [
+ {
+ "headline": "The Most Hilarious Parody Video I've Seen Yet: What Does The Farmer Say?",
+ "impressions": 9566,
+ "clicks": 208,
+ "ctr": 0.0217436755174576,
+ "ctr_select": 0.022370896926615095,
+ "ctr_holdout": 0.02111645410830023,
+ "n_select": 4783,
+ "n_holdout": 4783,
+ "eb": 0.020045901744286353,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Will No Longer Wonder What The Farmer Says After You See This",
+ "impressions": 9379,
+ "clicks": 129,
+ "ctr": 0.0137541315705299,
+ "ctr_select": 0.011940298507462687,
+ "ctr_holdout": 0.015568351460865857,
+ "n_select": 4690,
+ "n_holdout": 4689,
+ "eb": 0.013034222130126042,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Will Never Look At Farming The Same Way Agains After Watching This",
+ "impressions": 10265,
+ "clicks": 174,
+ "ctr": 0.0169508037018996,
+ "ctr_select": 0.0161730319563523,
+ "ctr_holdout": 0.017728423923631404,
+ "n_select": 5132,
+ "n_holdout": 5133,
+ "eb": 0.015885193358834076,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Forget What The Fox Says, These Farmers Are Actually Using Real Words Like 'Work' And 'Corn'",
+ "impressions": 9484,
+ "clicks": 92,
+ "ctr": 0.0097005482918599,
+ "ctr_select": 0.01033319274567693,
+ "ctr_holdout": 0.009067903838043019,
+ "n_select": 4742,
+ "n_holdout": 4742,
+ "eb": 0.011946671377672906,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.015247842042693957,
+ "pool_holdout": 0.015919780844575387,
+ "lift_select": 0.46715167064143137,
+ "lift_holdout": 0.3264286936145602,
+ "eb_lift": 0.31467139337867156,
+ "p_value": 3.5942110764181427e-06,
+ "held_up": true,
+ "z_path": [
+ 0.723,
+ 1.917,
+ 1.424,
+ 1.426,
+ 0.931,
+ 1.869,
+ 2.064,
+ 1.838,
+ 2.469,
+ 2.454
+ ],
+ "first_cross": 6,
+ "final_z": 2.454
+ },
+ {
+ "id": "52af71ac99f9bdaee6000ee8",
+ "variation": "headline",
+ "week": 201350,
+ "impressions": 37837,
+ "pooled_ctr": 0.00827232603007638,
+ "arms": [
+ {
+ "headline": "What Are Humans Actually \"Wired\" For? It's Possible The Answer Won't Surprise You.",
+ "impressions": 9179,
+ "clicks": 84,
+ "ctr": 0.0091513236736027,
+ "ctr_select": 0.009368191721132898,
+ "ctr_holdout": 0.00893440836783613,
+ "n_select": 4590,
+ "n_holdout": 4589,
+ "eb": 0.00895623670245161,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch This Total Genius Prove In Just 10 Minutes That Being Nicer Is Better",
+ "impressions": 10169,
+ "clicks": 78,
+ "ctr": 0.0076703707345855,
+ "ctr_select": 0.00885129819040126,
+ "ctr_holdout": 0.006489675516224189,
+ "n_select": 5084,
+ "n_holdout": 5085,
+ "eb": 0.008628692802692883,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Are Humans Actually \"Wired\" For? (Hint: It's Not Narcissism, Violence, Or Materialism)",
+ "impressions": 9029,
+ "clicks": 80,
+ "ctr": 0.0088603389079632,
+ "ctr_select": 0.007532122286220647,
+ "ctr_holdout": 0.010188261351052049,
+ "n_select": 4514,
+ "n_holdout": 4515,
+ "eb": 0.0077384828775172295,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The History (And Science) Behind Being Nice To Your Extended Family",
+ "impressions": 9460,
+ "clicks": 71,
+ "ctr": 0.0075052854122621,
+ "ctr_select": 0.006765327695560253,
+ "ctr_holdout": 0.00824524312896406,
+ "n_select": 4730,
+ "n_holdout": 4730,
+ "eb": 0.007217530865184245,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.00814039539063326,
+ "pool_holdout": 0.008404249696072731,
+ "lift_select": 0.15082760376878013,
+ "lift_holdout": 0.06308221327730634,
+ "eb_lift": 0.100221337253114,
+ "p_value": 0.28745298465617475,
+ "held_up": false,
+ "z_path": [
+ -0.8,
+ 0.074,
+ 0.961,
+ 1.288,
+ 1.13,
+ 0.817,
+ 0.833,
+ 1.204,
+ 1.451,
+ 1.129
+ ],
+ "first_cross": null,
+ "final_z": 1.129
+ },
+ {
+ "id": "54220c5ea369ffe8c6000017",
+ "variation": "headline",
+ "week": 201438,
+ "impressions": 35866,
+ "pooled_ctr": 0.0012546701611554118,
+ "arms": [
+ {
+ "headline": "He Visited One Of The World's Oldest Cultures, And Came Away With These",
+ "impressions": 7025,
+ "clicks": 11,
+ "ctr": 0.0015658362989323,
+ "ctr_select": 0.0011389521640091116,
+ "ctr_holdout": 0.0019925989183034445,
+ "n_select": 3512,
+ "n_holdout": 3513,
+ "eb": 0.0010187044302131009,
+ "labelled_winner": false
+ },
+ {
+ "headline": "For Him, A Journey Through The Desert Mountains Of Ethiopia Was A Journey Through The Old Testament",
+ "impressions": 7325,
+ "clicks": 6,
+ "ctr": 0.0008191126279863,
+ "ctr_select": 0.0005461496450027307,
+ "ctr_holdout": 0.001092001092001092,
+ "n_select": 3662,
+ "n_holdout": 3663,
+ "eb": 0.0006586852836158442,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Where He Went, The Mountains Have Names Like Golgotha and Sina, And The River Is Called Jordanus",
+ "impressions": 7288,
+ "clicks": 10,
+ "ctr": 0.0013721185510428,
+ "ctr_select": 0.0013721185510428102,
+ "ctr_holdout": 0.0013721185510428102,
+ "n_select": 3644,
+ "n_holdout": 3644,
+ "eb": 0.0011638863998622954,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Couldn't Find A Map For Such A Remote Area, But Farmers And Villagers Helped Guide Him On His Way",
+ "impressions": 7128,
+ "clicks": 6,
+ "ctr": 0.0008417508417508,
+ "ctr_select": 0.0002805836139169473,
+ "ctr_holdout": 0.0014029180695847362,
+ "n_select": 3564,
+ "n_holdout": 3564,
+ "eb": 0.0004996412644938463,
+ "labelled_winner": false
+ },
+ {
+ "headline": "We Live In A World Where Everyone Is Connected \u2014 But Not Here.",
+ "impressions": 7100,
+ "clicks": 12,
+ "ctr": 0.0016901408450704,
+ "ctr_select": 0.0008450704225352113,
+ "ctr_holdout": 0.0025352112676056337,
+ "n_select": 3550,
+ "n_holdout": 3550,
+ "eb": 0.0008416825690617075,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.0008364934195850993,
+ "pool_holdout": 0.0016728002676480427,
+ "lift_select": 0.6403219904866448,
+ "lift_holdout": -0.17974753018660805,
+ "eb_lift": 0.39138739482204543,
+ "p_value": 0.21022864171444344,
+ "held_up": false,
+ "z_path": [
+ 1.967,
+ 1.59,
+ 0.947,
+ 0.576,
+ 0.767,
+ 0.467,
+ -0.074,
+ 0.167,
+ 0.149,
+ -0.303
+ ],
+ "first_cross": 0,
+ "final_z": -0.303
+ },
+ {
+ "id": "5432e069a1243c5276000050",
+ "variation": "headline",
+ "week": 201440,
+ "impressions": 35834,
+ "pooled_ctr": 0.011720712172796785,
+ "arms": [
+ {
+ "headline": "Here Is The Most Thoughtful And Well-Reasoned Debate On Religion You\u2019ll Ever See",
+ "impressions": 7176,
+ "clicks": 106,
+ "ctr": 0.0147714604236343,
+ "ctr_select": 0.013656633221850614,
+ "ctr_holdout": 0.01588628762541806,
+ "n_select": 3588,
+ "n_holdout": 3588,
+ "eb": 0.01309374568466064,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Here Is The Most Thoughtful And Well-Reasoned And Informative Debate On Religion I've Ever Seen",
+ "impressions": 7098,
+ "clicks": 97,
+ "ctr": 0.013665821358129,
+ "ctr_select": 0.014652014652014652,
+ "ctr_holdout": 0.01267962806424345,
+ "n_select": 3549,
+ "n_holdout": 3549,
+ "eb": 0.013692149805419863,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Screw All The Shouting You See On TV. Watch This Thoughtful Debate On Religion With Actual Experts.",
+ "impressions": 7215,
+ "clicks": 73,
+ "ctr": 0.0101178101178101,
+ "ctr_select": 0.0102549889135255,
+ "ctr_holdout": 0.0099805932908234,
+ "n_select": 3608,
+ "n_holdout": 3607,
+ "eb": 0.011024474600497188,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Is What A Debate On Religion Should Look Like. No Shouting, Just Thoughtfulness And Experts.",
+ "impressions": 7175,
+ "clicks": 74,
+ "ctr": 0.0103135888501742,
+ "ctr_select": 0.010869565217391304,
+ "ctr_holdout": 0.009757457485363814,
+ "n_select": 3588,
+ "n_holdout": 3587,
+ "eb": 0.011400465652370287,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You've Seen Lots Of Shout-Matches On Religion. Maybe You've Been In Some. This Debate Is Different.",
+ "impressions": 7170,
+ "clicks": 70,
+ "ctr": 0.00976290097629,
+ "ctr_select": 0.011715481171548118,
+ "ctr_holdout": 0.007810320781032078,
+ "n_select": 3585,
+ "n_holdout": 3585,
+ "eb": 0.011914502038576373,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.012222346244000447,
+ "pool_holdout": 0.011219022103148023,
+ "lift_select": 0.1987890344054728,
+ "lift_holdout": 0.13019013133823698,
+ "eb_lift": 0.12025543476490008,
+ "p_value": 0.14128018213105395,
+ "held_up": false,
+ "z_path": [
+ -2.122,
+ -1.875,
+ -2.337,
+ -2.029,
+ -1.775,
+ -1.071,
+ -1.235,
+ -0.01,
+ -0.227,
+ -0.558
+ ],
+ "first_cross": 0,
+ "final_z": -0.558
+ },
+ {
+ "id": "53f7ab00029b3462d3000024",
+ "variation": "image",
+ "week": 201433,
+ "impressions": 35820,
+ "pooled_ctr": 0.0049692908989391405,
+ "arms": [
+ {
+ "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.",
+ "impressions": 5239,
+ "clicks": 35,
+ "ctr": 0.0066806642489024,
+ "ctr_select": 0.004961832061068702,
+ "ctr_holdout": 0.008400152730049638,
+ "n_select": 2620,
+ "n_holdout": 2619,
+ "eb": 0.004834292299664501,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.",
+ "impressions": 5062,
+ "clicks": 14,
+ "ctr": 0.0027657052548399,
+ "ctr_select": 0.0031608060055314103,
+ "ctr_holdout": 0.002370604504148558,
+ "n_select": 2531,
+ "n_holdout": 2531,
+ "eb": 0.0038918231535755824,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.",
+ "impressions": 5078,
+ "clicks": 30,
+ "ctr": 0.0059078377313903,
+ "ctr_select": 0.004332414336352895,
+ "ctr_holdout": 0.007483261126427727,
+ "n_select": 2539,
+ "n_holdout": 2539,
+ "eb": 0.004503116175156597,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.",
+ "impressions": 5086,
+ "clicks": 23,
+ "ctr": 0.0045222178529296,
+ "ctr_select": 0.005112072355485647,
+ "ctr_holdout": 0.003932363350373574,
+ "n_select": 2543,
+ "n_holdout": 2543,
+ "eb": 0.004910874002476275,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.",
+ "impressions": 5068,
+ "clicks": 36,
+ "ctr": 0.0071033938437253,
+ "ctr_select": 0.006314127861089187,
+ "ctr_holdout": 0.007892659826361484,
+ "n_select": 2534,
+ "n_holdout": 2534,
+ "eb": 0.005538324031589709,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.",
+ "impressions": 5174,
+ "clicks": 27,
+ "ctr": 0.0052183996907614,
+ "ctr_select": 0.005798221878623889,
+ "ctr_holdout": 0.004638577502899111,
+ "n_select": 2587,
+ "n_holdout": 2587,
+ "eb": 0.005274590922727144,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.",
+ "impressions": 5113,
+ "clicks": 13,
+ "ctr": 0.0025425386270291,
+ "ctr_select": 0.003129890453834116,
+ "ctr_holdout": 0.0019554165037152915,
+ "n_select": 2556,
+ "n_holdout": 2557,
+ "eb": 0.0038718597531482483,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.004690117252931323,
+ "pool_holdout": 0.005248464544946957,
+ "lift_select": 0.3462622618108018,
+ "lift_holdout": 0.5038035903205764,
+ "eb_lift": 0.18084980244966298,
+ "p_value": 0.19657893949011507,
+ "held_up": false,
+ "z_path": [
+ 0.015,
+ 0.407,
+ 0.338,
+ 1.045,
+ 2.098,
+ 2.254,
+ 2.401,
+ 1.972,
+ 1.15,
+ 1.22
+ ],
+ "first_cross": 4,
+ "final_z": 1.22
+ },
+ {
+ "id": "53fd0b577861f02488000064",
+ "variation": "headline",
+ "week": 201434,
+ "impressions": 33250,
+ "pooled_ctr": 0.013353383458646617,
+ "arms": [
+ {
+ "headline": "Leave It To Matt Damon To Put A New Unexpected Twist On The Ice Bucket Challenge",
+ "impressions": 5493,
+ "clicks": 83,
+ "ctr": 0.0151101401784088,
+ "ctr_select": 0.01529497450837582,
+ "ctr_holdout": 0.014925373134328358,
+ "n_select": 2746,
+ "n_holdout": 2747,
+ "eb": 0.014268994368627795,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Matt Damon Did The Ice Bucket Challenge? Not Surprising. How He Did It? Pretty Impressive.",
+ "impressions": 5620,
+ "clicks": 67,
+ "ctr": 0.0119217081850533,
+ "ctr_select": 0.012455516014234875,
+ "ctr_holdout": 0.011387900355871887,
+ "n_select": 2810,
+ "n_holdout": 2810,
+ "eb": 0.012725764620033098,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Matt Damon Did With The Toilet That Disgusted His Wife But Made Him Feel Great",
+ "impressions": 5544,
+ "clicks": 64,
+ "ctr": 0.0115440115440115,
+ "ctr_select": 0.011904761904761904,
+ "ctr_holdout": 0.011183261183261184,
+ "n_select": 2772,
+ "n_holdout": 2772,
+ "eb": 0.012427825325316628,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Very Good Reason Matt Damon Risked Disgusting His Wife To Help Strangers",
+ "impressions": 5468,
+ "clicks": 81,
+ "ctr": 0.0148134601316752,
+ "ctr_select": 0.012801755669348939,
+ "ctr_holdout": 0.016825164594001463,
+ "n_select": 2734,
+ "n_holdout": 2734,
+ "eb": 0.012917216715481189,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Ice Bucket Challenge Posed A Problem For Matt Damon. Here's How He Fixed It.",
+ "impressions": 5577,
+ "clicks": 101,
+ "ctr": 0.0181100950331719,
+ "ctr_select": 0.01721664275466284,
+ "ctr_holdout": 0.0190032269630692,
+ "n_select": 2788,
+ "n_holdout": 2789,
+ "eb": 0.015326773843432132,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Newer, More Nervy Twist To The Ice Bucket Challenge. First Up ... Matt Damon.",
+ "impressions": 5548,
+ "clicks": 48,
+ "ctr": 0.0086517664023071,
+ "ctr_select": 0.008651766402307137,
+ "ctr_holdout": 0.008651766402307137,
+ "n_select": 2774,
+ "n_holdout": 2774,
+ "eb": 0.010655360388437767,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.013053416746871993,
+ "pool_holdout": 0.013653314086370745,
+ "lift_select": 0.31893764586873297,
+ "lift_holdout": 0.3918398743964251,
+ "eb_lift": 0.17415801093647798,
+ "p_value": 0.03376206928772575,
+ "held_up": true,
+ "z_path": [
+ -0.808,
+ -0.568,
+ -0.456,
+ -0.066,
+ 0.123,
+ 0.287,
+ 0.424,
+ 0.789,
+ 0.828,
+ 1.234
+ ],
+ "first_cross": null,
+ "final_z": 1.234
+ },
+ {
+ "id": "53f4ef85ba722bdd33000008",
+ "variation": "headline",
+ "week": 201433,
+ "impressions": 33006,
+ "pooled_ctr": 0.013452099618251227,
+ "arms": [
+ {
+ "headline": "A Boy Called Her \u2018Whore\u2019 And She Thought It Was Love. Harry Potter\u2019s Penis Taught Her Otherwise.",
+ "impressions": 5485,
+ "clicks": 80,
+ "ctr": 0.0145852324521422,
+ "ctr_select": 0.016046681254558718,
+ "ctr_holdout": 0.013124316441851986,
+ "n_select": 2742,
+ "n_holdout": 2743,
+ "eb": 0.015219018567841035,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here\u2019s Why This Girl\u2019s Harry Potter Erotica Is Better Than Whatever Porn *You\u2019re* Watching",
+ "impressions": 5436,
+ "clicks": 68,
+ "ctr": 0.0125091979396615,
+ "ctr_select": 0.012877115526122149,
+ "ctr_holdout": 0.012141280353200883,
+ "n_select": 2718,
+ "n_holdout": 2718,
+ "eb": 0.013504312074155844,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In 3 Minutes She Covers Everything From Harry Potter Erotica To Rape Culture And It. Is. Awesome.",
+ "impressions": 5611,
+ "clicks": 71,
+ "ctr": 0.0126537159151666,
+ "ctr_select": 0.011404133998574484,
+ "ctr_holdout": 0.013903743315508022,
+ "n_select": 2806,
+ "n_holdout": 2805,
+ "eb": 0.012686859581059402,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Harry Potter Fan Issues A Smackdown To Anyone Who Thinks Their \u2018Normal\u2019 Porn Is Better Than Hers",
+ "impressions": 5389,
+ "clicks": 81,
+ "ctr": 0.0150306179254036,
+ "ctr_select": 0.016332590942835932,
+ "ctr_holdout": 0.013729128014842301,
+ "n_select": 2694,
+ "n_holdout": 2695,
+ "eb": 0.015364780130949857,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A 3-Minute Poem That\u2019ll Convince You To Throw Out Your Porn And Pick Up Her Harry Potter Erotica",
+ "impressions": 5527,
+ "clicks": 70,
+ "ctr": 0.0126650986068391,
+ "ctr_select": 0.015195369030390739,
+ "ctr_holdout": 0.010133912414042706,
+ "n_select": 2764,
+ "n_holdout": 2763,
+ "eb": 0.014759562192393258,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In 3 Mins She Argues That Her Harry Potter Erotica Is Better Than Your Porn. Holy $#*% She\u2019s Right.",
+ "impressions": 5558,
+ "clicks": 74,
+ "ctr": 0.0133141417776178,
+ "ctr_select": 0.013673983447283196,
+ "ctr_holdout": 0.012954300107952501,
+ "n_select": 2779,
+ "n_holdout": 2779,
+ "eb": 0.013931301683819716,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.014239835181482155,
+ "pool_holdout": 0.0126643640550203,
+ "lift_select": 0.14696488650902717,
+ "lift_holdout": 0.08407559631072958,
+ "eb_lift": 0.07899985745134241,
+ "p_value": 0.31621896596095755,
+ "held_up": false,
+ "z_path": [
+ 1.261,
+ 1.281,
+ 1.936,
+ 2.007,
+ 1.556,
+ 1.543,
+ 1.238,
+ 0.711,
+ 0.439,
+ 0.192
+ ],
+ "first_cross": 3,
+ "final_z": 0.192
+ },
+ {
+ "id": "5282632083121c4e7e00961f",
+ "variation": "headline",
+ "week": 201345,
+ "impressions": 32340,
+ "pooled_ctr": 0.014625850340136054,
+ "arms": [
+ {
+ "headline": "Some Call It The Miracle Of Life. They Call It An Inescapable Curse.",
+ "impressions": 7467,
+ "clicks": 139,
+ "ctr": 0.0186152403910539,
+ "ctr_select": 0.019550080342795928,
+ "ctr_holdout": 0.017680150013394052,
+ "n_select": 3734,
+ "n_holdout": 3733,
+ "eb": 0.017959878851117935,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Some Call It The Miracle Of Life. They Call It An Inescapable Curse. But It Doesn't Have To Be.",
+ "impressions": 8892,
+ "clicks": 130,
+ "ctr": 0.0146198830409356,
+ "ctr_select": 0.014619883040935672,
+ "ctr_holdout": 0.014619883040935672,
+ "n_select": 4446,
+ "n_holdout": 4446,
+ "eb": 0.014886495821631944,
+ "labelled_winner": false
+ },
+ {
+ "headline": "MUST WATCH: It's Called The Miracle Of Life But For Some It Is More Like An Inescapable Curse.",
+ "impressions": 7702,
+ "clicks": 101,
+ "ctr": 0.0131134770189561,
+ "ctr_select": 0.013243313425084394,
+ "ctr_holdout": 0.012983640612827837,
+ "n_select": 3851,
+ "n_holdout": 3851,
+ "eb": 0.014052845798181174,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Miracle Of Life Has Become An Inescapable Curse For Them. But It Doesn't Have To Be.",
+ "impressions": 8279,
+ "clicks": 103,
+ "ctr": 0.0124411160768208,
+ "ctr_select": 0.014492753623188406,
+ "ctr_holdout": 0.01038898284609809,
+ "n_select": 4140,
+ "n_holdout": 4139,
+ "eb": 0.014817627271426896,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.015397934574237833,
+ "pool_holdout": 0.013853670604242687,
+ "lift_select": 0.2696560209773211,
+ "lift_holdout": 0.27620689985075186,
+ "eb_lift": 0.1663823329374623,
+ "p_value": 0.018789544921063144,
+ "held_up": true,
+ "z_path": [
+ 0.085,
+ 0.246,
+ -0.033,
+ -0.849,
+ -0.233,
+ 0.145,
+ 0.354,
+ 1.098,
+ 1.506,
+ 2.001
+ ],
+ "first_cross": 9,
+ "final_z": 2.001
+ },
+ {
+ "id": "52729dad576e674866000cdd",
+ "variation": "headline",
+ "week": 201343,
+ "impressions": 32308,
+ "pooled_ctr": 0.025071189798192398,
+ "arms": [
+ {
+ "headline": "The Disappointing Difference Between Men And Women, According To The World\u2019s Biggest Search Engine",
+ "impressions": 8098,
+ "clicks": 218,
+ "ctr": 0.0269202272165966,
+ "ctr_select": 0.027167201778216844,
+ "ctr_holdout": 0.026673252654976536,
+ "n_select": 4049,
+ "n_holdout": 4049,
+ "eb": 0.026021081735781063,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Frustrating Between Men And Women, According To The World\u2019s Biggest Search Engine",
+ "impressions": 8261,
+ "clicks": 159,
+ "ctr": 0.0192470645200338,
+ "ctr_select": 0.016707021791767553,
+ "ctr_holdout": 0.02178649237472767,
+ "n_select": 4130,
+ "n_holdout": 4131,
+ "eb": 0.019335329767351247,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Upsetting Difference Between Men And Women, According To The World\u2019s Biggest Search Engine",
+ "impressions": 8283,
+ "clicks": 224,
+ "ctr": 0.0270433417843776,
+ "ctr_select": 0.02583293095123129,
+ "ctr_holdout": 0.02825404491668679,
+ "n_select": 4142,
+ "n_holdout": 4141,
+ "eb": 0.025182039283757437,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Difference Between Men And Women, According To The World\u2019s Biggest Search Engine",
+ "impressions": 7666,
+ "clicks": 209,
+ "ctr": 0.0272632402817636,
+ "ctr_select": 0.026611009653013305,
+ "ctr_holdout": 0.027915470910513957,
+ "n_select": 3833,
+ "n_holdout": 3833,
+ "eb": 0.02563422377927988,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.02401881886839173,
+ "pool_holdout": 0.026123560727993066,
+ "lift_select": 0.1310798389827703,
+ "lift_holdout": 0.021041998550926477,
+ "eb_lift": 0.08336225350465787,
+ "p_value": 0.13064836721119705,
+ "held_up": false,
+ "z_path": [
+ -0.042,
+ -0.054,
+ -0.585,
+ -0.974,
+ -0.957,
+ -0.189,
+ 0.348,
+ 0.53,
+ 0.333,
+ -0.133
+ ],
+ "first_cross": null,
+ "final_z": -0.133
+ },
+ {
+ "id": "5393c9cb73ead54d2c000016",
+ "variation": "image",
+ "week": 201423,
+ "impressions": 30752,
+ "pooled_ctr": 0.018080124869927158,
+ "arms": [
+ {
+ "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.",
+ "impressions": 5072,
+ "clicks": 104,
+ "ctr": 0.0205047318611987,
+ "ctr_select": 0.02168769716088328,
+ "ctr_holdout": 0.019321766561514197,
+ "n_select": 2536,
+ "n_holdout": 2536,
+ "eb": 0.02012031726422287,
+ "labelled_winner": false
+ },
+ {
+ "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.",
+ "impressions": 5019,
+ "clicks": 78,
+ "ctr": 0.0155409444112372,
+ "ctr_select": 0.015537848605577689,
+ "ctr_holdout": 0.015544041450777202,
+ "n_select": 2510,
+ "n_holdout": 2509,
+ "eb": 0.016914467872229298,
+ "labelled_winner": false
+ },
+ {
+ "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.",
+ "impressions": 5133,
+ "clicks": 109,
+ "ctr": 0.0212351451392947,
+ "ctr_select": 0.021823850350740453,
+ "ctr_holdout": 0.020646669263731984,
+ "n_select": 2566,
+ "n_holdout": 2567,
+ "eb": 0.02020148332036455,
+ "labelled_winner": false
+ },
+ {
+ "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.",
+ "impressions": 5183,
+ "clicks": 86,
+ "ctr": 0.0165927069264904,
+ "ctr_select": 0.018132716049382717,
+ "ctr_holdout": 0.015052103434967195,
+ "n_select": 2592,
+ "n_holdout": 2591,
+ "eb": 0.01826139675305129,
+ "labelled_winner": false
+ },
+ {
+ "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.",
+ "impressions": 5140,
+ "clicks": 91,
+ "ctr": 0.017704280155642,
+ "ctr_select": 0.017509727626459144,
+ "ctr_holdout": 0.0178988326848249,
+ "n_select": 2570,
+ "n_holdout": 2570,
+ "eb": 0.017934402456755608,
+ "labelled_winner": false
+ },
+ {
+ "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.",
+ "impressions": 5205,
+ "clicks": 88,
+ "ctr": 0.0169068203650336,
+ "ctr_select": 0.015757109915449655,
+ "ctr_holdout": 0.018056089127929314,
+ "n_select": 2602,
+ "n_holdout": 2603,
+ "eb": 0.017004690683142417,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.018405306971904265,
+ "pool_holdout": 0.017754942767950053,
+ "lift_select": 0.18573683036390531,
+ "lift_holdout": 0.16286881538147613,
+ "eb_lift": 0.09759013262871141,
+ "p_value": 0.1580991264810938,
+ "held_up": true,
+ "z_path": [
+ 0.367,
+ 0.244,
+ -0.405,
+ -0.705,
+ -0.848,
+ -0.426,
+ 0.256,
+ -0.004,
+ -0.157,
+ 0.258
+ ],
+ "first_cross": null,
+ "final_z": 0.258
+ },
+ {
+ "id": "53fa81f12a06a92e4900000c",
+ "variation": "headline",
+ "week": 201434,
+ "impressions": 30652,
+ "pooled_ctr": 0.013506459611118361,
+ "arms": [
+ {
+ "headline": "Does Your Best Friend Feel Like Family? That's Because They Kinda Are.",
+ "impressions": 5055,
+ "clicks": 41,
+ "ctr": 0.0081107814045499,
+ "ctr_select": 0.010680379746835443,
+ "ctr_holdout": 0.00554016620498615,
+ "n_select": 2528,
+ "n_holdout": 2527,
+ "eb": 0.012468111852896136,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Scientific Reason Kissing Your Friend Is Kinda Like Kissing Your Cousin",
+ "impressions": 5183,
+ "clicks": 58,
+ "ctr": 0.0111904302527493,
+ "ctr_select": 0.011188271604938271,
+ "ctr_holdout": 0.011192589733693555,
+ "n_select": 2592,
+ "n_holdout": 2591,
+ "eb": 0.0127129417617274,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Bizarre Scientific Reason You Might Wanna Keep That Friend 'In The Friend Zone'",
+ "impressions": 5063,
+ "clicks": 61,
+ "ctr": 0.0120481927710843,
+ "ctr_select": 0.014612954186413903,
+ "ctr_holdout": 0.009482418016594232,
+ "n_select": 2532,
+ "n_holdout": 2531,
+ "eb": 0.014519812424476196,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Bizarre Reason You Were Drawn To Your Best Friend ... Long Before You Met",
+ "impressions": 5188,
+ "clicks": 82,
+ "ctr": 0.0158057054741711,
+ "ctr_select": 0.015420200462606014,
+ "ctr_holdout": 0.016191210485736313,
+ "n_select": 2594,
+ "n_holdout": 2594,
+ "eb": 0.014947316671673932,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Scientific Reason Someone Chooses To Be Your Friend Might Freak You Out ... Just A Bit",
+ "impressions": 5031,
+ "clicks": 111,
+ "ctr": 0.0220632081097197,
+ "ctr_select": 0.023052464228934817,
+ "ctr_holdout": 0.02107355864811133,
+ "n_select": 2516,
+ "n_holdout": 2515,
+ "eb": 0.018912374453838473,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Best Friends Share Almost Everything, But Things Just Got Weird",
+ "impressions": 5132,
+ "clicks": 61,
+ "ctr": 0.0118862042088854,
+ "ctr_select": 0.011691348402182385,
+ "ctr_holdout": 0.012081060015588464,
+ "n_select": 2566,
+ "n_holdout": 2566,
+ "eb": 0.012985386242585084,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.014418058455114822,
+ "pool_holdout": 0.012594622813886713,
+ "lift_select": 0.5988605054348999,
+ "lift_holdout": 0.6732187187754304,
+ "eb_lift": 0.3117143693594395,
+ "p_value": 7.069069302376094e-05,
+ "held_up": true,
+ "z_path": [
+ 2.063,
+ 1.643,
+ 1.994,
+ 2.638,
+ 2.176,
+ 2.633,
+ 3.033,
+ 3.106,
+ 2.349,
+ 2.323
+ ],
+ "first_cross": 0,
+ "final_z": 2.323
+ },
+ {
+ "id": "5391cd299c9fb52c18000018",
+ "variation": "headline",
+ "week": 201422,
+ "impressions": 28169,
+ "pooled_ctr": 0.004792502396251198,
+ "arms": [
+ {
+ "headline": "It\u2019s Not A Revolution If You Can\u2019t Dance To It",
+ "impressions": 5816,
+ "clicks": 24,
+ "ctr": 0.0041265474552957,
+ "ctr_select": 0.002407152682255846,
+ "ctr_holdout": 0.005845942228335626,
+ "n_select": 2908,
+ "n_holdout": 2908,
+ "eb": 0.003417940235709462,
+ "labelled_winner": false
+ },
+ {
+ "headline": "We Spent An Afternoon In The Virtual Record Stacks And Made These Playlists For You",
+ "impressions": 5599,
+ "clicks": 26,
+ "ctr": 0.0046436863725665,
+ "ctr_select": 0.005,
+ "ctr_holdout": 0.004287245444801715,
+ "n_select": 2800,
+ "n_holdout": 2799,
+ "eb": 0.0048578724026257965,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Bob Your Head To Our 3 Songza Playlists",
+ "impressions": 5804,
+ "clicks": 26,
+ "ctr": 0.0044796691936595,
+ "ctr_select": 0.00585802894555479,
+ "ctr_holdout": 0.0031013094417643005,
+ "n_select": 2902,
+ "n_holdout": 2902,
+ "eb": 0.005337688889451963,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You've Ever Wondered What We Listened To In Our Offices, The Wait Is Over.",
+ "impressions": 5457,
+ "clicks": 26,
+ "ctr": 0.0047645226314824,
+ "ctr_select": 0.004398826979472141,
+ "ctr_holdout": 0.005130084279956028,
+ "n_select": 2728,
+ "n_holdout": 2729,
+ "eb": 0.0045308149204672235,
+ "labelled_winner": false
+ },
+ {
+ "headline": "We Tore Through Some Virtual Record Stacks And Made 3 Playlists That'll Get You Through The Summer",
+ "impressions": 5493,
+ "clicks": 33,
+ "ctr": 0.00600764609503,
+ "ctr_select": 0.005826656955571741,
+ "ctr_holdout": 0.006188569348380051,
+ "n_select": 2746,
+ "n_holdout": 2747,
+ "eb": 0.005304645737390898,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.004686168702073275,
+ "pool_holdout": 0.004898828541001065,
+ "lift_select": 0.25006787377566164,
+ "lift_holdout": -0.3669283552572439,
+ "eb_lift": 0.13903045937941583,
+ "p_value": 0.29955775065097434,
+ "held_up": false,
+ "z_path": [
+ -0.626,
+ -0.087,
+ -0.365,
+ -0.351,
+ -0.136,
+ 0.026,
+ -0.672,
+ -1.393,
+ -1.034,
+ -1.126
+ ],
+ "first_cross": null,
+ "final_z": -1.126
+ },
+ {
+ "id": "541f8fa29d99f1ec4d000032",
+ "variation": "headline",
+ "week": 201438,
+ "impressions": 27970,
+ "pooled_ctr": 0.0039685377189846265,
+ "arms": [
+ {
+ "headline": "Quit Your Job, Sell Everything, And Live Like A Boss By Calling The Entire Planet Your Home",
+ "impressions": 6973,
+ "clicks": 38,
+ "ctr": 0.0054495912806539,
+ "ctr_select": 0.005737234652897304,
+ "ctr_holdout": 0.005162030398623459,
+ "n_select": 3486,
+ "n_holdout": 3487,
+ "eb": 0.005159530469016437,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here's A Guy Who's Been Earth Hopping For 10 Years And He's Ready To Teach You How You Can Do It Too",
+ "impressions": 6757,
+ "clicks": 31,
+ "ctr": 0.0045878348379458,
+ "ctr_select": 0.004440497335701598,
+ "ctr_holdout": 0.004735128736312519,
+ "n_select": 3378,
+ "n_holdout": 3379,
+ "eb": 0.004379508259317443,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A World Traveler Teaches Us How To Have A Vacation That Never Has To End",
+ "impressions": 7097,
+ "clicks": 34,
+ "ctr": 0.0047907566577427,
+ "ctr_select": 0.005355129650507328,
+ "ctr_holdout": 0.00422654268808115,
+ "n_select": 3548,
+ "n_holdout": 3549,
+ "eb": 0.004934511902506225,
+ "labelled_winner": false
+ },
+ {
+ "headline": "One Guy Explains How To Be A Continent Bounding Nomad In Search Of Unforgettable Adventure",
+ "impressions": 7143,
+ "clicks": 8,
+ "ctr": 0.0011199776004479,
+ "ctr_select": 0.0016797312430011197,
+ "ctr_holdout": 0.0005600672080649678,
+ "n_select": 3572,
+ "n_holdout": 3571,
+ "eb": 0.00270716018709675,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.004290617848970252,
+ "pool_holdout": 0.0036465036465036463,
+ "lift_select": 0.33715815643526503,
+ "lift_holdout": 0.4156109246107391,
+ "eb_lift": 0.2025145679787641,
+ "p_value": 0.1315082638851518,
+ "held_up": true,
+ "z_path": [
+ 1.019,
+ 0.865,
+ 1.647,
+ 1.782,
+ 1.5,
+ 0.948,
+ 0.492,
+ 0.478,
+ 0.857,
+ 0.548
+ ],
+ "first_cross": null,
+ "final_z": 0.548
+ },
+ {
+ "id": "5420bab11afbe8fce400007e",
+ "variation": "image",
+ "week": 201438,
+ "impressions": 27932,
+ "pooled_ctr": 0.01202921380495489,
+ "arms": [
+ {
+ "headline": "The Messages These Women Read Would Be Funny If They Were From A Movie. Except They're Not.",
+ "impressions": 6955,
+ "clicks": 132,
+ "ctr": 0.018979151689432,
+ "ctr_select": 0.019838987924094307,
+ "ctr_holdout": 0.0181190681622088,
+ "n_select": 3478,
+ "n_holdout": 3477,
+ "eb": 0.016744495492083924,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Messages These Women Read Would Be Funny If They Were From A Movie. Except They're Not.",
+ "impressions": 7038,
+ "clicks": 62,
+ "ctr": 0.0088093208297811,
+ "ctr_select": 0.007956805910770106,
+ "ctr_holdout": 0.00966183574879227,
+ "n_select": 3519,
+ "n_holdout": 3519,
+ "eb": 0.009602353406829517,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Messages These Women Read Would Be Funny If They Were From A Movie. Except They're Not.",
+ "impressions": 6877,
+ "clicks": 76,
+ "ctr": 0.0110513305220299,
+ "ctr_select": 0.011052937754508435,
+ "ctr_holdout": 0.011049723756906077,
+ "n_select": 3438,
+ "n_holdout": 3439,
+ "eb": 0.011474896264192496,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Messages These Women Read Would Be Funny If They Were From A Movie. Except They're Not.",
+ "impressions": 7062,
+ "clicks": 66,
+ "ctr": 0.0093457943925233,
+ "ctr_select": 0.009629000283205891,
+ "ctr_holdout": 0.009062588501840838,
+ "n_select": 3531,
+ "n_holdout": 3531,
+ "eb": 0.010608521284929166,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.012100816268079621,
+ "pool_holdout": 0.01195761134183016,
+ "lift_select": 0.6394751795733793,
+ "lift_holdout": 0.5152748859485514,
+ "eb_lift": 0.3837492546890182,
+ "p_value": 1.4611751204554249e-06,
+ "held_up": true,
+ "z_path": [
+ 0.206,
+ -0.034,
+ 0.372,
+ 0.8,
+ 1.006,
+ 2.033,
+ 1.864,
+ 1.952,
+ 2.777,
+ 3.831
+ ],
+ "first_cross": 5,
+ "final_z": 3.831
+ },
+ {
+ "id": "53e271645fba149a24000001",
+ "variation": "image",
+ "week": 201431,
+ "impressions": 27615,
+ "pooled_ctr": 0.009053050878145934,
+ "arms": [
+ {
+ "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something",
+ "impressions": 4595,
+ "clicks": 43,
+ "ctr": 0.0093579978237214,
+ "ctr_select": 0.009138381201044387,
+ "ctr_holdout": 0.00957771005659556,
+ "n_select": 2298,
+ "n_holdout": 2297,
+ "eb": 0.009059003931114687,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something",
+ "impressions": 4643,
+ "clicks": 57,
+ "ctr": 0.0122765453370665,
+ "ctr_select": 0.014211886304909561,
+ "ctr_holdout": 0.01034037052994399,
+ "n_select": 2322,
+ "n_holdout": 2321,
+ "eb": 0.011598519359864435,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something",
+ "impressions": 4585,
+ "clicks": 28,
+ "ctr": 0.0061068702290076,
+ "ctr_select": 0.005235602094240838,
+ "ctr_holdout": 0.006977758395115569,
+ "n_select": 2292,
+ "n_holdout": 2293,
+ "eb": 0.007118389480969683,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something",
+ "impressions": 4619,
+ "clicks": 34,
+ "ctr": 0.0073609006278415,
+ "ctr_select": 0.008225108225108226,
+ "ctr_holdout": 0.0064963187527068,
+ "n_select": 2310,
+ "n_holdout": 2309,
+ "eb": 0.008603332843294554,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something",
+ "impressions": 4561,
+ "clicks": 40,
+ "ctr": 0.0087700065775049,
+ "ctr_select": 0.007894736842105263,
+ "ctr_holdout": 0.009644892590968873,
+ "n_select": 2280,
+ "n_holdout": 2281,
+ "eb": 0.008441969469196823,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something",
+ "impressions": 4612,
+ "clicks": 48,
+ "ctr": 0.01040763226366,
+ "ctr_select": 0.009106678230702514,
+ "ctr_holdout": 0.011708586296617519,
+ "n_select": 2306,
+ "n_holdout": 2306,
+ "eb": 0.00904332988923771,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.008980301274623407,
+ "pool_holdout": 0.009125805750706163,
+ "lift_select": 0.5825623072434776,
+ "lift_holdout": 0.1330912373566402,
+ "eb_lift": 0.2915512525887751,
+ "p_value": 0.003390270502562463,
+ "held_up": false,
+ "z_path": [
+ -0.019,
+ 0.832,
+ 0.512,
+ 0.294,
+ -0.311,
+ -0.289,
+ 0.197,
+ 0.397,
+ 0.688,
+ 1.355
+ ],
+ "first_cross": null,
+ "final_z": 1.355
+ },
+ {
+ "id": "544d103de8a6ed45d400000c",
+ "variation": "image",
+ "week": 201443,
+ "impressions": 27386,
+ "pooled_ctr": 0.008215876725334113,
+ "arms": [
+ {
+ "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised",
+ "impressions": 3795,
+ "clicks": 37,
+ "ctr": 0.0097496706192358,
+ "ctr_select": 0.007903055848261328,
+ "ctr_holdout": 0.011597258829731154,
+ "n_select": 1898,
+ "n_holdout": 1897,
+ "eb": 0.007533299221811757,
+ "labelled_winner": true
+ },
+ {
+ "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised",
+ "impressions": 3925,
+ "clicks": 29,
+ "ctr": 0.0073885350318471,
+ "ctr_select": 0.009683995922528032,
+ "ctr_holdout": 0.005094243504839531,
+ "n_select": 1962,
+ "n_holdout": 1963,
+ "eb": 0.008355289253022327,
+ "labelled_winner": false
+ },
+ {
+ "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised",
+ "impressions": 3945,
+ "clicks": 38,
+ "ctr": 0.0096324461343472,
+ "ctr_select": 0.006592292089249493,
+ "ctr_holdout": 0.012671059300557527,
+ "n_select": 1972,
+ "n_holdout": 1973,
+ "eb": 0.006937111723645805,
+ "labelled_winner": false
+ },
+ {
+ "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised",
+ "impressions": 4045,
+ "clicks": 28,
+ "ctr": 0.0069221260815822,
+ "ctr_select": 0.006429277942631058,
+ "ctr_holdout": 0.007414730598121601,
+ "n_select": 2022,
+ "n_holdout": 2023,
+ "eb": 0.006857185514543291,
+ "labelled_winner": false
+ },
+ {
+ "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised",
+ "impressions": 3922,
+ "clicks": 26,
+ "ctr": 0.0066292707802141,
+ "ctr_select": 0.006119326874043855,
+ "ctr_holdout": 0.0071392146863844975,
+ "n_select": 1961,
+ "n_holdout": 1961,
+ "eb": 0.006721230202023124,
+ "labelled_winner": false
+ },
+ {
+ "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised",
+ "impressions": 3792,
+ "clicks": 31,
+ "ctr": 0.008175105485232,
+ "ctr_select": 0.008438818565400843,
+ "ctr_holdout": 0.007911392405063292,
+ "n_select": 1896,
+ "n_holdout": 1896,
+ "eb": 0.007774196094249299,
+ "labelled_winner": false
+ },
+ {
+ "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised",
+ "impressions": 3962,
+ "clicks": 36,
+ "ctr": 0.0090863200403836,
+ "ctr_select": 0.005552751135790005,
+ "ctr_holdout": 0.012619888944977285,
+ "n_select": 1981,
+ "n_holdout": 1981,
+ "eb": 0.006457330602193736,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.007230499561787906,
+ "pool_holdout": 0.009201109975171607,
+ "lift_select": 0.33932598152781623,
+ "lift_holdout": -0.4463446781327576,
+ "eb_lift": 0.15556182275133024,
+ "p_value": 0.16579781078707745,
+ "held_up": false,
+ "z_path": [
+ 0.285,
+ -0.359,
+ -0.085,
+ 0.092,
+ -0.455,
+ -0.613,
+ -0.27,
+ 0.033,
+ -0.401,
+ -0.393
+ ],
+ "first_cross": null,
+ "final_z": -0.393
+ },
+ {
+ "id": "52a65dc3476fb047c10009c0",
+ "variation": "headline",
+ "week": 201349,
+ "impressions": 27366,
+ "pooled_ctr": 0.03749177811883359,
+ "arms": [
+ {
+ "headline": "This Ad Totally Nails 5 Ridiculous Double Standards Women Face In Less Than 60 Seconds",
+ "impressions": 6717,
+ "clicks": 251,
+ "ctr": 0.0373678725621557,
+ "ctr_select": 0.03871351995235259,
+ "ctr_holdout": 0.03602262578148258,
+ "n_select": 3358,
+ "n_holdout": 3359,
+ "eb": 0.03785648863959225,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Ad Totally Nails 5 Unfortunate Double Standards Women Face In Less Than 60 Seconds",
+ "impressions": 6990,
+ "clicks": 263,
+ "ctr": 0.0376251788268955,
+ "ctr_select": 0.03547925608011445,
+ "ctr_holdout": 0.03977110157367668,
+ "n_select": 3495,
+ "n_holdout": 3495,
+ "eb": 0.035932022666640334,
+ "labelled_winner": false
+ },
+ {
+ "headline": "NAILED IT: This Ad Calls Out 5 Ridiculous Double Standards Women Face In Less Than 60 Seconds",
+ "impressions": 7423,
+ "clicks": 279,
+ "ctr": 0.0375858817189815,
+ "ctr_select": 0.03529094827586207,
+ "ctr_holdout": 0.039881433575855565,
+ "n_select": 3712,
+ "n_holdout": 3711,
+ "eb": 0.03579980228715958,
+ "labelled_winner": true
+ },
+ {
+ "headline": "5 Ridiculous Double Standards Women Face In Less Than 60 Seconds",
+ "impressions": 6236,
+ "clicks": 233,
+ "ctr": 0.0373636946760744,
+ "ctr_select": 0.03720333547145606,
+ "ctr_holdout": 0.03752405388069275,
+ "n_select": 3118,
+ "n_holdout": 3118,
+ "eb": 0.03695238340903314,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.036614777461083096,
+ "pool_holdout": 0.03836877877658408,
+ "lift_select": 0.05731954792024041,
+ "lift_holdout": -0.06114745034661673,
+ "eb_lift": 0.03391284242622894,
+ "p_value": 0.4560013598018976,
+ "held_up": false,
+ "z_path": [
+ -0.702,
+ -0.464,
+ -0.27,
+ -0.358,
+ -0.841,
+ -0.2,
+ 0.392,
+ 0.448,
+ 0.555,
+ 0.001
+ ],
+ "first_cross": null,
+ "final_z": 0.001
+ },
+ {
+ "id": "53fb622a2a06a94b42000029",
+ "variation": "headline",
+ "week": 201434,
+ "impressions": 26961,
+ "pooled_ctr": 0.006565038388783799,
+ "arms": [
+ {
+ "headline": "Something That No Parent Should Ever Have To Teach Their Child",
+ "impressions": 4564,
+ "clicks": 29,
+ "ctr": 0.0063540753724802,
+ "ctr_select": 0.006573181419807186,
+ "ctr_holdout": 0.006134969325153374,
+ "n_select": 2282,
+ "n_holdout": 2282,
+ "eb": 0.006625270787771982,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here\u2019s Something I Sincerely Never Want To Have To Teach My Child",
+ "impressions": 4490,
+ "clicks": 25,
+ "ctr": 0.0055679287305122,
+ "ctr_select": 0.004008908685968819,
+ "ctr_holdout": 0.0071269487750556795,
+ "n_select": 2245,
+ "n_holdout": 2245,
+ "eb": 0.005363985172683472,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Mary Engelbreit: No One Should Have To Teach Their Children This In The USA",
+ "impressions": 4558,
+ "clicks": 36,
+ "ctr": 0.0078982009653356,
+ "ctr_select": 0.009653356735410267,
+ "ctr_holdout": 0.00614304519526108,
+ "n_select": 2279,
+ "n_holdout": 2279,
+ "eb": 0.008152427077663794,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Have No Words After Understanding What It Means To Have To Teach Kids These 4 Words.",
+ "impressions": 4553,
+ "clicks": 41,
+ "ctr": 0.0090050516143202,
+ "ctr_select": 0.007908611599297012,
+ "ctr_holdout": 0.010101010101010102,
+ "n_select": 2276,
+ "n_holdout": 2277,
+ "eb": 0.007286992960230153,
+ "labelled_winner": false
+ },
+ {
+ "headline": "4 Words That Leave Me With No Words",
+ "impressions": 4366,
+ "clicks": 27,
+ "ctr": 0.0061841502519468,
+ "ctr_select": 0.007787448465414567,
+ "ctr_holdout": 0.004580852038479157,
+ "n_select": 2183,
+ "n_holdout": 2183,
+ "eb": 0.0072153796958318965,
+ "labelled_winner": false
+ },
+ {
+ "headline": "No One Should Have To Teach Their Children This In The USA",
+ "impressions": 4430,
+ "clicks": 19,
+ "ctr": 0.0042889390519187,
+ "ctr_select": 0.004063205417607223,
+ "ctr_holdout": 0.004514672686230248,
+ "n_select": 2215,
+ "n_holdout": 2215,
+ "eb": 0.0053994871111904185,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.0066765578635014835,
+ "pool_holdout": 0.006453527186410504,
+ "lift_select": 0.4458583199258934,
+ "lift_holdout": -0.048110433594084956,
+ "eb_lift": 0.221052411187866,
+ "p_value": 0.055578146416599694,
+ "held_up": false,
+ "z_path": [
+ -0.578,
+ 0.302,
+ -0.449,
+ -0.603,
+ -0.672,
+ -0.623,
+ -0.288,
+ -0.783,
+ -0.126,
+ -0.577
+ ],
+ "first_cross": null,
+ "final_z": -0.577
+ },
+ {
+ "id": "534eb20dfcf40f8485000075",
+ "variation": "headline",
+ "week": 201415,
+ "impressions": 26798,
+ "pooled_ctr": 0.021792671094857826,
+ "arms": [
+ {
+ "headline": "The Most Honest And Awful Corporate Ad I Have Ever Laid My Eyes On. No, They Aren't Drunk. I Swear.",
+ "impressions": 3786,
+ "clicks": 138,
+ "ctr": 0.0364500792393026,
+ "ctr_select": 0.036978341257263604,
+ "ctr_holdout": 0.03592181722134179,
+ "n_select": 1893,
+ "n_holdout": 1893,
+ "eb": 0.02853752070455732,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest Corporate Ad I Have Ever Laid My Eyes On. No, They Aren't Drunk. I Swear.",
+ "impressions": 3871,
+ "clicks": 97,
+ "ctr": 0.025058124515629,
+ "ctr_select": 0.027892561983471075,
+ "ctr_holdout": 0.022222222222222223,
+ "n_select": 1936,
+ "n_holdout": 1935,
+ "eb": 0.02448760588067879,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest Coal Company Ad I Have Ever Laid My Eyes On. No, They Aren't Drunk. I Swear.",
+ "impressions": 3766,
+ "clicks": 62,
+ "ctr": 0.0164630908125331,
+ "ctr_select": 0.014338821030270845,
+ "ctr_holdout": 0.01858736059479554,
+ "n_select": 1883,
+ "n_holdout": 1883,
+ "eb": 0.018369061130369884,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest Energy Company Ad I Have Ever Laid My Eyes On. No, They Aren't Drunk. I Swear.",
+ "impressions": 3833,
+ "clicks": 74,
+ "ctr": 0.0193060266110096,
+ "ctr_select": 0.019311064718162838,
+ "ctr_holdout": 0.019300991131977047,
+ "n_select": 1916,
+ "n_holdout": 1917,
+ "eb": 0.020587882982580876,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest Energy Company Ad I Have Ever Laid My Eyes On",
+ "impressions": 3866,
+ "clicks": 45,
+ "ctr": 0.011639937920331,
+ "ctr_select": 0.012933264355923435,
+ "ctr_holdout": 0.010346611484738748,
+ "n_select": 1933,
+ "n_holdout": 1933,
+ "eb": 0.01768247900684302,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest, In-Your-Face, Company Ad I Have Ever Laid My Eyes On. And No, They Aren't Drunk.",
+ "impressions": 3876,
+ "clicks": 93,
+ "ctr": 0.0239938080495356,
+ "ctr_select": 0.021671826625386997,
+ "ctr_holdout": 0.02631578947368421,
+ "n_select": 1938,
+ "n_holdout": 1938,
+ "eb": 0.021656348576232366,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Extremely Honest And Rude Energy Company Ad That Doesn't Mix Words About Not Caring About You",
+ "impressions": 3800,
+ "clicks": 75,
+ "ctr": 0.0197368421052631,
+ "ctr_select": 0.018421052631578946,
+ "ctr_holdout": 0.021052631578947368,
+ "n_select": 1900,
+ "n_holdout": 1900,
+ "eb": 0.02019179156274224,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.021643406224345102,
+ "pool_holdout": 0.02194193596537055,
+ "lift_select": 0.7085268776071552,
+ "lift_holdout": 0.6371307107100632,
+ "eb_lift": 0.3185318617943569,
+ "p_value": 7.502304424942291e-07,
+ "held_up": true,
+ "z_path": [
+ 3.374,
+ 1.994,
+ 2.56,
+ 3.039,
+ 2.588,
+ 3.049,
+ 3.243,
+ 3.396,
+ 3.372,
+ 2.89
+ ],
+ "first_cross": 0,
+ "final_z": 2.89
+ },
+ {
+ "id": "5277d8c0ae0d4e881d000140",
+ "variation": "headline",
+ "week": 201344,
+ "impressions": 26603,
+ "pooled_ctr": 0.03213923241739654,
+ "arms": [
+ {
+ "headline": "Something Truly Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It",
+ "impressions": 5278,
+ "clicks": 163,
+ "ctr": 0.030882910193255,
+ "ctr_select": 0.028798787419477074,
+ "ctr_holdout": 0.03296703296703297,
+ "n_select": 2639,
+ "n_holdout": 2639,
+ "eb": 0.030518361180697322,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Something Really Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It",
+ "impressions": 5428,
+ "clicks": 180,
+ "ctr": 0.0331613854089904,
+ "ctr_select": 0.03500368459837878,
+ "ctr_holdout": 0.03131908621960206,
+ "n_select": 2714,
+ "n_holdout": 2714,
+ "eb": 0.03383953107054521,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Something Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It",
+ "impressions": 5339,
+ "clicks": 168,
+ "ctr": 0.0314665667728039,
+ "ctr_select": 0.03258426966292135,
+ "ctr_holdout": 0.030348445110528286,
+ "n_select": 2670,
+ "n_holdout": 2669,
+ "eb": 0.03253410700147092,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Something Very Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It",
+ "impressions": 5590,
+ "clicks": 191,
+ "ctr": 0.0341681574239713,
+ "ctr_select": 0.03398926654740608,
+ "ctr_holdout": 0.03434704830053667,
+ "n_select": 2795,
+ "n_holdout": 2795,
+ "eb": 0.033303414488837335,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Something Way Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It",
+ "impressions": 4968,
+ "clicks": 153,
+ "ctr": 0.0307971014492753,
+ "ctr_select": 0.03180354267310789,
+ "ctr_holdout": 0.029790660225442835,
+ "n_select": 2484,
+ "n_holdout": 2484,
+ "eb": 0.03212828051725769,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.03247631935047361,
+ "pool_holdout": 0.03180212014134275,
+ "lift_select": 0.07782178825841313,
+ "lift_holdout": -0.015188733316957248,
+ "eb_lift": 0.04197556088053789,
+ "p_value": 0.4050986674098218,
+ "held_up": false,
+ "z_path": [
+ 1.186,
+ 1.803,
+ 2.255,
+ 0.51,
+ -0.074,
+ -0.101,
+ -0.133,
+ -0.097,
+ -0.396,
+ -0.293
+ ],
+ "first_cross": 2,
+ "final_z": -0.293
+ },
+ {
+ "id": "52318ead117f64b609005c31",
+ "variation": "image",
+ "week": 201336,
+ "impressions": 26332,
+ "pooled_ctr": 0.028938174084763787,
+ "arms": [
+ {
+ "headline": "5 Reasons Why my Girlfriend Thinks She's Not Beautiful Enough, No Matter What Anyone Tells Her",
+ "impressions": 9151,
+ "clicks": 233,
+ "ctr": 0.0254616981750628,
+ "ctr_select": 0.024912587412587412,
+ "ctr_holdout": 0.026010928961748635,
+ "n_select": 4576,
+ "n_holdout": 4575,
+ "eb": 0.02608725987833294,
+ "labelled_winner": false
+ },
+ {
+ "headline": "5 Reasons Why my Girlfriend Thinks She's Not Beautiful Enough, No Matter What Anyone Tells Her",
+ "impressions": 8488,
+ "clicks": 285,
+ "ctr": 0.0335768143261074,
+ "ctr_select": 0.033930254476908575,
+ "ctr_holdout": 0.03322337417530632,
+ "n_select": 4244,
+ "n_holdout": 4244,
+ "eb": 0.03197917566291753,
+ "labelled_winner": false
+ },
+ {
+ "headline": "5 Reasons Why my Girlfriend Thinks She's Not Beautiful Enough, No Matter What Anyone Tells Her",
+ "impressions": 8693,
+ "clicks": 244,
+ "ctr": 0.0280685609110778,
+ "ctr_select": 0.026691210308329497,
+ "ctr_holdout": 0.029445594662985967,
+ "n_select": 4346,
+ "n_holdout": 4347,
+ "eb": 0.027287805000068135,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.02840650159501747,
+ "pool_holdout": 0.0294698465745101,
+ "lift_select": 0.19445382471384565,
+ "lift_holdout": 0.12736841338165727,
+ "eb_lift": 0.12576959031543367,
+ "p_value": 0.008506467301024528,
+ "held_up": true,
+ "z_path": [
+ 2.756,
+ 2.552,
+ 2.231,
+ 2.421,
+ 2.251,
+ 2.629,
+ 2.375,
+ 2.086,
+ 2.233,
+ 2.09
+ ],
+ "first_cross": 0,
+ "final_z": 2.09
+ },
+ {
+ "id": "527f702108d1715c87007d77",
+ "variation": "headline",
+ "week": 201345,
+ "impressions": 26268,
+ "pooled_ctr": 0.0063194761687224,
+ "arms": [
+ {
+ "headline": "How Can We Do This To 2 Million Of Our Fellow Americans?",
+ "impressions": 6363,
+ "clicks": 46,
+ "ctr": 0.0072292943580072,
+ "ctr_select": 0.009428032683846637,
+ "ctr_holdout": 0.005029864822382898,
+ "n_select": 3182,
+ "n_holdout": 3181,
+ "eb": 0.008310317238059441,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Congress Is About To Let 2 Million Of These People Fall Off The Edge",
+ "impressions": 6574,
+ "clicks": 41,
+ "ctr": 0.0062366899908731,
+ "ctr_select": 0.0060845756008518406,
+ "ctr_holdout": 0.006388804380894433,
+ "n_select": 3287,
+ "n_holdout": 3287,
+ "eb": 0.006370411843874417,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Congress Is About To Go Home For The Holidays, While These People May Not HAVE A Home...",
+ "impressions": 6487,
+ "clicks": 26,
+ "ctr": 0.0040080160320641,
+ "ctr_select": 0.0052404438964241675,
+ "ctr_holdout": 0.0027752081406105457,
+ "n_select": 3244,
+ "n_holdout": 3243,
+ "eb": 0.0058802607361574406,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Happened To An America Where We Give A Damn About Our Fellow Americans?",
+ "impressions": 6844,
+ "clicks": 53,
+ "ctr": 0.0077440093512565,
+ "ctr_select": 0.006428988895382817,
+ "ctr_holdout": 0.009059029807130333,
+ "n_select": 3422,
+ "n_holdout": 3422,
+ "eb": 0.006569027600268092,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.006775789874381424,
+ "pool_holdout": 0.00586309297190284,
+ "lift_select": 0.3914293180036581,
+ "lift_holdout": -0.14211409464474534,
+ "eb_lift": 0.22647210024618825,
+ "p_value": 0.03616509041168524,
+ "held_up": false,
+ "z_path": [
+ -0.147,
+ -0.232,
+ -0.315,
+ 0.096,
+ -0.014,
+ -0.235,
+ -0.288,
+ 0.11,
+ 0.123,
+ -0.343
+ ],
+ "first_cross": null,
+ "final_z": -0.343
+ },
+ {
+ "id": "5419f47d3088cf4a200000c0",
+ "variation": "headline",
+ "week": 201437,
+ "impressions": 25835,
+ "pooled_ctr": 0.004296497000193536,
+ "arms": [
+ {
+ "headline": "Morgan Freeman Is Speaking. Clouds Are Parting. I'm Over Here Feeling Inspired About Our Planet.",
+ "impressions": 4431,
+ "clicks": 20,
+ "ctr": 0.0045136538027533,
+ "ctr_select": 0.004963898916967509,
+ "ctr_holdout": 0.004063205417607223,
+ "n_select": 2216,
+ "n_holdout": 2215,
+ "eb": 0.004404817864750659,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Rallying Cry The Environmental Movement Has Been Waiting For",
+ "impressions": 4320,
+ "clicks": 9,
+ "ctr": 0.0020833333333333,
+ "ctr_select": 0.001851851851851852,
+ "ctr_holdout": 0.0023148148148148147,
+ "n_select": 2160,
+ "n_holdout": 2160,
+ "eb": 0.0028966030085792274,
+ "labelled_winner": false
+ },
+ {
+ "headline": "62 Seconds That Will Motivate You To Participate In The Future Of Our Planet, Via Morgan Freeman",
+ "impressions": 4360,
+ "clicks": 26,
+ "ctr": 0.0059633027522935,
+ "ctr_select": 0.005963302752293578,
+ "ctr_holdout": 0.005963302752293578,
+ "n_select": 2180,
+ "n_holdout": 2180,
+ "eb": 0.00488474639371937,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Morgan Freeman Has Something To Say About The Environment. And Yes It's An Inspiring Voiceover.",
+ "impressions": 4263,
+ "clicks": 24,
+ "ctr": 0.0056298381421534,
+ "ctr_select": 0.005628517823639775,
+ "ctr_holdout": 0.005631159080244017,
+ "n_select": 2132,
+ "n_holdout": 2131,
+ "eb": 0.00471270500486183,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Know That 62 Second Speech You Gave In 2nd Grade About Trees? This Is The PHD Version Of That",
+ "impressions": 4255,
+ "clicks": 17,
+ "ctr": 0.0039952996474735,
+ "ctr_select": 0.0018796992481203006,
+ "ctr_holdout": 0.0061118946873530795,
+ "n_select": 2128,
+ "n_holdout": 2127,
+ "eb": 0.0029174526709793303,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Well, Well, Well. Pessimists Beware Because I've Got An Inspiring Video On The FUTURE OF OUR PLANET!",
+ "impressions": 4206,
+ "clicks": 15,
+ "ctr": 0.0035663338088445,
+ "ctr_select": 0.0028530670470756064,
+ "ctr_holdout": 0.0042796005706134095,
+ "n_select": 2103,
+ "n_holdout": 2103,
+ "eb": 0.0033863687686807066,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.0038702685966406068,
+ "pool_holdout": 0.004722824403840198,
+ "lift_select": 0.5407981651376148,
+ "lift_holdout": 0.26265603850203045,
+ "eb_lift": 0.2621207732092108,
+ "p_value": 0.08429894800707505,
+ "held_up": false,
+ "z_path": [
+ -0.015,
+ -1.164,
+ -0.316,
+ -0.544,
+ -0.706,
+ -0.256,
+ -0.059,
+ -0.409,
+ -0.229,
+ 0.204
+ ],
+ "first_cross": null,
+ "final_z": 0.204
+ },
+ {
+ "id": "5388de3a5f5ab749ca00000a",
+ "variation": "image",
+ "week": 201421,
+ "impressions": 25592,
+ "pooled_ctr": 0.003594873397936855,
+ "arms": [
+ {
+ "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley",
+ "impressions": 5071,
+ "clicks": 10,
+ "ctr": 0.0019719976336028,
+ "ctr_select": 0.002365930599369085,
+ "ctr_holdout": 0.0015779092702169625,
+ "n_select": 2536,
+ "n_holdout": 2535,
+ "eb": 0.0031765171984118497,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley",
+ "impressions": 5138,
+ "clicks": 35,
+ "ctr": 0.0068119891008174,
+ "ctr_select": 0.009342156481121059,
+ "ctr_holdout": 0.004281821720513819,
+ "n_select": 2569,
+ "n_holdout": 2569,
+ "eb": 0.006838535193812306,
+ "labelled_winner": true
+ },
+ {
+ "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley",
+ "impressions": 5179,
+ "clicks": 17,
+ "ctr": 0.0032824869665958,
+ "ctr_select": 0.0015444015444015444,
+ "ctr_holdout": 0.005021243723445346,
+ "n_select": 2590,
+ "n_holdout": 2589,
+ "eb": 0.0027340425271084926,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley",
+ "impressions": 5023,
+ "clicks": 20,
+ "ctr": 0.0039816842524387,
+ "ctr_select": 0.0051751592356687895,
+ "ctr_holdout": 0.0027877339705296694,
+ "n_select": 2512,
+ "n_holdout": 2511,
+ "eb": 0.004641668196573108,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley",
+ "impressions": 5181,
+ "clicks": 10,
+ "ctr": 0.0019301293186643,
+ "ctr_select": 0.0019305019305019305,
+ "ctr_holdout": 0.0019297568506368198,
+ "n_select": 2590,
+ "n_holdout": 2591,
+ "eb": 0.002937804058519797,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.004063452371649606,
+ "pool_holdout": 0.0031262211801484957,
+ "lift_select": 1.2990687786328112,
+ "lift_holdout": 0.36964772284935754,
+ "eb_lift": 0.6829372091387707,
+ "p_value": 2.546077688100374e-06,
+ "held_up": false,
+ "z_path": [
+ -2.026,
+ -0.407,
+ -0.301,
+ 0.168,
+ 1.122,
+ 1.654,
+ 2.016,
+ 2.165,
+ 2.025,
+ 1.944
+ ],
+ "first_cross": 0,
+ "final_z": 1.944
+ },
+ {
+ "id": "529520d47a0573ac2e00093e",
+ "variation": "headline",
+ "week": 201347,
+ "impressions": 25571,
+ "pooled_ctr": 0.023190332798873725,
+ "arms": [
+ {
+ "headline": "Wondering What Happens To Divorced, Dark-Skinned, Single Mothers In This Country?",
+ "impressions": 6376,
+ "clicks": 171,
+ "ctr": 0.026819322459222,
+ "ctr_select": 0.027603513174404015,
+ "ctr_holdout": 0.026035131744040152,
+ "n_select": 3188,
+ "n_holdout": 3188,
+ "eb": 0.02576180550900934,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Thought This Ad Was Trying To Sell Me Jewelry, And Then I Realized It Had A Powerful Message",
+ "impressions": 5851,
+ "clicks": 138,
+ "ctr": 0.0235857118441292,
+ "ctr_select": 0.024265208475734792,
+ "ctr_holdout": 0.022905982905982905,
+ "n_select": 2926,
+ "n_holdout": 2925,
+ "eb": 0.023807000420306253,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Marriage Gets A Shiny Reboot In This Fantastic Ad From India",
+ "impressions": 6612,
+ "clicks": 146,
+ "ctr": 0.0220810647307924,
+ "ctr_select": 0.02026618269812462,
+ "ctr_holdout": 0.023895946763460376,
+ "n_select": 3306,
+ "n_holdout": 3306,
+ "eb": 0.021487057058314128,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Every Marriage Isn't The Same, Especially For Women In This Country",
+ "impressions": 6732,
+ "clicks": 138,
+ "ctr": 0.0204991087344028,
+ "ctr_select": 0.02109328579916815,
+ "ctr_holdout": 0.01990493166963755,
+ "n_select": 3366,
+ "n_holdout": 3366,
+ "eb": 0.021963995966287883,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.02322853120600657,
+ "pool_holdout": 0.02315213140398905,
+ "lift_select": 0.18834518332636274,
+ "lift_holdout": 0.12452418698497758,
+ "eb_lift": 0.10905873817573553,
+ "p_value": 0.05838428814795746,
+ "held_up": true,
+ "z_path": [
+ 0.671,
+ 0.92,
+ 0.736,
+ 1.086,
+ 0.741,
+ 0.868,
+ 1.108,
+ 1.351,
+ 1.015,
+ 1.138
+ ],
+ "first_cross": null,
+ "final_z": 1.138
+ },
+ {
+ "id": "540903f9844567e9bc000036",
+ "variation": "headline",
+ "week": 201435,
+ "impressions": 24691,
+ "pooled_ctr": 0.00486007047102183,
+ "arms": [
+ {
+ "headline": "The Outrageous Truth About What Happens To Thousands Of Rape Kits",
+ "impressions": 6062,
+ "clicks": 21,
+ "ctr": 0.0034642032332563,
+ "ctr_select": 0.0026393929396238865,
+ "ctr_holdout": 0.004289013526888816,
+ "n_select": 3031,
+ "n_holdout": 3031,
+ "eb": 0.0037421456332816377,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Thousands Of Rape Victims Nationwide May Have Subjected Themselves To This Exam For Nothing",
+ "impressions": 6168,
+ "clicks": 31,
+ "ctr": 0.0050259403372243,
+ "ctr_select": 0.005512321660181583,
+ "ctr_holdout": 0.004539559014267186,
+ "n_select": 3084,
+ "n_holdout": 3084,
+ "eb": 0.005371571558743007,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Could This Be One Reason Why There Are So Many Unsolved Crimes In The U.S.?",
+ "impressions": 6224,
+ "clicks": 28,
+ "ctr": 0.0044987146529562,
+ "ctr_select": 0.005784061696658098,
+ "ctr_holdout": 0.003213367609254499,
+ "n_select": 3112,
+ "n_holdout": 3112,
+ "eb": 0.005528043620830382,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Unbelievable. Just When You Thought The News About Rape In This Country Couldn't Get Worse.",
+ "impressions": 6237,
+ "clicks": 40,
+ "ctr": 0.006413339746673,
+ "ctr_select": 0.006735086593970494,
+ "ctr_holdout": 0.006091696056428343,
+ "n_select": 3118,
+ "n_holdout": 3119,
+ "eb": 0.006073848759284948,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.005184285135682463,
+ "pool_holdout": 0.004535882067066256,
+ "lift_select": 0.2991350625400897,
+ "lift_holdout": 0.3430014198690057,
+ "eb_lift": 0.17158848333394805,
+ "p_value": 0.16309374620995776,
+ "held_up": true,
+ "z_path": [
+ -0.003,
+ 0.24,
+ -0.197,
+ -0.371,
+ -0.694,
+ -0.167,
+ 0.431,
+ 0.681,
+ 0.782,
+ 1.451
+ ],
+ "first_cross": null,
+ "final_z": 1.451
+ },
+ {
+ "id": "541c94937bc1ef0682000055",
+ "variation": "image",
+ "week": 201437,
+ "impressions": 24644,
+ "pooled_ctr": 0.004463561110209382,
+ "arms": [
+ {
+ "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected",
+ "impressions": 4182,
+ "clicks": 23,
+ "ctr": 0.0054997608799617,
+ "ctr_select": 0.0052606408417025345,
+ "ctr_holdout": 0.005738880918220947,
+ "n_select": 2091,
+ "n_holdout": 2091,
+ "eb": 0.004926936558748172,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected",
+ "impressions": 4126,
+ "clicks": 11,
+ "ctr": 0.0026660203587009,
+ "ctr_select": 0.004362578768783325,
+ "ctr_holdout": 0.0009694619486185168,
+ "n_select": 2063,
+ "n_holdout": 2063,
+ "eb": 0.004501879660988422,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected",
+ "impressions": 4059,
+ "clicks": 20,
+ "ctr": 0.0049273220004927,
+ "ctr_select": 0.006896551724137931,
+ "ctr_holdout": 0.002957121734844751,
+ "n_select": 2030,
+ "n_holdout": 2029,
+ "eb": 0.005686083894719949,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected",
+ "impressions": 4178,
+ "clicks": 20,
+ "ctr": 0.0047869794159885,
+ "ctr_select": 0.005265677357587362,
+ "ctr_holdout": 0.00430828147438966,
+ "n_select": 2089,
+ "n_holdout": 2089,
+ "eb": 0.004929172671542369,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected",
+ "impressions": 4002,
+ "clicks": 18,
+ "ctr": 0.0044977511244377,
+ "ctr_select": 0.004497751124437781,
+ "ctr_holdout": 0.004497751124437781,
+ "n_select": 2001,
+ "n_holdout": 2001,
+ "eb": 0.004566509454058166,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected",
+ "impressions": 4097,
+ "clicks": 18,
+ "ctr": 0.0043934586282645,
+ "ctr_select": 0.00146484375,
+ "ctr_holdout": 0.007320644216691069,
+ "n_select": 2048,
+ "n_holdout": 2049,
+ "eb": 0.0031429967958766466,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.004625872423307905,
+ "pool_holdout": 0.004301249797110858,
+ "lift_select": 0.49086509376890497,
+ "lift_holdout": -0.3124970940234524,
+ "eb_lift": 0.22919167983752997,
+ "p_value": 0.09900465270977789,
+ "held_up": false,
+ "z_path": [
+ 1.761,
+ 1.672,
+ 1.458,
+ 1.313,
+ 0.954,
+ 0.559,
+ 0.065,
+ -0.108,
+ 0.082,
+ 0.092
+ ],
+ "first_cross": null,
+ "final_z": 0.092
+ },
+ {
+ "id": "53dbf3892866d1c05b000019",
+ "variation": "other",
+ "week": 201430,
+ "impressions": 24600,
+ "pooled_ctr": 0.017195121951219514,
+ "arms": [
+ {
+ "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.",
+ "impressions": 4985,
+ "clicks": 74,
+ "ctr": 0.0148445336008024,
+ "ctr_select": 0.01565008025682183,
+ "ctr_holdout": 0.014039310068190935,
+ "n_select": 2492,
+ "n_holdout": 2493,
+ "eb": 0.016649951646199795,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.",
+ "impressions": 4830,
+ "clicks": 74,
+ "ctr": 0.0153209109730848,
+ "ctr_select": 0.015320910973084885,
+ "ctr_holdout": 0.015320910973084885,
+ "n_select": 2415,
+ "n_holdout": 2415,
+ "eb": 0.016498250910583486,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.",
+ "impressions": 4966,
+ "clicks": 88,
+ "ctr": 0.017720499395892,
+ "ctr_select": 0.019331453886427707,
+ "ctr_holdout": 0.016109544905356425,
+ "n_select": 2483,
+ "n_holdout": 2483,
+ "eb": 0.0185558932671403,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.",
+ "impressions": 4882,
+ "clicks": 110,
+ "ctr": 0.0225317492830807,
+ "ctr_select": 0.02498975829578042,
+ "ctr_holdout": 0.02007374027038099,
+ "n_select": 2441,
+ "n_holdout": 2441,
+ "eb": 0.021451506389150794,
+ "labelled_winner": true
+ },
+ {
+ "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.",
+ "impressions": 4937,
+ "clicks": 77,
+ "ctr": 0.0155965161028964,
+ "ctr_select": 0.013371150729335495,
+ "ctr_holdout": 0.017820980153908466,
+ "n_select": 2468,
+ "n_holdout": 2469,
+ "eb": 0.015479714513363734,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.017725018294170256,
+ "pool_holdout": 0.016665311763271278,
+ "lift_select": 0.4098579691734101,
+ "lift_holdout": 0.2045223369071052,
+ "eb_lift": 0.2102388856888331,
+ "p_value": 0.0023789532901485836,
+ "held_up": true,
+ "z_path": [
+ 0.488,
+ 0.842,
+ 0.455,
+ 0.646,
+ 1.408,
+ 1.784,
+ 1.122,
+ 0.74,
+ 1.088,
+ 1.701
+ ],
+ "first_cross": null,
+ "final_z": 1.701
+ },
+ {
+ "id": "54482589b8fbabf5e4000003",
+ "variation": "other",
+ "week": 201442,
+ "impressions": 24558,
+ "pooled_ctr": 0.012134538643211987,
+ "arms": [
+ {
+ "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'",
+ "impressions": 3069,
+ "clicks": 39,
+ "ctr": 0.0127077223851417,
+ "ctr_select": 0.013689700130378096,
+ "ctr_holdout": 0.011726384364820847,
+ "n_select": 1534,
+ "n_holdout": 1535,
+ "eb": 0.012312301579783124,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'",
+ "impressions": 3095,
+ "clicks": 34,
+ "ctr": 0.0109854604200323,
+ "ctr_select": 0.009689922480620155,
+ "ctr_holdout": 0.012281835811247576,
+ "n_select": 1548,
+ "n_holdout": 1547,
+ "eb": 0.010715598295220138,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'",
+ "impressions": 3152,
+ "clicks": 49,
+ "ctr": 0.0155456852791878,
+ "ctr_select": 0.015228426395939087,
+ "ctr_holdout": 0.015862944162436547,
+ "n_select": 1576,
+ "n_holdout": 1576,
+ "eb": 0.012949968785096484,
+ "labelled_winner": true
+ },
+ {
+ "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'",
+ "impressions": 3059,
+ "clicks": 44,
+ "ctr": 0.0143837855508336,
+ "ctr_select": 0.01568627450980392,
+ "ctr_holdout": 0.013080444735120994,
+ "n_select": 1530,
+ "n_holdout": 1529,
+ "eb": 0.013104788266730132,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'",
+ "impressions": 3084,
+ "clicks": 46,
+ "ctr": 0.0149156939040207,
+ "ctr_select": 0.011673151750972763,
+ "ctr_holdout": 0.018158236057068743,
+ "n_select": 1542,
+ "n_holdout": 1542,
+ "eb": 0.011509518913949466,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'",
+ "impressions": 3080,
+ "clicks": 37,
+ "ctr": 0.012012987012987,
+ "ctr_select": 0.00909090909090909,
+ "ctr_holdout": 0.014935064935064935,
+ "n_select": 1540,
+ "n_holdout": 1540,
+ "eb": 0.010478598072901022,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'",
+ "impressions": 2940,
+ "clicks": 23,
+ "ctr": 0.0078231292517006,
+ "ctr_select": 0.007482993197278911,
+ "ctr_holdout": 0.00816326530612245,
+ "n_select": 1470,
+ "n_holdout": 1470,
+ "eb": 0.009880213958951296,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'",
+ "impressions": 3079,
+ "clicks": 26,
+ "ctr": 0.0084443000974342,
+ "ctr_select": 0.008441558441558441,
+ "ctr_holdout": 0.008447043534762833,
+ "n_select": 1540,
+ "n_holdout": 1539,
+ "eb": 0.0102193761025536,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.011400651465798045,
+ "pool_holdout": 0.012868545365694739,
+ "lift_select": 0.37591036414565826,
+ "lift_holdout": 0.016466458593769273,
+ "eb_lift": 0.14947714225318598,
+ "p_value": 0.09148048238766168,
+ "held_up": false,
+ "z_path": [
+ -1.224,
+ -1.445,
+ -1.465,
+ -1.337,
+ -1.539,
+ -0.901,
+ -0.54,
+ -0.697,
+ -0.627,
+ -0.377
+ ],
+ "first_cross": null,
+ "final_z": -0.377
+ },
+ {
+ "id": "542d3b2952a3f66b70000008",
+ "variation": "headline",
+ "week": 201439,
+ "impressions": 24451,
+ "pooled_ctr": 0.004907774733139749,
+ "arms": [
+ {
+ "headline": "The Perfect Response When Your Mom Tells You To Fix Your Hair",
+ "impressions": 4099,
+ "clicks": 27,
+ "ctr": 0.0065869724323005,
+ "ctr_select": 0.006829268292682927,
+ "ctr_holdout": 0.00634455832113226,
+ "n_select": 2050,
+ "n_holdout": 2049,
+ "eb": 0.005679321480686004,
+ "labelled_winner": false
+ },
+ {
+ "headline": "For Moms, Grandmothers, BFFs And Anyone In Between Who Has Something To Say About Your Hair",
+ "impressions": 4068,
+ "clicks": 19,
+ "ctr": 0.0046705998033431,
+ "ctr_select": 0.004424778761061947,
+ "ctr_holdout": 0.004916420845624385,
+ "n_select": 2034,
+ "n_holdout": 2034,
+ "eb": 0.004551225921318163,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why \u2018Nappy\u2019 Hair Is Not A Bad Thing",
+ "impressions": 4130,
+ "clicks": 12,
+ "ctr": 0.0029055690072639,
+ "ctr_select": 0.003389830508474576,
+ "ctr_holdout": 0.002421307506053269,
+ "n_select": 2065,
+ "n_holdout": 2065,
+ "eb": 0.004062693937627633,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Something That Should Be Played In Beauty Salons Across The Nation",
+ "impressions": 4102,
+ "clicks": 23,
+ "ctr": 0.0056070209653827,
+ "ctr_select": 0.0048756704046806435,
+ "ctr_holdout": 0.006338371526084837,
+ "n_select": 2051,
+ "n_holdout": 2051,
+ "eb": 0.004762416811994553,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Don\u2019t Have To Have Curly Hair To Understand Why What She Says Hits Home",
+ "impressions": 4049,
+ "clicks": 20,
+ "ctr": 0.004939491232403,
+ "ctr_select": 0.004940711462450593,
+ "ctr_holdout": 0.0049382716049382715,
+ "n_select": 2024,
+ "n_holdout": 2025,
+ "eb": 0.004792033165474578,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Next Time You Feel Like You\u2019re Having A 'Bad Hair Day,' Listen To This",
+ "impressions": 4003,
+ "clicks": 19,
+ "ctr": 0.0047464401698725,
+ "ctr_select": 0.0034965034965034965,
+ "ctr_holdout": 0.005997001499250375,
+ "n_select": 2002,
+ "n_holdout": 2001,
+ "eb": 0.004121945703739868,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.004662195321446098,
+ "pool_holdout": 0.0051533742331288344,
+ "lift_select": 0.4648181429182714,
+ "lift_holdout": 0.23114643612447416,
+ "eb_lift": 0.21816463899766836,
+ "p_value": 0.11438484724503767,
+ "held_up": true,
+ "z_path": [
+ 1.123,
+ 1.138,
+ 1.585,
+ 1.865,
+ 1.608,
+ 1.935,
+ 2.015,
+ 1.3,
+ 1.407,
+ 0.982
+ ],
+ "first_cross": 6,
+ "final_z": 0.982
+ },
+ {
+ "id": "542c3500d9a91cd011000037",
+ "variation": "image",
+ "week": 201439,
+ "impressions": 23766,
+ "pooled_ctr": 0.010982075233526888,
+ "arms": [
+ {
+ "headline": "CNN Asks An Expert A Dumb Question. Here's His Epic Response.",
+ "impressions": 5909,
+ "clicks": 69,
+ "ctr": 0.0116771027246573,
+ "ctr_select": 0.010832769126607989,
+ "ctr_holdout": 0.012521150592216581,
+ "n_select": 2954,
+ "n_holdout": 2955,
+ "eb": 0.01069531075049896,
+ "labelled_winner": false
+ },
+ {
+ "headline": "CNN Asks An Expert A Dumb Question. Here's His Epic Response.",
+ "impressions": 5825,
+ "clicks": 67,
+ "ctr": 0.0115021459227467,
+ "ctr_select": 0.01201923076923077,
+ "ctr_holdout": 0.010985238585650533,
+ "n_select": 2912,
+ "n_holdout": 2913,
+ "eb": 0.011354852323707776,
+ "labelled_winner": false
+ },
+ {
+ "headline": "CNN Asks An Expert A Dumb Question. Here's His Epic Response.",
+ "impressions": 6079,
+ "clicks": 63,
+ "ctr": 0.0103635466359598,
+ "ctr_select": 0.008881578947368421,
+ "ctr_holdout": 0.011846001974333662,
+ "n_select": 3040,
+ "n_holdout": 3039,
+ "eb": 0.00959039646624911,
+ "labelled_winner": false
+ },
+ {
+ "headline": "CNN Asks An Expert A Dumb Question. Here's His Epic Response.",
+ "impressions": 5953,
+ "clicks": 62,
+ "ctr": 0.0104149168486477,
+ "ctr_select": 0.010416666666666666,
+ "ctr_holdout": 0.010413167618407793,
+ "n_select": 2976,
+ "n_holdout": 2977,
+ "eb": 0.010461958393519908,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.010520114458845313,
+ "pool_holdout": 0.01144395826321104,
+ "lift_select": 0.14250000000000007,
+ "lift_holdout": -0.04008400476565488,
+ "eb_lift": 0.07934684248236623,
+ "p_value": 0.3614691120953939,
+ "held_up": false,
+ "z_path": [
+ -0.886,
+ 0.032,
+ 0.355,
+ 1.109,
+ 0.293,
+ -0.043,
+ 0.172,
+ -0.214,
+ -0.1,
+ -0.089
+ ],
+ "first_cross": null,
+ "final_z": -0.089
+ },
+ {
+ "id": "53c05eac5d80f0f0b4000029",
+ "variation": "headline",
+ "week": 201427,
+ "impressions": 23313,
+ "pooled_ctr": 0.01282546218847853,
+ "arms": [
+ {
+ "headline": "A Straight Kid Stands Up To Some Bullying Politicians For His Two Gay Moms And Wins",
+ "impressions": 3910,
+ "clicks": 57,
+ "ctr": 0.0145780051150895,
+ "ctr_select": 0.01687979539641944,
+ "ctr_holdout": 0.012276214833759591,
+ "n_select": 1955,
+ "n_holdout": 1955,
+ "eb": 0.014051501564002556,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Straight Kid Stands Up To Some Bullying Politicians For His Two Gay Moms And Makes Us All Proud",
+ "impressions": 3911,
+ "clicks": 56,
+ "ctr": 0.0143185885962669,
+ "ctr_select": 0.009713701431492843,
+ "ctr_holdout": 0.018925831202046037,
+ "n_select": 1956,
+ "n_holdout": 1955,
+ "eb": 0.010772361935954475,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Straight College Kid Stands Up To Some Bullying Politicians For His Two Gay Moms And Wins",
+ "impressions": 3846,
+ "clicks": 52,
+ "ctr": 0.0135205408216328,
+ "ctr_select": 0.010400416016640665,
+ "ctr_holdout": 0.016640665626625067,
+ "n_select": 1923,
+ "n_holdout": 1923,
+ "eb": 0.011091999891379885,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Straight Kid Stands Up To Some Bullying Politicians For His Two Moms And Shuts Down Their Hate",
+ "impressions": 3855,
+ "clicks": 64,
+ "ctr": 0.0166018158236057,
+ "ctr_select": 0.017634854771784232,
+ "ctr_holdout": 0.015568240788790867,
+ "n_select": 1928,
+ "n_holdout": 1927,
+ "eb": 0.014376392933617135,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Straight College Kid Stands Up To Bullying Politicians Who Don't Want His Moms To Stay Married",
+ "impressions": 3960,
+ "clicks": 36,
+ "ctr": 0.009090909090909,
+ "ctr_select": 0.007575757575757576,
+ "ctr_holdout": 0.010606060606060607,
+ "n_select": 1980,
+ "n_holdout": 1980,
+ "eb": 0.009781474028256599,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Straight Kid Stands Up For His 2 Moms When Some Politicians Decide To Ban Their Marriage",
+ "impressions": 3831,
+ "clicks": 34,
+ "ctr": 0.0088749673714434,
+ "ctr_select": 0.007828810020876827,
+ "ctr_holdout": 0.009921671018276762,
+ "n_select": 1916,
+ "n_holdout": 1915,
+ "eb": 0.009929338703885942,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.011665808886601476,
+ "pool_holdout": 0.013985413985413985,
+ "lift_select": 0.5116701244813278,
+ "lift_holdout": 0.11317697173838992,
+ "eb_lift": 0.23235285897138636,
+ "p_value": 0.007544455804351582,
+ "held_up": false,
+ "z_path": [
+ 0.02,
+ 1.004,
+ 2.003,
+ 1.955,
+ 0.687,
+ 1.133,
+ 1.253,
+ 0.99,
+ 0.655,
+ 0.72
+ ],
+ "first_cross": 2,
+ "final_z": 0.72
+ },
+ {
+ "id": "541b59e02a027740b30000ba",
+ "variation": "image",
+ "week": 201437,
+ "impressions": 23099,
+ "pooled_ctr": 0.009697389497380839,
+ "arms": [
+ {
+ "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living",
+ "impressions": 4686,
+ "clicks": 48,
+ "ctr": 0.0102432778489116,
+ "ctr_select": 0.011096884336320957,
+ "ctr_holdout": 0.009389671361502348,
+ "n_select": 2343,
+ "n_holdout": 2343,
+ "eb": 0.010315027018294181,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living",
+ "impressions": 4653,
+ "clicks": 44,
+ "ctr": 0.0094562647754137,
+ "ctr_select": 0.009888220120378332,
+ "ctr_holdout": 0.009024495058014611,
+ "n_select": 2326,
+ "n_holdout": 2327,
+ "eb": 0.009706752164857843,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living",
+ "impressions": 4494,
+ "clicks": 70,
+ "ctr": 0.0155763239875389,
+ "ctr_select": 0.013796172674677348,
+ "ctr_holdout": 0.017356475300400534,
+ "n_select": 2247,
+ "n_holdout": 2247,
+ "eb": 0.0116273247828701,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living",
+ "impressions": 4622,
+ "clicks": 35,
+ "ctr": 0.0075724794461272,
+ "ctr_select": 0.006490696668109044,
+ "ctr_holdout": 0.00865426222414539,
+ "n_select": 2311,
+ "n_holdout": 2311,
+ "eb": 0.00800986046470135,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living",
+ "impressions": 4644,
+ "clicks": 27,
+ "ctr": 0.005813953488372,
+ "ctr_select": 0.006459948320413436,
+ "ctr_holdout": 0.00516795865633075,
+ "n_select": 2322,
+ "n_holdout": 2322,
+ "eb": 0.007990870334153898,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.009524634167460386,
+ "pool_holdout": 0.009870129870129871,
+ "lift_select": 0.44847271108953346,
+ "lift_holdout": 0.7584849975405803,
+ "eb_lift": 0.22076339924878896,
+ "p_value": 0.020186427831472026,
+ "held_up": true,
+ "z_path": [
+ 0.588,
+ 0.851,
+ 1.406,
+ 2.073,
+ 2.435,
+ 1.821,
+ 1.408,
+ 1.741,
+ 2.082,
+ 2.268
+ ],
+ "first_cross": 3,
+ "final_z": 2.268
+ },
+ {
+ "id": "542dbdbbd3047b688f000060",
+ "variation": "headline",
+ "week": 201439,
+ "impressions": 22583,
+ "pooled_ctr": 0.016826816631979807,
+ "arms": [
+ {
+ "headline": "After You See These Images From Space, You'll Realize That The US Map Is *Incorrect*",
+ "impressions": 5625,
+ "clicks": 111,
+ "ctr": 0.0197333333333333,
+ "ctr_select": 0.02027027027027027,
+ "ctr_holdout": 0.019196587273373623,
+ "n_select": 2812,
+ "n_holdout": 2813,
+ "eb": 0.018474691857848717,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These Images From Space Prove That The US Map Is *WRONG*",
+ "impressions": 5650,
+ "clicks": 87,
+ "ctr": 0.0153982300884955,
+ "ctr_select": 0.01663716814159292,
+ "ctr_holdout": 0.01415929203539823,
+ "n_select": 2825,
+ "n_holdout": 2825,
+ "eb": 0.016483485766749007,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Is The US Map *Incorrect*? After Looking At These Images From Space, Yes, It Is.",
+ "impressions": 5613,
+ "clicks": 84,
+ "ctr": 0.0149652592196686,
+ "ctr_select": 0.012829650748396294,
+ "ctr_holdout": 0.017100106875667972,
+ "n_select": 2806,
+ "n_holdout": 2807,
+ "eb": 0.014397723806052927,
+ "labelled_winner": false
+ },
+ {
+ "headline": "After Seeing These 8 Satellite Images I'm Like, \"Nooooooooo!\"",
+ "impressions": 5695,
+ "clicks": 98,
+ "ctr": 0.017208077260755,
+ "ctr_select": 0.01544943820224719,
+ "ctr_holdout": 0.018967334035827187,
+ "n_select": 2848,
+ "n_holdout": 2847,
+ "eb": 0.015829339828320632,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.01629616508723762,
+ "pool_holdout": 0.017357421183138506,
+ "lift_select": 0.24386750881316122,
+ "lift_holdout": 0.10595848719864764,
+ "eb_lift": 0.13368340090744502,
+ "p_value": 0.05476592727539358,
+ "held_up": true,
+ "z_path": [
+ 1.014,
+ 1.534,
+ 1.216,
+ 2.049,
+ 1.505,
+ 1.055,
+ 1.041,
+ 1.067,
+ 1.39,
+ 1.752
+ ],
+ "first_cross": 3,
+ "final_z": 1.752
+ },
+ {
+ "id": "5330e65300d2ea8cf0000001",
+ "variation": "image",
+ "week": 201412,
+ "impressions": 22392,
+ "pooled_ctr": 0.00817256162915327,
+ "arms": [
+ {
+ "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of",
+ "impressions": 3691,
+ "clicks": 37,
+ "ctr": 0.010024383635871,
+ "ctr_select": 0.010834236186348862,
+ "ctr_holdout": 0.00921409214092141,
+ "n_select": 1846,
+ "n_holdout": 1845,
+ "eb": 0.009526644208933437,
+ "labelled_winner": false
+ },
+ {
+ "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of",
+ "impressions": 3798,
+ "clicks": 33,
+ "ctr": 0.0086887835703001,
+ "ctr_select": 0.006845708267509215,
+ "ctr_holdout": 0.010531858873091101,
+ "n_select": 1899,
+ "n_holdout": 1899,
+ "eb": 0.00774683624811482,
+ "labelled_winner": false
+ },
+ {
+ "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of",
+ "impressions": 3717,
+ "clicks": 31,
+ "ctr": 0.0083400591875168,
+ "ctr_select": 0.011302475780409042,
+ "ctr_holdout": 0.0053792361484669175,
+ "n_select": 1858,
+ "n_holdout": 1859,
+ "eb": 0.009738747749979973,
+ "labelled_winner": false
+ },
+ {
+ "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of",
+ "impressions": 3705,
+ "clicks": 23,
+ "ctr": 0.0062078272604588,
+ "ctr_select": 0.004859611231101512,
+ "ctr_holdout": 0.007555315704263357,
+ "n_select": 1852,
+ "n_holdout": 1853,
+ "eb": 0.0068748548830634155,
+ "labelled_winner": false
+ },
+ {
+ "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of",
+ "impressions": 3682,
+ "clicks": 29,
+ "ctr": 0.0078761542639869,
+ "ctr_select": 0.008690928843020097,
+ "ctr_holdout": 0.007061379684953829,
+ "n_select": 1841,
+ "n_holdout": 1841,
+ "eb": 0.008576258520992933,
+ "labelled_winner": false
+ },
+ {
+ "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of",
+ "impressions": 3799,
+ "clicks": 30,
+ "ctr": 0.0078968149513029,
+ "ctr_select": 0.008421052631578947,
+ "ctr_holdout": 0.00737230121116377,
+ "n_select": 1900,
+ "n_holdout": 1899,
+ "eb": 0.008456288032784967,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.008485173276170061,
+ "pool_holdout": 0.007859949982136478,
+ "lift_select": 0.33202651407852235,
+ "lift_holdout": -0.31561445547459543,
+ "eb_lift": 0.14773705061869236,
+ "p_value": 0.147139641184714,
+ "held_up": false,
+ "z_path": [
+ -0.387,
+ -0.014,
+ -1.065,
+ -0.897,
+ -1.296,
+ -1.265,
+ -1.211,
+ -1.545,
+ -0.783,
+ -0.76
+ ],
+ "first_cross": null,
+ "final_z": -0.76
+ },
+ {
+ "id": "5245cbe3eb36284c7700014b",
+ "variation": "other",
+ "week": 201338,
+ "impressions": 22254,
+ "pooled_ctr": 0.014963602049069831,
+ "arms": [
+ {
+ "headline": "6 Things We Could Be Doing To Kick-Start This Country",
+ "impressions": 6033,
+ "clicks": 91,
+ "ctr": 0.015083706282115,
+ "ctr_select": 0.01293103448275862,
+ "ctr_holdout": 0.01723566456745111,
+ "n_select": 3016,
+ "n_holdout": 3017,
+ "eb": 0.013951544529054918,
+ "labelled_winner": false
+ },
+ {
+ "headline": "6 Things We Could Be Doing To Kick-Start This Country",
+ "impressions": 5373,
+ "clicks": 63,
+ "ctr": 0.0117252931323283,
+ "ctr_select": 0.012658227848101266,
+ "ctr_holdout": 0.010792705619650168,
+ "n_select": 2686,
+ "n_holdout": 2687,
+ "eb": 0.013872404994896982,
+ "labelled_winner": false
+ },
+ {
+ "headline": "6 Things We Could Be Doing To Kick-Start This Country",
+ "impressions": 5296,
+ "clicks": 101,
+ "ctr": 0.0190709969788519,
+ "ctr_select": 0.020770392749244714,
+ "ctr_holdout": 0.017371601208459216,
+ "n_select": 2648,
+ "n_holdout": 2648,
+ "eb": 0.018207576539817284,
+ "labelled_winner": false
+ },
+ {
+ "headline": "6 Things We Could Be Doing To Kick-Start This Country",
+ "impressions": 5552,
+ "clicks": 78,
+ "ctr": 0.0140489913544668,
+ "ctr_select": 0.015129682997118156,
+ "ctr_holdout": 0.012968299711815562,
+ "n_select": 2776,
+ "n_holdout": 2776,
+ "eb": 0.015197863226216222,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.015279525435915873,
+ "pool_holdout": 0.014647735442127965,
+ "lift_select": 0.35936111604762755,
+ "lift_holdout": 0.18595814875910532,
+ "eb_lift": 0.19163233283533576,
+ "p_value": 0.00831927894562641,
+ "held_up": true,
+ "z_path": [
+ 0.749,
+ 1.655,
+ 2.27,
+ 2.695,
+ 2.659,
+ 1.941,
+ 1.997,
+ 1.529,
+ 2.121,
+ 2.052
+ ],
+ "first_cross": 2,
+ "final_z": 2.052
+ },
+ {
+ "id": "5450f8327ccd9d83d8000055",
+ "variation": "headline",
+ "week": 201443,
+ "impressions": 22029,
+ "pooled_ctr": 0.01411775386989877,
+ "arms": [
+ {
+ "headline": "Only 7 Countries In The World Don't Have Paid Maternity Leave. Bet You Can Guess 1 Of Them.",
+ "impressions": 3682,
+ "clicks": 37,
+ "ctr": 0.0100488864747419,
+ "ctr_select": 0.00977729494839761,
+ "ctr_holdout": 0.010320478001086366,
+ "n_select": 1841,
+ "n_holdout": 1841,
+ "eb": 0.012625998218564444,
+ "labelled_winner": false
+ },
+ {
+ "headline": "That Moment You Realize That Something Normal For The U.S. Would Seem Ridiculous Anywhere Else",
+ "impressions": 3659,
+ "clicks": 94,
+ "ctr": 0.0256900792566274,
+ "ctr_select": 0.029508196721311476,
+ "ctr_holdout": 0.02186987424822307,
+ "n_select": 1830,
+ "n_holdout": 1829,
+ "eb": 0.021338997225224633,
+ "labelled_winner": false
+ },
+ {
+ "headline": "One Thing's For Sure, I've Never Wanted To Be Swedish More Than I Did At 1:40",
+ "impressions": 3690,
+ "clicks": 59,
+ "ctr": 0.0159891598915989,
+ "ctr_select": 0.016260162601626018,
+ "ctr_holdout": 0.015718157181571817,
+ "n_select": 1845,
+ "n_holdout": 1845,
+ "eb": 0.015496611665662187,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You're A New Mom In America, Good Luck! You're Gonna Need It.",
+ "impressions": 3626,
+ "clicks": 21,
+ "ctr": 0.0057915057915057,
+ "ctr_select": 0.007722007722007722,
+ "ctr_holdout": 0.003861003861003861,
+ "n_select": 1813,
+ "n_holdout": 1813,
+ "eb": 0.011743224336130667,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The More She Explains, The Madder I Get. Why Can't America Have Nice Things?",
+ "impressions": 3698,
+ "clicks": 54,
+ "ctr": 0.0146024878312601,
+ "ctr_select": 0.014602487831260141,
+ "ctr_holdout": 0.014602487831260141,
+ "n_select": 1849,
+ "n_holdout": 1849,
+ "eb": 0.014761740392601717,
+ "labelled_winner": false
+ },
+ {
+ "headline": "American Parents Are At 1 Huge Disadvantage Compared To Moms And Dads Around The World",
+ "impressions": 3674,
+ "clicks": 46,
+ "ctr": 0.0125204137180185,
+ "ctr_select": 0.011431682090364725,
+ "ctr_holdout": 0.013609145345672292,
+ "n_select": 1837,
+ "n_holdout": 1837,
+ "eb": 0.0133602282900981,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.014888788016341352,
+ "pool_holdout": 0.01334664971854004,
+ "lift_select": 0.981907237105158,
+ "lift_holdout": 0.6386040474144823,
+ "eb_lift": 0.4332259416820081,
+ "p_value": 1.5580954056943683e-08,
+ "held_up": true,
+ "z_path": [
+ 0.495,
+ 0.381,
+ 0.888,
+ 1.03,
+ 1.073,
+ 1.769,
+ 2.182,
+ 1.89,
+ 2.878,
+ 2.912
+ ],
+ "first_cross": 6,
+ "final_z": 2.912
+ },
+ {
+ "id": "5296566858f57f80560003ad",
+ "variation": "headline",
+ "week": 201347,
+ "impressions": 21949,
+ "pooled_ctr": 0.01075219827782587,
+ "arms": [
+ {
+ "headline": "He Starts With 17 Lego Pieces And Ends Up With 6. The Reason Why Involves All Of Us.",
+ "impressions": 5195,
+ "clicks": 71,
+ "ctr": 0.0136669874879692,
+ "ctr_select": 0.013471901462663588,
+ "ctr_holdout": 0.013862148633038121,
+ "n_select": 2598,
+ "n_holdout": 2597,
+ "eb": 0.012275280755923995,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Why Does This Guy Start With 17 Lego Pieces And End Up With 6?",
+ "impressions": 5277,
+ "clicks": 50,
+ "ctr": 0.0094750805381845,
+ "ctr_select": 0.01023502653525398,
+ "ctr_holdout": 0.008715422508525956,
+ "n_select": 2638,
+ "n_holdout": 2639,
+ "eb": 0.010561897144511043,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why Are Only 6 Lego Pieces Left In His Version Of The Future?",
+ "impressions": 5750,
+ "clicks": 45,
+ "ctr": 0.0078260869565217,
+ "ctr_select": 0.008347826086956521,
+ "ctr_holdout": 0.007304347826086957,
+ "n_select": 2875,
+ "n_holdout": 2875,
+ "eb": 0.00950210704382063,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Starts With 17 Legos And Ends Up With 6. The Missing Pieces Are Not Stuck In Couch Cushions.",
+ "impressions": 5727,
+ "clicks": 70,
+ "ctr": 0.0122228042605203,
+ "ctr_select": 0.011871508379888268,
+ "ctr_holdout": 0.012574222843171499,
+ "n_select": 2864,
+ "n_holdout": 2863,
+ "eb": 0.011452148083158996,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.010933940774487472,
+ "pool_holdout": 0.010570439219974486,
+ "lift_select": 0.23211765460610723,
+ "lift_holdout": 0.3114070611979338,
+ "eb_lift": 0.12267671913554867,
+ "p_value": 0.15449253407685468,
+ "held_up": true,
+ "z_path": [
+ 0.45,
+ -0.541,
+ -0.352,
+ 0.345,
+ -0.347,
+ -0.328,
+ 0.382,
+ 0.131,
+ 0.731,
+ 0.668
+ ],
+ "first_cross": null,
+ "final_z": 0.668
+ },
+ {
+ "id": "533845f710ea6c8ea300000e",
+ "variation": "headline",
+ "week": 201413,
+ "impressions": 21821,
+ "pooled_ctr": 0.008798863480133817,
+ "arms": [
+ {
+ "headline": "How 1 Little Studio Is Saving A Rich Culture, Rebuilding A City, And Changing 2 Young Women\u2019s Lives",
+ "impressions": 3647,
+ "clicks": 23,
+ "ctr": 0.0063065533315053,
+ "ctr_select": 0.005482456140350877,
+ "ctr_holdout": 0.007131102578167855,
+ "n_select": 1824,
+ "n_holdout": 1823,
+ "eb": 0.0071325114870102214,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Making Jewelry Makes Her Happy. It\u2019s Also Helping To Heal Her War-Torn Home.",
+ "impressions": 3615,
+ "clicks": 14,
+ "ctr": 0.0038727524204702,
+ "ctr_select": 0.00497787610619469,
+ "ctr_holdout": 0.002767017155506364,
+ "n_select": 1808,
+ "n_holdout": 1807,
+ "eb": 0.0069177890619491755,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I\u2019m Not Sure Which Is More Beautiful, This Woman\u2019s Jewelry Or The Hope And Freedom It Represents",
+ "impressions": 3711,
+ "clicks": 41,
+ "ctr": 0.0110482349770951,
+ "ctr_select": 0.011853448275862068,
+ "ctr_holdout": 0.010242587601078167,
+ "n_select": 1856,
+ "n_holdout": 1855,
+ "eb": 0.009952974411132682,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Lovely Reminder That Freedom Is The Freedom To Pursue Your Very Individual Dream",
+ "impressions": 3682,
+ "clicks": 20,
+ "ctr": 0.0054318305268875,
+ "ctr_select": 0.005431830526887561,
+ "ctr_holdout": 0.005431830526887561,
+ "n_select": 1841,
+ "n_holdout": 1841,
+ "eb": 0.007103355077025418,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Life Under The Taliban Was Ugly For Storai. Her Life After It Is All About Making Beauty.",
+ "impressions": 3562,
+ "clicks": 48,
+ "ctr": 0.0134755755193711,
+ "ctr_select": 0.01347557551937114,
+ "ctr_holdout": 0.01347557551937114,
+ "n_select": 1781,
+ "n_holdout": 1781,
+ "eb": 0.010623058735168014,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Hopes And Dreams Of One Young Woman After The Taliban Are As Beautiful As Her Art",
+ "impressions": 3604,
+ "clicks": 46,
+ "ctr": 0.0127635960044395,
+ "ctr_select": 0.009433962264150943,
+ "ctr_holdout": 0.01609322974472808,
+ "n_select": 1802,
+ "n_holdout": 1802,
+ "eb": 0.008869754305740236,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.00843108504398827,
+ "pool_holdout": 0.009166743056192136,
+ "lift_select": 0.5983204355149772,
+ "lift_holdout": 0.47005053340819747,
+ "eb_lift": 0.2599871404147107,
+ "p_value": 0.010918368974714003,
+ "held_up": false,
+ "z_path": [
+ 0.915,
+ 0.99,
+ 1.287,
+ 0.292,
+ 0.754,
+ 1.594,
+ 1.062,
+ 1.503,
+ 0.968,
+ 0.941
+ ],
+ "first_cross": null,
+ "final_z": 0.941
+ },
+ {
+ "id": "5292bcd69c05ba48da002063",
+ "variation": "image",
+ "week": 201347,
+ "impressions": 21666,
+ "pooled_ctr": 0.04107818702113911,
+ "arms": [
+ {
+ "headline": "The Day After His Dad Dies, He Waits Until The End Of His Show To Tell Us About It. Just Beautiful.",
+ "impressions": 5141,
+ "clicks": 211,
+ "ctr": 0.041042598716203,
+ "ctr_select": 0.042801556420233464,
+ "ctr_holdout": 0.03928432516530533,
+ "n_select": 2570,
+ "n_holdout": 2571,
+ "eb": 0.04233636426928261,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Day After His Dad Dies, He Waits Until The End Of His Show To Tell Us About It. Just Beautiful.",
+ "impressions": 5511,
+ "clicks": 224,
+ "ctr": 0.0406459807657412,
+ "ctr_select": 0.040638606676342524,
+ "ctr_holdout": 0.040653357531760435,
+ "n_select": 2756,
+ "n_holdout": 2755,
+ "eb": 0.0411785176461539,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Day After His Dad Dies, He Waits Until The End Of His Show To Tell Us About It. Just Beautiful.",
+ "impressions": 5437,
+ "clicks": 235,
+ "ctr": 0.0432223652749678,
+ "ctr_select": 0.044518027961736574,
+ "ctr_holdout": 0.0419271791099669,
+ "n_select": 2718,
+ "n_holdout": 2719,
+ "eb": 0.043276495779517914,
+ "labelled_winner": true
+ },
+ {
+ "headline": "The Day After His Dad Dies, He Waits Until The End Of His Show To Tell Us About It. Just Beautiful.",
+ "impressions": 5577,
+ "clicks": 220,
+ "ctr": 0.039447731755424,
+ "ctr_select": 0.03945480631276901,
+ "ctr_holdout": 0.03944065973467192,
+ "n_select": 2788,
+ "n_holdout": 2789,
+ "eb": 0.040528711731936604,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.0418205317577548,
+ "pool_holdout": 0.04033597932434927,
+ "lift_select": 0.06450171938527727,
+ "lift_holdout": 0.03944864640133039,
+ "eb_lift": 0.03481457457778814,
+ "p_value": 0.4169545252779342,
+ "held_up": true,
+ "z_path": [
+ 0.782,
+ 0.437,
+ 1.137,
+ 1.361,
+ 1.063,
+ 1.5,
+ 1.065,
+ 0.506,
+ 0.767,
+ 0.558
+ ],
+ "first_cross": null,
+ "final_z": 0.558
+ },
+ {
+ "id": "540ff3111aec8007c5000045",
+ "variation": "headline",
+ "week": 201436,
+ "impressions": 21553,
+ "pooled_ctr": 0.00909386164339071,
+ "arms": [
+ {
+ "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Being Run On Americans Today",
+ "impressions": 3606,
+ "clicks": 40,
+ "ctr": 0.0110926234054353,
+ "ctr_select": 0.011092623405435386,
+ "ctr_holdout": 0.011092623405435386,
+ "n_select": 1803,
+ "n_holdout": 1803,
+ "eb": 0.009394085945374355,
+ "labelled_winner": false
+ },
+ {
+ "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Being Run On People Today",
+ "impressions": 3612,
+ "clicks": 23,
+ "ctr": 0.0063676633444075,
+ "ctr_select": 0.005537098560354375,
+ "ctr_holdout": 0.007198228128460687,
+ "n_select": 1806,
+ "n_holdout": 1806,
+ "eb": 0.006962243706655797,
+ "labelled_winner": false
+ },
+ {
+ "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Run On College Kids Today",
+ "impressions": 3544,
+ "clicks": 27,
+ "ctr": 0.0076185101580135,
+ "ctr_select": 0.006772009029345372,
+ "ctr_holdout": 0.008465011286681716,
+ "n_select": 1772,
+ "n_holdout": 1772,
+ "eb": 0.007509158527155074,
+ "labelled_winner": false
+ },
+ {
+ "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Being Run On American Teens",
+ "impressions": 3656,
+ "clicks": 32,
+ "ctr": 0.0087527352297593,
+ "ctr_select": 0.007111597374179431,
+ "ctr_holdout": 0.010393873085339168,
+ "n_select": 1828,
+ "n_holdout": 1828,
+ "eb": 0.007648938888330842,
+ "labelled_winner": false
+ },
+ {
+ "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Being Run On Americans Today.",
+ "impressions": 3595,
+ "clicks": 47,
+ "ctr": 0.0130737134909596,
+ "ctr_select": 0.01167964404894327,
+ "ctr_holdout": 0.014468558708959377,
+ "n_select": 1798,
+ "n_holdout": 1797,
+ "eb": 0.009648470622010838,
+ "labelled_winner": false
+ },
+ {
+ "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams In The World, Student Loans",
+ "impressions": 3540,
+ "clicks": 27,
+ "ctr": 0.0076271186440677,
+ "ctr_select": 0.0062146892655367235,
+ "ctr_holdout": 0.00903954802259887,
+ "n_select": 1770,
+ "n_holdout": 1770,
+ "eb": 0.007268196070017152,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.008072747517862114,
+ "pool_holdout": 0.010115070527097253,
+ "lift_select": 0.44679912546507605,
+ "lift_holdout": 0.4303962261261123,
+ "eb_lift": 0.195190435556446,
+ "p_value": 0.06114278912043014,
+ "held_up": true,
+ "z_path": [
+ 0.004,
+ 0.612,
+ 0.86,
+ 1.039,
+ 1.313,
+ 1.138,
+ 0.736,
+ 0.827,
+ 0.806,
+ 0.769
+ ],
+ "first_cross": null,
+ "final_z": 0.769
+ },
+ {
+ "id": "53ff424c6fd2bfb3da000004",
+ "variation": "other",
+ "week": 201434,
+ "impressions": 21459,
+ "pooled_ctr": 0.017428584742998274,
+ "arms": [
+ {
+ "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.",
+ "impressions": 3524,
+ "clicks": 72,
+ "ctr": 0.0204313280363223,
+ "ctr_select": 0.019863791146424517,
+ "ctr_holdout": 0.020998864926220204,
+ "n_select": 1762,
+ "n_holdout": 1762,
+ "eb": 0.01885049473520291,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.",
+ "impressions": 3614,
+ "clicks": 69,
+ "ctr": 0.0190924183729939,
+ "ctr_select": 0.022689540675152185,
+ "ctr_holdout": 0.01549529607083564,
+ "n_select": 1807,
+ "n_holdout": 1807,
+ "eb": 0.020099489960023344,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.",
+ "impressions": 3576,
+ "clicks": 60,
+ "ctr": 0.0167785234899328,
+ "ctr_select": 0.013982102908277404,
+ "ctr_holdout": 0.019574944071588368,
+ "n_select": 1788,
+ "n_holdout": 1788,
+ "eb": 0.016295481283707218,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.",
+ "impressions": 3670,
+ "clicks": 59,
+ "ctr": 0.0160762942779291,
+ "ctr_select": 0.015258855585831062,
+ "ctr_holdout": 0.016893732970027248,
+ "n_select": 1835,
+ "n_holdout": 1835,
+ "eb": 0.01683347189573097,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.",
+ "impressions": 3567,
+ "clicks": 68,
+ "ctr": 0.0190636389122511,
+ "ctr_select": 0.02242152466367713,
+ "ctr_holdout": 0.015703869882220976,
+ "n_select": 1784,
+ "n_holdout": 1783,
+ "eb": 0.019968395040397272,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.",
+ "impressions": 3508,
+ "clicks": 46,
+ "ctr": 0.0131128848346636,
+ "ctr_select": 0.014253135689851768,
+ "ctr_holdout": 0.011972633979475485,
+ "n_select": 1754,
+ "n_holdout": 1754,
+ "eb": 0.01643155385191764,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.018080149114631874,
+ "pool_holdout": 0.016776959642091527,
+ "lift_select": 0.2549421208473348,
+ "lift_holdout": -0.07639426920002446,
+ "eb_lift": 0.11168828490232197,
+ "p_value": 0.10683029735720204,
+ "held_up": false,
+ "z_path": [
+ 0.202,
+ -1.31,
+ -1.175,
+ -1.604,
+ -1.847,
+ -1.463,
+ -1.125,
+ -0.762,
+ -0.633,
+ 0.009
+ ],
+ "first_cross": null,
+ "final_z": 0.009
+ },
+ {
+ "id": "5476066d6163610018790000",
+ "variation": "headline",
+ "week": 201447,
+ "impressions": 21384,
+ "pooled_ctr": 0.01440329218106996,
+ "arms": [
+ {
+ "headline": "The Gap Between The Rich And The Poor Is So Large, You Can Literally See It From Space",
+ "impressions": 3511,
+ "clicks": 47,
+ "ctr": 0.0133864995727712,
+ "ctr_select": 0.014236902050113895,
+ "ctr_holdout": 0.012535612535612535,
+ "n_select": 1756,
+ "n_holdout": 1755,
+ "eb": 0.014543629857968816,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's So Obvious, You Can Even See It From Outer Space",
+ "impressions": 3626,
+ "clicks": 51,
+ "ctr": 0.0140650854936569,
+ "ctr_select": 0.013237727523441808,
+ "ctr_holdout": 0.014892443463872035,
+ "n_select": 1813,
+ "n_holdout": 1813,
+ "eb": 0.014100850687793202,
+ "labelled_winner": false
+ },
+ {
+ "headline": "6 Satellite Photos That Reveal The Striking Reality Of Rich And Poor Neighborhoods Around The World",
+ "impressions": 3578,
+ "clicks": 58,
+ "ctr": 0.0162101732811626,
+ "ctr_select": 0.01732811626607043,
+ "ctr_holdout": 0.01509223029625489,
+ "n_select": 1789,
+ "n_holdout": 1789,
+ "eb": 0.015887790205699786,
+ "labelled_winner": false
+ },
+ {
+ "headline": "6 Pictures From Space That Identify A Striking Similarity In Cities Across The Globe",
+ "impressions": 3518,
+ "clicks": 29,
+ "ctr": 0.0082433200682205,
+ "ctr_select": 0.008527572484366117,
+ "ctr_holdout": 0.007959067652075043,
+ "n_select": 1759,
+ "n_holdout": 1759,
+ "eb": 0.012079961561406904,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These 6 Images Have Me Shaking My Head. Is This The World We Want To Live In?",
+ "impressions": 3583,
+ "clicks": 62,
+ "ctr": 0.0173039352497906,
+ "ctr_select": 0.01953125,
+ "ctr_holdout": 0.01507537688442211,
+ "n_select": 1792,
+ "n_holdout": 1791,
+ "eb": 0.016849500023170344,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These 6 Images Don't Shock Me. But I'm Still Shaking My Head.",
+ "impressions": 3568,
+ "clicks": 61,
+ "ctr": 0.0170964125560538,
+ "ctr_select": 0.01569506726457399,
+ "ctr_holdout": 0.018497757847533634,
+ "n_select": 1784,
+ "n_holdout": 1784,
+ "eb": 0.015175753078811617,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.01477602169643692,
+ "pool_holdout": 0.014030492937985221,
+ "lift_select": 0.32182060917721533,
+ "lift_holdout": 0.07447236180904526,
+ "eb_lift": 0.14032723890987664,
+ "p_value": 0.06745589406513958,
+ "held_up": false,
+ "z_path": [
+ -0.715,
+ -0.799,
+ -1.169,
+ -1.331,
+ -0.801,
+ -0.495,
+ -0.35,
+ 0.207,
+ 0.093,
+ 0.361
+ ],
+ "first_cross": null,
+ "final_z": 0.361
+ },
+ {
+ "id": "545c301174bfe77ac7000003",
+ "variation": "headline",
+ "week": 201444,
+ "impressions": 21339,
+ "pooled_ctr": 0.012699751628473687,
+ "arms": [
+ {
+ "headline": "The U.S. Is The Only Country In The World That Does This To 13-Year-Olds. Hint: It's Not Good.",
+ "impressions": 3544,
+ "clicks": 61,
+ "ctr": 0.0172121896162528,
+ "ctr_select": 0.016930022573363433,
+ "ctr_holdout": 0.017494356659142212,
+ "n_select": 1772,
+ "n_holdout": 1772,
+ "eb": 0.013922139821731523,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The U.S. Is The Only Country That Does This To 13-Year-Olds. Um ... Doesn't That Tell Us Something?",
+ "impressions": 3606,
+ "clicks": 56,
+ "ctr": 0.0155296727676095,
+ "ctr_select": 0.013865779256794232,
+ "ctr_holdout": 0.017193566278424846,
+ "n_select": 1803,
+ "n_holdout": 1803,
+ "eb": 0.01260401684261435,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Only 1 Country Does This Terrifying Thing To 13-Year-Olds. It's The U.S.",
+ "impressions": 3437,
+ "clicks": 66,
+ "ctr": 0.0192027931335466,
+ "ctr_select": 0.017462165308498253,
+ "ctr_holdout": 0.020942408376963352,
+ "n_select": 1718,
+ "n_holdout": 1719,
+ "eb": 0.014108426210206611,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The U.S. Is The Only Country That Sentences 13-Year-Olds To Die In Prison. That's Terrifying.",
+ "impressions": 3577,
+ "clicks": 22,
+ "ctr": 0.0061504053676265,
+ "ctr_select": 0.0044742729306487695,
+ "ctr_holdout": 0.007825600894354388,
+ "n_select": 1788,
+ "n_holdout": 1789,
+ "eb": 0.008509477751636945,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Only 1 Country Sentences Kids To Die In Prison. I Wish This Surprised Me.",
+ "impressions": 3551,
+ "clicks": 28,
+ "ctr": 0.007885102787947,
+ "ctr_select": 0.007882882882882882,
+ "ctr_holdout": 0.007887323943661971,
+ "n_select": 1776,
+ "n_holdout": 1775,
+ "eb": 0.0100000894609754,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The U.S. Is The Only Country In The World That Does This To Kids. Hint: It's Not Good.",
+ "impressions": 3624,
+ "clicks": 38,
+ "ctr": 0.010485651214128,
+ "ctr_select": 0.009381898454746136,
+ "ctr_holdout": 0.011589403973509934,
+ "n_select": 1812,
+ "n_holdout": 1812,
+ "eb": 0.010639360654225792,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.011622457587402756,
+ "pool_holdout": 0.013776944704779756,
+ "lift_select": 0.5024503360997408,
+ "lift_holdout": 0.5201054243687004,
+ "eb_lift": 0.2138935422314059,
+ "p_value": 0.013679324670803135,
+ "held_up": true,
+ "z_path": [
+ 1.22,
+ 0.649,
+ 1.183,
+ 1.491,
+ 1.002,
+ 0.902,
+ 0.874,
+ 0.739,
+ 0.448,
+ 0.622
+ ],
+ "first_cross": null,
+ "final_z": 0.622
+ },
+ {
+ "id": "548634da3831610015740000",
+ "variation": "headline",
+ "week": 201449,
+ "impressions": 21282,
+ "pooled_ctr": 0.00817592331547787,
+ "arms": [
+ {
+ "headline": "Country Music Is Actually Pretty Progressive, And These 11 Moments In 2014 Prove It",
+ "impressions": 3011,
+ "clicks": 23,
+ "ctr": 0.007638658253072,
+ "ctr_select": 0.004648074369189907,
+ "ctr_holdout": 0.0106312292358804,
+ "n_select": 1506,
+ "n_holdout": 1505,
+ "eb": 0.006388135364749224,
+ "labelled_winner": false
+ },
+ {
+ "headline": "11 Reasons You Should Definitely Love (Or At Least Respect) Country Music",
+ "impressions": 3041,
+ "clicks": 25,
+ "ctr": 0.0082209799408089,
+ "ctr_select": 0.0059210526315789476,
+ "ctr_holdout": 0.01051939513477975,
+ "n_select": 1520,
+ "n_holdout": 1521,
+ "eb": 0.006885977105276738,
+ "labelled_winner": false
+ },
+ {
+ "headline": "11 Times Country Music Proved It Ain't What You Thought It Was In 2014",
+ "impressions": 2866,
+ "clicks": 10,
+ "ctr": 0.0034891835310537,
+ "ctr_select": 0.0027913468248429866,
+ "ctr_holdout": 0.00418702023726448,
+ "n_select": 1433,
+ "n_holdout": 1433,
+ "eb": 0.005712616749841165,
+ "labelled_winner": false
+ },
+ {
+ "headline": "11 Times The Most Popular Music Genre Totally Surprised Us In 2014",
+ "impressions": 3121,
+ "clicks": 21,
+ "ctr": 0.0067286126241589,
+ "ctr_select": 0.007692307692307693,
+ "ctr_holdout": 0.005765534913516977,
+ "n_select": 1560,
+ "n_holdout": 1561,
+ "eb": 0.007588600446062911,
+ "labelled_winner": false
+ },
+ {
+ "headline": "11 Times Country Acted Like It Was 2014 In 2014",
+ "impressions": 3172,
+ "clicks": 26,
+ "ctr": 0.0081967213114754,
+ "ctr_select": 0.008827238335435058,
+ "ctr_holdout": 0.007566204287515763,
+ "n_select": 1586,
+ "n_holdout": 1586,
+ "eb": 0.00805039245066981,
+ "labelled_winner": false
+ },
+ {
+ "headline": "That Time Garth Brooks Took Us All By Surprise \u2014 And 10 Other Great Moments In 2014 Country",
+ "impressions": 2989,
+ "clicks": 14,
+ "ctr": 0.0046838407494145,
+ "ctr_select": 0.004016064257028112,
+ "ctr_holdout": 0.005351170568561873,
+ "n_select": 1494,
+ "n_holdout": 1495,
+ "eb": 0.006145896224299961,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When Two Famous Singers Just Came Out As Gay \u2014 And 10 Other Great Moments In 2014",
+ "impressions": 3082,
+ "clicks": 55,
+ "ctr": 0.017845554834523,
+ "ctr_select": 0.018170019467878003,
+ "ctr_holdout": 0.01752109020116807,
+ "n_select": 1541,
+ "n_holdout": 1541,
+ "eb": 0.011772442934964435,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 6,
+ "pool_select": 0.007518796992481203,
+ "pool_holdout": 0.008832926141702687,
+ "lift_select": 1.4166125892277743,
+ "lift_holdout": 0.9836110842641554,
+ "eb_lift": 0.56573491035027,
+ "p_value": 1.6581862847817923e-07,
+ "held_up": true,
+ "z_path": [
+ 0.035,
+ 0.956,
+ 1.603,
+ 2.217,
+ 2.638,
+ 2.888,
+ 3.119,
+ 3.238,
+ 3.108,
+ 3.374
+ ],
+ "first_cross": 3,
+ "final_z": 3.374
+ },
+ {
+ "id": "53ffc120bd37b6a7da00002a",
+ "variation": "headline",
+ "week": 201434,
+ "impressions": 20624,
+ "pooled_ctr": 0.01134600465477114,
+ "arms": [
+ {
+ "headline": "Come For The Major \u201890s Fashion Swag, Stay For The Story Of A Little Girl Called 'Mike'",
+ "impressions": 5237,
+ "clicks": 20,
+ "ctr": 0.0038189803322512,
+ "ctr_select": 0.004201680672268907,
+ "ctr_holdout": 0.003436426116838488,
+ "n_select": 2618,
+ "n_holdout": 2619,
+ "eb": 0.00769311316082316,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It Begins With A Girl Sweeping Away The Gender Binary And Ends With A Man Saying 'Thank You'",
+ "impressions": 5170,
+ "clicks": 32,
+ "ctr": 0.0061895551257253,
+ "ctr_select": 0.005415860735009671,
+ "ctr_holdout": 0.006963249516441006,
+ "n_select": 2585,
+ "n_holdout": 2585,
+ "eb": 0.008356803424906037,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Is His Fifth Year Taking Testosterone. But It\u2019s Just One Part Of Why He Couldn\u2019t Be Happier.",
+ "impressions": 5147,
+ "clicks": 43,
+ "ctr": 0.0083543811929279,
+ "ctr_select": 0.009324009324009324,
+ "ctr_holdout": 0.007384376214535562,
+ "n_select": 2574,
+ "n_holdout": 2573,
+ "eb": 0.010419875480260838,
+ "labelled_winner": false
+ },
+ {
+ "headline": "5 Years' Worth Of Photos Show What Testosterone Did To One Guy's Life",
+ "impressions": 5070,
+ "clicks": 139,
+ "ctr": 0.0274161735700197,
+ "ctr_select": 0.028007889546351084,
+ "ctr_holdout": 0.026824457593688362,
+ "n_select": 2535,
+ "n_holdout": 2535,
+ "eb": 0.020188951987635012,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.011636927851047323,
+ "pool_holdout": 0.011055081458494957,
+ "lift_select": 1.40681130834977,
+ "lift_holdout": 1.426436900930828,
+ "eb_lift": 0.734903940804102,
+ "p_value": 8.73934596872334e-19,
+ "held_up": true,
+ "z_path": [
+ 2.119,
+ 2.151,
+ 3.368,
+ 3.843,
+ 4.406,
+ 5.619,
+ 6.182,
+ 6.445,
+ 6.577,
+ 7.283
+ ],
+ "first_cross": 0,
+ "final_z": 7.283
+ },
+ {
+ "id": "53a993bbf98fb2657b00007d",
+ "variation": "headline",
+ "week": 201425,
+ "impressions": 20420,
+ "pooled_ctr": 0.011998041136141039,
+ "arms": [
+ {
+ "headline": "The List She's Rattling Off Made Me Exhausted, But I Think That's The Point",
+ "impressions": 4119,
+ "clicks": 50,
+ "ctr": 0.0121388686574411,
+ "ctr_select": 0.01262135922330097,
+ "ctr_holdout": 0.011656143759106362,
+ "n_select": 2060,
+ "n_holdout": 2059,
+ "eb": 0.011746314498081327,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The List She's Rattling Off Is Exhausting, But I Think That's The Point",
+ "impressions": 4027,
+ "clicks": 57,
+ "ctr": 0.0141544574124658,
+ "ctr_select": 0.013902681231380337,
+ "ctr_holdout": 0.014406358668653751,
+ "n_select": 2014,
+ "n_holdout": 2013,
+ "eb": 0.012332765963564255,
+ "labelled_winner": true
+ },
+ {
+ "headline": "The List She's Rattling Off Is Sorta Overwhelming, But Spoiler Alert: That's The Point",
+ "impressions": 4050,
+ "clicks": 56,
+ "ctr": 0.0138271604938271,
+ "ctr_select": 0.012839506172839505,
+ "ctr_holdout": 0.014814814814814815,
+ "n_select": 2025,
+ "n_holdout": 2025,
+ "eb": 0.011840983995107135,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Stats She's Rattling Off Are A Bit Overwhelming, But I Think That's The Point",
+ "impressions": 4085,
+ "clicks": 35,
+ "ctr": 0.0085679314565483,
+ "ctr_select": 0.007345739471106758,
+ "ctr_holdout": 0.00978952520802741,
+ "n_select": 2042,
+ "n_holdout": 2043,
+ "eb": 0.009271701102852851,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Just When You Think There Can't Possibly Be More Stats On Body Image, She Keeps Going",
+ "impressions": 4139,
+ "clicks": 47,
+ "ctr": 0.0113553998550374,
+ "ctr_select": 0.00821256038647343,
+ "ctr_holdout": 0.014499758337361043,
+ "n_select": 2070,
+ "n_holdout": 2069,
+ "eb": 0.009668353854769185,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.010968563314073058,
+ "pool_holdout": 0.013027720638652169,
+ "lift_select": 0.26750248262164833,
+ "lift_holdout": 0.10582342592696348,
+ "eb_lift": 0.12437386833888042,
+ "p_value": 0.15823787741797757,
+ "held_up": false,
+ "z_path": [
+ -0.83,
+ -0.889,
+ -0.657,
+ -0.28,
+ -0.789,
+ -0.604,
+ -0.56,
+ 0.135,
+ -0.074,
+ 0.125
+ ],
+ "first_cross": null,
+ "final_z": 0.125
+ },
+ {
+ "id": "5351565b6948478edc00002d",
+ "variation": "headline",
+ "week": 201415,
+ "impressions": 20325,
+ "pooled_ctr": 0.011168511685116851,
+ "arms": [
+ {
+ "headline": "The Numbers Don't Lie, These Students Are Keeping Their Classmates Out Of Juvie",
+ "impressions": 4092,
+ "clicks": 26,
+ "ctr": 0.0063538611925708,
+ "ctr_select": 0.003421309872922776,
+ "ctr_holdout": 0.009286412512218964,
+ "n_select": 2046,
+ "n_holdout": 2046,
+ "eb": 0.00745740970711921,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How Some Kids Put Their Faculty To Shame AND Made Them Proud With A New Kind Of School Discipline",
+ "impressions": 4114,
+ "clicks": 53,
+ "ctr": 0.0128828390860476,
+ "ctr_select": 0.013612056392805057,
+ "ctr_holdout": 0.012153621779290228,
+ "n_select": 2057,
+ "n_holdout": 2057,
+ "eb": 0.012238990036901327,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This School Struggled With Detentions, So They Asked For The Students Help. Guess What? It's Working.",
+ "impressions": 4044,
+ "clicks": 78,
+ "ctr": 0.0192878338278931,
+ "ctr_select": 0.02126607319485658,
+ "ctr_holdout": 0.01730959446092977,
+ "n_select": 2022,
+ "n_holdout": 2022,
+ "eb": 0.015794160396997214,
+ "labelled_winner": false
+ },
+ {
+ "headline": "In One School, Students Serve As Judge, Jury, And Encouragers For Their Troubled Classmates",
+ "impressions": 4000,
+ "clicks": 31,
+ "ctr": 0.00775,
+ "ctr_select": 0.005,
+ "ctr_holdout": 0.0105,
+ "n_select": 2000,
+ "n_holdout": 2000,
+ "eb": 0.008231674390572943,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Instead Of Detention, These Students Are Going To Trial With Their Fellow Classmates As Jury",
+ "impressions": 4075,
+ "clicks": 39,
+ "ctr": 0.0095705521472392,
+ "ctr_select": 0.011776251226692836,
+ "ctr_holdout": 0.007363770250368188,
+ "n_select": 2038,
+ "n_holdout": 2037,
+ "eb": 0.011374040390535703,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.011020368001574339,
+ "pool_holdout": 0.011316669946860853,
+ "lift_select": 0.9297062667797089,
+ "lift_holdout": 0.529566077495377,
+ "eb_lift": 0.4331790367382382,
+ "p_value": 8.191541128689619e-07,
+ "held_up": true,
+ "z_path": [
+ 1.429,
+ 1.571,
+ 1.473,
+ 2.019,
+ 1.808,
+ 2.156,
+ 1.917,
+ 2.528,
+ 2.443,
+ 2.301
+ ],
+ "first_cross": 3,
+ "final_z": 2.301
+ },
+ {
+ "id": "5283e278a4e712a50a001a2d",
+ "variation": "image",
+ "week": 201345,
+ "impressions": 20297,
+ "pooled_ctr": 0.018968320441444548,
+ "arms": [
+ {
+ "headline": "A Dude Dresses Up As A Superhero and New Yorkers Can\u2019t Even Handle It",
+ "impressions": 5184,
+ "clicks": 109,
+ "ctr": 0.0210262345679012,
+ "ctr_select": 0.019675925925925927,
+ "ctr_holdout": 0.022376543209876542,
+ "n_select": 2592,
+ "n_holdout": 2592,
+ "eb": 0.01917951794211326,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Dude Dresses Up As A Superhero and New Yorkers Can\u2019t Even Handle It",
+ "impressions": 5289,
+ "clicks": 91,
+ "ctr": 0.0172055208924182,
+ "ctr_select": 0.01664145234493192,
+ "ctr_holdout": 0.01776937618147448,
+ "n_select": 2644,
+ "n_holdout": 2645,
+ "eb": 0.017567703718622683,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Dude Dresses Up As A Superhero and New Yorkers Can\u2019t Even Handle It",
+ "impressions": 4793,
+ "clicks": 79,
+ "ctr": 0.0164823701230961,
+ "ctr_select": 0.017529215358931552,
+ "ctr_holdout": 0.01543596161869003,
+ "n_select": 2396,
+ "n_holdout": 2397,
+ "eb": 0.018067691344027977,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Dude Dresses Up As A Superhero and New Yorkers Can\u2019t Even Handle It",
+ "impressions": 5031,
+ "clicks": 106,
+ "ctr": 0.0210693699065792,
+ "ctr_select": 0.02066772655007949,
+ "ctr_holdout": 0.02147117296222664,
+ "n_select": 2516,
+ "n_holdout": 2515,
+ "eb": 0.019687957549589005,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.018624359479700432,
+ "pool_holdout": 0.019312247512070155,
+ "lift_select": 0.10971475677358034,
+ "lift_holdout": 0.11179048160019467,
+ "eb_lift": 0.05710790059909643,
+ "p_value": 0.3820053508156698,
+ "held_up": false,
+ "z_path": [
+ 0.486,
+ -0.203,
+ -0.838,
+ -1.147,
+ -0.701,
+ -0.779,
+ -0.299,
+ -0.334,
+ 0.07,
+ 0.015
+ ],
+ "first_cross": null,
+ "final_z": 0.015
+ },
+ {
+ "id": "54b86431356636001f4b0000",
+ "variation": "other",
+ "week": 201502,
+ "impressions": 20135,
+ "pooled_ctr": 0.007648373479016638,
+ "arms": [
+ {
+ "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.",
+ "impressions": 3411,
+ "clicks": 34,
+ "ctr": 0.0099677513925535,
+ "ctr_select": 0.009378663540445486,
+ "ctr_holdout": 0.010557184750733138,
+ "n_select": 1706,
+ "n_holdout": 1705,
+ "eb": 0.008153343393340997,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.",
+ "impressions": 3434,
+ "clicks": 19,
+ "ctr": 0.0055329062317996,
+ "ctr_select": 0.0052417006406523005,
+ "ctr_holdout": 0.005824111822947,
+ "n_select": 1717,
+ "n_holdout": 1717,
+ "eb": 0.006396164219995819,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.",
+ "impressions": 3313,
+ "clicks": 27,
+ "ctr": 0.00814971325083,
+ "ctr_select": 0.007246376811594203,
+ "ctr_holdout": 0.009052504526252263,
+ "n_select": 1656,
+ "n_holdout": 1657,
+ "eb": 0.0072493156848055604,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.",
+ "impressions": 3350,
+ "clicks": 32,
+ "ctr": 0.0095522388059701,
+ "ctr_select": 0.00955223880597015,
+ "ctr_holdout": 0.00955223880597015,
+ "n_select": 1675,
+ "n_holdout": 1675,
+ "eb": 0.008216647370724563,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.",
+ "impressions": 3261,
+ "clicks": 13,
+ "ctr": 0.0039865072063784,
+ "ctr_select": 0.005521472392638037,
+ "ctr_holdout": 0.002452483139178418,
+ "n_select": 1630,
+ "n_holdout": 1631,
+ "eb": 0.006537123920756696,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.",
+ "impressions": 3366,
+ "clicks": 29,
+ "ctr": 0.0086155674390968,
+ "ctr_select": 0.006535947712418301,
+ "ctr_holdout": 0.0106951871657754,
+ "n_select": 1683,
+ "n_holdout": 1683,
+ "eb": 0.006950434936553902,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.007251415516042515,
+ "pool_holdout": 0.008045292014302742,
+ "lift_select": 0.3172929871191985,
+ "lift_holdout": 0.18730790491984517,
+ "eb_lift": 0.13310943946690657,
+ "p_value": 0.2241520111019315,
+ "held_up": false,
+ "z_path": [
+ 0.033,
+ -0.222,
+ 0.255,
+ 0.247,
+ 0.784,
+ 0.382,
+ -0.084,
+ -0.207,
+ -0.331,
+ -0.174
+ ],
+ "first_cross": null,
+ "final_z": -0.174
+ },
+ {
+ "id": "5343745daf7ae25546000034",
+ "variation": "image",
+ "week": 201414,
+ "impressions": 20124,
+ "pooled_ctr": 0.030659908566885313,
+ "arms": [
+ {
+ "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return",
+ "impressions": 2560,
+ "clicks": 69,
+ "ctr": 0.026953125,
+ "ctr_select": 0.0265625,
+ "ctr_holdout": 0.02734375,
+ "n_select": 1280,
+ "n_holdout": 1280,
+ "eb": 0.030258528066759122,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return",
+ "impressions": 2494,
+ "clicks": 92,
+ "ctr": 0.036888532477947,
+ "ctr_select": 0.03608660785886127,
+ "ctr_holdout": 0.03769045709703288,
+ "n_select": 1247,
+ "n_holdout": 1247,
+ "eb": 0.033624460445991385,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return",
+ "impressions": 2427,
+ "clicks": 57,
+ "ctr": 0.0234857849196538,
+ "ctr_select": 0.0214168039538715,
+ "ctr_holdout": 0.025556471558120363,
+ "n_select": 1214,
+ "n_holdout": 1213,
+ "eb": 0.028558796761976986,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return",
+ "impressions": 2537,
+ "clicks": 83,
+ "ctr": 0.0327158060701616,
+ "ctr_select": 0.031545741324921134,
+ "ctr_holdout": 0.0338849487785658,
+ "n_select": 1268,
+ "n_holdout": 1269,
+ "eb": 0.0320331066455884,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return",
+ "impressions": 2542,
+ "clicks": 94,
+ "ctr": 0.036978756884343,
+ "ctr_select": 0.03540519276160504,
+ "ctr_holdout": 0.03855232100708104,
+ "n_select": 1271,
+ "n_holdout": 1271,
+ "eb": 0.033399591394005834,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return",
+ "impressions": 2547,
+ "clicks": 78,
+ "ctr": 0.0306242638398115,
+ "ctr_select": 0.033751962323390894,
+ "ctr_holdout": 0.02749410840534171,
+ "n_select": 1274,
+ "n_holdout": 1273,
+ "eb": 0.032814854329003884,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return",
+ "impressions": 2401,
+ "clicks": 71,
+ "ctr": 0.0295710120783007,
+ "ctr_select": 0.041666666666666664,
+ "ctr_holdout": 0.017485428809325562,
+ "n_select": 1200,
+ "n_holdout": 1201,
+ "eb": 0.03549510147131108,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return",
+ "impressions": 2616,
+ "clicks": 73,
+ "ctr": 0.0279051987767584,
+ "ctr_select": 0.03211009174311927,
+ "ctr_holdout": 0.023700305810397553,
+ "n_select": 1308,
+ "n_holdout": 1308,
+ "eb": 0.03223132386961618,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 6,
+ "pool_select": 0.03229974160206718,
+ "pool_holdout": 0.029020075531703438,
+ "lift_select": 0.29000000000000004,
+ "lift_holdout": -0.3974712853444048,
+ "eb_lift": 0.0989283415517912,
+ "p_value": 0.05050487627726831,
+ "held_up": false,
+ "z_path": [
+ 0.859,
+ 0.115,
+ -0.448,
+ -0.72,
+ -1.099,
+ -1.602,
+ -2.035,
+ -1.955,
+ -1.695,
+ -1.426
+ ],
+ "first_cross": 6,
+ "final_z": -1.426
+ },
+ {
+ "id": "54cc4259343963000c710000",
+ "variation": "both",
+ "week": 201504,
+ "impressions": 19858,
+ "pooled_ctr": 0.007855776009668648,
+ "arms": [
+ {
+ "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was Money.",
+ "impressions": 2838,
+ "clicks": 24,
+ "ctr": 0.0084566596194503,
+ "ctr_select": 0.010570824524312896,
+ "ctr_holdout": 0.006342494714587738,
+ "n_select": 1419,
+ "n_holdout": 1419,
+ "eb": 0.009324089476860154,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.",
+ "impressions": 2859,
+ "clicks": 24,
+ "ctr": 0.0083945435466946,
+ "ctr_select": 0.00909090909090909,
+ "ctr_holdout": 0.007697690692792162,
+ "n_select": 1430,
+ "n_holdout": 1429,
+ "eb": 0.008763061040216088,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.",
+ "impressions": 2843,
+ "clicks": 19,
+ "ctr": 0.0066830812521983,
+ "ctr_select": 0.006329113924050633,
+ "ctr_holdout": 0.007037297677691766,
+ "n_select": 1422,
+ "n_holdout": 1421,
+ "eb": 0.00771220190077056,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.",
+ "impressions": 2908,
+ "clicks": 23,
+ "ctr": 0.0079092159559834,
+ "ctr_select": 0.006189821182943604,
+ "ctr_holdout": 0.009628610729023384,
+ "n_select": 1454,
+ "n_holdout": 1454,
+ "eb": 0.007646769705469904,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.",
+ "impressions": 2745,
+ "clicks": 19,
+ "ctr": 0.0069216757741347,
+ "ctr_select": 0.008746355685131196,
+ "ctr_holdout": 0.005098324836125273,
+ "n_select": 1372,
+ "n_holdout": 1373,
+ "eb": 0.008629786582785645,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.",
+ "impressions": 2860,
+ "clicks": 24,
+ "ctr": 0.0083916083916083,
+ "ctr_select": 0.009790209790209791,
+ "ctr_holdout": 0.006993006993006993,
+ "n_select": 1430,
+ "n_holdout": 1430,
+ "eb": 0.009029891526240471,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.",
+ "impressions": 2805,
+ "clicks": 23,
+ "ctr": 0.0081996434937611,
+ "ctr_select": 0.009272467902995721,
+ "ctr_holdout": 0.007127583749109052,
+ "n_select": 1402,
+ "n_holdout": 1403,
+ "eb": 0.008829024924544742,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.008560781548997885,
+ "pool_holdout": 0.00715077047033941,
+ "lift_select": 0.23479666708120872,
+ "lift_holdout": -0.11303337998392049,
+ "eb_lift": 0.08916334606758203,
+ "p_value": 0.37467075637699543,
+ "held_up": false,
+ "z_path": [
+ 1.434,
+ 1.282,
+ 1.405,
+ 2.206,
+ 1.907,
+ 1.548,
+ 0.984,
+ 0.021,
+ 0.188,
+ 0.027
+ ],
+ "first_cross": 3,
+ "final_z": 0.027
+ },
+ {
+ "id": "542aea408de8acc10600004b",
+ "variation": "image",
+ "week": 201439,
+ "impressions": 19685,
+ "pooled_ctr": 0.0063500127000254,
+ "arms": [
+ {
+ "headline": "There's An Incredibly Simple Reason Kids Are Dying Of Preventable Things. Hint: Rhymes With Bunny.",
+ "impressions": 4929,
+ "clicks": 23,
+ "ctr": 0.0046662609048488,
+ "ctr_select": 0.00487012987012987,
+ "ctr_holdout": 0.004462474645030426,
+ "n_select": 2464,
+ "n_holdout": 2465,
+ "eb": 0.0063509369453476645,
+ "labelled_winner": false
+ },
+ {
+ "headline": "There's An Incredibly Simple Reason Kids Are Dying Of Preventable Things. Hint: Rhymes With Bunny.",
+ "impressions": 4886,
+ "clicks": 24,
+ "ctr": 0.0049119934506753,
+ "ctr_select": 0.005321326238231682,
+ "ctr_holdout": 0.0045026606631191155,
+ "n_select": 2443,
+ "n_holdout": 2443,
+ "eb": 0.006589004923230203,
+ "labelled_winner": false
+ },
+ {
+ "headline": "There's An Incredibly Simple Reason Kids Are Dying Of Preventable Things. Hint: Rhymes With Bunny.",
+ "impressions": 4922,
+ "clicks": 49,
+ "ctr": 0.0099553027224705,
+ "ctr_select": 0.01381552214546932,
+ "ctr_holdout": 0.006095083299471759,
+ "n_select": 2461,
+ "n_holdout": 2461,
+ "eb": 0.01095868871618744,
+ "labelled_winner": false
+ },
+ {
+ "headline": "There's An Incredibly Simple Reason Kids Are Dying Of Preventable Things. Hint: Rhymes With Bunny.",
+ "impressions": 4948,
+ "clicks": 29,
+ "ctr": 0.005860953920776,
+ "ctr_select": 0.007679870654810024,
+ "ctr_holdout": 0.004042037186742118,
+ "n_select": 2474,
+ "n_holdout": 2474,
+ "eb": 0.007798543004435674,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.007925218451534242,
+ "pool_holdout": 0.004774966981611298,
+ "lift_select": 0.7432354994321673,
+ "lift_holdout": 0.27646606205745794,
+ "eb_lift": 0.38276172236816386,
+ "p_value": 0.0001415780554587344,
+ "held_up": true,
+ "z_path": [
+ 1.426,
+ 1.106,
+ 1.241,
+ 1.847,
+ 1.831,
+ 1.834,
+ 1.78,
+ 1.667,
+ 1.943,
+ 2.297
+ ],
+ "first_cross": 9,
+ "final_z": 2.297
+ },
+ {
+ "id": "544e5bc36fba94112e000026",
+ "variation": "headline",
+ "week": 201443,
+ "impressions": 19533,
+ "pooled_ctr": 0.01668970460246762,
+ "arms": [
+ {
+ "headline": "John Oliver Just Came Up With The Best Way To Hold A Massively Shady Industry Accountable",
+ "impressions": 3941,
+ "clicks": 71,
+ "ctr": 0.0180157320477036,
+ "ctr_select": 0.02182741116751269,
+ "ctr_holdout": 0.014205986808726534,
+ "n_select": 1970,
+ "n_holdout": 1971,
+ "eb": 0.019604204303222503,
+ "labelled_winner": false
+ },
+ {
+ "headline": "John Oliver Just Found The Best Way To Call Out An Industry That Sells Cocaine-Like Products To Kids",
+ "impressions": 3902,
+ "clicks": 61,
+ "ctr": 0.0156330087134802,
+ "ctr_select": 0.01742696053305997,
+ "ctr_holdout": 0.013839056893900564,
+ "n_select": 1951,
+ "n_holdout": 1951,
+ "eb": 0.017583091374829236,
+ "labelled_winner": false
+ },
+ {
+ "headline": "John Oliver Just Started #ShowUsYourPeanuts To Call Out An Massively Shady Industry. Sign Me Up!",
+ "impressions": 3914,
+ "clicks": 59,
+ "ctr": 0.015074092999489,
+ "ctr_select": 0.013796627491057742,
+ "ctr_holdout": 0.016351558507920288,
+ "n_select": 1957,
+ "n_holdout": 1957,
+ "eb": 0.015920868911578567,
+ "labelled_winner": false
+ },
+ {
+ "headline": "John Oliver Wants YOU To Tweet A Massively Shady Industry #ShowUsYourPeanuts For The Best Reason",
+ "impressions": 3877,
+ "clicks": 60,
+ "ctr": 0.0154758834150116,
+ "ctr_select": 0.01805985552115583,
+ "ctr_holdout": 0.012893243940175348,
+ "n_select": 1938,
+ "n_holdout": 1939,
+ "eb": 0.017871782036887995,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Just In Time For Halloween, John Oliver Came Up With A Genius Way To Expose This Shady Industry",
+ "impressions": 3899,
+ "clicks": 75,
+ "ctr": 0.0192357014619133,
+ "ctr_select": 0.017435897435897435,
+ "ctr_holdout": 0.02103642893791688,
+ "n_select": 1950,
+ "n_holdout": 1949,
+ "eb": 0.01758721141626416,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.017714519762441122,
+ "pool_holdout": 0.015664994368792873,
+ "lift_select": 0.2321762859071037,
+ "lift_holdout": -0.09313808391613032,
+ "eb_lift": 0.10667433078191313,
+ "p_value": 0.12140930497342717,
+ "held_up": false,
+ "z_path": [
+ 1.479,
+ 2.579,
+ 2.819,
+ 2.179,
+ 1.842,
+ 1.488,
+ 0.899,
+ 0.832,
+ 1.323,
+ 0.875
+ ],
+ "first_cross": 1,
+ "final_z": 0.875
+ },
+ {
+ "id": "5258cf57b018770da500029f",
+ "variation": "image",
+ "week": 201340,
+ "impressions": 19259,
+ "pooled_ctr": 0.012980943974245807,
+ "arms": [
+ {
+ "headline": "Heaven Caught On Tape. More Beautiful Than You Imagined, Closer Than You\u2019d Expect.",
+ "impressions": 4797,
+ "clicks": 71,
+ "ctr": 0.0148009172399416,
+ "ctr_select": 0.013761467889908258,
+ "ctr_holdout": 0.015839933305543976,
+ "n_select": 2398,
+ "n_holdout": 2399,
+ "eb": 0.013225056189516589,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Heaven Caught On Tape. More Beautiful Than You Imagined, Closer Than You\u2019d Expect.",
+ "impressions": 4816,
+ "clicks": 55,
+ "ctr": 0.0114202657807308,
+ "ctr_select": 0.009966777408637873,
+ "ctr_holdout": 0.01287375415282392,
+ "n_select": 2408,
+ "n_holdout": 2408,
+ "eb": 0.011292590100342115,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Heaven Caught On Tape. More Beautiful Than You Imagined, Closer Than You\u2019d Expect.",
+ "impressions": 4904,
+ "clicks": 51,
+ "ctr": 0.0103996737357259,
+ "ctr_select": 0.010603588907014683,
+ "ctr_holdout": 0.010195758564437194,
+ "n_select": 2452,
+ "n_holdout": 2452,
+ "eb": 0.011607730834339688,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Heaven Caught On Tape. More Beautiful Than You Imagined, Closer Than You\u2019d Expect.",
+ "impressions": 4742,
+ "clicks": 73,
+ "ctr": 0.0153943483762125,
+ "ctr_select": 0.016448755799240825,
+ "ctr_holdout": 0.01433994095318431,
+ "n_select": 2371,
+ "n_holdout": 2371,
+ "eb": 0.014580886084635187,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.012670059196178212,
+ "pool_holdout": 0.013291796469366563,
+ "lift_select": 0.2982382753351631,
+ "lift_holdout": 0.07885649514972592,
+ "eb_lift": 0.1508143615487887,
+ "p_value": 0.05811539921166236,
+ "held_up": false,
+ "z_path": [
+ -1.492,
+ -0.683,
+ -0.252,
+ 0.045,
+ 0.168,
+ 0.383,
+ 0.882,
+ 1.0,
+ 0.155,
+ 0.238
+ ],
+ "first_cross": null,
+ "final_z": 0.238
+ },
+ {
+ "id": "51a92833b8852eea050083c4",
+ "variation": "image",
+ "week": 201321,
+ "impressions": 19205,
+ "pooled_ctr": 0.016974746159854205,
+ "arms": [
+ {
+ "headline": "Obama's Speech Writer/Stand-Up Comedian Gives A Graduation Speech. Not Surprisingly, It's Badass.",
+ "impressions": 4831,
+ "clicks": 88,
+ "ctr": 0.0182156903332643,
+ "ctr_select": 0.018625827814569538,
+ "ctr_holdout": 0.01780538302277433,
+ "n_select": 2416,
+ "n_holdout": 2415,
+ "eb": 0.017919845598968823,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Obama's Speech Writer/Stand-Up Comedian Gives A Graduation Speech. Not Surprisingly, It's Badass.",
+ "impressions": 4852,
+ "clicks": 99,
+ "ctr": 0.0204039571310799,
+ "ctr_select": 0.018961253091508656,
+ "ctr_holdout": 0.021846661170651278,
+ "n_select": 2426,
+ "n_holdout": 2426,
+ "eb": 0.018092875499927406,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Obama's Speech Writer/Stand-Up Comedian Gives A Graduation Speech. Not Surprisingly, It's Badass.",
+ "impressions": 4697,
+ "clicks": 69,
+ "ctr": 0.0146902278049819,
+ "ctr_select": 0.014054514480408859,
+ "ctr_holdout": 0.01532567049808429,
+ "n_select": 2348,
+ "n_holdout": 2349,
+ "eb": 0.015609054977330061,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Obama's Speech Writer/Stand-Up Comedian Gives A Graduation Speech. Not Surprisingly, It's Badass.",
+ "impressions": 4825,
+ "clicks": 70,
+ "ctr": 0.0145077720207253,
+ "ctr_select": 0.016998341625207296,
+ "ctr_holdout": 0.012018234562784915,
+ "n_select": 2412,
+ "n_holdout": 2413,
+ "eb": 0.017089280758904066,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.017183920016663193,
+ "pool_holdout": 0.016765594085181714,
+ "lift_select": 0.10343001324040069,
+ "lift_holdout": 0.3030651380233802,
+ "eb_lift": 0.052895700304866455,
+ "p_value": 0.4358538102105517,
+ "held_up": true,
+ "z_path": [
+ -0.441,
+ -0.913,
+ -0.407,
+ -0.375,
+ -0.231,
+ -0.409,
+ 0.067,
+ 0.99,
+ 0.993,
+ 0.782
+ ],
+ "first_cross": null,
+ "final_z": 0.782
+ },
+ {
+ "id": "5342b02edf3e0e1d32000008",
+ "variation": "headline",
+ "week": 201414,
+ "impressions": 19139,
+ "pooled_ctr": 0.011547102774439625,
+ "arms": [
+ {
+ "headline": "Half Of All Human Deaths Since The Stone Age Can Be Linked To ... Mosquitoes?",
+ "impressions": 3835,
+ "clicks": 17,
+ "ctr": 0.0044328552803129,
+ "ctr_select": 0.0031282586027111575,
+ "ctr_holdout": 0.005738132498695879,
+ "n_select": 1918,
+ "n_holdout": 1917,
+ "eb": 0.007991864434868825,
+ "labelled_winner": false
+ },
+ {
+ "headline": "We Now Have The Technology To Eliminate The Mosquito. Should We?",
+ "impressions": 3792,
+ "clicks": 33,
+ "ctr": 0.0087025316455696,
+ "ctr_select": 0.00949367088607595,
+ "ctr_holdout": 0.007911392405063292,
+ "n_select": 1896,
+ "n_holdout": 1896,
+ "eb": 0.01088144548008152,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Could Press A Button And Murder Ever Mosquito, Would You? Because That's Kinda Possible.",
+ "impressions": 3802,
+ "clicks": 66,
+ "ctr": 0.0173592845870594,
+ "ctr_select": 0.016833245660178853,
+ "ctr_holdout": 0.01788532351394003,
+ "n_select": 1901,
+ "n_holdout": 1901,
+ "eb": 0.014187108447418717,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's Not Often That Humanity Has The Chance To Cause The Extinction Of Something That's Actually Bad",
+ "impressions": 3880,
+ "clicks": 50,
+ "ctr": 0.0128865979381443,
+ "ctr_select": 0.014948453608247423,
+ "ctr_holdout": 0.010824742268041237,
+ "n_select": 1940,
+ "n_holdout": 1940,
+ "eb": 0.01335255008373267,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Suck. They Itch. They Murder. Now We Can Murder Them All. Should We?",
+ "impressions": 3830,
+ "clicks": 55,
+ "ctr": 0.0143603133159268,
+ "ctr_select": 0.015665796344647518,
+ "ctr_holdout": 0.013054830287206266,
+ "n_select": 1915,
+ "n_holdout": 1915,
+ "eb": 0.013667671513414026,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.012016718913270637,
+ "pool_holdout": 0.011077437558783572,
+ "lift_select": 0.4008187910253185,
+ "lift_holdout": 0.6145722708008694,
+ "eb_lift": 0.18061415514606205,
+ "p_value": 0.03131873465938484,
+ "held_up": true,
+ "z_path": [
+ -0.373,
+ 0.011,
+ 1.283,
+ 1.264,
+ 0.717,
+ 0.525,
+ 0.493,
+ 0.035,
+ 0.625,
+ 1.049
+ ],
+ "first_cross": null,
+ "final_z": 1.049
+ },
+ {
+ "id": "54b7c3d465656200132d0000",
+ "variation": "headline",
+ "week": 201502,
+ "impressions": 19032,
+ "pooled_ctr": 0.007356031946195881,
+ "arms": [
+ {
+ "headline": "The Video That Made You Cry, Laugh, Spit Nails, Or Go 'Awwwwww' Is Being Missed By A Lot Of People",
+ "impressions": 3782,
+ "clicks": 29,
+ "ctr": 0.0076679005817028,
+ "ctr_select": 0.011634056054997356,
+ "ctr_holdout": 0.0037017451084082496,
+ "n_select": 1891,
+ "n_holdout": 1891,
+ "eb": 0.009914752502363839,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Videos That Make You Cry, Laugh, Spit Nails, Or Go 'Awwwwww' Are Being Missed By A Lot Of People",
+ "impressions": 3939,
+ "clicks": 29,
+ "ctr": 0.0073622746890073,
+ "ctr_select": 0.007614213197969543,
+ "ctr_holdout": 0.007110208227526663,
+ "n_select": 1970,
+ "n_holdout": 1969,
+ "eb": 0.00809949732724397,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What If The Videos That Make You Cry, Laugh, Or Spit Nails Are Being Missed By A Lot Of People?",
+ "impressions": 3790,
+ "clicks": 16,
+ "ctr": 0.004221635883905,
+ "ctr_select": 0.005277044854881266,
+ "ctr_holdout": 0.0031662269129287598,
+ "n_select": 1895,
+ "n_holdout": 1895,
+ "eb": 0.007056807343667719,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Know, It Hadn't Even Occurred To Me That Doing Something So Simple Would Include So Many People",
+ "impressions": 3760,
+ "clicks": 24,
+ "ctr": 0.0063829787234042,
+ "ctr_select": 0.005319148936170213,
+ "ctr_holdout": 0.007446808510638298,
+ "n_select": 1880,
+ "n_holdout": 1880,
+ "eb": 0.007082024049967974,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Pretty Much A No-Brainer. So Why Are So Many People WITH Brains Forgetting Do To This?",
+ "impressions": 3761,
+ "clicks": 42,
+ "ctr": 0.0111672427545865,
+ "ctr_select": 0.01276595744680851,
+ "ctr_holdout": 0.009569377990430622,
+ "n_select": 1880,
+ "n_holdout": 1881,
+ "eb": 0.01041718560401326,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.008511979823455234,
+ "pool_holdout": 0.006200084068936528,
+ "lift_select": 0.4997635933806144,
+ "lift_holdout": 0.5434271348633524,
+ "eb_lift": 0.22382639762703915,
+ "p_value": 0.02500352120313054,
+ "held_up": true,
+ "z_path": [
+ 0.008,
+ 0.591,
+ 0.443,
+ 0.775,
+ 1.022,
+ 1.117,
+ 1.157,
+ 1.438,
+ 1.892,
+ 1.574
+ ],
+ "first_cross": null,
+ "final_z": 1.574
+ },
+ {
+ "id": "52791ac5ea4d32dc8b004160",
+ "variation": "image",
+ "week": 201344,
+ "impressions": 18713,
+ "pooled_ctr": 0.016352268476460215,
+ "arms": [
+ {
+ "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.",
+ "impressions": 3979,
+ "clicks": 51,
+ "ctr": 0.012817290776577,
+ "ctr_select": 0.012060301507537688,
+ "ctr_holdout": 0.013574660633484163,
+ "n_select": 1990,
+ "n_holdout": 1989,
+ "eb": 0.015290119061016731,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.",
+ "impressions": 3861,
+ "clicks": 56,
+ "ctr": 0.0145040145040145,
+ "ctr_select": 0.018134715025906734,
+ "ctr_holdout": 0.010875194199896427,
+ "n_select": 1930,
+ "n_holdout": 1931,
+ "eb": 0.018095734707972804,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.",
+ "impressions": 3813,
+ "clicks": 54,
+ "ctr": 0.014162077104642,
+ "ctr_select": 0.014690451206715634,
+ "ctr_holdout": 0.013633980073413739,
+ "n_select": 1906,
+ "n_holdout": 1907,
+ "eb": 0.016541243163796977,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.",
+ "impressions": 3415,
+ "clicks": 80,
+ "ctr": 0.0234260614934114,
+ "ctr_select": 0.026346604215456676,
+ "ctr_holdout": 0.020503807850029292,
+ "n_select": 1708,
+ "n_holdout": 1707,
+ "eb": 0.021577678493526813,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.",
+ "impressions": 3645,
+ "clicks": 65,
+ "ctr": 0.017832647462277,
+ "ctr_select": 0.02030735455543359,
+ "ctr_holdout": 0.015359297860669226,
+ "n_select": 1822,
+ "n_holdout": 1823,
+ "eb": 0.01905095888872262,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.018063274903805045,
+ "pool_holdout": 0.014641444907555841,
+ "lift_select": 0.4585729528982998,
+ "lift_holdout": 0.4003951098738985,
+ "eb_lift": 0.19456070997299912,
+ "p_value": 0.004468816016140567,
+ "held_up": true,
+ "z_path": [
+ 1.433,
+ 0.74,
+ 0.061,
+ 0.786,
+ 0.758,
+ 1.29,
+ 0.931,
+ 0.73,
+ 1.087,
+ 1.656
+ ],
+ "first_cross": null,
+ "final_z": 1.656
+ },
+ {
+ "id": "5272e33f6a50c7c8ed0058f7",
+ "variation": "image",
+ "week": 201343,
+ "impressions": 18616,
+ "pooled_ctr": 0.033358401375161155,
+ "arms": [
+ {
+ "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop",
+ "impressions": 4087,
+ "clicks": 208,
+ "ctr": 0.0508930756055786,
+ "ctr_select": 0.053816046966731895,
+ "ctr_holdout": 0.047968673519334315,
+ "n_select": 2044,
+ "n_holdout": 2043,
+ "eb": 0.0429149176927707,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop",
+ "impressions": 3574,
+ "clicks": 80,
+ "ctr": 0.0223838836038052,
+ "ctr_select": 0.020705092333519866,
+ "ctr_holdout": 0.024062674874090653,
+ "n_select": 1787,
+ "n_holdout": 1787,
+ "eb": 0.027817370781339574,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop",
+ "impressions": 3783,
+ "clicks": 95,
+ "ctr": 0.0251123446999735,
+ "ctr_select": 0.025898520084566595,
+ "ctr_holdout": 0.024325753569539928,
+ "n_select": 1892,
+ "n_holdout": 1891,
+ "eb": 0.02997409953123644,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop",
+ "impressions": 3158,
+ "clicks": 101,
+ "ctr": 0.031982267257758,
+ "ctr_select": 0.03356554781507283,
+ "ctr_holdout": 0.030398986700443317,
+ "n_select": 1579,
+ "n_holdout": 1579,
+ "eb": 0.03340826203693005,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop",
+ "impressions": 4014,
+ "clicks": 137,
+ "ctr": 0.0341305430991529,
+ "ctr_select": 0.030393622321873443,
+ "ctr_holdout": 0.037867463876432486,
+ "n_select": 2007,
+ "n_holdout": 2007,
+ "eb": 0.03195180505843267,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.03330110645611774,
+ "pool_holdout": 0.03341570860642527,
+ "lift_select": 0.616043810365507,
+ "lift_holdout": 0.4355126831011076,
+ "eb_lift": 0.2886934477483949,
+ "p_value": 4.870700208930216e-09,
+ "held_up": true,
+ "z_path": [
+ 0.928,
+ 0.911,
+ 1.441,
+ 2.529,
+ 3.176,
+ 3.563,
+ 3.79,
+ 3.649,
+ 3.75,
+ 3.95
+ ],
+ "first_cross": 3,
+ "final_z": 3.95
+ },
+ {
+ "id": "5411b14a5856e96e050000d5",
+ "variation": "image",
+ "week": 201436,
+ "impressions": 18513,
+ "pooled_ctr": 0.01226165397288392,
+ "arms": [
+ {
+ "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive",
+ "impressions": 3621,
+ "clicks": 50,
+ "ctr": 0.0138083402375034,
+ "ctr_select": 0.012154696132596685,
+ "ctr_holdout": 0.015461071231363888,
+ "n_select": 1810,
+ "n_holdout": 1811,
+ "eb": 0.012609435780192224,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive",
+ "impressions": 3734,
+ "clicks": 56,
+ "ctr": 0.0149973219068023,
+ "ctr_select": 0.017675415104445636,
+ "ctr_holdout": 0.012319228709159078,
+ "n_select": 1867,
+ "n_holdout": 1867,
+ "eb": 0.015066306714774547,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive",
+ "impressions": 3717,
+ "clicks": 44,
+ "ctr": 0.011837503362927,
+ "ctr_select": 0.010764262648008612,
+ "ctr_holdout": 0.012910166756320602,
+ "n_select": 1858,
+ "n_holdout": 1859,
+ "eb": 0.011985527369734747,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive",
+ "impressions": 3710,
+ "clicks": 32,
+ "ctr": 0.0086253369272237,
+ "ctr_select": 0.009164420485175202,
+ "ctr_holdout": 0.008086253369272238,
+ "n_select": 1855,
+ "n_holdout": 1855,
+ "eb": 0.011275185133679892,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive",
+ "impressions": 3731,
+ "clicks": 45,
+ "ctr": 0.0120611096220852,
+ "ctr_select": 0.015005359056806002,
+ "ctr_holdout": 0.009115281501340482,
+ "n_select": 1866,
+ "n_holdout": 1865,
+ "eb": 0.013874792880549967,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.01296456352636128,
+ "pool_holdout": 0.01155882035216593,
+ "lift_select": 0.36336368505624006,
+ "lift_holdout": 0.0657859828101457,
+ "eb_lift": 0.16211445793294343,
+ "p_value": 0.04401652968976107,
+ "held_up": false,
+ "z_path": [
+ 1.139,
+ 1.047,
+ 1.063,
+ 0.926,
+ 1.576,
+ 1.699,
+ 1.84,
+ 1.363,
+ 0.64,
+ 1.098
+ ],
+ "first_cross": null,
+ "final_z": 1.098
+ },
+ {
+ "id": "5395561485b18f33ec000011",
+ "variation": "image",
+ "week": 201423,
+ "impressions": 18473,
+ "pooled_ctr": 0.04065392735343474,
+ "arms": [
+ {
+ "headline": "Have You Ever Wondered What Makes You Sexually Attracted To Certain People? Here Are Some Answers.",
+ "impressions": 4540,
+ "clicks": 144,
+ "ctr": 0.0317180616740088,
+ "ctr_select": 0.032158590308370046,
+ "ctr_holdout": 0.03127753303964758,
+ "n_select": 2270,
+ "n_holdout": 2270,
+ "eb": 0.036752376746970086,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Have You Ever Wondered What Makes You Sexually Attracted To Certain People? Here Are Some Answers.",
+ "impressions": 4625,
+ "clicks": 221,
+ "ctr": 0.0477837837837837,
+ "ctr_select": 0.04801038062283737,
+ "ctr_holdout": 0.04755728491137051,
+ "n_select": 2312,
+ "n_holdout": 2313,
+ "eb": 0.04462684339738134,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Have You Ever Wondered What Makes You Sexually Attracted To Certain People? Here Are Some Answers.",
+ "impressions": 4671,
+ "clicks": 220,
+ "ctr": 0.0470991222436309,
+ "ctr_select": 0.04708904109589041,
+ "ctr_holdout": 0.047109207708779445,
+ "n_select": 2336,
+ "n_holdout": 2335,
+ "eb": 0.04418181143076374,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Have You Ever Wondered What Makes You Sexually Attracted To Certain People? Here Are Some Answers.",
+ "impressions": 4637,
+ "clicks": 166,
+ "ctr": 0.0357990079792969,
+ "ctr_select": 0.037532355478861086,
+ "ctr_holdout": 0.03406640793445451,
+ "n_select": 2318,
+ "n_holdout": 2319,
+ "eb": 0.03939186851191274,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.04125162407968817,
+ "pool_holdout": 0.040056295333982896,
+ "lift_select": 0.16384219273628875,
+ "lift_holdout": 0.1872611911522417,
+ "eb_lift": 0.08182027721316043,
+ "p_value": 0.05911393220439828,
+ "held_up": true,
+ "z_path": [
+ 0.525,
+ 0.161,
+ -0.131,
+ 0.139,
+ 1.115,
+ 0.969,
+ 0.492,
+ -0.186,
+ 0.1,
+ 0.155
+ ],
+ "first_cross": null,
+ "final_z": 0.155
+ },
+ {
+ "id": "543d6d2bdd6e021068000040",
+ "variation": "headline",
+ "week": 201441,
+ "impressions": 18472,
+ "pooled_ctr": 0.00698354265915981,
+ "arms": [
+ {
+ "headline": "Thanks, Joss Whedon! 16 Really Important Life Lessons 'Buffy The Vampire Slayer' Taught Me.",
+ "impressions": 3090,
+ "clicks": 25,
+ "ctr": 0.0080906148867313,
+ "ctr_select": 0.007766990291262136,
+ "ctr_holdout": 0.008414239482200648,
+ "n_select": 1545,
+ "n_holdout": 1545,
+ "eb": 0.007329388374407745,
+ "labelled_winner": false
+ },
+ {
+ "headline": "16 Things That Got Drilled Into My Skull Because Of Buffy The Vampire Slayer",
+ "impressions": 3150,
+ "clicks": 23,
+ "ctr": 0.0073015873015873,
+ "ctr_select": 0.006349206349206349,
+ "ctr_holdout": 0.008253968253968255,
+ "n_select": 1575,
+ "n_holdout": 1575,
+ "eb": 0.006759120190317952,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Make A Little Room For The Cheese Slices And These 16 Important Lessons From Buffy",
+ "impressions": 3074,
+ "clicks": 12,
+ "ctr": 0.0039037085230969,
+ "ctr_select": 0.004554326610279766,
+ "ctr_holdout": 0.0032530904359141183,
+ "n_select": 1537,
+ "n_holdout": 1537,
+ "eb": 0.006047481135076326,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Buffy The Vampire Slayer Taught Me Countless Things That Shaped My Attitudes. Here Are 16.",
+ "impressions": 3081,
+ "clicks": 23,
+ "ctr": 0.0074651087309315,
+ "ctr_select": 0.007792207792207792,
+ "ctr_holdout": 0.007138221933809215,
+ "n_select": 1540,
+ "n_holdout": 1541,
+ "eb": 0.007338888066887023,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Don't Live On A Hellmouth But These 16 Things I Learned From Buffy Came In Pretty Handy Anyway",
+ "impressions": 3055,
+ "clicks": 28,
+ "ctr": 0.009165302782324,
+ "ctr_select": 0.00981675392670157,
+ "ctr_holdout": 0.008513425016371971,
+ "n_select": 1528,
+ "n_holdout": 1527,
+ "eb": 0.008141880630000198,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Got The Mustard Out + 15 Other GIF-Lessons That Will Make You Miss Buffy The Vampire Slayer",
+ "impressions": 3022,
+ "clicks": 18,
+ "ctr": 0.0059563203176704,
+ "ctr_select": 0.005956320317670417,
+ "ctr_holdout": 0.005956320317670417,
+ "n_select": 1511,
+ "n_holdout": 1511,
+ "eb": 0.006610919326318461,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.0070376786487656995,
+ "pool_holdout": 0.006929406669553919,
+ "lift_select": 0.3948852194925494,
+ "lift_holdout": 0.2285936476751802,
+ "eb_lift": 0.15689860767202823,
+ "p_value": 0.15488141639581227,
+ "held_up": true,
+ "z_path": [
+ 0.587,
+ -0.368,
+ 0.592,
+ 0.745,
+ 1.111,
+ 1.825,
+ 1.159,
+ 0.872,
+ 0.487,
+ 0.733
+ ],
+ "first_cross": null,
+ "final_z": 0.733
+ },
+ {
+ "id": "53da7b0e4d33e277ec00001c",
+ "variation": "image",
+ "week": 201430,
+ "impressions": 18426,
+ "pooled_ctr": 0.005047215890589384,
+ "arms": [
+ {
+ "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.",
+ "impressions": 3076,
+ "clicks": 19,
+ "ctr": 0.0061768530559167,
+ "ctr_select": 0.007802340702210663,
+ "ctr_holdout": 0.004551365409622887,
+ "n_select": 1538,
+ "n_holdout": 1538,
+ "eb": 0.006439814380415746,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.",
+ "impressions": 3146,
+ "clicks": 20,
+ "ctr": 0.0063572790845518,
+ "ctr_select": 0.008264462809917356,
+ "ctr_holdout": 0.004450095359186269,
+ "n_select": 1573,
+ "n_holdout": 1573,
+ "eb": 0.0066389063056924965,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.",
+ "impressions": 3078,
+ "clicks": 16,
+ "ctr": 0.0051981806367771,
+ "ctr_select": 0.004548408057179987,
+ "ctr_holdout": 0.005847953216374269,
+ "n_select": 1539,
+ "n_holdout": 1539,
+ "eb": 0.005141698687198532,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.",
+ "impressions": 3001,
+ "clicks": 10,
+ "ctr": 0.0033322225924691,
+ "ctr_select": 0.004,
+ "ctr_holdout": 0.002664890073284477,
+ "n_select": 1500,
+ "n_holdout": 1501,
+ "eb": 0.004932286152982215,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.",
+ "impressions": 3007,
+ "clicks": 13,
+ "ctr": 0.0043232457598935,
+ "ctr_select": 0.005319148936170213,
+ "ctr_holdout": 0.00332667997338656,
+ "n_select": 1504,
+ "n_holdout": 1503,
+ "eb": 0.005450451379129435,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.",
+ "impressions": 3118,
+ "clicks": 15,
+ "ctr": 0.0048107761385503,
+ "ctr_select": 0.003207184092366902,
+ "ctr_holdout": 0.006414368184733804,
+ "n_select": 1559,
+ "n_holdout": 1559,
+ "eb": 0.0045992695028878065,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.005535656138065777,
+ "pool_holdout": 0.004558775643112993,
+ "lift_select": 0.4929508993680116,
+ "lift_holdout": -0.02383979656706925,
+ "eb_lift": 0.19929889792833277,
+ "p_value": 0.10919695828262725,
+ "held_up": false,
+ "z_path": [
+ 1.322,
+ 1.234,
+ 0.494,
+ 1.297,
+ 0.992,
+ 0.907,
+ 0.907,
+ 0.871,
+ 0.278,
+ 0.09
+ ],
+ "first_cross": null,
+ "final_z": 0.09
+ },
+ {
+ "id": "53f27699f1592f712e000032",
+ "variation": "headline",
+ "week": 201433,
+ "impressions": 18404,
+ "pooled_ctr": 0.012442947185394479,
+ "arms": [
+ {
+ "headline": "What's The One Thing White Women Are Never Supposed To Do (At Least Not On Camera)?",
+ "impressions": 3095,
+ "clicks": 42,
+ "ctr": 0.0135702746365105,
+ "ctr_select": 0.013565891472868217,
+ "ctr_holdout": 0.013574660633484163,
+ "n_select": 1548,
+ "n_holdout": 1547,
+ "eb": 0.012924388058133559,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Think Racism Doesn't Exist? Well Eff You. (Although Apparently I Shouldn't If We're Different Races)",
+ "impressions": 3146,
+ "clicks": 44,
+ "ctr": 0.0139860139860139,
+ "ctr_select": 0.012714558169103624,
+ "ctr_holdout": 0.015257469802924348,
+ "n_select": 1573,
+ "n_holdout": 1573,
+ "eb": 0.012584318023939589,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here's A Thing White Women Can Do That Could Cost Them Money At Work. Hint: It's Sexist And Racist.",
+ "impressions": 3007,
+ "clicks": 45,
+ "ctr": 0.0149650814765547,
+ "ctr_select": 0.016622340425531915,
+ "ctr_holdout": 0.01330671989354624,
+ "n_select": 1504,
+ "n_holdout": 1503,
+ "eb": 0.014119844482909892,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Too Many People Have Told Her Who She Can't Have Sex With. So She's Speaking Out.",
+ "impressions": 2984,
+ "clicks": 33,
+ "ctr": 0.0110589812332439,
+ "ctr_select": 0.010723860589812333,
+ "ctr_holdout": 0.011394101876675604,
+ "n_select": 1492,
+ "n_holdout": 1492,
+ "eb": 0.011801927800055114,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Turns Out There Aren't Just Outdated Stereotypes About Porn. They're Also IN Porn. Go Figure.",
+ "impressions": 3067,
+ "clicks": 41,
+ "ctr": 0.0133681121617215,
+ "ctr_select": 0.012385919165580182,
+ "ctr_holdout": 0.014350945857795172,
+ "n_select": 1534,
+ "n_holdout": 1533,
+ "eb": 0.012452113568627922,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Lot Of People Aren't Cool With The 'Relations' In Interracial Relationships. Especially On Film.",
+ "impressions": 3105,
+ "clicks": 24,
+ "ctr": 0.0077294685990338,
+ "ctr_select": 0.00902061855670103,
+ "ctr_holdout": 0.00643915003219575,
+ "n_select": 1552,
+ "n_holdout": 1553,
+ "eb": 0.01110210164444606,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.012495925241768988,
+ "pool_holdout": 0.012389957613302902,
+ "lift_select": 0.3302208603145236,
+ "lift_holdout": 0.07399236614490312,
+ "eb_lift": 0.1299559024019108,
+ "p_value": 0.11524792580880683,
+ "held_up": false,
+ "z_path": [
+ 0.635,
+ 0.563,
+ -0.302,
+ 0.413,
+ 0.837,
+ 0.809,
+ 0.625,
+ 0.241,
+ 0.242,
+ 0.459
+ ],
+ "first_cross": null,
+ "final_z": 0.459
+ },
+ {
+ "id": "53f2409127e3e436d300000f",
+ "variation": "headline",
+ "week": 201433,
+ "impressions": 18398,
+ "pooled_ctr": 0.007392107837808457,
+ "arms": [
+ {
+ "headline": "The Number Of Black Men Getting Killed By White Police Officers In America Per Week Is How Many?",
+ "impressions": 3070,
+ "clicks": 21,
+ "ctr": 0.0068403908794788,
+ "ctr_select": 0.006514657980456026,
+ "ctr_holdout": 0.0071661237785016286,
+ "n_select": 1535,
+ "n_holdout": 1535,
+ "eb": 0.007239205222555899,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If This Many Police Officers Were Killed Each Week, People Would Freak Out",
+ "impressions": 2997,
+ "clicks": 11,
+ "ctr": 0.0036703370036703,
+ "ctr_select": 0.004005340453938585,
+ "ctr_holdout": 0.00333555703802535,
+ "n_select": 1498,
+ "n_holdout": 1499,
+ "eb": 0.006261101165142563,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How Many Black Men Are Killed Every Week By The Police?",
+ "impressions": 3132,
+ "clicks": 14,
+ "ctr": 0.0044699872286079,
+ "ctr_select": 0.005747126436781609,
+ "ctr_holdout": 0.0031928480204342275,
+ "n_select": 1566,
+ "n_holdout": 1566,
+ "eb": 0.00692393471104358,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Folks In America Have A Less American Dream-Like Experience Than Others. Please Hear Her Out.",
+ "impressions": 3030,
+ "clicks": 11,
+ "ctr": 0.0036303630363036,
+ "ctr_select": 0.0026402640264026403,
+ "ctr_holdout": 0.0046204620462046205,
+ "n_select": 1515,
+ "n_holdout": 1515,
+ "eb": 0.005711504280091988,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Ferguson Has Opened Some Raw Wounds That Most People Are Afraid To Look At, Let Alone Heal",
+ "impressions": 3120,
+ "clicks": 22,
+ "ctr": 0.007051282051282,
+ "ctr_select": 0.0057692307692307696,
+ "ctr_holdout": 0.008333333333333333,
+ "n_select": 1560,
+ "n_holdout": 1560,
+ "eb": 0.00693464818368396,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Aren't Black, You May Want To Watch This. It's Painful. It Also Happens To Be True.",
+ "impressions": 3049,
+ "clicks": 57,
+ "ctr": 0.018694653984913,
+ "ctr_select": 0.021653543307086614,
+ "ctr_holdout": 0.015737704918032787,
+ "n_select": 1524,
+ "n_holdout": 1525,
+ "eb": 0.013246869810430063,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 5,
+ "pool_select": 0.007719069362904979,
+ "pool_holdout": 0.007065217391304348,
+ "lift_select": 1.805201286458911,
+ "lift_holdout": 1.2274905422446407,
+ "eb_lift": 0.716122655159658,
+ "p_value": 1.0114889966721197e-11,
+ "held_up": true,
+ "z_path": [
+ 1.689,
+ 1.625,
+ 2.782,
+ 3.935,
+ 4.061,
+ 4.28,
+ 4.256,
+ 4.304,
+ 4.299,
+ 4.133
+ ],
+ "first_cross": 2,
+ "final_z": 4.133
+ },
+ {
+ "id": "54836027663239001e260000",
+ "variation": "other",
+ "week": 201448,
+ "impressions": 18301,
+ "pooled_ctr": 0.01021802087317633,
+ "arms": [
+ {
+ "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.",
+ "impressions": 3065,
+ "clicks": 32,
+ "ctr": 0.0104404567699836,
+ "ctr_select": 0.013054830287206266,
+ "ctr_holdout": 0.007827788649706457,
+ "n_select": 1532,
+ "n_holdout": 1533,
+ "eb": 0.012497932049296142,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.",
+ "impressions": 3054,
+ "clicks": 30,
+ "ctr": 0.0098231827111984,
+ "ctr_select": 0.010478061558611657,
+ "ctr_holdout": 0.0091683038637852,
+ "n_select": 1527,
+ "n_holdout": 1527,
+ "eb": 0.011473791624677716,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.",
+ "impressions": 2943,
+ "clicks": 28,
+ "ctr": 0.0095141012572205,
+ "ctr_select": 0.010190217391304348,
+ "ctr_holdout": 0.008837525492861998,
+ "n_select": 1472,
+ "n_holdout": 1471,
+ "eb": 0.011376437827779464,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.",
+ "impressions": 3018,
+ "clicks": 27,
+ "ctr": 0.0089463220675944,
+ "ctr_select": 0.008614976805831677,
+ "ctr_holdout": 0.00927766732935719,
+ "n_select": 1509,
+ "n_holdout": 1509,
+ "eb": 0.01074379621705174,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.",
+ "impressions": 3183,
+ "clicks": 36,
+ "ctr": 0.0113100848256361,
+ "ctr_select": 0.013190954773869347,
+ "ctr_holdout": 0.009428032683846637,
+ "n_select": 1592,
+ "n_holdout": 1591,
+ "eb": 0.012561907347628225,
+ "labelled_winner": true
+ },
+ {
+ "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.",
+ "impressions": 3038,
+ "clicks": 34,
+ "ctr": 0.0111915734035549,
+ "ctr_select": 0.017116524028966424,
+ "ctr_holdout": 0.005266622778143515,
+ "n_select": 1519,
+ "n_holdout": 1519,
+ "eb": 0.014104124038098777,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 5,
+ "pool_select": 0.01212982187739045,
+ "pool_holdout": 0.008306010928961749,
+ "lift_select": 0.4111109134150608,
+ "lift_holdout": -0.36592633657877416,
+ "eb_lift": 0.16276431596974694,
+ "p_value": 0.051877098634353976,
+ "held_up": false,
+ "z_path": [
+ 0.444,
+ 0.091,
+ 0.108,
+ 0.131,
+ 0.64,
+ 0.456,
+ 0.742,
+ 0.174,
+ -0.058,
+ -0.044
+ ],
+ "first_cross": null,
+ "final_z": -0.044
+ },
+ {
+ "id": "553ffd1c316638000c290200",
+ "variation": "image",
+ "week": 201517,
+ "impressions": 18290,
+ "pooled_ctr": 0.004264625478403499,
+ "arms": [
+ {
+ "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.",
+ "impressions": 3104,
+ "clicks": 15,
+ "ctr": 0.0048324742268041,
+ "ctr_select": 0.003865979381443299,
+ "ctr_holdout": 0.005798969072164949,
+ "n_select": 1552,
+ "n_holdout": 1552,
+ "eb": 0.003973756164608848,
+ "labelled_winner": false
+ },
+ {
+ "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.",
+ "impressions": 2961,
+ "clicks": 8,
+ "ctr": 0.0027017899358324,
+ "ctr_select": 0.0013513513513513514,
+ "ctr_holdout": 0.004051316677920324,
+ "n_select": 1480,
+ "n_holdout": 1481,
+ "eb": 0.0029958243108922484,
+ "labelled_winner": false
+ },
+ {
+ "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.",
+ "impressions": 3107,
+ "clicks": 17,
+ "ctr": 0.0054715159317669,
+ "ctr_select": 0.007722007722007722,
+ "ctr_holdout": 0.003219575016097875,
+ "n_select": 1554,
+ "n_holdout": 1553,
+ "eb": 0.005521411210223256,
+ "labelled_winner": false
+ },
+ {
+ "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.",
+ "impressions": 3066,
+ "clicks": 17,
+ "ctr": 0.0055446836268754,
+ "ctr_select": 0.006523157208088715,
+ "ctr_holdout": 0.0045662100456621,
+ "n_select": 1533,
+ "n_holdout": 1533,
+ "eb": 0.005032136146559357,
+ "labelled_winner": false
+ },
+ {
+ "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.",
+ "impressions": 3063,
+ "clicks": 12,
+ "ctr": 0.0039177277179236,
+ "ctr_select": 0.0026109660574412533,
+ "ctr_holdout": 0.005225342913128674,
+ "n_select": 1532,
+ "n_holdout": 1531,
+ "eb": 0.003474879363348337,
+ "labelled_winner": false
+ },
+ {
+ "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.",
+ "impressions": 2989,
+ "clicks": 9,
+ "ctr": 0.0030110404817664,
+ "ctr_select": 0.002008032128514056,
+ "ctr_holdout": 0.004013377926421404,
+ "n_select": 1494,
+ "n_holdout": 1495,
+ "eb": 0.003247171226069369,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.004045926735921268,
+ "pool_holdout": 0.00448332422088573,
+ "lift_select": 0.908588124804341,
+ "lift_holdout": -0.2818777189703643,
+ "eb_lift": 0.3646839329051803,
+ "p_value": 0.012221883065665079,
+ "held_up": false,
+ "z_path": [
+ -0.016,
+ -0.93,
+ -0.561,
+ -0.802,
+ -0.973,
+ -0.886,
+ -0.968,
+ -0.769,
+ -0.392,
+ -0.039
+ ],
+ "first_cross": null,
+ "final_z": -0.039
+ },
+ {
+ "id": "51cdd24c558059c42800309f",
+ "variation": "headline",
+ "week": 201325,
+ "impressions": 18288,
+ "pooled_ctr": 0.003937007874015748,
+ "arms": [
+ {
+ "headline": "I Never Thought I'd See Something So Hardcore on Sesame Street's Lesson Plan",
+ "impressions": 4225,
+ "clicks": 24,
+ "ctr": 0.005680473372781,
+ "ctr_select": 0.007102272727272727,
+ "ctr_holdout": 0.004259346900141978,
+ "n_select": 2112,
+ "n_holdout": 2113,
+ "eb": 0.005503368907174159,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Sesame Street Takes The Phrase Educational Programming To A Whole New Level",
+ "impressions": 4975,
+ "clicks": 12,
+ "ctr": 0.0024120603015075,
+ "ctr_select": 0.0028135048231511255,
+ "ctr_holdout": 0.002010454362685967,
+ "n_select": 2488,
+ "n_holdout": 2487,
+ "eb": 0.0034080921486323913,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Can You Tell Me How To Get, How To Get To... Jail?",
+ "impressions": 4113,
+ "clicks": 9,
+ "ctr": 0.0021881838074398,
+ "ctr_select": 0.0029182879377431907,
+ "ctr_holdout": 0.0014584346135148275,
+ "n_select": 2056,
+ "n_holdout": 2057,
+ "eb": 0.0035160776199896757,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Whoa. Apparently No Topic Is Off Limits For Sesame Street.",
+ "impressions": 4975,
+ "clicks": 27,
+ "ctr": 0.0054271356783919,
+ "ctr_select": 0.003617363344051447,
+ "ctr_holdout": 0.007237635705669481,
+ "n_select": 2488,
+ "n_holdout": 2487,
+ "eb": 0.0038242648092788128,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.004046369203849519,
+ "pool_holdout": 0.0038276465441819773,
+ "lift_select": 0.75522113022113,
+ "lift_holdout": 0.11278480156852133,
+ "eb_lift": 0.36007581857298687,
+ "p_value": 0.011646279577591007,
+ "held_up": false,
+ "z_path": [
+ 1.023,
+ 1.133,
+ 1.512,
+ 1.266,
+ 2.044,
+ 1.394,
+ 0.819,
+ 0.333,
+ 0.544,
+ 0.163
+ ],
+ "first_cross": 4,
+ "final_z": 0.163
+ },
+ {
+ "id": "5362dbb2f0b630c49100002f",
+ "variation": "headline",
+ "week": 201417,
+ "impressions": 18200,
+ "pooled_ctr": 0.0035164835164835165,
+ "arms": [
+ {
+ "headline": "The NFL Makes Over 9 Billion Dollars Annually. Guess How Much Money It Pays In Taxes?",
+ "impressions": 3050,
+ "clicks": 16,
+ "ctr": 0.0052459016393442,
+ "ctr_select": 0.003934426229508197,
+ "ctr_holdout": 0.006557377049180328,
+ "n_select": 1525,
+ "n_holdout": 1525,
+ "eb": 0.0033509493816641766,
+ "labelled_winner": false
+ },
+ {
+ "headline": "UNFAIR: The NFL Gets Taxpayers To Pay For Stadiums, But Doesn't Pay Any Taxes Itself",
+ "impressions": 3088,
+ "clicks": 7,
+ "ctr": 0.0022668393782383,
+ "ctr_select": 0.0019430051813471502,
+ "ctr_holdout": 0.0025906735751295338,
+ "n_select": 1544,
+ "n_holdout": 1544,
+ "eb": 0.00255760192865792,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Financially Successful Sports League In The World Pays No Taxes. None.",
+ "impressions": 3011,
+ "clicks": 11,
+ "ctr": 0.0036532713384257,
+ "ctr_select": 0.0033200531208499337,
+ "ctr_holdout": 0.003986710963455149,
+ "n_select": 1506,
+ "n_holdout": 1505,
+ "eb": 0.003106073339132471,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The NFL Has Avoided Paying Taxes For Over The Past 40 Years",
+ "impressions": 2986,
+ "clicks": 12,
+ "ctr": 0.0040187541862022,
+ "ctr_select": 0.003348961821835231,
+ "ctr_holdout": 0.004688546550569324,
+ "n_select": 1493,
+ "n_holdout": 1493,
+ "eb": 0.003116669549206972,
+ "labelled_winner": false
+ },
+ {
+ "headline": "NFL Ticket Prices Are High, But The NFL Itself? Get's A Free Ride From The Government.",
+ "impressions": 3068,
+ "clicks": 7,
+ "ctr": 0.0022816166883963,
+ "ctr_select": 0.001303780964797914,
+ "ctr_holdout": 0.003259452411994785,
+ "n_select": 1534,
+ "n_holdout": 1534,
+ "eb": 0.0023046163478993255,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Do The NFL And Organized Crime Have In Common?",
+ "impressions": 2997,
+ "clicks": 11,
+ "ctr": 0.0036703370036703,
+ "ctr_select": 0.004005340453938585,
+ "ctr_holdout": 0.00333555703802535,
+ "n_select": 1498,
+ "n_holdout": 1499,
+ "eb": 0.003374660807544442,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 5,
+ "pool_select": 0.0029670329670329672,
+ "pool_holdout": 0.004065934065934066,
+ "lift_select": 0.34994807892004154,
+ "lift_holdout": -0.1796332690261977,
+ "eb_lift": 0.13738567957979342,
+ "p_value": 0.4188642021490535,
+ "held_up": false,
+ "z_path": [
+ 0.402,
+ 0.401,
+ 0.359,
+ 0.934,
+ 0.567,
+ -0.194,
+ -0.411,
+ -0.586,
+ -0.959,
+ -0.919
+ ],
+ "first_cross": null,
+ "final_z": -0.919
+ },
+ {
+ "id": "5358433c1a09bfcf21000036",
+ "variation": "image",
+ "week": 201416,
+ "impressions": 18140,
+ "pooled_ctr": 0.0185226019845645,
+ "arms": [
+ {
+ "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him",
+ "impressions": 3030,
+ "clicks": 53,
+ "ctr": 0.0174917491749174,
+ "ctr_select": 0.01782178217821782,
+ "ctr_holdout": 0.017161716171617162,
+ "n_select": 1515,
+ "n_holdout": 1515,
+ "eb": 0.018245579874176965,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him",
+ "impressions": 2994,
+ "clicks": 45,
+ "ctr": 0.0150300601202404,
+ "ctr_select": 0.013360053440213761,
+ "ctr_holdout": 0.016700066800267203,
+ "n_select": 1497,
+ "n_holdout": 1497,
+ "eb": 0.01649666552267801,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him",
+ "impressions": 3130,
+ "clicks": 70,
+ "ctr": 0.0223642172523961,
+ "ctr_select": 0.021086261980830672,
+ "ctr_holdout": 0.023642172523961662,
+ "n_select": 1565,
+ "n_holdout": 1565,
+ "eb": 0.01955593710343505,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him",
+ "impressions": 3017,
+ "clicks": 47,
+ "ctr": 0.0155783891282731,
+ "ctr_select": 0.015915119363395226,
+ "ctr_holdout": 0.015241882041086813,
+ "n_select": 1508,
+ "n_holdout": 1509,
+ "eb": 0.01749479370061142,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him",
+ "impressions": 2978,
+ "clicks": 73,
+ "ctr": 0.0245130960376091,
+ "ctr_select": 0.0241773002014775,
+ "ctr_holdout": 0.024848891873740765,
+ "n_select": 1489,
+ "n_holdout": 1489,
+ "eb": 0.020734452333157554,
+ "labelled_winner": true
+ },
+ {
+ "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him",
+ "impressions": 2991,
+ "clicks": 48,
+ "ctr": 0.0160481444332999,
+ "ctr_select": 0.01871657754010695,
+ "ctr_holdout": 0.013377926421404682,
+ "n_select": 1496,
+ "n_holdout": 1495,
+ "eb": 0.01859869281649879,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.0185226019845645,
+ "pool_holdout": 0.0185226019845645,
+ "lift_select": 0.30528638587738643,
+ "lift_holdout": 0.3415443410406471,
+ "eb_lift": 0.11941358727225593,
+ "p_value": 0.07670489980645687,
+ "held_up": true,
+ "z_path": [
+ 0.41,
+ 0.92,
+ 1.265,
+ 1.538,
+ 1.58,
+ 1.426,
+ 1.139,
+ 1.116,
+ 0.642,
+ 0.555
+ ],
+ "first_cross": null,
+ "final_z": 0.555
+ },
+ {
+ "id": "53396e3a09ccd48b4900002c",
+ "variation": "image",
+ "week": 201413,
+ "impressions": 18131,
+ "pooled_ctr": 0.007059731950802493,
+ "arms": [
+ {
+ "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening",
+ "impressions": 3573,
+ "clicks": 23,
+ "ctr": 0.0064371676462356,
+ "ctr_select": 0.007278835386338186,
+ "ctr_holdout": 0.005595970900951315,
+ "n_select": 1786,
+ "n_holdout": 1787,
+ "eb": 0.007342681014997483,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening",
+ "impressions": 3581,
+ "clicks": 40,
+ "ctr": 0.0111700642278693,
+ "ctr_select": 0.012849162011173185,
+ "ctr_holdout": 0.009491903964265773,
+ "n_select": 1790,
+ "n_holdout": 1791,
+ "eb": 0.009769984578424628,
+ "labelled_winner": true
+ },
+ {
+ "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening",
+ "impressions": 3625,
+ "clicks": 23,
+ "ctr": 0.0063448275862068,
+ "ctr_select": 0.00772626931567329,
+ "ctr_holdout": 0.004964147821290678,
+ "n_select": 1812,
+ "n_holdout": 1813,
+ "eb": 0.007538600995460494,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening",
+ "impressions": 3648,
+ "clicks": 20,
+ "ctr": 0.0054824561403508,
+ "ctr_select": 0.00712719298245614,
+ "ctr_holdout": 0.003837719298245614,
+ "n_select": 1824,
+ "n_holdout": 1824,
+ "eb": 0.007275312052202901,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening",
+ "impressions": 3704,
+ "clicks": 22,
+ "ctr": 0.0059395248380129,
+ "ctr_select": 0.0021598272138228943,
+ "ctr_holdout": 0.009719222462203024,
+ "n_select": 1852,
+ "n_holdout": 1852,
+ "eb": 0.005068027642296455,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.0073918799646954985,
+ "pool_holdout": 0.006727693834785485,
+ "lift_select": 0.7382806637205037,
+ "lift_holdout": 0.41087038104914364,
+ "eb_lift": 0.32171851072896773,
+ "p_value": 0.0026218286752617594,
+ "held_up": false,
+ "z_path": [
+ 1.023,
+ 1.181,
+ 0.72,
+ 1.497,
+ 1.87,
+ 1.713,
+ 2.079,
+ 1.523,
+ 1.994,
+ 2.2
+ ],
+ "first_cross": 6,
+ "final_z": 2.2
+ },
+ {
+ "id": "52474008bf43c26f6f0000ee",
+ "variation": "image",
+ "week": 201338,
+ "impressions": 17532,
+ "pooled_ctr": 0.03165639972621492,
+ "arms": [
+ {
+ "headline": "A Man Who Clearly Hasn't Seen Senator Warren In Action Tries To One Up Her. Good Luck With That.",
+ "impressions": 4246,
+ "clicks": 139,
+ "ctr": 0.032736693358455,
+ "ctr_select": 0.037211493170042395,
+ "ctr_holdout": 0.02826189354686764,
+ "n_select": 2123,
+ "n_holdout": 2123,
+ "eb": 0.03499687040791555,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Man Who Clearly Hasn't Seen Senator Warren In Action Tries To One Up Her. Good Luck With That.",
+ "impressions": 4229,
+ "clicks": 148,
+ "ctr": 0.0349964530621896,
+ "ctr_select": 0.03500473036896878,
+ "ctr_holdout": 0.034988179669030735,
+ "n_select": 2114,
+ "n_holdout": 2115,
+ "eb": 0.03393970710875151,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Man Who Clearly Hasn't Seen Senator Warren In Action Tries To One Up Her. Good Luck With That.",
+ "impressions": 4250,
+ "clicks": 121,
+ "ctr": 0.0284705882352941,
+ "ctr_select": 0.027294117647058823,
+ "ctr_holdout": 0.029647058823529412,
+ "n_select": 2125,
+ "n_holdout": 2125,
+ "eb": 0.030254257891462362,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Man Who Clearly Hasn't Seen Senator Warren In Action Tries To One Up Her. Good Luck With That.",
+ "impressions": 4807,
+ "clicks": 147,
+ "ctr": 0.0305804035781152,
+ "ctr_select": 0.0324459234608985,
+ "ctr_holdout": 0.02871410736579276,
+ "n_select": 2404,
+ "n_holdout": 2403,
+ "eb": 0.03270233118445129,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.03296828656171572,
+ "pool_holdout": 0.03034451289071412,
+ "lift_select": 0.1287057063273067,
+ "lift_holdout": -0.0686324855945799,
+ "eb_lift": 0.06153137022763899,
+ "p_value": 0.2084560854860471,
+ "held_up": false,
+ "z_path": [
+ 0.574,
+ 0.264,
+ 0.789,
+ 0.078,
+ 0.765,
+ 0.523,
+ -0.032,
+ -0.233,
+ -0.286,
+ -0.575
+ ],
+ "first_cross": null,
+ "final_z": -0.575
+ },
+ {
+ "id": "54755e743762650015310000",
+ "variation": "headline",
+ "week": 201447,
+ "impressions": 17127,
+ "pooled_ctr": 0.002218718981724762,
+ "arms": [
+ {
+ "headline": "If You Take Playing In The Dirt Seriously You'll Love This Message",
+ "impressions": 4339,
+ "clicks": 13,
+ "ctr": 0.0029960820465545,
+ "ctr_select": 0.002304147465437788,
+ "ctr_holdout": 0.0036883356385431073,
+ "n_select": 2170,
+ "n_holdout": 2169,
+ "eb": 0.0021997818449777467,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Is A Reminder Of Who We All Need To Thank For That Groaning T-Day Table",
+ "impressions": 4214,
+ "clicks": 8,
+ "ctr": 0.0018984337921214,
+ "ctr_select": 0.0023730422401518746,
+ "ctr_holdout": 0.0014238253440911248,
+ "n_select": 2107,
+ "n_holdout": 2107,
+ "eb": 0.002231102912649432,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Earth's Skin Is A Lot Like Ours. It's Not Very Thick, It's Alive, And It Needs Proper Treatment.",
+ "impressions": 4305,
+ "clicks": 3,
+ "ctr": 0.0006968641114982,
+ "ctr_select": 0.00046468401486988845,
+ "ctr_holdout": 0.0009289363678588017,
+ "n_select": 2152,
+ "n_holdout": 2153,
+ "eb": 0.0013137255189125028,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Think About The Best Food You've Ever Tasted. Now, Here's The Backstory.",
+ "impressions": 4269,
+ "clicks": 14,
+ "ctr": 0.0032794565472007,
+ "ctr_select": 0.0032802249297094657,
+ "ctr_holdout": 0.003278688524590164,
+ "n_select": 2134,
+ "n_holdout": 2135,
+ "eb": 0.0026668378502080177,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.002102067032582039,
+ "pool_holdout": 0.002335357309668379,
+ "lift_select": 0.5604758929501197,
+ "lift_holdout": 0.40393442622950815,
+ "eb_lift": 0.26867402840729193,
+ "p_value": 0.1702390050023943,
+ "held_up": false,
+ "z_path": [
+ 1.408,
+ 0.568,
+ 1.33,
+ 0.984,
+ 0.885,
+ 1.134,
+ 1.134,
+ 1.187,
+ 1.12,
+ 1.25
+ ],
+ "first_cross": null,
+ "final_z": 1.25
+ },
+ {
+ "id": "54b06a2f3663330012c00000",
+ "variation": "image",
+ "week": 201501,
+ "impressions": 17083,
+ "pooled_ctr": 0.016273488263185622,
+ "arms": [
+ {
+ "headline": "What Happens If You Try To Kill Yourself And Survive?",
+ "impressions": 2869,
+ "clicks": 45,
+ "ctr": 0.0156849076333217,
+ "ctr_select": 0.01394700139470014,
+ "ctr_holdout": 0.017421602787456445,
+ "n_select": 1434,
+ "n_holdout": 1435,
+ "eb": 0.015530685598719212,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Happens If You Try To Kill Yourself And Survive?",
+ "impressions": 2734,
+ "clicks": 64,
+ "ctr": 0.0234089246525237,
+ "ctr_select": 0.02121433796634967,
+ "ctr_holdout": 0.025603511338697878,
+ "n_select": 1367,
+ "n_holdout": 1367,
+ "eb": 0.018255619023376114,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Happens If You Try To Kill Yourself And Survive?",
+ "impressions": 2860,
+ "clicks": 45,
+ "ctr": 0.0157342657342657,
+ "ctr_select": 0.016083916083916083,
+ "ctr_holdout": 0.015384615384615385,
+ "n_select": 1430,
+ "n_holdout": 1430,
+ "eb": 0.016347753298338756,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Happens If You Try To Kill Yourself And Survive?",
+ "impressions": 2890,
+ "clicks": 32,
+ "ctr": 0.0110726643598615,
+ "ctr_select": 0.011072664359861591,
+ "ctr_holdout": 0.011072664359861591,
+ "n_select": 1445,
+ "n_holdout": 1445,
+ "eb": 0.014422215594219759,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Happens If You Try To Kill Yourself And Survive?",
+ "impressions": 2821,
+ "clicks": 30,
+ "ctr": 0.010634526763559,
+ "ctr_select": 0.01347517730496454,
+ "ctr_holdout": 0.007795889440113395,
+ "n_select": 1410,
+ "n_holdout": 1411,
+ "eb": 0.015362414565139707,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Happens If You Try To Kill Yourself And Survive?",
+ "impressions": 2909,
+ "clicks": 62,
+ "ctr": 0.0213131660364386,
+ "ctr_select": 0.023383768913342505,
+ "ctr_holdout": 0.019243986254295534,
+ "n_select": 1454,
+ "n_holdout": 1455,
+ "eb": 0.01916018810927458,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 5,
+ "pool_select": 0.01651053864168618,
+ "pool_holdout": 0.016036521128409222,
+ "lift_select": 0.41629352141804965,
+ "lift_holdout": 0.20001003336092515,
+ "eb_lift": 0.1604823152709569,
+ "p_value": 0.02395147998986142,
+ "held_up": false,
+ "z_path": [
+ 0.1,
+ 0.012,
+ -0.054,
+ -0.108,
+ 0.084,
+ 0.336,
+ -0.517,
+ -0.522,
+ -0.622,
+ -0.533
+ ],
+ "first_cross": null,
+ "final_z": -0.533
+ },
+ {
+ "id": "54babe753032350013290000",
+ "variation": "other",
+ "week": 201502,
+ "impressions": 17082,
+ "pooled_ctr": 0.006029738906451235,
+ "arms": [
+ {
+ "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These",
+ "impressions": 2839,
+ "clicks": 16,
+ "ctr": 0.0056357872490313,
+ "ctr_select": 0.004225352112676056,
+ "ctr_holdout": 0.007047216349541931,
+ "n_select": 1420,
+ "n_holdout": 1419,
+ "eb": 0.00530792827337932,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These",
+ "impressions": 2794,
+ "clicks": 11,
+ "ctr": 0.0039370078740157,
+ "ctr_select": 0.005010737294201861,
+ "ctr_holdout": 0.002863278453829635,
+ "n_select": 1397,
+ "n_holdout": 1397,
+ "eb": 0.0056099938624279,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These",
+ "impressions": 2801,
+ "clicks": 21,
+ "ctr": 0.0074973223848625,
+ "ctr_select": 0.007142857142857143,
+ "ctr_holdout": 0.007851534618129907,
+ "n_select": 1400,
+ "n_holdout": 1401,
+ "eb": 0.006412417904246618,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These",
+ "impressions": 2804,
+ "clicks": 17,
+ "ctr": 0.0060627674750356,
+ "ctr_select": 0.007132667617689016,
+ "ctr_holdout": 0.004992867332382311,
+ "n_select": 1402,
+ "n_holdout": 1402,
+ "eb": 0.006408970087561806,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These",
+ "impressions": 2930,
+ "clicks": 17,
+ "ctr": 0.0058020477815699,
+ "ctr_select": 0.005460750853242321,
+ "ctr_holdout": 0.0061433447098976105,
+ "n_select": 1465,
+ "n_holdout": 1465,
+ "eb": 0.005773506906728837,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These",
+ "impressions": 2914,
+ "clicks": 21,
+ "ctr": 0.0072065888812628,
+ "ctr_select": 0.0068634179821551134,
+ "ctr_holdout": 0.0075497597803706245,
+ "n_select": 1457,
+ "n_holdout": 1457,
+ "eb": 0.006315586886469134,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.00597119775201967,
+ "pool_holdout": 0.0060882800608828,
+ "lift_select": 0.19621848739495795,
+ "lift_holdout": 0.28961456102783734,
+ "eb_lift": 0.07389139843471293,
+ "p_value": 0.5337349863618881,
+ "held_up": true,
+ "z_path": [
+ 0.0,
+ 0.382,
+ 1.301,
+ 0.952,
+ 0.902,
+ 1.288,
+ 0.791,
+ 0.544,
+ 0.338,
+ 0.654
+ ],
+ "first_cross": null,
+ "final_z": 0.654
+ },
+ {
+ "id": "54b965fd3536310024ae0000",
+ "variation": "other",
+ "week": 201502,
+ "impressions": 17077,
+ "pooled_ctr": 0.020846752942554312,
+ "arms": [
+ {
+ "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore",
+ "impressions": 2817,
+ "clicks": 48,
+ "ctr": 0.0170394036208732,
+ "ctr_select": 0.020596590909090908,
+ "ctr_holdout": 0.0134847409510291,
+ "n_select": 1408,
+ "n_holdout": 1409,
+ "eb": 0.021262996745371805,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore",
+ "impressions": 2835,
+ "clicks": 55,
+ "ctr": 0.019400352733686,
+ "ctr_select": 0.021156558533145273,
+ "ctr_holdout": 0.017642907551164433,
+ "n_select": 1418,
+ "n_holdout": 1417,
+ "eb": 0.02147376594962769,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore",
+ "impressions": 2865,
+ "clicks": 54,
+ "ctr": 0.018848167539267,
+ "ctr_select": 0.017458100558659217,
+ "ctr_holdout": 0.020237264480111653,
+ "n_select": 1432,
+ "n_holdout": 1433,
+ "eb": 0.020060149935530378,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore",
+ "impressions": 2841,
+ "clicks": 84,
+ "ctr": 0.0295670538542766,
+ "ctr_select": 0.03169014084507042,
+ "ctr_holdout": 0.02744546094299789,
+ "n_select": 1420,
+ "n_holdout": 1421,
+ "eb": 0.02547544121875294,
+ "labelled_winner": true
+ },
+ {
+ "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore",
+ "impressions": 2908,
+ "clicks": 52,
+ "ctr": 0.0178817056396148,
+ "ctr_select": 0.02063273727647868,
+ "ctr_holdout": 0.015130674002751032,
+ "n_select": 1454,
+ "n_holdout": 1454,
+ "eb": 0.021268803719161348,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore",
+ "impressions": 2811,
+ "clicks": 63,
+ "ctr": 0.0224119530416221,
+ "ctr_select": 0.01849217638691323,
+ "ctr_holdout": 0.026334519572953737,
+ "n_select": 1406,
+ "n_holdout": 1405,
+ "eb": 0.020468766321449727,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.02166783790114781,
+ "pool_holdout": 0.020025764141000116,
+ "lift_select": 0.46254282451465545,
+ "lift_holdout": 0.37050754966233335,
+ "eb_lift": 0.17572603851736535,
+ "p_value": 0.0044982836010920854,
+ "held_up": true,
+ "z_path": [
+ 1.301,
+ 0.183,
+ -0.155,
+ 0.386,
+ 0.474,
+ 1.439,
+ 1.852,
+ 2.201,
+ 2.595,
+ 2.478
+ ],
+ "first_cross": 7,
+ "final_z": 2.478
+ },
+ {
+ "id": "549ae7786335660018720000",
+ "variation": "headline",
+ "week": 201451,
+ "impressions": 17068,
+ "pooled_ctr": 0.021092102179517225,
+ "arms": [
+ {
+ "headline": "She Tries Playing The Video Game Using A Female Avatar. What Happens Next Is Way Too Realistic.",
+ "impressions": 2850,
+ "clicks": 89,
+ "ctr": 0.0312280701754385,
+ "ctr_select": 0.03368421052631579,
+ "ctr_holdout": 0.028771929824561403,
+ "n_select": 1425,
+ "n_holdout": 1425,
+ "eb": 0.025378493382975018,
+ "labelled_winner": true
+ },
+ {
+ "headline": "She Tries To Play The Game Using A Female Avatar. How Real It Is Will Make You Laugh With Anger.",
+ "impressions": 2867,
+ "clicks": 60,
+ "ctr": 0.0209277990931287,
+ "ctr_select": 0.02092050209205021,
+ "ctr_holdout": 0.0209351011863224,
+ "n_select": 1434,
+ "n_holdout": 1433,
+ "eb": 0.02051978463948251,
+ "labelled_winner": false
+ },
+ {
+ "headline": "She Tries Playing A Video Game Using A Female Avatar. Brace Yourself. You'll Be Smacked By Reality.",
+ "impressions": 2849,
+ "clicks": 68,
+ "ctr": 0.0238680238680238,
+ "ctr_select": 0.025280898876404494,
+ "ctr_holdout": 0.02245614035087719,
+ "n_select": 1424,
+ "n_holdout": 1425,
+ "eb": 0.02217817565239346,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Amy Schumer Tries Playing A Video Game Using A Female Avatar. Get Ready To Be Smacked By Reality.",
+ "impressions": 2850,
+ "clicks": 44,
+ "ctr": 0.015438596491228,
+ "ctr_select": 0.01263157894736842,
+ "ctr_holdout": 0.018245614035087718,
+ "n_select": 1425,
+ "n_holdout": 1425,
+ "eb": 0.01736288475851829,
+ "labelled_winner": false
+ },
+ {
+ "headline": "She Tries To Play A Video Game Using A Female Character. It's Funny, But Not Ha-Ha Funny.",
+ "impressions": 2794,
+ "clicks": 39,
+ "ctr": 0.0139584824624194,
+ "ctr_select": 0.008589835361488905,
+ "ctr_holdout": 0.019327129563350035,
+ "n_select": 1397,
+ "n_holdout": 1397,
+ "eb": 0.015878554224735456,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Amy Schumer Tries Playing A Game Using A Female Avatar. Her Boyfriend Doesn't Make It Any Easier.",
+ "impressions": 2858,
+ "clicks": 60,
+ "ctr": 0.0209937018894331,
+ "ctr_select": 0.020293911826452064,
+ "ctr_holdout": 0.021693491952414275,
+ "n_select": 1429,
+ "n_holdout": 1429,
+ "eb": 0.020280266763329165,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.020271853761424888,
+ "pool_holdout": 0.021912350597609563,
+ "lift_select": 0.6616245816854274,
+ "lift_holdout": 0.31304625199362035,
+ "eb_lift": 0.2519078758977389,
+ "p_value": 8.275727817878576e-05,
+ "held_up": true,
+ "z_path": [
+ 1.522,
+ 1.85,
+ 2.391,
+ 2.32,
+ 1.885,
+ 1.944,
+ 2.117,
+ 1.821,
+ 1.442,
+ 1.697
+ ],
+ "first_cross": 2,
+ "final_z": 1.697
+ },
+ {
+ "id": "5342f20eaf7ae2a1fd000005",
+ "variation": "headline",
+ "week": 201414,
+ "impressions": 16972,
+ "pooled_ctr": 0.022507659674758426,
+ "arms": [
+ {
+ "headline": "The Scientific Truth About Why Some Dogs Are Mean",
+ "impressions": 2859,
+ "clicks": 85,
+ "ctr": 0.0297306750612102,
+ "ctr_select": 0.02727272727272727,
+ "ctr_holdout": 0.03219034289713086,
+ "n_select": 1430,
+ "n_holdout": 1429,
+ "eb": 0.023961471128794282,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here's A New Study On Dog Behavior That May Give You Paws",
+ "impressions": 2866,
+ "clicks": 75,
+ "ctr": 0.026168876482903,
+ "ctr_select": 0.02372644801116539,
+ "ctr_holdout": 0.028611304954640614,
+ "n_select": 1433,
+ "n_holdout": 1433,
+ "eb": 0.02260922021101975,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Internet TV Host Just Dropped Some Science On The Notion Of \u2018Aggressive Dog Breeds\u2019",
+ "impressions": 2820,
+ "clicks": 42,
+ "ctr": 0.0148936170212765,
+ "ctr_select": 0.01347517730496454,
+ "ctr_holdout": 0.016312056737588652,
+ "n_select": 1410,
+ "n_holdout": 1410,
+ "eb": 0.01872478829553866,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here's The Truth About Those 'Aggressive' Dog Breeds",
+ "impressions": 2817,
+ "clicks": 52,
+ "ctr": 0.0184593539226127,
+ "ctr_select": 0.022017045454545456,
+ "ctr_holdout": 0.014904187366926898,
+ "n_select": 1408,
+ "n_holdout": 1409,
+ "eb": 0.02195571322045096,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You're Afraid Of Pit Bulls And Not Poodles, It Might Be Time To Examine Your Prejudices",
+ "impressions": 2778,
+ "clicks": 73,
+ "ctr": 0.0262778977681785,
+ "ctr_select": 0.020158387329013677,
+ "ctr_holdout": 0.032397408207343416,
+ "n_select": 1389,
+ "n_holdout": 1389,
+ "eb": 0.021258909311790614,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Truth About Why Some Dogs Are Mean",
+ "impressions": 2832,
+ "clicks": 55,
+ "ctr": 0.0194209039548022,
+ "ctr_select": 0.024717514124293787,
+ "ctr_holdout": 0.014124293785310734,
+ "n_select": 1416,
+ "n_holdout": 1416,
+ "eb": 0.022979993865745226,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.0219184539241103,
+ "pool_holdout": 0.02309686542540655,
+ "lift_select": 0.2442815249266861,
+ "lift_holdout": 0.393710458291084,
+ "eb_lift": 0.09320991397284017,
+ "p_value": 0.12938778246600205,
+ "held_up": false,
+ "z_path": [
+ 2.541,
+ 2.571,
+ 3.145,
+ 2.892,
+ 3.588,
+ 3.5,
+ 3.345,
+ 3.12,
+ 2.708,
+ 2.51
+ ],
+ "first_cross": 0,
+ "final_z": 2.51
+ },
+ {
+ "id": "52881022a060cda2b7007f35",
+ "variation": "headline",
+ "week": 201346,
+ "impressions": 16919,
+ "pooled_ctr": 0.006915302322832319,
+ "arms": [
+ {
+ "headline": "Do You Think Video Games Are Racist?",
+ "impressions": 4608,
+ "clicks": 31,
+ "ctr": 0.0067274305555555,
+ "ctr_select": 0.007378472222222222,
+ "ctr_holdout": 0.006076388888888889,
+ "n_select": 2304,
+ "n_holdout": 2304,
+ "eb": 0.006880007164228276,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Yet Another Reason To Question The Thing We Spend 3 Billion Hours A Week Doing",
+ "impressions": 4373,
+ "clicks": 44,
+ "ctr": 0.0100617425108621,
+ "ctr_select": 0.00777676120768527,
+ "ctr_holdout": 0.012345679012345678,
+ "n_select": 2186,
+ "n_holdout": 2187,
+ "eb": 0.007060268118364603,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Standing In Line To Get A Video Game? Chew On This While You Wait.",
+ "impressions": 3781,
+ "clicks": 21,
+ "ctr": 0.0055540862205765,
+ "ctr_select": 0.006349206349206349,
+ "ctr_holdout": 0.004759386567953464,
+ "n_select": 1890,
+ "n_holdout": 1891,
+ "eb": 0.0063686403348955185,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Look Around You: Are The Video Games You\u2019re Playing Totally Racist?",
+ "impressions": 4157,
+ "clicks": 21,
+ "ctr": 0.0050517199903776,
+ "ctr_select": 0.0038498556304138597,
+ "ctr_holdout": 0.006253006253006253,
+ "n_select": 2078,
+ "n_holdout": 2079,
+ "eb": 0.005186278686713683,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.006384488058642705,
+ "pool_holdout": 0.0074459283772603715,
+ "lift_select": 0.21807122767781517,
+ "lift_holdout": 0.6580442876739172,
+ "eb_lift": 0.10584718046532982,
+ "p_value": 0.3425740359456335,
+ "held_up": true,
+ "z_path": [
+ 1.702,
+ 1.396,
+ 1.134,
+ 1.243,
+ 1.496,
+ 1.386,
+ 1.561,
+ 1.641,
+ 2.011,
+ 1.736
+ ],
+ "first_cross": 8,
+ "final_z": 1.736
+ },
+ {
+ "id": "53756c883a55adcf1c00000e",
+ "variation": "image",
+ "week": 201419,
+ "impressions": 16881,
+ "pooled_ctr": 0.009478111486286358,
+ "arms": [
+ {
+ "headline": "A Sweet Little Girl Reads A Sweet Story About How Her Two Dads Met And Created A Family",
+ "impressions": 5589,
+ "clicks": 51,
+ "ctr": 0.0091250670960815,
+ "ctr_select": 0.00894774516821761,
+ "ctr_holdout": 0.009302325581395349,
+ "n_select": 2794,
+ "n_holdout": 2795,
+ "eb": 0.009188473445826791,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Sweet Little Girl Reads A Sweet Story About How Her Two Dads Met And Created A Family",
+ "impressions": 5721,
+ "clicks": 57,
+ "ctr": 0.0099632931305715,
+ "ctr_select": 0.01048951048951049,
+ "ctr_holdout": 0.009437259699405802,
+ "n_select": 2860,
+ "n_holdout": 2861,
+ "eb": 0.010037028344781802,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Sweet Little Girl Reads A Sweet Story About How Her Two Dads Met And Created A Family",
+ "impressions": 5571,
+ "clicks": 52,
+ "ctr": 0.0093340513372823,
+ "ctr_select": 0.008973438621679828,
+ "ctr_holdout": 0.00969479353680431,
+ "n_select": 2786,
+ "n_holdout": 2785,
+ "eb": 0.009202876294863063,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.009478672985781991,
+ "pool_holdout": 0.00947755005331122,
+ "lift_select": 0.10664335664335667,
+ "lift_holdout": -0.00425113596644533,
+ "eb_lift": 0.058906490374480036,
+ "p_value": 0.4926247661873232,
+ "held_up": false,
+ "z_path": [
+ -0.747,
+ 0.675,
+ 0.488,
+ -0.082,
+ 0.198,
+ 0.916,
+ 0.33,
+ 1.032,
+ 0.263,
+ 0.342
+ ],
+ "first_cross": null,
+ "final_z": 0.342
+ },
+ {
+ "id": "537b91dde51c404662000015",
+ "variation": "headline",
+ "week": 201420,
+ "impressions": 16428,
+ "pooled_ctr": 0.008400292184075968,
+ "arms": [
+ {
+ "headline": "Guess What, Ronald? The Folks Who Work For You Have Had Enough.",
+ "impressions": 4126,
+ "clicks": 42,
+ "ctr": 0.0101793504604944,
+ "ctr_select": 0.0116335433834222,
+ "ctr_holdout": 0.00872515753756665,
+ "n_select": 2063,
+ "n_holdout": 2063,
+ "eb": 0.010051747351084175,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Ronald McDonald Now Has A Bigger Problem On His Rather Large, Clownish Hands",
+ "impressions": 4098,
+ "clicks": 37,
+ "ctr": 0.0090287945339189,
+ "ctr_select": 0.010248901903367497,
+ "ctr_holdout": 0.007808687164470474,
+ "n_select": 2049,
+ "n_holdout": 2049,
+ "eb": 0.009396956080772421,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Would You Like A Large Order Of Fries With Your Injustice?",
+ "impressions": 4118,
+ "clicks": 31,
+ "ctr": 0.0075279261777561,
+ "ctr_select": 0.006313744536182613,
+ "ctr_holdout": 0.008742107819329771,
+ "n_select": 2059,
+ "n_holdout": 2059,
+ "eb": 0.007547623522977165,
+ "labelled_winner": false
+ },
+ {
+ "headline": "That One Time When Some People Decided To Take On McDonald\u2019s Around The World",
+ "impressions": 4086,
+ "clicks": 28,
+ "ctr": 0.0068526676456191,
+ "ctr_select": 0.006363191385217817,
+ "ctr_holdout": 0.007342143906020558,
+ "n_select": 2043,
+ "n_holdout": 2043,
+ "eb": 0.0075753167923912085,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.008643778914049185,
+ "pool_holdout": 0.008156805454102752,
+ "lift_select": 0.3458862725553513,
+ "lift_holdout": 0.06967826885929052,
+ "eb_lift": 0.16288806678599177,
+ "p_value": 0.09003494119720921,
+ "held_up": false,
+ "z_path": [
+ 1.338,
+ 0.567,
+ -0.017,
+ 0.335,
+ 0.46,
+ 0.708,
+ 0.381,
+ 0.353,
+ 0.093,
+ 0.535
+ ],
+ "first_cross": null,
+ "final_z": 0.535
+ },
+ {
+ "id": "53fb519f019157fe6900001a",
+ "variation": "image",
+ "week": 201434,
+ "impressions": 16386,
+ "pooled_ctr": 0.02038325399731478,
+ "arms": [
+ {
+ "headline": "They Were At A Beauty Pageant, So She Decided To Tell A Room Full Of Strangers Her Deepest Secret",
+ "impressions": 4181,
+ "clicks": 100,
+ "ctr": 0.0239177230327672,
+ "ctr_select": 0.022488038277511963,
+ "ctr_holdout": 0.025346724055475847,
+ "n_select": 2090,
+ "n_holdout": 2091,
+ "eb": 0.021251627439111943,
+ "labelled_winner": true
+ },
+ {
+ "headline": "They Were At A Beauty Pageant, So She Decided To Tell A Room Full Of Strangers Her Deepest Secret",
+ "impressions": 4107,
+ "clicks": 98,
+ "ctr": 0.0238616995373752,
+ "ctr_select": 0.025803310613437196,
+ "ctr_holdout": 0.021919142717973697,
+ "n_select": 2054,
+ "n_holdout": 2053,
+ "eb": 0.022799094589147684,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Were At A Beauty Pageant, So She Decided To Tell A Room Full Of Strangers Her Deepest Secret",
+ "impressions": 4123,
+ "clicks": 85,
+ "ctr": 0.0206160562697065,
+ "ctr_select": 0.02133850630455868,
+ "ctr_holdout": 0.019893255701115962,
+ "n_select": 2062,
+ "n_holdout": 2061,
+ "eb": 0.02070251318650949,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Were At A Beauty Pageant, So She Decided To Tell A Room Full Of Strangers Her Deepest Secret",
+ "impressions": 3975,
+ "clicks": 51,
+ "ctr": 0.0128301886792452,
+ "ctr_select": 0.01056338028169014,
+ "ctr_holdout": 0.01509813789632612,
+ "n_select": 1988,
+ "n_holdout": 1987,
+ "eb": 0.015716558412886487,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.02013668537954601,
+ "pool_holdout": 0.0206298828125,
+ "lift_select": 0.2814080434333599,
+ "lift_holdout": 0.06249477600970721,
+ "eb_lift": 0.13221685493015833,
+ "p_value": 0.03467900442739771,
+ "held_up": false,
+ "z_path": [
+ -0.386,
+ -1.816,
+ -1.024,
+ -1.229,
+ -0.625,
+ -0.475,
+ -0.587,
+ -0.376,
+ -0.186,
+ -0.017
+ ],
+ "first_cross": null,
+ "final_z": -0.017
+ },
+ {
+ "id": "53c87edad52a7babcf000099",
+ "variation": "image",
+ "week": 201428,
+ "impressions": 16332,
+ "pooled_ctr": 0.012674504041146216,
+ "arms": [
+ {
+ "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them",
+ "impressions": 3291,
+ "clicks": 46,
+ "ctr": 0.0139775144333029,
+ "ctr_select": 0.015188335358444714,
+ "ctr_holdout": 0.01276595744680851,
+ "n_select": 1646,
+ "n_holdout": 1645,
+ "eb": 0.012965745738948178,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them",
+ "impressions": 3219,
+ "clicks": 34,
+ "ctr": 0.0105622864243553,
+ "ctr_select": 0.008695652173913044,
+ "ctr_holdout": 0.012430080795525171,
+ "n_select": 1610,
+ "n_holdout": 1609,
+ "eb": 0.010283962682943118,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them",
+ "impressions": 3342,
+ "clicks": 52,
+ "ctr": 0.0155595451825254,
+ "ctr_select": 0.013165769000598444,
+ "ctr_holdout": 0.017953321364452424,
+ "n_select": 1671,
+ "n_holdout": 1671,
+ "eb": 0.012132355021067593,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them",
+ "impressions": 3192,
+ "clicks": 38,
+ "ctr": 0.0119047619047619,
+ "ctr_select": 0.009398496240601503,
+ "ctr_holdout": 0.014411027568922305,
+ "n_select": 1596,
+ "n_holdout": 1596,
+ "eb": 0.01057626310160004,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them",
+ "impressions": 3288,
+ "clicks": 37,
+ "ctr": 0.0112530413625304,
+ "ctr_select": 0.010340632603406326,
+ "ctr_holdout": 0.012165450121654502,
+ "n_select": 1644,
+ "n_holdout": 1644,
+ "eb": 0.010952954985328965,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.011387290314681034,
+ "pool_holdout": 0.013962033067973056,
+ "lift_select": 0.3337971491657845,
+ "lift_holdout": -0.0856662933930572,
+ "eb_lift": 0.13861554247300822,
+ "p_value": 0.10383159141852126,
+ "held_up": false,
+ "z_path": [
+ 0.358,
+ 0.453,
+ 0.588,
+ 0.876,
+ 1.218,
+ 0.737,
+ 0.573,
+ -0.287,
+ -1.014,
+ -0.534
+ ],
+ "first_cross": null,
+ "final_z": -0.534
+ },
+ {
+ "id": "52b0d9903e602968240015ad",
+ "variation": "other",
+ "week": 201350,
+ "impressions": 16114,
+ "pooled_ctr": 0.008812212982499689,
+ "arms": [
+ {
+ "headline": "The Shocking Reality Of What Is 'Hidden In The Harvest'",
+ "impressions": 4190,
+ "clicks": 35,
+ "ctr": 0.0083532219570405,
+ "ctr_select": 0.009069212410501194,
+ "ctr_holdout": 0.0076372315035799524,
+ "n_select": 2095,
+ "n_holdout": 2095,
+ "eb": 0.00847789945920465,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Shocking Reality Of What Is 'Hidden In The Harvest'",
+ "impressions": 4162,
+ "clicks": 21,
+ "ctr": 0.0050456511292647,
+ "ctr_select": 0.0028832292167227293,
+ "ctr_holdout": 0.007208073041806823,
+ "n_select": 2081,
+ "n_holdout": 2081,
+ "eb": 0.005549462391569773,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Shocking Reality Of What Is 'Hidden In The Harvest'",
+ "impressions": 3754,
+ "clicks": 37,
+ "ctr": 0.0098561534363345,
+ "ctr_select": 0.008524240809802876,
+ "ctr_holdout": 0.011188066062866276,
+ "n_select": 1877,
+ "n_holdout": 1877,
+ "eb": 0.008203310442546574,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Shocking Reality Of What Is 'Hidden In The Harvest'",
+ "impressions": 4008,
+ "clicks": 49,
+ "ctr": 0.0122255489021956,
+ "ctr_select": 0.011477045908183632,
+ "ctr_holdout": 0.012974051896207584,
+ "n_select": 2004,
+ "n_holdout": 2004,
+ "eb": 0.009581976878981407,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.007943403251830706,
+ "pool_holdout": 0.009681022713168673,
+ "lift_select": 0.44485248253493026,
+ "lift_holdout": 0.34015302727877583,
+ "eb_lift": 0.20628105803051877,
+ "p_value": 0.03979246831994764,
+ "held_up": true,
+ "z_path": [
+ 1.395,
+ 1.973,
+ 2.074,
+ 1.702,
+ 1.218,
+ 1.293,
+ 1.488,
+ 1.387,
+ 1.335,
+ 1.74
+ ],
+ "first_cross": 1,
+ "final_z": 1.74
+ },
+ {
+ "id": "52b1ed8b196a3278ac0002cb",
+ "variation": "other",
+ "week": 201350,
+ "impressions": 16106,
+ "pooled_ctr": 0.0201788153483174,
+ "arms": [
+ {
+ "headline": "See Why We Have An Absolutely Ridiculous Standard Of Beauty In Just 37 Seconds",
+ "impressions": 3842,
+ "clicks": 75,
+ "ctr": 0.0195210827693909,
+ "ctr_select": 0.019260801665799063,
+ "ctr_holdout": 0.01978136387298282,
+ "n_select": 1921,
+ "n_holdout": 1921,
+ "eb": 0.019593017961662424,
+ "labelled_winner": false
+ },
+ {
+ "headline": "See Why We Have An Absolutely Ridiculous Standard Of Beauty In Just 37 Seconds",
+ "impressions": 4132,
+ "clicks": 76,
+ "ctr": 0.0183930300096805,
+ "ctr_select": 0.017909002904162634,
+ "ctr_holdout": 0.01887705711519845,
+ "n_select": 2066,
+ "n_holdout": 2066,
+ "eb": 0.0189449377800756,
+ "labelled_winner": false
+ },
+ {
+ "headline": "See Why We Have An Absolutely Ridiculous Standard Of Beauty In Just 37 Seconds",
+ "impressions": 3591,
+ "clicks": 73,
+ "ctr": 0.0203285992759676,
+ "ctr_select": 0.021158129175946547,
+ "ctr_holdout": 0.019498607242339833,
+ "n_select": 1796,
+ "n_holdout": 1795,
+ "eb": 0.0204314673307841,
+ "labelled_winner": false
+ },
+ {
+ "headline": "See Why We Have An Absolutely Ridiculous Standard Of Beauty In Just 37 Seconds",
+ "impressions": 4541,
+ "clicks": 101,
+ "ctr": 0.0222417969610218,
+ "ctr_select": 0.021145374449339206,
+ "ctr_holdout": 0.023337736679876705,
+ "n_select": 2270,
+ "n_holdout": 2271,
+ "eb": 0.020500234827145,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.01986837203526636,
+ "pool_holdout": 0.020489258661368435,
+ "lift_select": 0.0649150890868595,
+ "lift_holdout": -0.04834979319658994,
+ "eb_lift": 0.02834129009252706,
+ "p_value": 0.6567833803052845,
+ "held_up": false,
+ "z_path": [
+ -1.013,
+ -1.911,
+ -1.873,
+ -1.964,
+ -1.493,
+ -1.277,
+ -0.814,
+ -0.731,
+ -0.216,
+ -0.592
+ ],
+ "first_cross": 3,
+ "final_z": -0.592
+ },
+ {
+ "id": "53d014f6e54a0f95e8000088",
+ "variation": "image",
+ "week": 201429,
+ "impressions": 16064,
+ "pooled_ctr": 0.014379980079681274,
+ "arms": [
+ {
+ "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out",
+ "impressions": 2698,
+ "clicks": 34,
+ "ctr": 0.0126019273535952,
+ "ctr_select": 0.01111934766493699,
+ "ctr_holdout": 0.014084507042253521,
+ "n_select": 1349,
+ "n_holdout": 1349,
+ "eb": 0.013455813304447523,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out",
+ "impressions": 2640,
+ "clicks": 39,
+ "ctr": 0.0147727272727272,
+ "ctr_select": 0.015909090909090907,
+ "ctr_holdout": 0.013636363636363636,
+ "n_select": 1320,
+ "n_holdout": 1320,
+ "eb": 0.015212478949778167,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out",
+ "impressions": 2740,
+ "clicks": 41,
+ "ctr": 0.014963503649635,
+ "ctr_select": 0.016058394160583942,
+ "ctr_holdout": 0.013868613138686132,
+ "n_select": 1370,
+ "n_holdout": 1370,
+ "eb": 0.015277391009194069,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out",
+ "impressions": 2681,
+ "clicks": 33,
+ "ctr": 0.0123088399850801,
+ "ctr_select": 0.014925373134328358,
+ "ctr_holdout": 0.009694258016405667,
+ "n_select": 1340,
+ "n_holdout": 1341,
+ "eb": 0.014855902303943513,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out",
+ "impressions": 2658,
+ "clicks": 33,
+ "ctr": 0.0124153498871331,
+ "ctr_select": 0.013544018058690745,
+ "ctr_holdout": 0.011286681715575621,
+ "n_select": 1329,
+ "n_holdout": 1329,
+ "eb": 0.014352272657778239,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out",
+ "impressions": 2647,
+ "clicks": 51,
+ "ctr": 0.0192670948243294,
+ "ctr_select": 0.017371601208459216,
+ "ctr_holdout": 0.021164021164021163,
+ "n_select": 1324,
+ "n_holdout": 1323,
+ "eb": 0.01574496413703939,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 5,
+ "pool_select": 0.01481573705179283,
+ "pool_holdout": 0.013944223107569721,
+ "lift_select": 0.17251009164995312,
+ "lift_holdout": 0.5177626606198034,
+ "eb_lift": 0.06271892393865874,
+ "p_value": 0.3996092861143552,
+ "held_up": true,
+ "z_path": [
+ 0.903,
+ 0.571,
+ 0.481,
+ -0.071,
+ 0.557,
+ 1.24,
+ 2.068,
+ 2.284,
+ 1.597,
+ 1.219
+ ],
+ "first_cross": 6,
+ "final_z": 1.219
+ },
+ {
+ "id": "52f27c51c67e22334600001e",
+ "variation": "image",
+ "week": 201405,
+ "impressions": 15959,
+ "pooled_ctr": 0.022181840967479164,
+ "arms": [
+ {
+ "headline": "Here Is The Harrowing Footage Of Gay People Being Beaten On Camera In Russia",
+ "impressions": 3989,
+ "clicks": 77,
+ "ctr": 0.0193030834795688,
+ "ctr_select": 0.023069207622868605,
+ "ctr_holdout": 0.015538847117794486,
+ "n_select": 1994,
+ "n_holdout": 1995,
+ "eb": 0.022660331249792377,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here Is The Harrowing Footage Of Gay People Being Beaten On Camera In Russia",
+ "impressions": 4060,
+ "clicks": 109,
+ "ctr": 0.026847290640394,
+ "ctr_select": 0.026600985221674877,
+ "ctr_holdout": 0.027093596059113302,
+ "n_select": 2030,
+ "n_holdout": 2030,
+ "eb": 0.024312752499629935,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here Is The Harrowing Footage Of Gay People Being Beaten On Camera In Russia",
+ "impressions": 3982,
+ "clicks": 90,
+ "ctr": 0.0226017076845806,
+ "ctr_select": 0.020090406830738324,
+ "ctr_holdout": 0.025113008538422903,
+ "n_select": 1991,
+ "n_holdout": 1991,
+ "eb": 0.02128357673766438,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here Is The Harrowing Footage Of Gay People Being Beaten On Camera In Russia",
+ "impressions": 3928,
+ "clicks": 78,
+ "ctr": 0.0198574338085539,
+ "ctr_select": 0.019348268839103868,
+ "ctr_holdout": 0.020366598778004074,
+ "n_select": 1964,
+ "n_holdout": 1964,
+ "eb": 0.020950684617200017,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.02230855996992104,
+ "pool_holdout": 0.02205513784461153,
+ "lift_select": 0.19241157912215656,
+ "lift_holdout": 0.22844827586206895,
+ "eb_lift": 0.0898396190704902,
+ "p_value": 0.12937186818373697,
+ "held_up": true,
+ "z_path": [
+ 0.47,
+ 0.132,
+ -1.02,
+ -0.427,
+ -0.082,
+ 1.168,
+ 1.402,
+ 1.979,
+ 1.928,
+ 2.252
+ ],
+ "first_cross": 7,
+ "final_z": 2.252
+ },
+ {
+ "id": "52858f3907156c96cc001f74",
+ "variation": "headline",
+ "week": 201345,
+ "impressions": 15647,
+ "pooled_ctr": 0.01910909439509171,
+ "arms": [
+ {
+ "headline": "A Suicide Bomber Destroyed His Leg But An Artist Brought It Back To Life",
+ "impressions": 4117,
+ "clicks": 79,
+ "ctr": 0.0191887296575176,
+ "ctr_select": 0.019436345966958212,
+ "ctr_holdout": 0.01894123360854784,
+ "n_select": 2058,
+ "n_holdout": 2059,
+ "eb": 0.019501956180535008,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Think Your Scars Are Ugly? This Artist Thinks You're A Walking Work Of Art.",
+ "impressions": 3905,
+ "clicks": 94,
+ "ctr": 0.0240717029449423,
+ "ctr_select": 0.02612704918032787,
+ "ctr_holdout": 0.022017409114183307,
+ "n_select": 1952,
+ "n_holdout": 1953,
+ "eb": 0.022562408760340383,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How One Artist Is Telling The Story Of People's Lives Through Their Scars",
+ "impressions": 3824,
+ "clicks": 63,
+ "ctr": 0.0164748953974895,
+ "ctr_select": 0.013598326359832637,
+ "ctr_holdout": 0.019351464435146442,
+ "n_select": 1912,
+ "n_holdout": 1912,
+ "eb": 0.016865192237470544,
+ "labelled_winner": false
+ },
+ {
+ "headline": "After A Suicide Bomber Destroyed This Man's Leg, An Artist Transformed It Into A Work Of Art",
+ "impressions": 3801,
+ "clicks": 63,
+ "ctr": 0.0165745856353591,
+ "ctr_select": 0.018947368421052633,
+ "ctr_holdout": 0.014203051025775907,
+ "n_select": 1900,
+ "n_holdout": 1901,
+ "eb": 0.019284138075063375,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.019560214778828945,
+ "pool_holdout": 0.01865814696485623,
+ "lift_select": 0.3357240437158471,
+ "lift_holdout": 0.18004264601701636,
+ "eb_lift": 0.1534847145319116,
+ "p_value": 0.015586443794496661,
+ "held_up": true,
+ "z_path": [
+ -1.241,
+ -0.165,
+ 0.47,
+ 0.58,
+ 1.5,
+ 1.522,
+ 1.022,
+ 1.425,
+ 1.546,
+ 1.505
+ ],
+ "first_cross": null,
+ "final_z": 1.505
+ },
+ {
+ "id": "544ada1437d2889992000090",
+ "variation": "headline",
+ "week": 201442,
+ "impressions": 15611,
+ "pooled_ctr": 0.006597911728909103,
+ "arms": [
+ {
+ "headline": "Which Religions Are Protected In The United States?",
+ "impressions": 3876,
+ "clicks": 31,
+ "ctr": 0.0079979360165118,
+ "ctr_select": 0.008255933952528379,
+ "ctr_holdout": 0.007739938080495356,
+ "n_select": 1938,
+ "n_holdout": 1938,
+ "eb": 0.008085293481375159,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Religion Is Tops When It Comes To We The People?",
+ "impressions": 4001,
+ "clicks": 31,
+ "ctr": 0.0077480629842539,
+ "ctr_select": 0.011,
+ "ctr_holdout": 0.004497751124437781,
+ "n_select": 2000,
+ "n_holdout": 2001,
+ "eb": 0.009358822148806647,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Video About Church And State May Make You A Believer In Why It Exists",
+ "impressions": 3815,
+ "clicks": 28,
+ "ctr": 0.0073394495412844,
+ "ctr_select": 0.00890985324947589,
+ "ctr_holdout": 0.0057682223387519665,
+ "n_select": 1908,
+ "n_holdout": 1907,
+ "eb": 0.00837934168152885,
+ "labelled_winner": false
+ },
+ {
+ "headline": "We Protect Religion But We Don't Recognize It. How Do We Separate Church And State?",
+ "impressions": 3919,
+ "clicks": 13,
+ "ctr": 0.00331717274815,
+ "ctr_select": 0.0035714285714285713,
+ "ctr_holdout": 0.0030627871362940277,
+ "n_select": 1960,
+ "n_holdout": 1959,
+ "eb": 0.0059397757074923585,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.007942608250064053,
+ "pool_holdout": 0.005253042921204356,
+ "lift_select": 0.38493548387096777,
+ "lift_holdout": -0.14378176765275907,
+ "eb_lift": 0.1783058982836241,
+ "p_value": 0.07409231981375168,
+ "held_up": false,
+ "z_path": [
+ -0.503,
+ -0.381,
+ -0.094,
+ 0.112,
+ 0.434,
+ -0.31,
+ -0.309,
+ -0.161,
+ -0.041,
+ 0.209
+ ],
+ "first_cross": null,
+ "final_z": 0.209
+ },
+ {
+ "id": "5447163bcdf1931e1400002b",
+ "variation": "headline",
+ "week": 201442,
+ "impressions": 15580,
+ "pooled_ctr": 0.00853658536585366,
+ "arms": [
+ {
+ "headline": "Like Coke Or Pepsi? You Might Not Like Them As Much After You Hear What They're Doing.",
+ "impressions": 3923,
+ "clicks": 40,
+ "ctr": 0.0101962783583991,
+ "ctr_select": 0.008664627930682976,
+ "ctr_holdout": 0.011728709841917389,
+ "n_select": 1962,
+ "n_holdout": 1961,
+ "eb": 0.008560525941624401,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Coke And Pepsi Might Not Want Us To Know About This Little Secret. Ahh, Well \u2026 We Do Now.",
+ "impressions": 3921,
+ "clicks": 38,
+ "ctr": 0.0096914052537617,
+ "ctr_select": 0.010714285714285714,
+ "ctr_holdout": 0.008669046404895462,
+ "n_select": 1960,
+ "n_holdout": 1961,
+ "eb": 0.009499610778669192,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Pepsi And Coke Don\u2019t Want Us To Know About How They\u2019re Playing Voters. Secret\u2019s Out. Bah Dah BOOM!",
+ "impressions": 3956,
+ "clicks": 23,
+ "ctr": 0.005813953488372,
+ "ctr_select": 0.005055611729019211,
+ "ctr_holdout": 0.006572295247724975,
+ "n_select": 1978,
+ "n_holdout": 1978,
+ "eb": 0.006899103535814354,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Pepsi And Coke Don\u2019t Want Us To Know A Secret About How They\u2019re Playing Voters. Secret\u2019s Out.",
+ "impressions": 3780,
+ "clicks": 32,
+ "ctr": 0.0084656084656084,
+ "ctr_select": 0.009523809523809525,
+ "ctr_holdout": 0.007407407407407408,
+ "n_select": 1890,
+ "n_holdout": 1890,
+ "eb": 0.008944669000669916,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.008472400513478818,
+ "pool_holdout": 0.008600770218228498,
+ "lift_select": 0.26461038961038974,
+ "lift_holdout": 0.007938380509487253,
+ "eb_lift": 0.12124193887625778,
+ "p_value": 0.2106582277897937,
+ "held_up": false,
+ "z_path": [
+ 0.277,
+ 0.365,
+ -0.291,
+ 0.068,
+ -0.114,
+ -0.123,
+ 0.138,
+ 0.387,
+ 0.452,
+ 0.567
+ ],
+ "first_cross": null,
+ "final_z": 0.567
+ },
+ {
+ "id": "52a5592e0fe0295fc000816d",
+ "variation": "image",
+ "week": 201349,
+ "impressions": 15396,
+ "pooled_ctr": 0.014744089373863341,
+ "arms": [
+ {
+ "headline": "Say Hello To The Ingenious Invention That's Maybe Gonna Be Around Longer Than All Of Us",
+ "impressions": 3688,
+ "clicks": 28,
+ "ctr": 0.0075921908893709,
+ "ctr_select": 0.009219088937093275,
+ "ctr_holdout": 0.00596529284164859,
+ "n_select": 1844,
+ "n_holdout": 1844,
+ "eb": 0.012043852240492764,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Say Hello To The Ingenious Invention That's Maybe Gonna Be Around Longer Than All Of Us",
+ "impressions": 4001,
+ "clicks": 65,
+ "ctr": 0.0162459385153711,
+ "ctr_select": 0.016,
+ "ctr_holdout": 0.01649175412293853,
+ "n_select": 2000,
+ "n_holdout": 2001,
+ "eb": 0.01508277713706444,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Say Hello To The Ingenious Invention That's Maybe Gonna Be Around Longer Than All Of Us",
+ "impressions": 4001,
+ "clicks": 72,
+ "ctr": 0.0179955011247188,
+ "ctr_select": 0.0155,
+ "ctr_holdout": 0.02048975512243878,
+ "n_select": 2000,
+ "n_holdout": 2001,
+ "eb": 0.014851172224352116,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Say Hello To The Ingenious Invention That's Maybe Gonna Be Around Longer Than All Of Us",
+ "impressions": 3706,
+ "clicks": 62,
+ "ctr": 0.0167296276308688,
+ "ctr_select": 0.016189962223421478,
+ "ctr_holdout": 0.017269293038316244,
+ "n_select": 1853,
+ "n_holdout": 1853,
+ "eb": 0.015134847139382557,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.01429128231778615,
+ "pool_holdout": 0.015196778802441876,
+ "lift_select": 0.13285581121522827,
+ "lift_holdout": 0.1363785222392886,
+ "eb_lift": 0.05902653119843215,
+ "p_value": 0.4293597867880051,
+ "held_up": false,
+ "z_path": [
+ -1.674,
+ -1.856,
+ -1.538,
+ -1.524,
+ -0.057,
+ -0.561,
+ -0.501,
+ -0.338,
+ -0.284,
+ 0.167
+ ],
+ "first_cross": null,
+ "final_z": 0.167
+ },
+ {
+ "id": "53a4685862e6f52d4f000069",
+ "variation": "both",
+ "week": 201424,
+ "impressions": 15394,
+ "pooled_ctr": 0.03488372093023256,
+ "arms": [
+ {
+ "headline": "A Customer Wants To Make A Waitress's Life Miserable. She Responds In The Best Way Possible.",
+ "impressions": 2564,
+ "clicks": 81,
+ "ctr": 0.031591263650546,
+ "ctr_select": 0.031981279251170044,
+ "ctr_holdout": 0.031201248049921998,
+ "n_select": 1282,
+ "n_holdout": 1282,
+ "eb": 0.03154703093723764,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Customer Wants To Make A Waitress's Life Miserable. So The Waitress Blurts Out The Truth.",
+ "impressions": 2574,
+ "clicks": 85,
+ "ctr": 0.033022533022533,
+ "ctr_select": 0.03418803418803419,
+ "ctr_holdout": 0.031857031857031856,
+ "n_select": 1287,
+ "n_holdout": 1287,
+ "eb": 0.032335520036243036,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Customer Wants To Make A Waitress's Life Miserable. She Responds In The Best Way Possible.",
+ "impressions": 2592,
+ "clicks": 106,
+ "ctr": 0.040895061728395,
+ "ctr_select": 0.030864197530864196,
+ "ctr_holdout": 0.05092592592592592,
+ "n_select": 1296,
+ "n_holdout": 1296,
+ "eb": 0.0311480882057323,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Customer Wants To Make A Waitress's Life Miserable. So The Waitress Blurts Out The Truth.",
+ "impressions": 2538,
+ "clicks": 72,
+ "ctr": 0.0283687943262411,
+ "ctr_select": 0.030732860520094562,
+ "ctr_holdout": 0.026004728132387706,
+ "n_select": 1269,
+ "n_holdout": 1269,
+ "eb": 0.031103757283101215,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Customer Wants To Make A Waitress's Life Miserable. She Responds In The Best Way Possible.",
+ "impressions": 2575,
+ "clicks": 81,
+ "ctr": 0.0314563106796116,
+ "ctr_select": 0.024844720496894408,
+ "ctr_holdout": 0.03807303807303807,
+ "n_select": 1288,
+ "n_holdout": 1287,
+ "eb": 0.028998486123341275,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Customer Wants To Make A Waitress's Life Miserable. So The Waitress Blurts Out The Truth.",
+ "impressions": 2551,
+ "clicks": 112,
+ "ctr": 0.0439043512348098,
+ "ctr_select": 0.03526645768025078,
+ "ctr_holdout": 0.05254901960784314,
+ "n_select": 1276,
+ "n_holdout": 1275,
+ "eb": 0.03271276120448333,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 5,
+ "pool_select": 0.03130683294362172,
+ "pool_holdout": 0.038461538461538464,
+ "lift_select": 0.1264779718778859,
+ "lift_holdout": 0.3662745098039215,
+ "eb_lift": 0.04490803216644257,
+ "p_value": 0.37387246759247594,
+ "held_up": true,
+ "z_path": [
+ 0.93,
+ 1.174,
+ 1.255,
+ 1.558,
+ 1.324,
+ 1.696,
+ 2.169,
+ 2.419,
+ 2.578,
+ 2.026
+ ],
+ "first_cross": 6,
+ "final_z": 2.026
+ },
+ {
+ "id": "54392e97f50d2746c0000001",
+ "variation": "image",
+ "week": 201440,
+ "impressions": 15393,
+ "pooled_ctr": 0.006756317806795297,
+ "arms": [
+ {
+ "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?",
+ "impressions": 3067,
+ "clicks": 20,
+ "ctr": 0.006521030322791,
+ "ctr_select": 0.007170795306388526,
+ "ctr_holdout": 0.005870841487279843,
+ "n_select": 1534,
+ "n_holdout": 1533,
+ "eb": 0.007078030003862936,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?",
+ "impressions": 3080,
+ "clicks": 21,
+ "ctr": 0.0068181818181818,
+ "ctr_select": 0.005844155844155844,
+ "ctr_holdout": 0.007792207792207792,
+ "n_select": 1540,
+ "n_holdout": 1540,
+ "eb": 0.006548577377865412,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?",
+ "impressions": 3093,
+ "clicks": 25,
+ "ctr": 0.0080827675396055,
+ "ctr_select": 0.01034928848641656,
+ "ctr_holdout": 0.005817711700064641,
+ "n_select": 1546,
+ "n_holdout": 1547,
+ "eb": 0.0083501439257307,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?",
+ "impressions": 3093,
+ "clicks": 23,
+ "ctr": 0.0074361461364371,
+ "ctr_select": 0.007761966364812419,
+ "ctr_holdout": 0.007110536522301228,
+ "n_select": 1546,
+ "n_holdout": 1547,
+ "eb": 0.007314866244663066,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?",
+ "impressions": 3060,
+ "clicks": 15,
+ "ctr": 0.0049019607843137,
+ "ctr_select": 0.00392156862745098,
+ "ctr_holdout": 0.0058823529411764705,
+ "n_select": 1530,
+ "n_holdout": 1530,
+ "eb": 0.0057859098162513535,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.007016632016632017,
+ "pool_holdout": 0.006496037417175523,
+ "lift_select": 0.47496526280484885,
+ "lift_holdout": -0.10442146089204907,
+ "eb_lift": 0.19005014171154566,
+ "p_value": 0.0790674108497883,
+ "held_up": false,
+ "z_path": [
+ 0.0,
+ -0.58,
+ 0.502,
+ 0.41,
+ 0.41,
+ 0.733,
+ 0.874,
+ 0.688,
+ 0.765,
+ 0.29
+ ],
+ "first_cross": null,
+ "final_z": 0.29
+ },
+ {
+ "id": "53e3e69f6dcda749e100001e",
+ "variation": "image",
+ "week": 201431,
+ "impressions": 15388,
+ "pooled_ctr": 0.012152326488172603,
+ "arms": [
+ {
+ "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.",
+ "impressions": 2518,
+ "clicks": 38,
+ "ctr": 0.0150913423351866,
+ "ctr_select": 0.013502779984114376,
+ "ctr_holdout": 0.016679904686258934,
+ "n_select": 1259,
+ "n_holdout": 1259,
+ "eb": 0.011828503496807205,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.",
+ "impressions": 2538,
+ "clicks": 35,
+ "ctr": 0.0137903861308116,
+ "ctr_select": 0.014972419227738377,
+ "ctr_holdout": 0.01260835303388495,
+ "n_select": 1269,
+ "n_holdout": 1269,
+ "eb": 0.012353140675625112,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.",
+ "impressions": 2545,
+ "clicks": 31,
+ "ctr": 0.012180746561886,
+ "ctr_select": 0.008647798742138365,
+ "ctr_holdout": 0.015710919088766692,
+ "n_select": 1272,
+ "n_holdout": 1273,
+ "eb": 0.010114216932366358,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.",
+ "impressions": 2582,
+ "clicks": 23,
+ "ctr": 0.0089078233927188,
+ "ctr_select": 0.006971340046475601,
+ "ctr_holdout": 0.010844306738962044,
+ "n_select": 1291,
+ "n_holdout": 1291,
+ "eb": 0.00950674844895755,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.",
+ "impressions": 2560,
+ "clicks": 33,
+ "ctr": 0.012890625,
+ "ctr_select": 0.01015625,
+ "ctr_holdout": 0.015625,
+ "n_select": 1280,
+ "n_holdout": 1280,
+ "eb": 0.010647637586933614,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.",
+ "impressions": 2645,
+ "clicks": 27,
+ "ctr": 0.0102079395085066,
+ "ctr_select": 0.0113464447806354,
+ "ctr_holdout": 0.009070294784580499,
+ "n_select": 1322,
+ "n_holdout": 1323,
+ "eb": 0.011074266210341466,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.01091901728844404,
+ "pool_holdout": 0.013385315139701105,
+ "lift_select": 0.37122406094037297,
+ "lift_holdout": -0.05804585829374087,
+ "eb_lift": 0.13134180020933317,
+ "p_value": 0.12838407947755892,
+ "held_up": false,
+ "z_path": [
+ 1.411,
+ 0.567,
+ 0.883,
+ 0.521,
+ 0.294,
+ 0.123,
+ -0.313,
+ -0.543,
+ -0.402,
+ -0.388
+ ],
+ "first_cross": null,
+ "final_z": -0.388
+ },
+ {
+ "id": "538fd67daa2af376fb000061",
+ "variation": "headline",
+ "week": 201422,
+ "impressions": 15387,
+ "pooled_ctr": 0.007733801260804575,
+ "arms": [
+ {
+ "headline": "Neil DeGrasse Tyson: 'If People Begin To Lose Their Wealth, They Change Their Mind Real Fast'",
+ "impressions": 3765,
+ "clicks": 31,
+ "ctr": 0.0082337317397078,
+ "ctr_select": 0.005844845908607864,
+ "ctr_holdout": 0.010621348911311737,
+ "n_select": 1882,
+ "n_holdout": 1883,
+ "eb": 0.0064930325025082955,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Neil DeGrasse Tyson: 'If People Begin To Lose Their Wealth, They Change Their Mind Real Fast'",
+ "impressions": 3838,
+ "clicks": 23,
+ "ctr": 0.0059927045336112,
+ "ctr_select": 0.005211047420531527,
+ "ctr_holdout": 0.006774361646690985,
+ "n_select": 1919,
+ "n_holdout": 1919,
+ "eb": 0.006200294552090872,
+ "labelled_winner": false
+ },
+ {
+ "headline": "'If People Begin To Lose Their Wealth, They Change Their Mind Real Fast.' Oh, Yes.",
+ "impressions": 3839,
+ "clicks": 29,
+ "ctr": 0.0075540505339932,
+ "ctr_select": 0.008333333333333333,
+ "ctr_holdout": 0.006774361646690985,
+ "n_select": 1920,
+ "n_holdout": 1919,
+ "eb": 0.00761469456908438,
+ "labelled_winner": false
+ },
+ {
+ "headline": "'If People Begin To Lose Their Wealth, They Change Their Mind Real Fast.' Oh, Yes ... What He Said.",
+ "impressions": 3945,
+ "clicks": 36,
+ "ctr": 0.0091254752851711,
+ "ctr_select": 0.008620689655172414,
+ "ctr_holdout": 0.00963000506842372,
+ "n_select": 1972,
+ "n_holdout": 1973,
+ "eb": 0.007755505396326373,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.007019368256856883,
+ "pool_holdout": 0.008448141408890044,
+ "lift_select": 0.2281289910600255,
+ "lift_holdout": 0.1398962922531093,
+ "eb_lift": 0.10487227803590349,
+ "p_value": 0.3233002646613852,
+ "held_up": false,
+ "z_path": [
+ 0.671,
+ 0.723,
+ 0.593,
+ 1.16,
+ 0.951,
+ 1.091,
+ 0.908,
+ 1.039,
+ 1.118,
+ 0.762
+ ],
+ "first_cross": null,
+ "final_z": 0.762
+ },
+ {
+ "id": "5387a93cd006a99c9f00003a",
+ "variation": "headline",
+ "week": 201421,
+ "impressions": 15347,
+ "pooled_ctr": 0.01407441193718642,
+ "arms": [
+ {
+ "headline": "If A Restaurant Gets An \u2018A\u2019 That Means It\u2019s Clean. If It Gets One Of These That Means It\u2019s Awesome.",
+ "impressions": 3166,
+ "clicks": 50,
+ "ctr": 0.0157927984838913,
+ "ctr_select": 0.017056222362602652,
+ "ctr_holdout": 0.014529374605180037,
+ "n_select": 1583,
+ "n_holdout": 1583,
+ "eb": 0.016445365507732353,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Leftover Food, Meet Hungry People. There, That Was Easy, Wasn\u2019t It?",
+ "impressions": 3043,
+ "clicks": 21,
+ "ctr": 0.0069010844561288,
+ "ctr_select": 0.00788436268068331,
+ "ctr_holdout": 0.005917159763313609,
+ "n_select": 1522,
+ "n_holdout": 1521,
+ "eb": 0.012800070023367676,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Did Your Mom Ever Tell You To Finish Your Meal Because Of Starving Kids? This Makes Way More Sense.",
+ "impressions": 3034,
+ "clicks": 70,
+ "ctr": 0.023071852340145,
+ "ctr_select": 0.024390243902439025,
+ "ctr_holdout": 0.021753460777851022,
+ "n_select": 1517,
+ "n_holdout": 1517,
+ "eb": 0.019336178546296993,
+ "labelled_winner": false
+ },
+ {
+ "headline": "We Care A Lot About Where Our Food Comes From, But What About Where The Uneaten Stuff Goes?",
+ "impressions": 3051,
+ "clicks": 13,
+ "ctr": 0.0042608980662078,
+ "ctr_select": 0.007208387942332897,
+ "ctr_holdout": 0.0013114754098360656,
+ "n_select": 1526,
+ "n_holdout": 1525,
+ "eb": 0.012526583302800112,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Best Restaurant Window Sticker Since The One With The Big \u2018A\u2019 On It",
+ "impressions": 3053,
+ "clicks": 62,
+ "ctr": 0.0203078938748771,
+ "ctr_select": 0.023591087811271297,
+ "ctr_holdout": 0.017026850032743943,
+ "n_select": 1526,
+ "n_holdout": 1527,
+ "eb": 0.019030736834400347,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.01602814698983581,
+ "pool_holdout": 0.012120422259872279,
+ "lift_select": 0.5217132659131469,
+ "lift_holdout": 0.794777468262913,
+ "eb_lift": 0.20638889564457807,
+ "p_value": 0.0037873404799747736,
+ "held_up": true,
+ "z_path": [
+ 1.532,
+ 0.917,
+ 0.386,
+ 1.21,
+ 1.066,
+ 0.469,
+ 0.235,
+ 0.798,
+ 1.054,
+ 0.74
+ ],
+ "first_cross": null,
+ "final_z": 0.74
+ },
+ {
+ "id": "546d11cd545240e554000041",
+ "variation": "image",
+ "week": 201446,
+ "impressions": 15294,
+ "pooled_ctr": 0.004184647574212109,
+ "arms": [
+ {
+ "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead",
+ "impressions": 3049,
+ "clicks": 13,
+ "ctr": 0.0042636930141029,
+ "ctr_select": 0.003937007874015748,
+ "ctr_holdout": 0.0045901639344262295,
+ "n_select": 1524,
+ "n_holdout": 1525,
+ "eb": 0.004559771405450846,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead",
+ "impressions": 3000,
+ "clicks": 22,
+ "ctr": 0.0073333333333333,
+ "ctr_select": 0.008666666666666666,
+ "ctr_holdout": 0.006,
+ "n_select": 1500,
+ "n_holdout": 1500,
+ "eb": 0.006422002274359335,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead",
+ "impressions": 3092,
+ "clicks": 10,
+ "ctr": 0.0032341526520051,
+ "ctr_select": 0.00646830530401035,
+ "ctr_holdout": 0.0,
+ "n_select": 1546,
+ "n_holdout": 1546,
+ "eb": 0.005569085623900896,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead",
+ "impressions": 3098,
+ "clicks": 11,
+ "ctr": 0.0035506778566817,
+ "ctr_select": 0.0032278889606197547,
+ "ctr_holdout": 0.003873466752743706,
+ "n_select": 1549,
+ "n_holdout": 1549,
+ "eb": 0.004271671747750383,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead",
+ "impressions": 3055,
+ "clicks": 8,
+ "ctr": 0.0026186579378068,
+ "ctr_select": 0.002617801047120419,
+ "ctr_holdout": 0.0026195153896529143,
+ "n_select": 1528,
+ "n_holdout": 1527,
+ "eb": 0.004034967003068992,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.00496926899437688,
+ "pool_holdout": 0.0034000261540473387,
+ "lift_select": 0.7440526315789473,
+ "lift_holdout": 0.7646923076923078,
+ "eb_lift": 0.29234345768489045,
+ "p_value": 0.023123530464628953,
+ "held_up": true,
+ "z_path": [
+ 1.184,
+ 1.322,
+ 0.566,
+ 0.795,
+ 0.758,
+ 0.727,
+ 1.119,
+ 2.131,
+ 2.066,
+ 2.213
+ ],
+ "first_cross": 7,
+ "final_z": 2.213
+ },
+ {
+ "id": "5474a87ea341b993aa00004f",
+ "variation": "headline",
+ "week": 201447,
+ "impressions": 15278,
+ "pooled_ctr": 0.014072522581489723,
+ "arms": [
+ {
+ "headline": "1 Ugly Article That Should Never Exist And 1 Beautiful Response To It",
+ "impressions": 3042,
+ "clicks": 42,
+ "ctr": 0.0138067061143984,
+ "ctr_select": 0.015779092702169626,
+ "ctr_holdout": 0.011834319526627219,
+ "n_select": 1521,
+ "n_holdout": 1521,
+ "eb": 0.015025346934253475,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Ugly Things Are Said About Women With Eating Disorders. I've Waited For This Kind Of Response.",
+ "impressions": 3070,
+ "clicks": 35,
+ "ctr": 0.011400651465798,
+ "ctr_select": 0.011074918566775244,
+ "ctr_holdout": 0.011726384364820847,
+ "n_select": 1535,
+ "n_holdout": 1535,
+ "eb": 0.013153838861246171,
+ "labelled_winner": false
+ },
+ {
+ "headline": "1 Ugly Article About Body Issues That Should Never Exist And 1 Beautiful Response To It",
+ "impressions": 2950,
+ "clicks": 36,
+ "ctr": 0.0122033898305084,
+ "ctr_select": 0.009491525423728813,
+ "ctr_holdout": 0.014915254237288136,
+ "n_select": 1475,
+ "n_holdout": 1475,
+ "eb": 0.01257093723277629,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Misunderstood Truth About Dating A Woman With An Ugly Eating Disorder",
+ "impressions": 3109,
+ "clicks": 27,
+ "ctr": 0.008684464458025,
+ "ctr_select": 0.009009009009009009,
+ "ctr_holdout": 0.008360128617363344,
+ "n_select": 1554,
+ "n_holdout": 1555,
+ "eb": 0.012314433725232133,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Beautiful And Real Response To A Disgusting Article That Should Never Exist",
+ "impressions": 3107,
+ "clicks": 75,
+ "ctr": 0.0241390408754425,
+ "ctr_select": 0.02702702702702703,
+ "ctr_holdout": 0.021249195106245976,
+ "n_select": 1554,
+ "n_holdout": 1553,
+ "eb": 0.01954640329932514,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.014530697735305667,
+ "pool_holdout": 0.013614347427673779,
+ "lift_select": 0.8599951302654008,
+ "lift_holdout": 0.5607942443905098,
+ "eb_lift": 0.3451799531850879,
+ "p_value": 3.979609515560447e-06,
+ "held_up": true,
+ "z_path": [
+ 1.792,
+ 1.946,
+ 1.938,
+ 2.123,
+ 2.545,
+ 2.553,
+ 2.822,
+ 2.78,
+ 2.72,
+ 2.965
+ ],
+ "first_cross": 3,
+ "final_z": 2.965
+ },
+ {
+ "id": "5494308437323200121e0000",
+ "variation": "headline",
+ "week": 201450,
+ "impressions": 15266,
+ "pooled_ctr": 0.01978252325429058,
+ "arms": [
+ {
+ "headline": "A Tampon Commercial That Shows Just How Confusing Actual Tampon Commercials Are",
+ "impressions": 3077,
+ "clicks": 61,
+ "ctr": 0.0198245043873903,
+ "ctr_select": 0.01950585175552666,
+ "ctr_holdout": 0.02014294996751137,
+ "n_select": 1538,
+ "n_holdout": 1539,
+ "eb": 0.019202645797161494,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Tampon Commercial That Totally Explains Why Actual Tampon Commercials Are So Confusing",
+ "impressions": 3085,
+ "clicks": 66,
+ "ctr": 0.0213938411669367,
+ "ctr_select": 0.017509727626459144,
+ "ctr_holdout": 0.025275437459494492,
+ "n_select": 1542,
+ "n_holdout": 1543,
+ "eb": 0.018405482180048564,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Tampon Commercial That Shows Just How Mystifying Actual Tampon Commercials Are",
+ "impressions": 3121,
+ "clicks": 68,
+ "ctr": 0.0217878884972765,
+ "ctr_select": 0.01730769230769231,
+ "ctr_holdout": 0.02626521460602178,
+ "n_select": 1560,
+ "n_holdout": 1561,
+ "eb": 0.018320045226351802,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why Tampon Commercials Are So Confusing",
+ "impressions": 2981,
+ "clicks": 38,
+ "ctr": 0.0127474002012747,
+ "ctr_select": 0.012080536912751677,
+ "ctr_holdout": 0.01341381623071764,
+ "n_select": 1490,
+ "n_holdout": 1491,
+ "eb": 0.01629320448614213,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Tampon Commercial That Shows Just How Baffling Actual Tampon Commercials Are",
+ "impressions": 3002,
+ "clicks": 69,
+ "ctr": 0.0229846768820786,
+ "ctr_select": 0.028647568287808126,
+ "ctr_holdout": 0.017321785476349102,
+ "n_select": 1501,
+ "n_holdout": 1501,
+ "eb": 0.022793005379135388,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.019001441488664658,
+ "pool_holdout": 0.020563195808775376,
+ "lift_select": 0.5076523696845778,
+ "lift_holdout": -0.15763164259920126,
+ "eb_lift": 0.19954085550470446,
+ "p_value": 0.00225773388675334,
+ "held_up": false,
+ "z_path": [
+ -0.497,
+ 0.975,
+ 0.823,
+ 0.892,
+ 1.187,
+ 1.245,
+ 1.354,
+ 0.91,
+ 0.79,
+ 0.852
+ ],
+ "first_cross": null,
+ "final_z": 0.852
+ },
+ {
+ "id": "53626ef4d3d68209a800009a",
+ "variation": "image",
+ "week": 201417,
+ "impressions": 15226,
+ "pooled_ctr": 0.017470116905293577,
+ "arms": [
+ {
+ "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness",
+ "impressions": 2947,
+ "clicks": 44,
+ "ctr": 0.014930437733288,
+ "ctr_select": 0.016960651289009497,
+ "ctr_holdout": 0.012898845892735914,
+ "n_select": 1474,
+ "n_holdout": 1473,
+ "eb": 0.01783262634375151,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness",
+ "impressions": 3083,
+ "clicks": 54,
+ "ctr": 0.0175154070710347,
+ "ctr_select": 0.017509727626459144,
+ "ctr_holdout": 0.01752109020116807,
+ "n_select": 1542,
+ "n_holdout": 1541,
+ "eb": 0.01803662714093299,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness",
+ "impressions": 3012,
+ "clicks": 50,
+ "ctr": 0.0166002656042496,
+ "ctr_select": 0.016600265604249667,
+ "ctr_holdout": 0.016600265604249667,
+ "n_select": 1506,
+ "n_holdout": 1506,
+ "eb": 0.01768338754741349,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness",
+ "impressions": 3107,
+ "clicks": 73,
+ "ctr": 0.023495333118764,
+ "ctr_select": 0.027670527670527672,
+ "ctr_holdout": 0.019317450096587252,
+ "n_select": 1554,
+ "n_holdout": 1553,
+ "eb": 0.022113278180970787,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness",
+ "impressions": 3077,
+ "clicks": 45,
+ "ctr": 0.0146246343841403,
+ "ctr_select": 0.013003901170351105,
+ "ctr_holdout": 0.016244314489928524,
+ "n_select": 1538,
+ "n_holdout": 1539,
+ "eb": 0.016239843777156916,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.018387181507748884,
+ "pool_holdout": 0.01655281135049921,
+ "lift_select": 0.5048814120242693,
+ "lift_holdout": 0.16701928678747757,
+ "eb_lift": 0.202646429070797,
+ "p_value": 0.0022632359549176876,
+ "held_up": true,
+ "z_path": [
+ -0.553,
+ -0.218,
+ -0.021,
+ -0.025,
+ 0.379,
+ 0.472,
+ 0.758,
+ 0.98,
+ 1.763,
+ 1.659
+ ],
+ "first_cross": null,
+ "final_z": 1.659
+ },
+ {
+ "id": "54709a23bc296ffe23000024",
+ "variation": "other",
+ "week": 201446,
+ "impressions": 15219,
+ "pooled_ctr": 0.04914908995334779,
+ "arms": [
+ {
+ "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.",
+ "impressions": 3045,
+ "clicks": 132,
+ "ctr": 0.0433497536945812,
+ "ctr_select": 0.04204993429697766,
+ "ctr_holdout": 0.04464871963230466,
+ "n_select": 1522,
+ "n_holdout": 1523,
+ "eb": 0.044040093291538455,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.",
+ "impressions": 3089,
+ "clicks": 147,
+ "ctr": 0.0475882162512139,
+ "ctr_select": 0.04145077720207254,
+ "ctr_holdout": 0.05372168284789644,
+ "n_select": 1544,
+ "n_holdout": 1545,
+ "eb": 0.043789197927240224,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.",
+ "impressions": 3070,
+ "clicks": 170,
+ "ctr": 0.0553745928338762,
+ "ctr_select": 0.05472312703583062,
+ "ctr_holdout": 0.056026058631921824,
+ "n_select": 1535,
+ "n_holdout": 1535,
+ "eb": 0.04908265833987898,
+ "labelled_winner": true
+ },
+ {
+ "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.",
+ "impressions": 2993,
+ "clicks": 163,
+ "ctr": 0.0544604076177748,
+ "ctr_select": 0.0427807486631016,
+ "ctr_holdout": 0.06613226452905811,
+ "n_select": 1496,
+ "n_holdout": 1497,
+ "eb": 0.04434033796593638,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.",
+ "impressions": 3022,
+ "clicks": 136,
+ "ctr": 0.0450033090668431,
+ "ctr_select": 0.045665122435473195,
+ "ctr_holdout": 0.04434149569821311,
+ "n_select": 1511,
+ "n_holdout": 1511,
+ "eb": 0.04547254929859503,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.045347003154574135,
+ "pool_holdout": 0.052949678097490474,
+ "lift_select": 0.2067639144597082,
+ "lift_holdout": 0.05810008001875189,
+ "eb_lift": 0.082379317825505,
+ "p_value": 0.0481407967967455,
+ "held_up": false,
+ "z_path": [
+ 1.875,
+ 1.941,
+ 1.103,
+ 1.156,
+ 1.526,
+ 1.198,
+ 1.986,
+ 1.874,
+ 1.843,
+ 1.853
+ ],
+ "first_cross": 6,
+ "final_z": 1.853
+ },
+ {
+ "id": "547fa599373665001e0a0000",
+ "variation": "image",
+ "week": 201448,
+ "impressions": 15200,
+ "pooled_ctr": 0.013421052631578948,
+ "arms": [
+ {
+ "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans",
+ "impressions": 3141,
+ "clicks": 45,
+ "ctr": 0.0143266475644699,
+ "ctr_select": 0.01592356687898089,
+ "ctr_holdout": 0.012730744748567792,
+ "n_select": 1570,
+ "n_holdout": 1571,
+ "eb": 0.014197357775596121,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans",
+ "impressions": 2984,
+ "clicks": 46,
+ "ctr": 0.0154155495978552,
+ "ctr_select": 0.014745308310991957,
+ "ctr_holdout": 0.0160857908847185,
+ "n_select": 1492,
+ "n_holdout": 1492,
+ "eb": 0.013700571353093015,
+ "labelled_winner": true
+ },
+ {
+ "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans",
+ "impressions": 3004,
+ "clicks": 31,
+ "ctr": 0.0103195739014647,
+ "ctr_select": 0.010652463382157125,
+ "ctr_holdout": 0.009986684420772303,
+ "n_select": 1502,
+ "n_holdout": 1502,
+ "eb": 0.012093898210847923,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans",
+ "impressions": 3037,
+ "clicks": 39,
+ "ctr": 0.0128416200197563,
+ "ctr_select": 0.011198945981554678,
+ "ctr_holdout": 0.014483212639894667,
+ "n_select": 1518,
+ "n_holdout": 1519,
+ "eb": 0.012304159209166965,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans",
+ "impressions": 3034,
+ "clicks": 43,
+ "ctr": 0.0141727092946605,
+ "ctr_select": 0.012524719841793012,
+ "ctr_holdout": 0.015820698747528016,
+ "n_select": 1517,
+ "n_holdout": 1517,
+ "eb": 0.012828921345378467,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.013028030003947888,
+ "pool_holdout": 0.013813971845809762,
+ "lift_select": 0.22225439104419986,
+ "lift_holdout": -0.07841532539177354,
+ "eb_lift": 0.08975476501772639,
+ "p_value": 0.25599554720899975,
+ "held_up": false,
+ "z_path": [
+ 0.257,
+ -1.543,
+ -1.213,
+ -0.641,
+ -0.469,
+ -1.01,
+ -0.826,
+ -0.572,
+ -0.456,
+ -0.352
+ ],
+ "first_cross": null,
+ "final_z": -0.352
+ },
+ {
+ "id": "547f42b734333600212d0000",
+ "variation": "headline",
+ "week": 201448,
+ "impressions": 15198,
+ "pooled_ctr": 0.016449532833267534,
+ "arms": [
+ {
+ "headline": "The Ancient Greeks, Pacific Islanders, and Maya All Ignored This One Thing. Big Mistake!",
+ "impressions": 3064,
+ "clicks": 48,
+ "ctr": 0.0156657963446475,
+ "ctr_select": 0.015013054830287207,
+ "ctr_holdout": 0.016318537859007835,
+ "n_select": 1532,
+ "n_holdout": 1532,
+ "eb": 0.016275372655567517,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Ancient Greeks, Pacific Islanders and Maya All Ignored This One Thing. Big Mistake!",
+ "impressions": 3049,
+ "clicks": 56,
+ "ctr": 0.0183666775992128,
+ "ctr_select": 0.01706036745406824,
+ "ctr_holdout": 0.019672131147540985,
+ "n_select": 1524,
+ "n_holdout": 1525,
+ "eb": 0.017090169451647708,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Ancient Greeks, Pacific Islanders and Maya All Ignored This One Thing. Big Mistake!",
+ "impressions": 3085,
+ "clicks": 32,
+ "ctr": 0.0103727714748784,
+ "ctr_select": 0.009079118028534372,
+ "ctr_holdout": 0.011665586519766688,
+ "n_select": 1542,
+ "n_holdout": 1543,
+ "eb": 0.013901416344026751,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Ancient Greeks, Pacific Islanders and Maya All Ignored This One Thing. Big Mistake!",
+ "impressions": 2972,
+ "clicks": 53,
+ "ctr": 0.0178331090174966,
+ "ctr_select": 0.01816958277254374,
+ "ctr_holdout": 0.017496635262449527,
+ "n_select": 1486,
+ "n_holdout": 1486,
+ "eb": 0.017523807076728445,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Ancient Greeks, Pacific Islanders and Maya All Ignored This One Thing. Big Mistake!",
+ "impressions": 3028,
+ "clicks": 61,
+ "ctr": 0.0201453104359313,
+ "ctr_select": 0.026420079260237782,
+ "ctr_holdout": 0.013870541611624834,
+ "n_select": 1514,
+ "n_holdout": 1514,
+ "eb": 0.020788504442264998,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.01710976572782311,
+ "pool_holdout": 0.015789473684210527,
+ "lift_select": 0.544152017071436,
+ "lift_holdout": -0.12153236459709382,
+ "eb_lift": 0.21500812886407283,
+ "p_value": 0.0017973729636608256,
+ "held_up": false,
+ "z_path": [
+ -0.02,
+ -0.031,
+ 1.311,
+ 1.395,
+ 1.486,
+ 1.604,
+ 0.923,
+ 0.667,
+ 0.517,
+ 0.656
+ ],
+ "first_cross": null,
+ "final_z": 0.656
+ },
+ {
+ "id": "54753ffe3762650018130000",
+ "variation": "headline",
+ "week": 201447,
+ "impressions": 14748,
+ "pooled_ctr": 0.03356387306753458,
+ "arms": [
+ {
+ "headline": "The Truth About The Ferguson Case That Most People Won't Acknowledge",
+ "impressions": 3747,
+ "clicks": 122,
+ "ctr": 0.0325593808380037,
+ "ctr_select": 0.0368196371398079,
+ "ctr_holdout": 0.02829684997330486,
+ "n_select": 1874,
+ "n_holdout": 1873,
+ "eb": 0.03430710130431008,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Truth About The Ferguson Case That Some People Won't Acknowledge",
+ "impressions": 3680,
+ "clicks": 125,
+ "ctr": 0.0339673913043478,
+ "ctr_select": 0.02717391304347826,
+ "ctr_holdout": 0.04076086956521739,
+ "n_select": 1840,
+ "n_holdout": 1840,
+ "eb": 0.030017814342495553,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Truth About The Ferguson Case That Some People Really Can't Accept",
+ "impressions": 3681,
+ "clicks": 129,
+ "ctr": 0.0350448247758761,
+ "ctr_select": 0.03206521739130435,
+ "ctr_holdout": 0.03802281368821293,
+ "n_select": 1840,
+ "n_holdout": 1841,
+ "eb": 0.03218247388230977,
+ "labelled_winner": true
+ },
+ {
+ "headline": "The Truth About The Ferguson Case That Many People Are Unwilling To Hear",
+ "impressions": 3640,
+ "clicks": 119,
+ "ctr": 0.0326923076923076,
+ "ctr_select": 0.03296703296703297,
+ "ctr_holdout": 0.03241758241758242,
+ "n_select": 1820,
+ "n_holdout": 1820,
+ "eb": 0.03257971157219836,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.032275562788174665,
+ "pool_holdout": 0.034852183346894494,
+ "lift_select": 0.14078993390312378,
+ "lift_holdout": -0.18808960426789867,
+ "eb_lift": 0.06294355049572498,
+ "p_value": 0.19746540669340096,
+ "held_up": false,
+ "z_path": [
+ 0.139,
+ 0.341,
+ 0.241,
+ 0.396,
+ -0.26,
+ -0.096,
+ 0.198,
+ 0.155,
+ -0.011,
+ -0.032
+ ],
+ "first_cross": null,
+ "final_z": -0.032
+ },
+ {
+ "id": "52a01fae8f48676d170045ff",
+ "variation": "image",
+ "week": 201348,
+ "impressions": 14385,
+ "pooled_ctr": 0.019256169621133124,
+ "arms": [
+ {
+ "headline": "You Know You Shouldn't Do It, But You Do Anyway. These People Will Convince You To Stop. Right Now.",
+ "impressions": 3576,
+ "clicks": 71,
+ "ctr": 0.0198545861297539,
+ "ctr_select": 0.01901565995525727,
+ "ctr_holdout": 0.02069351230425056,
+ "n_select": 1788,
+ "n_holdout": 1788,
+ "eb": 0.0184065202360505,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Know You Shouldn't Do It, But You Do Anyway. These People Will Convince You To Stop. Right Now.",
+ "impressions": 3454,
+ "clicks": 56,
+ "ctr": 0.0162130862767805,
+ "ctr_select": 0.015055008685581933,
+ "ctr_holdout": 0.017371163867979156,
+ "n_select": 1727,
+ "n_holdout": 1727,
+ "eb": 0.016706219554068397,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Know You Shouldn't Do It, But You Do Anyway. These People Will Convince You To Stop. Right Now.",
+ "impressions": 3645,
+ "clicks": 80,
+ "ctr": 0.0219478737997256,
+ "ctr_select": 0.021405049396267837,
+ "ctr_holdout": 0.022490400438837082,
+ "n_select": 1822,
+ "n_holdout": 1823,
+ "eb": 0.019463162199804915,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Know You Shouldn't Do It, But You Do Anyway. These People Will Convince You To Stop. Right Now.",
+ "impressions": 3710,
+ "clicks": 70,
+ "ctr": 0.0188679245283018,
+ "ctr_select": 0.016172506738544475,
+ "ctr_holdout": 0.0215633423180593,
+ "n_select": 1855,
+ "n_holdout": 1855,
+ "eb": 0.017152358725619637,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.01793659621802002,
+ "pool_holdout": 0.020575559571805922,
+ "lift_select": 0.1933729864958007,
+ "lift_holdout": 0.09306385376050774,
+ "eb_lift": 0.08510901194571274,
+ "p_value": 0.1967225439640914,
+ "held_up": true,
+ "z_path": [
+ -0.031,
+ 0.346,
+ 0.232,
+ 0.456,
+ 0.621,
+ 0.46,
+ 0.201,
+ 0.454,
+ 0.761,
+ 0.622
+ ],
+ "first_cross": null,
+ "final_z": 0.622
+ },
+ {
+ "id": "537c01c2db41d8a90c00006e",
+ "variation": "headline",
+ "week": 201420,
+ "impressions": 14342,
+ "pooled_ctr": 0.014642309301352671,
+ "arms": [
+ {
+ "headline": "The Whole Problem With Education In America, Explained By One Teacher In 6 Minutes",
+ "impressions": 3539,
+ "clicks": 67,
+ "ctr": 0.0189319016671376,
+ "ctr_select": 0.01864406779661017,
+ "ctr_holdout": 0.01921989824759751,
+ "n_select": 1770,
+ "n_holdout": 1769,
+ "eb": 0.016532052544685278,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Job Owes Her $11,000 ... So Why Does She Still Go In Everyday?",
+ "impressions": 3523,
+ "clicks": 45,
+ "ctr": 0.0127732046551234,
+ "ctr_select": 0.01362088535754824,
+ "ctr_holdout": 0.01192504258943782,
+ "n_select": 1762,
+ "n_holdout": 1761,
+ "eb": 0.014358424932627262,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Teacher Warns About A Crisis And I Hope Someone Is Listening",
+ "impressions": 3626,
+ "clicks": 48,
+ "ctr": 0.0132377275234418,
+ "ctr_select": 0.012134583563154992,
+ "ctr_holdout": 0.014340871483728626,
+ "n_select": 1813,
+ "n_holdout": 1813,
+ "eb": 0.013696967748651128,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Teachers Have Infinite Patience, So When They\u2019re Pissed, You Know Something Is Really Wrong",
+ "impressions": 3654,
+ "clicks": 50,
+ "ctr": 0.0136836343732895,
+ "ctr_select": 0.01532567049808429,
+ "ctr_holdout": 0.0120415982484948,
+ "n_select": 1827,
+ "n_holdout": 1827,
+ "eb": 0.015098334715661707,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.014919129949804797,
+ "pool_holdout": 0.014365411436541143,
+ "lift_select": 0.2496752732456835,
+ "lift_holdout": 0.3379288391774191,
+ "eb_lift": 0.10811103598582061,
+ "p_value": 0.13635546839561022,
+ "held_up": true,
+ "z_path": [
+ -0.196,
+ 0.639,
+ 1.212,
+ 0.816,
+ 1.377,
+ 1.327,
+ 1.723,
+ 1.82,
+ 1.928,
+ 1.759
+ ],
+ "first_cross": null,
+ "final_z": 1.759
+ },
+ {
+ "id": "51b7848d8c43697706000ef2",
+ "variation": "headline",
+ "week": 201323,
+ "impressions": 14315,
+ "pooled_ctr": 0.01683548725113517,
+ "arms": [
+ {
+ "headline": "Let The Daily Show Sum Up The NSA Scandal In Less Tham 5 Minutes",
+ "impressions": 3544,
+ "clicks": 71,
+ "ctr": 0.0200338600451467,
+ "ctr_select": 0.01636568848758465,
+ "ctr_holdout": 0.023702031602708805,
+ "n_select": 1772,
+ "n_holdout": 1772,
+ "eb": 0.015246886736382654,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The NSA Scandal Summed Up In Less Tham 5 Minutes",
+ "impressions": 3688,
+ "clicks": 64,
+ "ctr": 0.0173535791757049,
+ "ctr_select": 0.016811279826464208,
+ "ctr_holdout": 0.01789587852494577,
+ "n_select": 1844,
+ "n_holdout": 1844,
+ "eb": 0.015463679069467249,
+ "labelled_winner": false
+ },
+ {
+ "headline": "NAILED IT: Why You Should Feel Paranoid About The NSA",
+ "impressions": 3510,
+ "clicks": 60,
+ "ctr": 0.017094017094017,
+ "ctr_select": 0.013105413105413105,
+ "ctr_holdout": 0.021082621082621083,
+ "n_select": 1755,
+ "n_holdout": 1755,
+ "eb": 0.013837304355900729,
+ "labelled_winner": false
+ },
+ {
+ "headline": "That Guy Sitting In Jon Stewart's Seat Nails Why You SHOULD Be Paranoid About The NSA",
+ "impressions": 3573,
+ "clicks": 46,
+ "ctr": 0.0128743352924713,
+ "ctr_select": 0.011198208286674132,
+ "ctr_holdout": 0.014549524342473419,
+ "n_select": 1786,
+ "n_holdout": 1787,
+ "eb": 0.013001727107810373,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.014391504820455498,
+ "pool_holdout": 0.019279128248113998,
+ "lift_select": 0.1681391234757703,
+ "lift_holdout": -0.07174856172781274,
+ "eb_lift": 0.07450049611822429,
+ "p_value": 0.31124256988573895,
+ "held_up": false,
+ "z_path": [
+ 1.229,
+ 0.104,
+ -0.651,
+ -0.737,
+ -0.284,
+ 0.052,
+ -0.404,
+ -0.7,
+ -0.497,
+ -0.842
+ ],
+ "first_cross": null,
+ "final_z": -0.842
+ },
+ {
+ "id": "545d64fa81f41f555c000007",
+ "variation": "image",
+ "week": 201444,
+ "impressions": 14259,
+ "pooled_ctr": 0.01718213058419244,
+ "arms": [
+ {
+ "headline": "He Was Just 'Being Friendly', Until His Neighbor Started Being More Friendly To Him Than He Wanted",
+ "impressions": 3556,
+ "clicks": 59,
+ "ctr": 0.0165916760404949,
+ "ctr_select": 0.0140607424071991,
+ "ctr_holdout": 0.019122609673790775,
+ "n_select": 1778,
+ "n_holdout": 1778,
+ "eb": 0.0151530608001254,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Was Just 'Being Friendly', Until His Neighbor Started Being More Friendly To Him Than He Wanted",
+ "impressions": 3605,
+ "clicks": 62,
+ "ctr": 0.0171983356449375,
+ "ctr_select": 0.016648168701442843,
+ "ctr_holdout": 0.01774819744869662,
+ "n_select": 1802,
+ "n_holdout": 1803,
+ "eb": 0.01627846534997421,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Was Just 'Being Friendly', Until His Neighbor Started Being More Friendly To Him Than He Wanted",
+ "impressions": 3504,
+ "clicks": 42,
+ "ctr": 0.011986301369863,
+ "ctr_select": 0.014269406392694063,
+ "ctr_holdout": 0.009703196347031963,
+ "n_select": 1752,
+ "n_holdout": 1752,
+ "eb": 0.015249868870256662,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Was Just 'Being Friendly', Until His Neighbor Started Being More Friendly To Him Than He Wanted",
+ "impressions": 3594,
+ "clicks": 82,
+ "ctr": 0.0228158041179744,
+ "ctr_select": 0.018920422927100722,
+ "ctr_holdout": 0.02671118530884808,
+ "n_select": 1797,
+ "n_holdout": 1797,
+ "eb": 0.01727037110337943,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.015991022583812596,
+ "pool_holdout": 0.018373071528751752,
+ "lift_select": 0.18319030743246545,
+ "lift_holdout": 0.4538225286418842,
+ "eb_lift": 0.08000417189466624,
+ "p_value": 0.25234145506332073,
+ "held_up": true,
+ "z_path": [
+ 0.285,
+ 0.637,
+ 1.645,
+ 0.85,
+ 1.275,
+ 1.82,
+ 1.417,
+ 1.353,
+ 1.383,
+ 1.702
+ ],
+ "first_cross": null,
+ "final_z": 1.702
+ },
+ {
+ "id": "5535537b646339001c260000",
+ "variation": "image",
+ "week": 201516,
+ "impressions": 14246,
+ "pooled_ctr": 0.035518742103046466,
+ "arms": [
+ {
+ "headline": "Canceling out the harmful effects of sitting is easier than you think.",
+ "impressions": 2079,
+ "clicks": 17,
+ "ctr": 0.0081770081770081,
+ "ctr_select": 0.006730769230769231,
+ "ctr_holdout": 0.009624639076034648,
+ "n_select": 1040,
+ "n_holdout": 1039,
+ "eb": 0.023594952519395473,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Canceling out the harmful effects of sitting is easier than you think.",
+ "impressions": 2044,
+ "clicks": 19,
+ "ctr": 0.0092954990215264,
+ "ctr_select": 0.005870841487279843,
+ "ctr_holdout": 0.012720156555772993,
+ "n_select": 1022,
+ "n_holdout": 1022,
+ "eb": 0.023422694157851972,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Canceling out the harmful effects of sitting is easier than you think.",
+ "impressions": 2069,
+ "clicks": 22,
+ "ctr": 0.0106331561140647,
+ "ctr_select": 0.010638297872340425,
+ "ctr_holdout": 0.010628019323671498,
+ "n_select": 1034,
+ "n_holdout": 1035,
+ "eb": 0.024830615780252835,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Canceling out the harmful effects of sitting is easier than you think.",
+ "impressions": 2087,
+ "clicks": 44,
+ "ctr": 0.0210828941063727,
+ "ctr_select": 0.017241379310344827,
+ "ctr_holdout": 0.024928092042186004,
+ "n_select": 1044,
+ "n_holdout": 1043,
+ "eb": 0.02683903367960474,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Canceling out the harmful effects of sitting is easier than you think.",
+ "impressions": 2008,
+ "clicks": 25,
+ "ctr": 0.0124501992031872,
+ "ctr_select": 0.010956175298804782,
+ "ctr_holdout": 0.013944223107569721,
+ "n_select": 1004,
+ "n_holdout": 1004,
+ "eb": 0.025054874138371434,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Canceling out the harmful effects of sitting is easier than you think.",
+ "impressions": 2011,
+ "clicks": 29,
+ "ctr": 0.0144206862257583,
+ "ctr_select": 0.013916500994035786,
+ "ctr_holdout": 0.014925373134328358,
+ "n_select": 1006,
+ "n_holdout": 1005,
+ "eb": 0.025942406794428056,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Canceling out the harmful effects of sitting is easier than you think.",
+ "impressions": 1948,
+ "clicks": 350,
+ "ctr": 0.1796714579055441,
+ "ctr_select": 0.1591375770020534,
+ "ctr_holdout": 0.2002053388090349,
+ "n_select": 974,
+ "n_holdout": 974,
+ "eb": 0.0690296431520524,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 6,
+ "pool_select": 0.03116226838854576,
+ "pool_holdout": 0.03987643920247121,
+ "lift_select": 4.106739182714542,
+ "lift_holdout": 4.020642334499812,
+ "eb_lift": 1.2151674676361321,
+ "p_value": 4.027004748096685e-135,
+ "held_up": true,
+ "z_path": [
+ 5.567,
+ 8.105,
+ 9.51,
+ 10.963,
+ 11.702,
+ 12.623,
+ 13.745,
+ 15.22,
+ 16.132,
+ 16.959
+ ],
+ "first_cross": 0,
+ "final_z": 16.959
+ },
+ {
+ "id": "54ac7e253137331dd60f0000",
+ "variation": "image",
+ "week": 201501,
+ "impressions": 14240,
+ "pooled_ctr": 0.0125,
+ "arms": [
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2715,
+ "clicks": 31,
+ "ctr": 0.0114180478821362,
+ "ctr_select": 0.013991163475699559,
+ "ctr_holdout": 0.008843036109064112,
+ "n_select": 1358,
+ "n_holdout": 1357,
+ "eb": 0.013846735947562396,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2775,
+ "clicks": 35,
+ "ctr": 0.0126126126126126,
+ "ctr_select": 0.016570605187319884,
+ "ctr_holdout": 0.008651766402307137,
+ "n_select": 1388,
+ "n_holdout": 1387,
+ "eb": 0.014814056600051239,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2901,
+ "clicks": 39,
+ "ctr": 0.0134436401240951,
+ "ctr_select": 0.011724137931034483,
+ "ctr_holdout": 0.015161957270847692,
+ "n_select": 1450,
+ "n_holdout": 1451,
+ "eb": 0.012977796886143287,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2903,
+ "clicks": 39,
+ "ctr": 0.0134343782294178,
+ "ctr_select": 0.015151515151515152,
+ "ctr_holdout": 0.01171605789110958,
+ "n_select": 1452,
+ "n_holdout": 1451,
+ "eb": 0.014297277862061602,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2946,
+ "clicks": 34,
+ "ctr": 0.0115410726408689,
+ "ctr_select": 0.011541072640868975,
+ "ctr_holdout": 0.011541072640868975,
+ "n_select": 1473,
+ "n_holdout": 1473,
+ "eb": 0.012899054303210231,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.013762112062912512,
+ "pool_holdout": 0.011237533361427167,
+ "lift_select": 0.20407428100923375,
+ "lift_holdout": -0.2301009372746936,
+ "eb_lift": 0.07643772498943746,
+ "p_value": 0.3168489205325362,
+ "held_up": false,
+ "z_path": [
+ 0.877,
+ 1.821,
+ 0.503,
+ -0.046,
+ -0.329,
+ 0.013,
+ -0.108,
+ -0.078,
+ -0.184,
+ -0.273
+ ],
+ "first_cross": null,
+ "final_z": -0.273
+ },
+ {
+ "id": "54ac36206233650024a00000",
+ "variation": "image",
+ "week": 201501,
+ "impressions": 14220,
+ "pooled_ctr": 0.026160337552742614,
+ "arms": [
+ {
+ "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.",
+ "impressions": 2795,
+ "clicks": 84,
+ "ctr": 0.0300536672629695,
+ "ctr_select": 0.02575107296137339,
+ "ctr_holdout": 0.03435934144595562,
+ "n_select": 1398,
+ "n_holdout": 1397,
+ "eb": 0.024249712527628392,
+ "labelled_winner": false
+ },
+ {
+ "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.",
+ "impressions": 2942,
+ "clicks": 69,
+ "ctr": 0.0234534330387491,
+ "ctr_select": 0.016315431679129844,
+ "ctr_holdout": 0.030591434398368457,
+ "n_select": 1471,
+ "n_holdout": 1471,
+ "eb": 0.020615157836797883,
+ "labelled_winner": false
+ },
+ {
+ "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.",
+ "impressions": 2903,
+ "clicks": 65,
+ "ctr": 0.022390630382363,
+ "ctr_select": 0.02203856749311295,
+ "ctr_holdout": 0.022742935906271536,
+ "n_select": 1452,
+ "n_holdout": 1451,
+ "eb": 0.0228412482706762,
+ "labelled_winner": false
+ },
+ {
+ "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.",
+ "impressions": 2743,
+ "clicks": 86,
+ "ctr": 0.0313525337222019,
+ "ctr_select": 0.02696793002915452,
+ "ctr_holdout": 0.03574033552151714,
+ "n_select": 1372,
+ "n_holdout": 1371,
+ "eb": 0.024691616622825808,
+ "labelled_winner": true
+ },
+ {
+ "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.",
+ "impressions": 2837,
+ "clicks": 68,
+ "ctr": 0.0239689813182939,
+ "ctr_select": 0.02609308885754584,
+ "ctr_holdout": 0.021846370683579985,
+ "n_select": 1418,
+ "n_holdout": 1419,
+ "eb": 0.024387573187180305,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.023344114751792995,
+ "pool_holdout": 0.028977352651568436,
+ "lift_select": 0.1552346411886616,
+ "lift_holdout": 0.23338856904109395,
+ "eb_lift": 0.05772340846333934,
+ "p_value": 0.32240180654825834,
+ "held_up": true,
+ "z_path": [
+ 2.585,
+ 2.766,
+ 1.844,
+ 1.479,
+ 1.201,
+ 0.795,
+ 1.335,
+ 1.115,
+ 1.44,
+ 1.683
+ ],
+ "first_cross": 0,
+ "final_z": 1.683
+ },
+ {
+ "id": "51b7cf31ac05f91e9a0049e7",
+ "variation": "headline",
+ "week": 201323,
+ "impressions": 14217,
+ "pooled_ctr": 0.010199057466413449,
+ "arms": [
+ {
+ "headline": "You Know The Name Of One Of These Athletes But The Other Two Are The Ones You Should Remember",
+ "impressions": 3484,
+ "clicks": 50,
+ "ctr": 0.0143513203214695,
+ "ctr_select": 0.011481056257175661,
+ "ctr_holdout": 0.01722158438576349,
+ "n_select": 1742,
+ "n_holdout": 1742,
+ "eb": 0.010066137124591212,
+ "labelled_winner": false
+ },
+ {
+ "headline": "RESPECT: These Two Little Boys Came In Last But Won Gold At LIFE!",
+ "impressions": 3596,
+ "clicks": 42,
+ "ctr": 0.0116796440489432,
+ "ctr_select": 0.012235817575083427,
+ "ctr_holdout": 0.011123470522803115,
+ "n_select": 1798,
+ "n_holdout": 1798,
+ "eb": 0.010415117157649922,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When Faced With His Brother\u2019s Disability, This Big Brother Decided To Run With It.",
+ "impressions": 3494,
+ "clicks": 33,
+ "ctr": 0.0094447624499141,
+ "ctr_select": 0.005724098454493417,
+ "ctr_holdout": 0.01316542644533486,
+ "n_select": 1747,
+ "n_holdout": 1747,
+ "eb": 0.007593547176717703,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Agree With This Superstar, Love Watching These Two Brothers Bond Over Disability",
+ "impressions": 3643,
+ "clicks": 20,
+ "ctr": 0.0054899807850672,
+ "ctr_select": 0.006586169045005488,
+ "ctr_holdout": 0.004393190554640308,
+ "n_select": 1822,
+ "n_holdout": 1821,
+ "eb": 0.007939099935351461,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.009002672668448446,
+ "pool_holdout": 0.011395610579628587,
+ "lift_select": 0.35913167408231383,
+ "lift_holdout": -0.023881129924882116,
+ "eb_lift": 0.15689168552708277,
+ "p_value": 0.09310424148884665,
+ "held_up": false,
+ "z_path": [
+ -0.044,
+ -0.971,
+ -1.393,
+ -1.471,
+ -0.998,
+ -1.537,
+ -1.543,
+ -1.433,
+ -1.121,
+ -0.992
+ ],
+ "first_cross": null,
+ "final_z": -0.992
+ },
+ {
+ "id": "552c5646363535000c070000",
+ "variation": "image",
+ "week": 201515,
+ "impressions": 14217,
+ "pooled_ctr": 0.0046423296054019835,
+ "arms": [
+ {
+ "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.",
+ "impressions": 2039,
+ "clicks": 18,
+ "ctr": 0.0088278567925453,
+ "ctr_select": 0.011764705882352941,
+ "ctr_holdout": 0.005888125613346418,
+ "n_select": 1020,
+ "n_holdout": 1019,
+ "eb": 0.007306573200067666,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.",
+ "impressions": 1983,
+ "clicks": 6,
+ "ctr": 0.0030257186081694,
+ "ctr_select": 0.004032258064516129,
+ "ctr_holdout": 0.0020181634712411706,
+ "n_select": 992,
+ "n_holdout": 991,
+ "eb": 0.004951247534076134,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.",
+ "impressions": 2114,
+ "clicks": 3,
+ "ctr": 0.0014191106906338,
+ "ctr_select": 0.000946073793755913,
+ "ctr_holdout": 0.001892147587511826,
+ "n_select": 1057,
+ "n_holdout": 1057,
+ "eb": 0.003966913119425191,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.",
+ "impressions": 2051,
+ "clicks": 4,
+ "ctr": 0.0019502681618722,
+ "ctr_select": 0.0009746588693957114,
+ "ctr_holdout": 0.002926829268292683,
+ "n_select": 1026,
+ "n_holdout": 1025,
+ "eb": 0.00400369105737447,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.",
+ "impressions": 1950,
+ "clicks": 7,
+ "ctr": 0.0035897435897435,
+ "ctr_select": 0.003076923076923077,
+ "ctr_holdout": 0.0041025641025641026,
+ "n_select": 975,
+ "n_holdout": 975,
+ "eb": 0.004673108186321323,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.",
+ "impressions": 2029,
+ "clicks": 9,
+ "ctr": 0.0044356826022671,
+ "ctr_select": 0.004930966469428008,
+ "ctr_holdout": 0.003940886699507389,
+ "n_select": 1014,
+ "n_holdout": 1015,
+ "eb": 0.005218700536493161,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.",
+ "impressions": 2051,
+ "clicks": 19,
+ "ctr": 0.0092637737688932,
+ "ctr_select": 0.011695906432748537,
+ "ctr_holdout": 0.006829268292682927,
+ "n_select": 1026,
+ "n_holdout": 1025,
+ "eb": 0.007293462134692454,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.005344585091420534,
+ "pool_holdout": 0.003939777683973547,
+ "lift_select": 1.2012383900928794,
+ "lift_holdout": 0.4945324547876069,
+ "eb_lift": 0.3670983013810818,
+ "p_value": 0.0023766595616396833,
+ "held_up": false,
+ "z_path": [
+ -0.439,
+ 0.648,
+ 0.291,
+ -0.719,
+ -0.679,
+ -0.196,
+ 0.014,
+ -0.172,
+ -0.329,
+ -0.147
+ ],
+ "first_cross": null,
+ "final_z": -0.147
+ },
+ {
+ "id": "545d2b0674bfe7db21000063",
+ "variation": "other",
+ "week": 201444,
+ "impressions": 14212,
+ "pooled_ctr": 0.013720799324514494,
+ "arms": [
+ {
+ "headline": "If You've Ever Wondered What It's Like To Have ADD ... Here's The General Idea",
+ "impressions": 3455,
+ "clicks": 69,
+ "ctr": 0.0199710564399421,
+ "ctr_select": 0.022569444444444444,
+ "ctr_holdout": 0.017371163867979156,
+ "n_select": 1728,
+ "n_holdout": 1727,
+ "eb": 0.018427356224183487,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You've Ever Wondered What It's Like To Have ADD ... Here's The General Idea",
+ "impressions": 3598,
+ "clicks": 39,
+ "ctr": 0.0108393551973318,
+ "ctr_select": 0.012229016120066704,
+ "ctr_holdout": 0.009449694274596998,
+ "n_select": 1799,
+ "n_holdout": 1799,
+ "eb": 0.0139800191580425,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You've Ever Wondered What It's Like To Have ADD ... Here's The General Idea",
+ "impressions": 3586,
+ "clicks": 43,
+ "ctr": 0.0119910764082543,
+ "ctr_select": 0.012827663134411601,
+ "ctr_holdout": 0.011154489682097044,
+ "n_select": 1793,
+ "n_holdout": 1793,
+ "eb": 0.01424369218700942,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You've Ever Wondered What It's Like To Have ADD ... Here's The General Idea",
+ "impressions": 3573,
+ "clicks": 44,
+ "ctr": 0.0123145815841029,
+ "ctr_select": 0.013997760358342666,
+ "ctr_holdout": 0.010632344711807499,
+ "n_select": 1786,
+ "n_holdout": 1787,
+ "eb": 0.014755354106681998,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.015339150014072614,
+ "pool_holdout": 0.012102448634956376,
+ "lift_select": 0.4713621304791029,
+ "lift_holdout": 0.4353429121611614,
+ "eb_lift": 0.20132837916557667,
+ "p_value": 0.004936272800119368,
+ "held_up": true,
+ "z_path": [
+ 0.604,
+ 0.713,
+ 0.776,
+ 1.44,
+ 1.293,
+ 1.569,
+ 1.255,
+ 1.717,
+ 2.348,
+ 2.551
+ ],
+ "first_cross": 8,
+ "final_z": 2.551
+ },
+ {
+ "id": "54ad55df3236640022250000",
+ "variation": "image",
+ "week": 201501,
+ "impressions": 14199,
+ "pooled_ctr": 0.011761391647299106,
+ "arms": [
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2808,
+ "clicks": 42,
+ "ctr": 0.0149572649572649,
+ "ctr_select": 0.01282051282051282,
+ "ctr_holdout": 0.017094017094017096,
+ "n_select": 1404,
+ "n_holdout": 1404,
+ "eb": 0.012467417149813951,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2868,
+ "clicks": 43,
+ "ctr": 0.0149930264993026,
+ "ctr_select": 0.01394700139470014,
+ "ctr_holdout": 0.01603905160390516,
+ "n_select": 1434,
+ "n_holdout": 1434,
+ "eb": 0.012900814937566719,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2761,
+ "clicks": 16,
+ "ctr": 0.005795001810938,
+ "ctr_select": 0.007246376811594203,
+ "ctr_holdout": 0.004344677769732078,
+ "n_select": 1380,
+ "n_holdout": 1381,
+ "eb": 0.010384828846623723,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2891,
+ "clicks": 43,
+ "ctr": 0.0148737461086129,
+ "ctr_select": 0.017289073305670817,
+ "ctr_holdout": 0.01245674740484429,
+ "n_select": 1446,
+ "n_holdout": 1445,
+ "eb": 0.014188163338258787,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men",
+ "impressions": 2871,
+ "clicks": 23,
+ "ctr": 0.0080111459421804,
+ "ctr_select": 0.009749303621169917,
+ "ctr_holdout": 0.0062717770034843206,
+ "n_select": 1436,
+ "n_holdout": 1435,
+ "eb": 0.011295517413891351,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.012253521126760564,
+ "pool_holdout": 0.011269192844062545,
+ "lift_select": 0.4109473617271586,
+ "lift_holdout": 0.10538062283737015,
+ "eb_lift": 0.1578845942716942,
+ "p_value": 0.051125448545898705,
+ "held_up": false,
+ "z_path": [
+ 0.697,
+ 0.432,
+ 0.584,
+ 1.169,
+ 1.138,
+ 0.652,
+ 0.738,
+ 0.327,
+ 0.201,
+ -0.037
+ ],
+ "first_cross": null,
+ "final_z": -0.037
+ },
+ {
+ "id": "5243023a2dec5eddc9006a29",
+ "variation": "headline",
+ "week": 201338,
+ "impressions": 14190,
+ "pooled_ctr": 0.0817477096546864,
+ "arms": [
+ {
+ "headline": "A Girl Is Hit By A Car And Is Dying. The Response She's Met With Is A Shrug And \"Not My Problem.\"",
+ "impressions": 3627,
+ "clicks": 342,
+ "ctr": 0.0942928039702233,
+ "ctr_select": 0.0926130099228225,
+ "ctr_holdout": 0.09597352454495311,
+ "n_select": 1814,
+ "n_holdout": 1813,
+ "eb": 0.08493679287552869,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Young Girl Is Hit By A Car And Is Dying. The Response She's Met With Is: \"It's Not My Problem.\"",
+ "impressions": 3540,
+ "clicks": 326,
+ "ctr": 0.0920903954802259,
+ "ctr_select": 0.08870056497175141,
+ "ctr_holdout": 0.09548022598870057,
+ "n_select": 1770,
+ "n_holdout": 1770,
+ "eb": 0.0831600516294128,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Young Girl Is Hit By A Car And Is Dying. Instead Of Saving Her, They Say \"It's Not My Problem.\"",
+ "impressions": 3290,
+ "clicks": 267,
+ "ctr": 0.0811550151975683,
+ "ctr_select": 0.0729483282674772,
+ "ctr_holdout": 0.08936170212765958,
+ "n_select": 1645,
+ "n_holdout": 1645,
+ "eb": 0.07644619265520684,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Girl Is Hit By A Car And Is Dying. So Why Would Someone Shrug And Say \"It's Not My Problem\"?",
+ "impressions": 3733,
+ "clicks": 225,
+ "ctr": 0.0602732386820251,
+ "ctr_select": 0.06162915326902465,
+ "ctr_holdout": 0.05891805034815212,
+ "n_select": 1866,
+ "n_holdout": 1867,
+ "eb": 0.07121287750613461,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.07892882311486962,
+ "pool_holdout": 0.08456659619450317,
+ "lift_select": 0.1733737596471887,
+ "lift_holdout": 0.13488692774407052,
+ "eb_lift": 0.07611883116406437,
+ "p_value": 0.012228451244630794,
+ "held_up": true,
+ "z_path": [
+ 1.264,
+ 0.713,
+ -0.049,
+ 0.485,
+ 0.159,
+ 0.724,
+ 0.775,
+ 0.375,
+ 0.408,
+ 0.321
+ ],
+ "first_cross": null,
+ "final_z": 0.321
+ },
+ {
+ "id": "52f82fbcab954f6fba000013",
+ "variation": "other",
+ "week": 201406,
+ "impressions": 14145,
+ "pooled_ctr": 0.0272180982679392,
+ "arms": [
+ {
+ "headline": "Please Don't Ever Ask This Question. These Ladies Tell Us Why.",
+ "impressions": 3597,
+ "clicks": 101,
+ "ctr": 0.0280789546844592,
+ "ctr_select": 0.029477196885428252,
+ "ctr_holdout": 0.02668148971650917,
+ "n_select": 1798,
+ "n_holdout": 1799,
+ "eb": 0.028721448609586846,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Please Don't Ever Ask This Question. These Ladies Tell Us Why.",
+ "impressions": 3471,
+ "clicks": 112,
+ "ctr": 0.0322673581100547,
+ "ctr_select": 0.03341013824884793,
+ "ctr_holdout": 0.03112391930835735,
+ "n_select": 1736,
+ "n_holdout": 1735,
+ "eb": 0.03039417566223379,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Please Don't Ever Ask This Question. These Ladies Tell Us Why.",
+ "impressions": 3527,
+ "clicks": 101,
+ "ctr": 0.0286362347604196,
+ "ctr_select": 0.031746031746031744,
+ "ctr_holdout": 0.025524673851389675,
+ "n_select": 1764,
+ "n_holdout": 1763,
+ "eb": 0.029695682749516217,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Please Don't Ever Ask This Question. These Ladies Tell Us Why.",
+ "impressions": 3550,
+ "clicks": 71,
+ "ctr": 0.02,
+ "ctr_select": 0.018028169014084508,
+ "ctr_holdout": 0.021971830985915493,
+ "n_select": 1775,
+ "n_holdout": 1775,
+ "eb": 0.0237517664708476,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.02813516188321787,
+ "pool_holdout": 0.026300904977375566,
+ "lift_select": 0.18748697404071057,
+ "lift_holdout": 0.1833782653156084,
+ "eb_lift": 0.08029147969336492,
+ "p_value": 0.12599097812917776,
+ "held_up": true,
+ "z_path": [
+ 0.486,
+ 1.365,
+ 1.301,
+ 0.826,
+ 1.133,
+ 1.344,
+ 1.105,
+ 0.574,
+ 0.479,
+ 0.884
+ ],
+ "first_cross": null,
+ "final_z": 0.884
+ },
+ {
+ "id": "52b9d34721757196f500485b",
+ "variation": "headline",
+ "week": 201351,
+ "impressions": 14075,
+ "pooled_ctr": 0.013783303730017762,
+ "arms": [
+ {
+ "headline": "Parents Worrying About Their Kids Is Natural. So Is Trying To Make Money Off That Fear.",
+ "impressions": 3406,
+ "clicks": 42,
+ "ctr": 0.0123311802701115,
+ "ctr_select": 0.014092777451556078,
+ "ctr_holdout": 0.010569583088667059,
+ "n_select": 1703,
+ "n_holdout": 1703,
+ "eb": 0.014405244571863492,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Parents Worrying About Their Kids Is Natural. So Is A Corporation Getting Rich Off That Fear.",
+ "impressions": 3623,
+ "clicks": 64,
+ "ctr": 0.0176649185757659,
+ "ctr_select": 0.018211920529801324,
+ "ctr_holdout": 0.017117614577581448,
+ "n_select": 1812,
+ "n_holdout": 1811,
+ "eb": 0.016204366171812434,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's Natural To Worry About Your Kid. But Don't Fall For Every Ad Claiming To Want To Help.",
+ "impressions": 3552,
+ "clicks": 33,
+ "ctr": 0.0092905405405405,
+ "ctr_select": 0.009572072072072071,
+ "ctr_holdout": 0.009009009009009009,
+ "n_select": 1776,
+ "n_holdout": 1776,
+ "eb": 0.012438420581750237,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Kind Of Company Would Exploit Your Children For Profit? One That Could Make A Ton Of Money.",
+ "impressions": 3494,
+ "clicks": 55,
+ "ctr": 0.0157412707498568,
+ "ctr_select": 0.01659988551803091,
+ "ctr_holdout": 0.014882655981682884,
+ "n_select": 1747,
+ "n_holdout": 1747,
+ "eb": 0.015479412826457218,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.014634839443023587,
+ "pool_holdout": 0.012931647008668466,
+ "lift_select": 0.24442229794894876,
+ "lift_holdout": 0.3236994921147325,
+ "eb_lift": 0.10724591375937775,
+ "p_value": 0.14116056121565634,
+ "held_up": true,
+ "z_path": [
+ -0.646,
+ 1.006,
+ 0.054,
+ -0.271,
+ -0.27,
+ -0.272,
+ 0.068,
+ 0.246,
+ 0.502,
+ 0.633
+ ],
+ "first_cross": null,
+ "final_z": 0.633
+ },
+ {
+ "id": "52bf66c349f74f8c140017b8",
+ "variation": "headline",
+ "week": 201352,
+ "impressions": 13879,
+ "pooled_ctr": 0.032351033936162545,
+ "arms": [
+ {
+ "headline": "Feel Like Having Your Teeny-Little Mind Blown? Just Start Watching. It Won't Take Long.",
+ "impressions": 3245,
+ "clicks": 90,
+ "ctr": 0.0277349768875192,
+ "ctr_select": 0.02651048088779285,
+ "ctr_holdout": 0.02895871842267406,
+ "n_select": 1622,
+ "n_holdout": 1623,
+ "eb": 0.030159762151385584,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Feel Like Having Your Teeny-Little Mind Blown? Just Start Watching. It Won't Take Long.",
+ "impressions": 3577,
+ "clicks": 129,
+ "ctr": 0.036063740564719,
+ "ctr_select": 0.035794183445190156,
+ "ctr_holdout": 0.03633314700950251,
+ "n_select": 1788,
+ "n_holdout": 1789,
+ "eb": 0.0340551973815093,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Feel Like Having Your Teeny-Little Mind Blown? Just Start Watching. It Won't Take Long.",
+ "impressions": 3475,
+ "clicks": 102,
+ "ctr": 0.0293525179856115,
+ "ctr_select": 0.02991944764096663,
+ "ctr_holdout": 0.028785261945883708,
+ "n_select": 1738,
+ "n_holdout": 1737,
+ "eb": 0.0315162408613627,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Halfway Through You'll Feel Smarter. By The End, You'll Be Picking Up Your Brain Pieces.",
+ "impressions": 3582,
+ "clicks": 128,
+ "ctr": 0.0357342266890005,
+ "ctr_select": 0.03796761585706309,
+ "ctr_holdout": 0.03350083752093802,
+ "n_select": 1791,
+ "n_holdout": 1791,
+ "eb": 0.03500387618147532,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.03271364749963972,
+ "pool_holdout": 0.031988472622478385,
+ "lift_select": 0.16060478604476125,
+ "lift_holdout": 0.04727843421310762,
+ "eb_lift": 0.07000835604959121,
+ "p_value": 0.14672870169283841,
+ "held_up": false,
+ "z_path": [
+ -0.211,
+ -0.43,
+ -0.955,
+ -1.056,
+ 0.086,
+ 0.574,
+ -0.158,
+ -0.078,
+ -0.409,
+ -0.075
+ ],
+ "first_cross": null,
+ "final_z": -0.075
+ },
+ {
+ "id": "51c069a938e7046c89005a28",
+ "variation": "headline",
+ "week": 201324,
+ "impressions": 13828,
+ "pooled_ctr": 0.008244142319930576,
+ "arms": [
+ {
+ "headline": "Waaaaaiiiit, Shouldn't My Lunch Be Worth More Than Where It Came From?",
+ "impressions": 3505,
+ "clicks": 22,
+ "ctr": 0.0062767475035663,
+ "ctr_select": 0.00684931506849315,
+ "ctr_holdout": 0.005704506560182544,
+ "n_select": 1752,
+ "n_holdout": 1753,
+ "eb": 0.007726731210874642,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Waaaaaiiiit, Shouldn't My Lunch Be Worth More Than Where It Came From?",
+ "impressions": 3448,
+ "clicks": 34,
+ "ctr": 0.009860788863109,
+ "ctr_select": 0.009280742459396751,
+ "ctr_holdout": 0.010440835266821345,
+ "n_select": 1724,
+ "n_holdout": 1724,
+ "eb": 0.008769943440495396,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Holy Crap! Am I The Only One Who Thinks My Lunch Should Be Worth More Than Where It Came From?",
+ "impressions": 3350,
+ "clicks": 28,
+ "ctr": 0.0083582089552238,
+ "ctr_select": 0.008955223880597015,
+ "ctr_holdout": 0.007761194029850746,
+ "n_select": 1675,
+ "n_holdout": 1675,
+ "eb": 0.00862711450286913,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Holy Crap! Am I The Only One Who Thinks My Lunch Should Be Worth More Than Where It Came From?",
+ "impressions": 3525,
+ "clicks": 30,
+ "ctr": 0.0085106382978723,
+ "ctr_select": 0.00851305334846765,
+ "ctr_holdout": 0.008508224617129892,
+ "n_select": 1762,
+ "n_holdout": 1763,
+ "eb": 0.008443140342014748,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.008389989874150152,
+ "pool_holdout": 0.008098336948662329,
+ "lift_select": 0.10616849347947821,
+ "lift_holdout": 0.2892567119655285,
+ "eb_lift": 0.04528653455421838,
+ "p_value": 0.6397690669597684,
+ "held_up": true,
+ "z_path": [
+ -0.03,
+ 0.668,
+ 0.782,
+ -0.066,
+ 0.098,
+ 0.581,
+ 0.712,
+ 1.113,
+ 0.428,
+ 0.652
+ ],
+ "first_cross": null,
+ "final_z": 0.652
+ },
+ {
+ "id": "52b8930dd45683a980004370",
+ "variation": "image",
+ "week": 201351,
+ "impressions": 13713,
+ "pooled_ctr": 0.03405527601545978,
+ "arms": [
+ {
+ "headline": "What's It Like Being An Unmarried Woman Over 30? Looks Pretty Cool To Me.",
+ "impressions": 3467,
+ "clicks": 129,
+ "ctr": 0.0372079607730026,
+ "ctr_select": 0.03863898500576701,
+ "ctr_holdout": 0.03577611079053664,
+ "n_select": 1734,
+ "n_holdout": 1733,
+ "eb": 0.037141749711242,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What's It Like Being An Unmarried Woman Over 30? Looks Pretty Cool To Me.",
+ "impressions": 3522,
+ "clicks": 147,
+ "ctr": 0.0417376490630323,
+ "ctr_select": 0.04258943781942078,
+ "ctr_holdout": 0.04088586030664395,
+ "n_select": 1761,
+ "n_holdout": 1761,
+ "eb": 0.03885729061780303,
+ "labelled_winner": true
+ },
+ {
+ "headline": "What's It Like Being An Unmarried Woman Over 30? Looks Pretty Cool To Me.",
+ "impressions": 3313,
+ "clicks": 89,
+ "ctr": 0.0268638696045879,
+ "ctr_select": 0.029589371980676328,
+ "ctr_holdout": 0.024140012070006035,
+ "n_select": 1656,
+ "n_holdout": 1657,
+ "eb": 0.03334102186400039,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What's It Like Being An Unmarried Woman Over 30? Looks Pretty Cool To Me.",
+ "impressions": 3411,
+ "clicks": 102,
+ "ctr": 0.0299032541776605,
+ "ctr_select": 0.032825322391559206,
+ "ctr_holdout": 0.026979472140762465,
+ "n_select": 1706,
+ "n_holdout": 1705,
+ "eb": 0.0346664069446584,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.03602158378299548,
+ "pool_holdout": 0.03208868144690782,
+ "lift_select": 0.18233107339177468,
+ "lift_holdout": 0.27415208301068605,
+ "eb_lift": 0.07872243630070996,
+ "p_value": 0.08621759607191591,
+ "held_up": true,
+ "z_path": [
+ 2.114,
+ 1.324,
+ 0.973,
+ 0.71,
+ 0.528,
+ 0.774,
+ 0.995,
+ 0.842,
+ 1.036,
+ 0.972
+ ],
+ "first_cross": 0,
+ "final_z": 0.972
+ },
+ {
+ "id": "53a30adbb5839f8d24000056",
+ "variation": "image",
+ "week": 201424,
+ "impressions": 13398,
+ "pooled_ctr": 0.005896402448126586,
+ "arms": [
+ {
+ "headline": "An Infographic About How Your Tax Dollars Are Spent That Will Tick You Off",
+ "impressions": 3331,
+ "clicks": 20,
+ "ctr": 0.0060042029420594,
+ "ctr_select": 0.007202881152460984,
+ "ctr_holdout": 0.004804804804804805,
+ "n_select": 1666,
+ "n_holdout": 1665,
+ "eb": 0.006833594175026042,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Infographic About How Your Tax Dollars Are Spent That Will Tick You Off",
+ "impressions": 3388,
+ "clicks": 18,
+ "ctr": 0.0053128689492325,
+ "ctr_select": 0.007674144037780401,
+ "ctr_holdout": 0.0029515938606847697,
+ "n_select": 1694,
+ "n_holdout": 1694,
+ "eb": 0.007035169523578636,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Infographic About How Your Tax Dollars Are Spent That Will Tick You Off",
+ "impressions": 3314,
+ "clicks": 22,
+ "ctr": 0.0066385033192516,
+ "ctr_select": 0.004224502112251056,
+ "ctr_holdout": 0.009052504526252263,
+ "n_select": 1657,
+ "n_holdout": 1657,
+ "eb": 0.005591110338666137,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Infographic About How Your Tax Dollars Are Spent That Will Tick You Off",
+ "impressions": 3365,
+ "clicks": 19,
+ "ctr": 0.0056463595839524,
+ "ctr_select": 0.007134363852556481,
+ "ctr_holdout": 0.0041592394533571005,
+ "n_select": 1682,
+ "n_holdout": 1683,
+ "eb": 0.0068062577324059575,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.006568144499178982,
+ "pool_holdout": 0.00522466039707419,
+ "lift_select": 0.16838842975206614,
+ "lift_holdout": -0.43506493506493504,
+ "eb_lift": 0.07110455996484744,
+ "p_value": 0.5144238242612851,
+ "held_up": false,
+ "z_path": [
+ -0.015,
+ -0.028,
+ -0.278,
+ -0.509,
+ -1.134,
+ -0.669,
+ -0.645,
+ -0.978,
+ -0.573,
+ -0.378
+ ],
+ "first_cross": null,
+ "final_z": -0.378
+ },
+ {
+ "id": "5359593d67617a1d64000029",
+ "variation": "image",
+ "week": 201416,
+ "impressions": 13346,
+ "pooled_ctr": 0.014536190618912034,
+ "arms": [
+ {
+ "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.",
+ "impressions": 2248,
+ "clicks": 51,
+ "ctr": 0.0226868327402135,
+ "ctr_select": 0.017793594306049824,
+ "ctr_holdout": 0.027580071174377226,
+ "n_select": 1124,
+ "n_holdout": 1124,
+ "eb": 0.014893586581392228,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.",
+ "impressions": 2090,
+ "clicks": 12,
+ "ctr": 0.0057416267942583,
+ "ctr_select": 0.004784688995215311,
+ "ctr_holdout": 0.0066985645933014355,
+ "n_select": 1045,
+ "n_holdout": 1045,
+ "eb": 0.010782777976437981,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.",
+ "impressions": 2236,
+ "clicks": 27,
+ "ctr": 0.0120751341681574,
+ "ctr_select": 0.01073345259391771,
+ "ctr_holdout": 0.013416815742397137,
+ "n_select": 1118,
+ "n_holdout": 1118,
+ "eb": 0.012591102730778259,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.",
+ "impressions": 2246,
+ "clicks": 41,
+ "ctr": 0.0182546749777382,
+ "ctr_select": 0.018699910952804988,
+ "ctr_holdout": 0.017809439002671415,
+ "n_select": 1123,
+ "n_holdout": 1123,
+ "eb": 0.015188553961181062,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.",
+ "impressions": 2244,
+ "clicks": 35,
+ "ctr": 0.0155971479500891,
+ "ctr_select": 0.016042780748663103,
+ "ctr_holdout": 0.015151515151515152,
+ "n_select": 1122,
+ "n_holdout": 1122,
+ "eb": 0.014320799960003138,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.",
+ "impressions": 2282,
+ "clicks": 28,
+ "ctr": 0.0122699386503067,
+ "ctr_select": 0.012269938650306749,
+ "ctr_holdout": 0.012269938650306749,
+ "n_select": 1141,
+ "n_holdout": 1141,
+ "eb": 0.013085625462527957,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.013487187172186423,
+ "pool_holdout": 0.015585194065637645,
+ "lift_select": 0.38649450875630764,
+ "lift_holdout": 0.14271525446948408,
+ "eb_lift": 0.1261468953662359,
+ "p_value": 0.09680028677081567,
+ "held_up": false,
+ "z_path": [
+ 0.381,
+ -0.245,
+ -0.412,
+ -1.536,
+ -1.355,
+ -1.205,
+ -1.318,
+ -1.14,
+ -1.319,
+ -1.049
+ ],
+ "first_cross": null,
+ "final_z": -1.049
+ },
+ {
+ "id": "5236b8bf9d26831210005b30",
+ "variation": "headline",
+ "week": 201337,
+ "impressions": 13321,
+ "pooled_ctr": 0.021394790180917347,
+ "arms": [
+ {
+ "headline": "A Young Thief Gets Caught And Receives Nothing But Generosity, What He Learns is Invaluable",
+ "impressions": 3342,
+ "clicks": 85,
+ "ctr": 0.0254338719329742,
+ "ctr_select": 0.022142429682824656,
+ "ctr_holdout": 0.02872531418312388,
+ "n_select": 1671,
+ "n_holdout": 1671,
+ "eb": 0.021925153631637197,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Young Thief Gets Heart Punched With Generosity, Watch What Lesson He Learns",
+ "impressions": 3240,
+ "clicks": 79,
+ "ctr": 0.0243827160493827,
+ "ctr_select": 0.025308641975308643,
+ "ctr_holdout": 0.02345679012345679,
+ "n_select": 1620,
+ "n_holdout": 1620,
+ "eb": 0.023224944326385055,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Think Your Time Is Expensive? Learn The Value Of A Simple Act Of Kindness.",
+ "impressions": 3236,
+ "clicks": 54,
+ "ctr": 0.0166872682323856,
+ "ctr_select": 0.0203955500618047,
+ "ctr_holdout": 0.012978986402966625,
+ "n_select": 1618,
+ "n_holdout": 1618,
+ "eb": 0.02120407008546402,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Probably The Best Reason I've Seen To Release A Young Thief",
+ "impressions": 3503,
+ "clicks": 67,
+ "ctr": 0.0191264630316871,
+ "ctr_select": 0.019406392694063926,
+ "ctr_holdout": 0.018846373500856654,
+ "n_select": 1752,
+ "n_holdout": 1751,
+ "eb": 0.020751617464878604,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.021768503227743582,
+ "pool_holdout": 0.021021021021021023,
+ "lift_select": 0.16262664963814388,
+ "lift_holdout": 0.1158730158730159,
+ "eb_lift": 0.06690589074517828,
+ "p_value": 0.2616853798033174,
+ "held_up": false,
+ "z_path": [
+ -1.287,
+ -1.398,
+ -1.444,
+ -0.476,
+ 0.148,
+ 0.156,
+ 1.044,
+ 0.803,
+ 0.525,
+ -0.274
+ ],
+ "first_cross": null,
+ "final_z": -0.274
+ },
+ {
+ "id": "52e6b42bacd398e056000001",
+ "variation": "other",
+ "week": 201404,
+ "impressions": 13224,
+ "pooled_ctr": 0.05565638233514821,
+ "arms": [
+ {
+ "headline": "The Amusing Response SNL Give To Victoria's Secret After They Ban A Mom From Breastfeeding In Store",
+ "impressions": 3319,
+ "clicks": 170,
+ "ctr": 0.0512202470623681,
+ "ctr_select": 0.05,
+ "ctr_holdout": 0.05244122965641953,
+ "n_select": 1660,
+ "n_holdout": 1659,
+ "eb": 0.05241458630933749,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Amusing Response SNL Give To Victoria's Secret After They Ban A Mom From Breastfeeding In Store",
+ "impressions": 3301,
+ "clicks": 192,
+ "ctr": 0.0581641926688882,
+ "ctr_select": 0.052121212121212124,
+ "ctr_holdout": 0.06420351302241066,
+ "n_select": 1650,
+ "n_holdout": 1651,
+ "eb": 0.05330279557461928,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Amusing Response SNL Give To Victoria's Secret After They Ban A Mom From Breastfeeding In Store",
+ "impressions": 3332,
+ "clicks": 177,
+ "ctr": 0.0531212484993997,
+ "ctr_select": 0.05342136854741897,
+ "ctr_holdout": 0.05282112845138055,
+ "n_select": 1666,
+ "n_holdout": 1666,
+ "eb": 0.05384178106749651,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Amusing Response SNL Give To Victoria's Secret After They Ban A Mom From Breastfeeding In Store",
+ "impressions": 3272,
+ "clicks": 197,
+ "ctr": 0.0602078239608801,
+ "ctr_select": 0.061124694376528114,
+ "ctr_holdout": 0.059290953545232276,
+ "n_select": 1636,
+ "n_holdout": 1636,
+ "eb": 0.057032529165723246,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.05414398064125832,
+ "pool_holdout": 0.05716878402903811,
+ "lift_select": 0.12892871289833496,
+ "lift_holdout": 0.03712112391818989,
+ "eb_lift": 0.05334939341832978,
+ "p_value": 0.15036714809725585,
+ "held_up": false,
+ "z_path": [
+ 1.618,
+ 1.717,
+ 0.295,
+ -0.25,
+ -0.748,
+ -0.863,
+ -0.58,
+ 0.098,
+ 0.397,
+ 1.246
+ ],
+ "first_cross": null,
+ "final_z": 1.246
+ },
+ {
+ "id": "543e5ef84b9f3dd0fa000026",
+ "variation": "other",
+ "week": 201441,
+ "impressions": 13126,
+ "pooled_ctr": 0.01973182995581289,
+ "arms": [
+ {
+ "headline": "Her Bracelet Came From The Bottom Of The Sea, And It\u2019s A Big Friggin\u2019 Red Flag For All Of Humanity",
+ "impressions": 3389,
+ "clicks": 63,
+ "ctr": 0.018589554440838,
+ "ctr_select": 0.017119244391971666,
+ "ctr_holdout": 0.020058997050147492,
+ "n_select": 1694,
+ "n_holdout": 1695,
+ "eb": 0.018056437156611232,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Bracelet Came From The Bottom Of The Sea, And It\u2019s A Big Friggin\u2019 Red Flag For All Of Humanity",
+ "impressions": 3236,
+ "clicks": 61,
+ "ctr": 0.0188504326328801,
+ "ctr_select": 0.0173053152039555,
+ "ctr_holdout": 0.0203955500618047,
+ "n_select": 1618,
+ "n_holdout": 1618,
+ "eb": 0.018151030098407576,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Bracelet Came From The Bottom Of The Sea, And It\u2019s A Big Friggin\u2019 Red Flag For All Of Humanity",
+ "impressions": 3206,
+ "clicks": 73,
+ "ctr": 0.0227698066126013,
+ "ctr_select": 0.020586400499064253,
+ "ctr_holdout": 0.024953212726138492,
+ "n_select": 1603,
+ "n_holdout": 1603,
+ "eb": 0.019495756432509906,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Bracelet Came From The Bottom Of The Sea, And It\u2019s A Big Friggin\u2019 Red Flag For All Of Humanity",
+ "impressions": 3295,
+ "clicks": 62,
+ "ctr": 0.0188163884673748,
+ "ctr_select": 0.020024271844660196,
+ "ctr_holdout": 0.017607771706132362,
+ "n_select": 1648,
+ "n_holdout": 1647,
+ "eb": 0.019274532046587155,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.018741429224440043,
+ "pool_holdout": 0.020722230687185737,
+ "lift_select": 0.09844346727933884,
+ "lift_holdout": 0.2041759935415215,
+ "eb_lift": 0.04024918265497979,
+ "p_value": 0.5309373058246877,
+ "held_up": true,
+ "z_path": [
+ 0.959,
+ 1.717,
+ 1.008,
+ 0.881,
+ 1.198,
+ 1.124,
+ 0.854,
+ 0.826,
+ 1.606,
+ 1.118
+ ],
+ "first_cross": null,
+ "final_z": 1.118
+ },
+ {
+ "id": "52efd159f41748cff200001a",
+ "variation": "other",
+ "week": 201405,
+ "impressions": 13113,
+ "pooled_ctr": 0.014336917562724014,
+ "arms": [
+ {
+ "headline": "It's Amazing What People Can Do When They Expect Their Children To Live Past Kindergarten",
+ "impressions": 3295,
+ "clicks": 69,
+ "ctr": 0.0209408194233687,
+ "ctr_select": 0.02366504854368932,
+ "ctr_holdout": 0.018214936247723135,
+ "n_select": 1648,
+ "n_holdout": 1647,
+ "eb": 0.018390974546732667,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's Amazing What People Can Do When They Expect Their Children To Live Past Kindergarten",
+ "impressions": 3227,
+ "clicks": 44,
+ "ctr": 0.0136349550666253,
+ "ctr_select": 0.014250309789343246,
+ "ctr_holdout": 0.013019218846869188,
+ "n_select": 1614,
+ "n_holdout": 1613,
+ "eb": 0.014480524654491601,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's Amazing What People Can Do When They Expect Their Children To Live Past Kindergarten",
+ "impressions": 3329,
+ "clicks": 40,
+ "ctr": 0.0120156203063983,
+ "ctr_select": 0.010817307692307692,
+ "ctr_holdout": 0.013213213213213212,
+ "n_select": 1664,
+ "n_holdout": 1665,
+ "eb": 0.013042940323048615,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's Amazing What People Can Do When They Expect Their Children To Live Past Kindergarten",
+ "impressions": 3262,
+ "clicks": 35,
+ "ctr": 0.0107296137339055,
+ "ctr_select": 0.009809932556713672,
+ "ctr_holdout": 0.011649294911097487,
+ "n_select": 1631,
+ "n_holdout": 1631,
+ "eb": 0.012645446494261128,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.014640841848406283,
+ "pool_holdout": 0.014032946918852958,
+ "lift_select": 0.6163721177184467,
+ "lift_holdout": 0.29801219608774865,
+ "eb_lift": 0.256141876072147,
+ "p_value": 0.0004234271447894082,
+ "held_up": true,
+ "z_path": [
+ 2.121,
+ 2.383,
+ 2.035,
+ 1.761,
+ 1.362,
+ 2.009,
+ 1.992,
+ 1.933,
+ 2.07,
+ 2.261
+ ],
+ "first_cross": 0,
+ "final_z": 2.261
+ },
+ {
+ "id": "547cd788366131001b130000",
+ "variation": "headline",
+ "week": 201448,
+ "impressions": 13053,
+ "pooled_ctr": 0.004290201486248372,
+ "arms": [
+ {
+ "headline": "His Movie Is About To Be Released In Theaters. Here's Why It Means So Much To Him.",
+ "impressions": 3296,
+ "clicks": 17,
+ "ctr": 0.0051577669902912,
+ "ctr_select": 0.0054611650485436895,
+ "ctr_holdout": 0.0048543689320388345,
+ "n_select": 1648,
+ "n_holdout": 1648,
+ "eb": 0.004776619358260943,
+ "labelled_winner": false
+ },
+ {
+ "headline": "At His Movie Screening, Someone Asked Him About His Anger. His Answer Was Brilliant.",
+ "impressions": 3243,
+ "clicks": 14,
+ "ctr": 0.0043169904409497,
+ "ctr_select": 0.004315659679408138,
+ "ctr_holdout": 0.00431832202344232,
+ "n_select": 1622,
+ "n_holdout": 1621,
+ "eb": 0.004300489452537783,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How A Camera Helps Make One Black Man A Better Husband, Father And American",
+ "impressions": 3304,
+ "clicks": 13,
+ "ctr": 0.0039346246973365,
+ "ctr_select": 0.0036319612590799033,
+ "ctr_holdout": 0.00423728813559322,
+ "n_select": 1652,
+ "n_holdout": 1652,
+ "eb": 0.004016081226123429,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Everyone Has A Coping Mechanism. It Just So Happens That His Makes Beautiful Art For The Rest Of Us.",
+ "impressions": 3210,
+ "clicks": 12,
+ "ctr": 0.0037383177570093,
+ "ctr_select": 0.003738317757009346,
+ "ctr_holdout": 0.003738317757009346,
+ "n_select": 1605,
+ "n_holdout": 1605,
+ "eb": 0.004064200105486682,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.004289872835912364,
+ "pool_holdout": 0.004290530186944529,
+ "lift_select": 0.27303658113730944,
+ "lift_holdout": 0.13141470180305137,
+ "eb_lift": 0.11346409112032774,
+ "p_value": 0.40007606065102785,
+ "held_up": true,
+ "z_path": [
+ 1.723,
+ 1.875,
+ 1.711,
+ 1.042,
+ 0.744,
+ 0.193,
+ 0.388,
+ 0.351,
+ 0.15,
+ 0.495
+ ],
+ "first_cross": null,
+ "final_z": 0.495
+ },
+ {
+ "id": "53c7fbfa4273de563b000054",
+ "variation": "headline",
+ "week": 201428,
+ "impressions": 13018,
+ "pooled_ctr": 0.0202796128437548,
+ "arms": [
+ {
+ "headline": "These Sexy Young Pop Stars Wrote A Catchy Song. May Its Unsexy Message Never Ever Catch On.",
+ "impressions": 3268,
+ "clicks": 75,
+ "ctr": 0.0229498164014687,
+ "ctr_select": 0.020195838433292534,
+ "ctr_holdout": 0.025703794369645042,
+ "n_select": 1634,
+ "n_holdout": 1634,
+ "eb": 0.01889174740981986,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These Sexy Young Pop Stars Wrote A Catchy Song, But I Hope Its Unsexy Message Never Ever Catches On",
+ "impressions": 3243,
+ "clicks": 70,
+ "ctr": 0.0215849522047486,
+ "ctr_select": 0.02281134401972873,
+ "ctr_holdout": 0.020357803824799507,
+ "n_select": 1622,
+ "n_holdout": 1621,
+ "eb": 0.019964596467235123,
+ "labelled_winner": false
+ },
+ {
+ "headline": "May The Unsexy Message Behind These Sexy Young Pop Stars' Catchy Song Never Ever Catch On",
+ "impressions": 3259,
+ "clicks": 60,
+ "ctr": 0.0184105553850874,
+ "ctr_select": 0.016564417177914112,
+ "ctr_holdout": 0.020257826887661142,
+ "n_select": 1630,
+ "n_holdout": 1629,
+ "eb": 0.017391016235949557,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Sexy Young Pop Stars Wrote A Song With Some Profoundly Unsexy Advice",
+ "impressions": 3248,
+ "clicks": 59,
+ "ctr": 0.0181650246305418,
+ "ctr_select": 0.012315270935960592,
+ "ctr_holdout": 0.02401477832512315,
+ "n_select": 1624,
+ "n_holdout": 1624,
+ "eb": 0.015641604074160174,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.017972350230414748,
+ "pool_holdout": 0.02258758451137062,
+ "lift_select": 0.2692465775079831,
+ "lift_holdout": -0.09871709325309386,
+ "eb_lift": 0.1108506239461593,
+ "p_value": 0.09046724145172004,
+ "held_up": false,
+ "z_path": [
+ -0.772,
+ -1.13,
+ -0.454,
+ -0.815,
+ -0.34,
+ -0.506,
+ -1.078,
+ -1.003,
+ -0.849,
+ -0.373
+ ],
+ "first_cross": null,
+ "final_z": -0.373
+ },
+ {
+ "id": "547bd484346330001b090000",
+ "variation": "other",
+ "week": 201448,
+ "impressions": 13014,
+ "pooled_ctr": 0.013523897341324727,
+ "arms": [
+ {
+ "headline": "A Father Wrote His Kid 14 Things To Always Remember. And I Forget 9 Of Them Every Day.",
+ "impressions": 3227,
+ "clicks": 39,
+ "ctr": 0.0120855283545088,
+ "ctr_select": 0.01486988847583643,
+ "ctr_holdout": 0.009299442033477991,
+ "n_select": 1614,
+ "n_holdout": 1613,
+ "eb": 0.015162166775797372,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Father Wrote His Kid 14 Things To Always Remember. And I Forget 9 Of Them Every Day.",
+ "impressions": 3239,
+ "clicks": 48,
+ "ctr": 0.0148193887002161,
+ "ctr_select": 0.012962962962962963,
+ "ctr_holdout": 0.016676961087090797,
+ "n_select": 1620,
+ "n_holdout": 1619,
+ "eb": 0.014377197173631846,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Father Wrote His Kid 14 Things To Always Remember. And I Forget 9 Of Them Every Day.",
+ "impressions": 3304,
+ "clicks": 40,
+ "ctr": 0.0121065375302663,
+ "ctr_select": 0.01513317191283293,
+ "ctr_holdout": 0.009079903147699757,
+ "n_select": 1652,
+ "n_holdout": 1652,
+ "eb": 0.015268935022161551,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Father Wrote His Kid 14 Things To Always Remember. And I Forget 9 Of Them Every Day.",
+ "impressions": 3244,
+ "clicks": 49,
+ "ctr": 0.0151048088779284,
+ "ctr_select": 0.018495684340320593,
+ "ctr_holdout": 0.011713933415536375,
+ "n_select": 1622,
+ "n_holdout": 1622,
+ "eb": 0.01665433777634867,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.015365703749231715,
+ "pool_holdout": 0.01168152474638795,
+ "lift_select": 0.2036991368680643,
+ "lift_holdout": 0.0027743526510481953,
+ "eb_lift": 0.08386430248477161,
+ "p_value": 0.23690020250145705,
+ "held_up": false,
+ "z_path": [
+ 0.615,
+ 0.041,
+ 1.359,
+ 0.884,
+ 1.275,
+ 1.052,
+ 0.851,
+ 0.925,
+ 1.343,
+ 1.048
+ ],
+ "first_cross": null,
+ "final_z": 1.048
+ },
+ {
+ "id": "5398abbb326359ba6c00007f",
+ "variation": "image",
+ "week": 201423,
+ "impressions": 12997,
+ "pooled_ctr": 0.033007617142417484,
+ "arms": [
+ {
+ "headline": "What It Looks Like When We Judge People At First Glance",
+ "impressions": 4376,
+ "clicks": 132,
+ "ctr": 0.0301645338208409,
+ "ctr_select": 0.031992687385740404,
+ "ctr_holdout": 0.0283363802559415,
+ "n_select": 2188,
+ "n_holdout": 2188,
+ "eb": 0.032713970458522396,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What It Looks Like When We Judge People At First Glance",
+ "impressions": 4144,
+ "clicks": 135,
+ "ctr": 0.03257722007722,
+ "ctr_select": 0.032818532818532815,
+ "ctr_holdout": 0.032335907335907334,
+ "n_select": 2072,
+ "n_holdout": 2072,
+ "eb": 0.0331228415685521,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What It Looks Like When We Judge People At First Glance",
+ "impressions": 4477,
+ "clicks": 162,
+ "ctr": 0.0361849452758543,
+ "ctr_select": 0.035299374441465595,
+ "ctr_holdout": 0.03707012058954891,
+ "n_select": 2238,
+ "n_holdout": 2239,
+ "eb": 0.034330474000797105,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.03339489073561096,
+ "pool_holdout": 0.03262040313894445,
+ "lift_select": 0.0570291941043477,
+ "lift_holdout": 0.1364090269409357,
+ "eb_lift": 0.028015760632164044,
+ "p_value": 0.535693647889956,
+ "held_up": true,
+ "z_path": [
+ 0.501,
+ 1.392,
+ 0.919,
+ 1.022,
+ 0.9,
+ 0.644,
+ 0.204,
+ 0.745,
+ 0.965,
+ 0.918
+ ],
+ "first_cross": null,
+ "final_z": 0.918
+ },
+ {
+ "id": "51ccb50a38cf80cec6000270",
+ "variation": "both",
+ "week": 201325,
+ "impressions": 12903,
+ "pooled_ctr": 0.009455165465395644,
+ "arms": [
+ {
+ "headline": "Sometimes We Inspire Our Kids In Ways We Don't Even Realize",
+ "impressions": 3116,
+ "clicks": 43,
+ "ctr": 0.0137997432605905,
+ "ctr_select": 0.01540436456996149,
+ "ctr_holdout": 0.012195121951219513,
+ "n_select": 1558,
+ "n_holdout": 1558,
+ "eb": 0.01258774380554559,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Sometimes We Inspire Our Kids In Ways We Don't Even Realize",
+ "impressions": 3395,
+ "clicks": 20,
+ "ctr": 0.0058910162002945,
+ "ctr_select": 0.005889281507656066,
+ "ctr_holdout": 0.0058927519151443725,
+ "n_select": 1698,
+ "n_holdout": 1697,
+ "eb": 0.008662576800423039,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Just When We Think We're Not Enough For Our Kids, It Turns Out We're More Than We Imagined",
+ "impressions": 3315,
+ "clicks": 32,
+ "ctr": 0.0096530920060331,
+ "ctr_select": 0.013268998793727383,
+ "ctr_holdout": 0.006035003017501509,
+ "n_select": 1658,
+ "n_holdout": 1657,
+ "eb": 0.011768070223030487,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Just When We Think We're Not Enough For Our Kids, It Turns Out We're More Than We Imagined",
+ "impressions": 3077,
+ "clicks": 27,
+ "ctr": 0.0087747806304842,
+ "ctr_select": 0.008452535760728219,
+ "ctr_holdout": 0.009096816114359974,
+ "n_select": 1538,
+ "n_holdout": 1539,
+ "eb": 0.009800117298682572,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.01069435833849969,
+ "pool_holdout": 0.008215780499147418,
+ "lift_select": 0.44041971312161654,
+ "lift_holdout": 0.4843534284399449,
+ "eb_lift": 0.17704526135333554,
+ "p_value": 0.0379601069814802,
+ "held_up": true,
+ "z_path": [
+ 0.519,
+ 1.192,
+ 1.005,
+ 0.004,
+ 0.672,
+ 0.796,
+ 1.324,
+ 1.145,
+ 1.244,
+ 1.548
+ ],
+ "first_cross": null,
+ "final_z": 1.548
+ },
+ {
+ "id": "537e69c542275460e3000026",
+ "variation": "headline",
+ "week": 201420,
+ "impressions": 12793,
+ "pooled_ctr": 0.01438286562964121,
+ "arms": [
+ {
+ "headline": "The Reason These Girls Are Missing Up To 50 Days Of School A Year Shouldn't Be A Thing. Period.",
+ "impressions": 2505,
+ "clicks": 46,
+ "ctr": 0.0183632734530938,
+ "ctr_select": 0.01437699680511182,
+ "ctr_holdout": 0.0223463687150838,
+ "n_select": 1252,
+ "n_holdout": 1253,
+ "eb": 0.013266192293871151,
+ "labelled_winner": false
+ },
+ {
+ "headline": "You Can't Stick A Band-Aid On Global Poverty, But You Can Give It A Maxi Pad",
+ "impressions": 2533,
+ "clicks": 22,
+ "ctr": 0.0086853533359652,
+ "ctr_select": 0.0071090047393364926,
+ "ctr_holdout": 0.010260457774269928,
+ "n_select": 1266,
+ "n_holdout": 1267,
+ "eb": 0.010702994320940589,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Half The World Goes Through It Every Month. Why's It Taking 5 Years Off These Girls' Lives?",
+ "impressions": 2580,
+ "clicks": 45,
+ "ctr": 0.0174418604651162,
+ "ctr_select": 0.017829457364341085,
+ "ctr_holdout": 0.017054263565891473,
+ "n_select": 1290,
+ "n_holdout": 1290,
+ "eb": 0.01451238448094321,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They're Spinning Banana Fibers Into Gold For Women And Girls. No, It's Not A Fairytale.",
+ "impressions": 2637,
+ "clicks": 47,
+ "ctr": 0.0178232840348881,
+ "ctr_select": 0.012898330804248861,
+ "ctr_holdout": 0.022744503411675512,
+ "n_select": 1318,
+ "n_holdout": 1319,
+ "eb": 0.012750316430758317,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Can These Banana Fibers Change The World?",
+ "impressions": 2538,
+ "clicks": 24,
+ "ctr": 0.0094562647754137,
+ "ctr_select": 0.01103230890464933,
+ "ctr_holdout": 0.007880220646178092,
+ "n_select": 1269,
+ "n_holdout": 1269,
+ "eb": 0.0120880820607349,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.012666145426114151,
+ "pool_holdout": 0.01609878086902157,
+ "lift_select": 0.407646664752608,
+ "lift_holdout": 0.05935124557838489,
+ "eb_lift": 0.14576171303249175,
+ "p_value": 0.06344539675420757,
+ "held_up": false,
+ "z_path": [
+ 0.419,
+ 0.953,
+ 0.325,
+ 0.991,
+ 0.533,
+ -0.242,
+ 0.029,
+ -0.0,
+ -0.245,
+ -0.248
+ ],
+ "first_cross": null,
+ "final_z": -0.248
+ },
+ {
+ "id": "54b7e3856565620016530000",
+ "variation": "headline",
+ "week": 201502,
+ "impressions": 12619,
+ "pooled_ctr": 0.011886837308820033,
+ "arms": [
+ {
+ "headline": "Jay Walking Can Be Extremely Dangerous. So They Made Waiting For The Light More Interesting.",
+ "impressions": 3119,
+ "clicks": 29,
+ "ctr": 0.0092978518756011,
+ "ctr_select": 0.010897435897435897,
+ "ctr_holdout": 0.007697241821680564,
+ "n_select": 1560,
+ "n_holdout": 1559,
+ "eb": 0.011298792436400393,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Saw This At Your Crosswalk, You Might Not Make It Across. Point Made.",
+ "impressions": 3094,
+ "clicks": 44,
+ "ctr": 0.0142210730446024,
+ "ctr_select": 0.011635423400129283,
+ "ctr_holdout": 0.01680672268907563,
+ "n_select": 1547,
+ "n_holdout": 1547,
+ "eb": 0.011595551559448485,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Traffic Light Started Dancing, And Things Got A Whole Lots Safer",
+ "impressions": 3262,
+ "clicks": 37,
+ "ctr": 0.0113427345187001,
+ "ctr_select": 0.009809932556713672,
+ "ctr_holdout": 0.012875536480686695,
+ "n_select": 1631,
+ "n_holdout": 1631,
+ "eb": 0.010842385185759414,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Crosswalk Experiment Stops People In Their Tracks For All The Right Reasons",
+ "impressions": 3144,
+ "clicks": 40,
+ "ctr": 0.0127226463104325,
+ "ctr_select": 0.013994910941475827,
+ "ctr_holdout": 0.011450381679389313,
+ "n_select": 1572,
+ "n_holdout": 1572,
+ "eb": 0.012549381777820875,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.011568938193343899,
+ "pool_holdout": 0.012204786812490093,
+ "lift_select": 0.20969709644811596,
+ "lift_holdout": -0.06181223356795873,
+ "eb_lift": 0.08474793175410578,
+ "p_value": 0.299256653523877,
+ "held_up": false,
+ "z_path": [
+ 0.886,
+ 0.823,
+ -0.047,
+ -0.205,
+ -0.496,
+ -0.868,
+ -0.708,
+ -0.568,
+ 0.044,
+ -0.513
+ ],
+ "first_cross": null,
+ "final_z": -0.513
+ },
+ {
+ "id": "53988492326359ba6c00003f",
+ "variation": "headline",
+ "week": 201423,
+ "impressions": 12426,
+ "pooled_ctr": 0.005391920167390954,
+ "arms": [
+ {
+ "headline": "Why Raising The Minimum Wage Is Also Critical To Our Children",
+ "impressions": 3071,
+ "clicks": 7,
+ "ctr": 0.0022793878215564,
+ "ctr_select": 0.0026041666666666665,
+ "ctr_holdout": 0.001954397394136808,
+ "n_select": 1536,
+ "n_holdout": 1535,
+ "eb": 0.004425983132926598,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It Used To Be That When You Got To A Certain Age, Your Income Began To Go Up. Ummm\u2026",
+ "impressions": 3077,
+ "clicks": 15,
+ "ctr": 0.0048748781280467,
+ "ctr_select": 0.003250975292587776,
+ "ctr_holdout": 0.00649772579597141,
+ "n_select": 1538,
+ "n_holdout": 1539,
+ "eb": 0.004683043750656792,
+ "labelled_winner": false
+ },
+ {
+ "headline": "People Working At Minimum Wage Or Close To It Are Just High-School Kids, Right? In A Word, No.",
+ "impressions": 3176,
+ "clicks": 20,
+ "ctr": 0.0062972292191435,
+ "ctr_select": 0.00818639798488665,
+ "ctr_holdout": 0.004408060453400504,
+ "n_select": 1588,
+ "n_holdout": 1588,
+ "eb": 0.006671381917036867,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Hey, America! Don\u2019t Look Now, But We\u2019re Dooming An Entire Generation To Poverty.",
+ "impressions": 3102,
+ "clicks": 25,
+ "ctr": 0.0080593165699548,
+ "ctr_select": 0.008381689232753063,
+ "ctr_holdout": 0.007736943907156673,
+ "n_select": 1551,
+ "n_holdout": 1551,
+ "eb": 0.006735186625953432,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.005633349428617415,
+ "pool_holdout": 0.005150490906164494,
+ "lift_select": 0.4878695772312793,
+ "lift_holdout": 0.5021760154738879,
+ "eb_lift": 0.1955918430585335,
+ "p_value": 0.0950199860421498,
+ "held_up": true,
+ "z_path": [
+ 0.848,
+ 0.368,
+ 0.823,
+ 0.879,
+ 0.456,
+ 0.78,
+ 0.569,
+ 0.568,
+ 0.556,
+ 0.827
+ ],
+ "first_cross": null,
+ "final_z": 0.827
+ },
+ {
+ "id": "52d4d061d4d15ffc2f002a66",
+ "variation": "headline",
+ "week": 201402,
+ "impressions": 12369,
+ "pooled_ctr": 0.0063060878001455255,
+ "arms": [
+ {
+ "headline": "I'm Glad We Can Always Rely On Local News ... For Laughs",
+ "impressions": 3125,
+ "clicks": 20,
+ "ctr": 0.0064,
+ "ctr_select": 0.005761843790012804,
+ "ctr_holdout": 0.007037747920665387,
+ "n_select": 1562,
+ "n_holdout": 1563,
+ "eb": 0.0058946476417575736,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why Local News Is A Joke. A Really Unfunny One. =(",
+ "impressions": 3133,
+ "clicks": 28,
+ "ctr": 0.0089371209703159,
+ "ctr_select": 0.006385696040868455,
+ "ctr_holdout": 0.01148691767708998,
+ "n_select": 1566,
+ "n_holdout": 1567,
+ "eb": 0.006146063041125004,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Yuck. Do People Actually Call This News?",
+ "impressions": 3090,
+ "clicks": 18,
+ "ctr": 0.0058252427184466,
+ "ctr_select": 0.006472491909385114,
+ "ctr_holdout": 0.005177993527508091,
+ "n_select": 1545,
+ "n_holdout": 1545,
+ "eb": 0.006179476819197458,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Teleprompter Your Newscaster Reads Has A Secret",
+ "impressions": 3021,
+ "clicks": 12,
+ "ctr": 0.0039721946375372,
+ "ctr_select": 0.005298013245033113,
+ "ctr_holdout": 0.0026472534745201853,
+ "n_select": 1510,
+ "n_holdout": 1511,
+ "eb": 0.0057134738981285986,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.005984150088953582,
+ "pool_holdout": 0.00662786938247656,
+ "lift_select": 0.0816058777223827,
+ "lift_holdout": -0.21875443997158417,
+ "eb_lift": 0.03264068035399692,
+ "p_value": 0.7738375961909268,
+ "held_up": false,
+ "z_path": [
+ 1.75,
+ 1.158,
+ 0.03,
+ -1.499,
+ -2.161,
+ -1.882,
+ -2.3,
+ -2.307,
+ -1.944,
+ -1.433
+ ],
+ "first_cross": 4,
+ "final_z": -1.433
+ },
+ {
+ "id": "5441525e5c55ba2e07000045",
+ "variation": "headline",
+ "week": 201441,
+ "impressions": 12360,
+ "pooled_ctr": 0.017233009708737864,
+ "arms": [
+ {
+ "headline": "An Albino Model Had To Explain Why He's Black. And Not. At The Same Time.",
+ "impressions": 3030,
+ "clicks": 67,
+ "ctr": 0.0221122112211221,
+ "ctr_select": 0.02046204620462046,
+ "ctr_holdout": 0.023762376237623763,
+ "n_select": 1515,
+ "n_holdout": 1515,
+ "eb": 0.01963467529720976,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Model Explains Why He's Black \u2014 And Not Black \u2014 At The Same Time",
+ "impressions": 3136,
+ "clicks": 43,
+ "ctr": 0.0137117346938775,
+ "ctr_select": 0.01594387755102041,
+ "ctr_holdout": 0.011479591836734694,
+ "n_select": 1568,
+ "n_holdout": 1568,
+ "eb": 0.017822738722124337,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An Albino Model Has Had To Explain Why He's Black. And Not. At The Same Time.",
+ "impressions": 3070,
+ "clicks": 59,
+ "ctr": 0.0192182410423452,
+ "ctr_select": 0.02280130293159609,
+ "ctr_holdout": 0.015635179153094463,
+ "n_select": 1535,
+ "n_holdout": 1535,
+ "eb": 0.020570981950523568,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Model Has Had To Explain Why He's Black. And Not. At The Same Time.",
+ "impressions": 3124,
+ "clicks": 44,
+ "ctr": 0.0140845070422535,
+ "ctr_select": 0.017285531370038413,
+ "ctr_holdout": 0.01088348271446863,
+ "n_select": 1562,
+ "n_holdout": 1562,
+ "eb": 0.018365805886820206,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.019093851132686083,
+ "pool_holdout": 0.015372168284789644,
+ "lift_select": 0.19416993319715137,
+ "lift_holdout": 0.017109549117092504,
+ "eb_lift": 0.07736159706979362,
+ "p_value": 0.2208574281336798,
+ "held_up": false,
+ "z_path": [
+ -1.543,
+ -1.62,
+ -1.641,
+ -1.169,
+ -1.474,
+ -1.341,
+ -1.441,
+ -1.35,
+ -1.125,
+ -0.795
+ ],
+ "first_cross": null,
+ "final_z": -0.795
+ },
+ {
+ "id": "53b1ee6cd98e8a020400000f",
+ "variation": "headline",
+ "week": 201426,
+ "impressions": 12351,
+ "pooled_ctr": 0.01659784632823253,
+ "arms": [
+ {
+ "headline": "If They Stopped Walking, They'd Be Shot. So Her 4 Closest Friends Formed A Chair And Carried Her.",
+ "impressions": 3006,
+ "clicks": 57,
+ "ctr": 0.0189620758483033,
+ "ctr_select": 0.015968063872255488,
+ "ctr_holdout": 0.021956087824351298,
+ "n_select": 1503,
+ "n_holdout": 1503,
+ "eb": 0.016989459002276885,
+ "labelled_winner": false
+ },
+ {
+ "headline": "She Was Forced Into A Death March. She Survived By Leaning On Her Best Friends ... Literally.",
+ "impressions": 3138,
+ "clicks": 56,
+ "ctr": 0.0178457616316124,
+ "ctr_select": 0.0165710643722116,
+ "ctr_holdout": 0.019120458891013385,
+ "n_select": 1569,
+ "n_holdout": 1569,
+ "eb": 0.01721553670953002,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If They Stopped Walking, They'd Be Shot. But The Girls Used Their Unbreakable Bond To Survive.",
+ "impressions": 3089,
+ "clicks": 43,
+ "ctr": 0.0139203625768857,
+ "ctr_select": 0.019430051813471502,
+ "ctr_holdout": 0.008414239482200648,
+ "n_select": 1544,
+ "n_holdout": 1545,
+ "eb": 0.018362801139995836,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If The Girl Collapsed, Her Captors Would Shoot Her. But Her Best Friends Came To The Rescue.",
+ "impressions": 3118,
+ "clicks": 49,
+ "ctr": 0.0157152020525978,
+ "ctr_select": 0.01860166773572803,
+ "ctr_holdout": 0.012828736369467608,
+ "n_select": 1559,
+ "n_holdout": 1559,
+ "eb": 0.01803379895455969,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.017651821862348177,
+ "pool_holdout": 0.015544041450777202,
+ "lift_select": 0.10073917383657371,
+ "lift_holdout": -0.4586839266450916,
+ "eb_lift": 0.040277954490589796,
+ "p_value": 0.5400585915233771,
+ "held_up": false,
+ "z_path": [
+ -0.765,
+ -0.638,
+ 0.026,
+ -1.099,
+ -1.043,
+ 0.036,
+ -0.457,
+ -0.196,
+ -0.837,
+ -0.585
+ ],
+ "first_cross": null,
+ "final_z": -0.585
+ },
+ {
+ "id": "544104243637f0339500002e",
+ "variation": "headline",
+ "week": 201441,
+ "impressions": 12314,
+ "pooled_ctr": 0.007227545882735098,
+ "arms": [
+ {
+ "headline": "Listen To Some Of The Gross Things We Used To Do To Cure Diseases Before This Guy Invented Vaccines",
+ "impressions": 3038,
+ "clicks": 25,
+ "ctr": 0.0082290980908492,
+ "ctr_select": 0.008558262014483212,
+ "ctr_holdout": 0.007899934167215274,
+ "n_select": 1519,
+ "n_holdout": 1519,
+ "eb": 0.008195906277052547,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Listen To Some Of The Icky Things We Used To Do To Cure Diseases Before This Guy Invented Vaccines",
+ "impressions": 3123,
+ "clicks": 21,
+ "ctr": 0.0067243035542747,
+ "ctr_select": 0.0076824583866837385,
+ "ctr_holdout": 0.005765534913516977,
+ "n_select": 1562,
+ "n_holdout": 1561,
+ "eb": 0.007847316239148864,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Hear Some Of The Astonishing Things We Used To Do To Cure Diseases Before This Guy Invented Vaccines",
+ "impressions": 3117,
+ "clicks": 22,
+ "ctr": 0.0070580686557587,
+ "ctr_select": 0.00834403080872914,
+ "ctr_holdout": 0.005772931366260423,
+ "n_select": 1558,
+ "n_holdout": 1559,
+ "eb": 0.008113433295627073,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Hear Some Of The Ridiculous Things We Used To Do To Cure Diseases Before This Guy Invented Vaccines",
+ "impressions": 3036,
+ "clicks": 21,
+ "ctr": 0.0069169960474308,
+ "ctr_select": 0.007246376811594203,
+ "ctr_holdout": 0.006587615283267457,
+ "n_select": 1518,
+ "n_holdout": 1518,
+ "eb": 0.007676625497621855,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.007958421309079098,
+ "pool_holdout": 0.0064966704563911,
+ "lift_select": 0.0753718208810843,
+ "lift_holdout": 0.21599736668861103,
+ "eb_lift": 0.029840713220663728,
+ "p_value": 0.7617765873966197,
+ "held_up": true,
+ "z_path": [
+ -0.42,
+ 0.045,
+ -0.952,
+ -0.598,
+ 0.064,
+ -0.125,
+ 0.247,
+ 0.399,
+ 0.388,
+ 0.528
+ ],
+ "first_cross": null,
+ "final_z": 0.528
+ },
+ {
+ "id": "543c0908a9b5dab1e4000021",
+ "variation": "image",
+ "week": 201441,
+ "impressions": 12303,
+ "pooled_ctr": 0.01715028854750874,
+ "arms": [
+ {
+ "headline": "They Were Shooting A Beautiful Video From Space. Then They Sped It Up. Just WOW.",
+ "impressions": 3055,
+ "clicks": 49,
+ "ctr": 0.0160392798690671,
+ "ctr_select": 0.01767015706806283,
+ "ctr_holdout": 0.014407334643091027,
+ "n_select": 1528,
+ "n_holdout": 1527,
+ "eb": 0.018288493354124375,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Were Shooting A Beautiful Video From Space. Then They Sped It Up. Just WOW.",
+ "impressions": 3037,
+ "clicks": 76,
+ "ctr": 0.0250246954231149,
+ "ctr_select": 0.025691699604743084,
+ "ctr_holdout": 0.024358130348913758,
+ "n_select": 1518,
+ "n_holdout": 1519,
+ "eb": 0.021464678202795396,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Were Shooting A Beautiful Video From Space. Then They Sped It Up. Just WOW.",
+ "impressions": 3134,
+ "clicks": 48,
+ "ctr": 0.0153158902361199,
+ "ctr_select": 0.016592214422463305,
+ "ctr_holdout": 0.014039566049776643,
+ "n_select": 1567,
+ "n_holdout": 1567,
+ "eb": 0.017847467727145197,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Were Shooting A Beautiful Video From Space. Then They Sped It Up. Just WOW.",
+ "impressions": 3077,
+ "clicks": 38,
+ "ctr": 0.0123496912577185,
+ "ctr_select": 0.01495448634590377,
+ "ctr_holdout": 0.009746588693957114,
+ "n_select": 1538,
+ "n_holdout": 1539,
+ "eb": 0.017203635241800415,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.01869614696797269,
+ "pool_holdout": 0.015604681404421327,
+ "lift_select": 0.37417081972847566,
+ "lift_holdout": 0.5609501865262232,
+ "eb_lift": 0.1480803097860388,
+ "p_value": 0.020417897701711415,
+ "held_up": true,
+ "z_path": [
+ -1.669,
+ 0.623,
+ 1.332,
+ 2.31,
+ 1.99,
+ 1.985,
+ 2.346,
+ 2.631,
+ 2.492,
+ 2.474
+ ],
+ "first_cross": 3,
+ "final_z": 2.474
+ },
+ {
+ "id": "540e19a64c8041124a00000d",
+ "variation": "headline",
+ "week": 201436,
+ "impressions": 12301,
+ "pooled_ctr": 0.0042272985936102755,
+ "arms": [
+ {
+ "headline": "These People Are Using Their Bodies To Share Some Really Great Words About Sex",
+ "impressions": 3108,
+ "clicks": 10,
+ "ctr": 0.0032175032175032,
+ "ctr_select": 0.002574002574002574,
+ "ctr_holdout": 0.003861003861003861,
+ "n_select": 1554,
+ "n_holdout": 1554,
+ "eb": 0.003758588800317962,
+ "labelled_winner": false
+ },
+ {
+ "headline": "75 People Use Their Bodies To Share Some Terrific Messages About Sex",
+ "impressions": 3082,
+ "clicks": 18,
+ "ctr": 0.0058403634003893,
+ "ctr_select": 0.007138221933809215,
+ "ctr_holdout": 0.004542504866969501,
+ "n_select": 1541,
+ "n_holdout": 1541,
+ "eb": 0.005585335074310531,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Want Us To Talk About Sex, And They're Using Their Bodies To Do It",
+ "impressions": 3094,
+ "clicks": 19,
+ "ctr": 0.0061409179056237,
+ "ctr_select": 0.007110536522301228,
+ "ctr_holdout": 0.005171299288946348,
+ "n_select": 1547,
+ "n_holdout": 1547,
+ "eb": 0.005576663759499626,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These People Want Women To Get Equal Treatment For Sexual Health. You'll Love Where They Put It.",
+ "impressions": 3017,
+ "clicks": 5,
+ "ctr": 0.0016572754391779,
+ "ctr_select": 0.001326259946949602,
+ "ctr_holdout": 0.0019880715705765406,
+ "n_select": 1508,
+ "n_holdout": 1509,
+ "eb": 0.003281001427700372,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.0045528455284552845,
+ "pool_holdout": 0.003901804584620387,
+ "lift_select": 0.5678594604616669,
+ "lift_holdout": 0.16420614319705829,
+ "eb_lift": 0.22677895382177726,
+ "p_value": 0.08160632319437633,
+ "held_up": false,
+ "z_path": [
+ 0.582,
+ 0.004,
+ -0.703,
+ -1.063,
+ -0.22,
+ 0.011,
+ 0.011,
+ -0.158,
+ -0.153,
+ -0.153
+ ],
+ "first_cross": null,
+ "final_z": -0.153
+ },
+ {
+ "id": "53e9508caa1cffe874000027",
+ "variation": "headline",
+ "week": 201432,
+ "impressions": 12300,
+ "pooled_ctr": 0.007723577235772358,
+ "arms": [
+ {
+ "headline": "Big Companies Are Not Even Cuddling After They Screw Us. Geez.",
+ "impressions": 3064,
+ "clicks": 18,
+ "ctr": 0.0058746736292428,
+ "ctr_select": 0.0045691906005221935,
+ "ctr_holdout": 0.007180156657963447,
+ "n_select": 1532,
+ "n_holdout": 1532,
+ "eb": 0.006516461546893176,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Whole Tomfoolery Thing Happening Between You And Big Information Companies Is, Well, Insulting",
+ "impressions": 3110,
+ "clicks": 25,
+ "ctr": 0.0080385852090032,
+ "ctr_select": 0.00964630225080386,
+ "ctr_holdout": 0.006430868167202572,
+ "n_select": 1555,
+ "n_holdout": 1555,
+ "eb": 0.008543503671825317,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Everything I Hoped Wasn't True Is True. The Deal Is Sealed And Now We Have To Change It.",
+ "impressions": 3075,
+ "clicks": 31,
+ "ctr": 0.0100813008130081,
+ "ctr_select": 0.011053315994798439,
+ "ctr_holdout": 0.009108653220559532,
+ "n_select": 1538,
+ "n_holdout": 1537,
+ "eb": 0.009099885350773504,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Companies Are Trying To Keep You In The Slow Lane. You Can Stop It. So, Um, Let's Stop It.",
+ "impressions": 3051,
+ "clicks": 21,
+ "ctr": 0.0068829891838741,
+ "ctr_select": 0.005897771952817824,
+ "ctr_holdout": 0.007868852459016393,
+ "n_select": 1526,
+ "n_holdout": 1525,
+ "eb": 0.007046966005353298,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.007803609169240774,
+ "pool_holdout": 0.007643519271426248,
+ "lift_select": 0.4164363892501084,
+ "lift_holdout": 0.19168316283447995,
+ "eb_lift": 0.166112391512663,
+ "p_value": 0.0944327611699148,
+ "held_up": true,
+ "z_path": [
+ -1.629,
+ -0.929,
+ -0.671,
+ -0.797,
+ -0.875,
+ -0.315,
+ 0.034,
+ 0.63,
+ 0.181,
+ 0.848
+ ],
+ "first_cross": null,
+ "final_z": 0.848
+ },
+ {
+ "id": "53ed1533441a0bc7d2000046",
+ "variation": "other",
+ "week": 201432,
+ "impressions": 12271,
+ "pooled_ctr": 0.01026811180832858,
+ "arms": [
+ {
+ "headline": "This Kind Of Conversation Often Happens In Private \u2014 But Turns Out, It's Not So Rare",
+ "impressions": 3127,
+ "clicks": 41,
+ "ctr": 0.0131116085705148,
+ "ctr_select": 0.015345268542199489,
+ "ctr_holdout": 0.010876519513755598,
+ "n_select": 1564,
+ "n_holdout": 1563,
+ "eb": 0.012020406219054282,
+ "labelled_winner": true
+ },
+ {
+ "headline": "This Kind Of Conversation Often Happens In Private \u2014 But Turns Out, It's Not So Rare",
+ "impressions": 3090,
+ "clicks": 30,
+ "ctr": 0.0097087378640776,
+ "ctr_select": 0.009061488673139158,
+ "ctr_holdout": 0.010355987055016181,
+ "n_select": 1545,
+ "n_holdout": 1545,
+ "eb": 0.009490668447932626,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Kind Of Conversation Often Happens In Private \u2014 But Turns Out, It's Not So Rare",
+ "impressions": 3035,
+ "clicks": 29,
+ "ctr": 0.0095551894563426,
+ "ctr_select": 0.00922266139657444,
+ "ctr_holdout": 0.00988793671720501,
+ "n_select": 1518,
+ "n_holdout": 1517,
+ "eb": 0.009557474559317417,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Kind Of Conversation Often Happens In Private \u2014 But Turns Out, It's Not So Rare",
+ "impressions": 3019,
+ "clicks": 26,
+ "ctr": 0.0086121232196091,
+ "ctr_select": 0.005298013245033113,
+ "ctr_holdout": 0.011928429423459244,
+ "n_select": 1510,
+ "n_holdout": 1509,
+ "eb": 0.00800992805470346,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.009776763891152028,
+ "pool_holdout": 0.010759700032605152,
+ "lift_select": 0.5695652173913044,
+ "lift_holdout": 0.010857131778436946,
+ "eb_lift": 0.2294872161056023,
+ "p_value": 0.009519599171415954,
+ "held_up": false,
+ "z_path": [
+ 0.67,
+ 1.242,
+ 1.505,
+ 2.124,
+ 1.817,
+ 2.273,
+ 2.096,
+ 1.564,
+ 1.063,
+ 1.317
+ ],
+ "first_cross": 3,
+ "final_z": 1.317
+ },
+ {
+ "id": "53ed3cc90d60d0875600006e",
+ "variation": "headline",
+ "week": 201432,
+ "impressions": 12271,
+ "pooled_ctr": 0.004726591149865536,
+ "arms": [
+ {
+ "headline": "Another Unarmed Black Kid Was Shot By A Cop. These Stats Explain Why, Yes It's 'A Racial Thing'.",
+ "impressions": 3038,
+ "clicks": 8,
+ "ctr": 0.0026333113890717,
+ "ctr_select": 0.0026333113890717576,
+ "ctr_holdout": 0.0026333113890717576,
+ "n_select": 1519,
+ "n_holdout": 1519,
+ "eb": 0.004193463238629587,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How Does One City Turn Into A Racially Charged Powder Keg? Well, Uh, Just Look At The Numbers.",
+ "impressions": 3141,
+ "clicks": 10,
+ "ctr": 0.003183699458771,
+ "ctr_select": 0.0025477707006369425,
+ "ctr_holdout": 0.0038192234245703373,
+ "n_select": 1570,
+ "n_holdout": 1571,
+ "eb": 0.0041384521134294,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Numbers Don't Lie. Here's What It's Like To Live In One Of Most Segregated Cities In The US.",
+ "impressions": 3019,
+ "clicks": 23,
+ "ctr": 0.0076184166942696,
+ "ctr_select": 0.007947019867549669,
+ "ctr_holdout": 0.00728959575878065,
+ "n_select": 1510,
+ "n_holdout": 1509,
+ "eb": 0.006293352464316464,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here Are Some Statics That Give Context To Why Ferguson Residents Are So Upset About Mike Brown",
+ "impressions": 3073,
+ "clicks": 17,
+ "ctr": 0.0055320533680442,
+ "ctr_select": 0.0078125,
+ "ctr_holdout": 0.0032530904359141183,
+ "n_select": 1536,
+ "n_holdout": 1537,
+ "eb": 0.006250892682409995,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.0052159739201303994,
+ "pool_holdout": 0.00423728813559322,
+ "lift_select": 0.5235927152317881,
+ "lift_holdout": 0.7203445990722332,
+ "eb_lift": 0.20655366776817208,
+ "p_value": 0.08972994717979814,
+ "held_up": true,
+ "z_path": [
+ -0.984,
+ -0.305,
+ 0.335,
+ 0.038,
+ 0.699,
+ 0.246,
+ 0.607,
+ 0.575,
+ 0.548,
+ 1.008
+ ],
+ "first_cross": null,
+ "final_z": 1.008
+ },
+ {
+ "id": "543b622147df54d1ac00002f",
+ "variation": "image",
+ "week": 201441,
+ "impressions": 12257,
+ "pooled_ctr": 0.018193685241086725,
+ "arms": [
+ {
+ "headline": "The Dirty Little Secret About What It Means When Men Are Silent",
+ "impressions": 3012,
+ "clicks": 43,
+ "ctr": 0.0142762284196547,
+ "ctr_select": 0.01593625498007968,
+ "ctr_holdout": 0.012616201859229747,
+ "n_select": 1506,
+ "n_holdout": 1506,
+ "eb": 0.016957535139036268,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Dirty Little Secret About What It Means When Men Are Silent",
+ "impressions": 3120,
+ "clicks": 62,
+ "ctr": 0.0198717948717948,
+ "ctr_select": 0.019230769230769232,
+ "ctr_holdout": 0.020512820512820513,
+ "n_select": 1560,
+ "n_holdout": 1560,
+ "eb": 0.018268697970635,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Dirty Little Secret About What It Means When Men Are Silent",
+ "impressions": 3090,
+ "clicks": 68,
+ "ctr": 0.0220064724919093,
+ "ctr_select": 0.018770226537216828,
+ "ctr_holdout": 0.02524271844660194,
+ "n_select": 1545,
+ "n_holdout": 1545,
+ "eb": 0.01808075431416675,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Dirty Little Secret About What It Means When Men Are Silent",
+ "impressions": 3035,
+ "clicks": 50,
+ "ctr": 0.0164744645799011,
+ "ctr_select": 0.016469038208168644,
+ "ctr_holdout": 0.016479894528675015,
+ "n_select": 1518,
+ "n_holdout": 1517,
+ "eb": 0.01716519216512809,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.01762114537444934,
+ "pool_holdout": 0.018766318537859008,
+ "lift_select": 0.09134615384615374,
+ "lift_holdout": 0.0930657748049053,
+ "eb_lift": 0.03674860983353612,
+ "p_value": 0.575720080785344,
+ "held_up": false,
+ "z_path": [
+ -1.25,
+ -1.563,
+ -0.741,
+ -1.416,
+ -0.171,
+ -0.043,
+ -0.258,
+ -0.051,
+ -0.513,
+ -0.588
+ ],
+ "first_cross": null,
+ "final_z": -0.588
+ },
+ {
+ "id": "52e91de0b06959e12f00000b",
+ "variation": "headline",
+ "week": 201404,
+ "impressions": 12250,
+ "pooled_ctr": 0.012489795918367347,
+ "arms": [
+ {
+ "headline": "PRETTY URGENT STUFF: How One Big Oil Company Is Trying To Make It A Crime For People To Speak Up",
+ "impressions": 3047,
+ "clicks": 49,
+ "ctr": 0.016081391532655,
+ "ctr_select": 0.015091863517060367,
+ "ctr_holdout": 0.017071569271175313,
+ "n_select": 1524,
+ "n_holdout": 1523,
+ "eb": 0.013373082198286484,
+ "labelled_winner": false
+ },
+ {
+ "headline": "THIS IS NOT GOOD: How One Big Oil Company Is Trying To Make It A Crime For People To Speak Up",
+ "impressions": 3174,
+ "clicks": 31,
+ "ctr": 0.0097668557025834,
+ "ctr_select": 0.010081915563957152,
+ "ctr_holdout": 0.00945179584120983,
+ "n_select": 1587,
+ "n_holdout": 1587,
+ "eb": 0.011364602983830413,
+ "labelled_winner": false
+ },
+ {
+ "headline": "UH OH: How One Big Oil Company Is Trying To Make It A Crime For People To Speak Up",
+ "impressions": 3086,
+ "clicks": 40,
+ "ctr": 0.0129617627997407,
+ "ctr_select": 0.012313674659753726,
+ "ctr_holdout": 0.013609850939727802,
+ "n_select": 1543,
+ "n_holdout": 1543,
+ "eb": 0.012271185886471893,
+ "labelled_winner": false
+ },
+ {
+ "headline": "AWFUL STUFF ALERT: How One Big Oil Company Is Trying To Make It A Crime For People To Speak Up",
+ "impressions": 2943,
+ "clicks": 33,
+ "ctr": 0.0112130479102956,
+ "ctr_select": 0.01154891304347826,
+ "ctr_holdout": 0.010876954452753228,
+ "n_select": 1472,
+ "n_holdout": 1471,
+ "eb": 0.011973340018533606,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.012242899118511263,
+ "pool_holdout": 0.012736773350751143,
+ "lift_select": 0.23270341207349077,
+ "lift_holdout": 0.3403370540599695,
+ "eb_lift": 0.09231335395604012,
+ "p_value": 0.2432574415910621,
+ "held_up": true,
+ "z_path": [
+ 0.326,
+ 0.242,
+ 1.509,
+ 1.494,
+ 0.902,
+ 0.969,
+ 1.292,
+ 1.352,
+ 0.938,
+ 1.021
+ ],
+ "first_cross": null,
+ "final_z": 1.021
+ },
+ {
+ "id": "53d9554b45ba39bd63000056",
+ "variation": "headline",
+ "week": 201430,
+ "impressions": 12249,
+ "pooled_ctr": 0.01665442076904237,
+ "arms": [
+ {
+ "headline": "When It\u2019s Appropriate To Use The Word \u2018Rape\u2019 And Other Facts About Rape Culture",
+ "impressions": 3061,
+ "clicks": 44,
+ "ctr": 0.01437438745508,
+ "ctr_select": 0.01241830065359477,
+ "ctr_holdout": 0.016329196603527107,
+ "n_select": 1530,
+ "n_holdout": 1531,
+ "eb": 0.015265868857133958,
+ "labelled_winner": false
+ },
+ {
+ "headline": "There's Only One Cause For Sexual Assault",
+ "impressions": 3115,
+ "clicks": 38,
+ "ctr": 0.012199036918138,
+ "ctr_select": 0.012195121951219513,
+ "ctr_holdout": 0.012202954399486191,
+ "n_select": 1558,
+ "n_holdout": 1557,
+ "eb": 0.01515558050118487,
+ "labelled_winner": false
+ },
+ {
+ "headline": "\u2018What Are Women\u2019s Bodies For?' Plus 3 Other Thoughts On Rape Culture",
+ "impressions": 3000,
+ "clicks": 42,
+ "ctr": 0.014,
+ "ctr_select": 0.017333333333333333,
+ "ctr_holdout": 0.010666666666666666,
+ "n_select": 1500,
+ "n_holdout": 1500,
+ "eb": 0.017219395993832355,
+ "labelled_winner": false
+ },
+ {
+ "headline": "3 Facts About Rape Culture That Are Totally Worth Sharing",
+ "impressions": 3073,
+ "clicks": 80,
+ "ctr": 0.0260331923202082,
+ "ctr_select": 0.026692708333333332,
+ "ctr_holdout": 0.025374105400130124,
+ "n_select": 1536,
+ "n_holdout": 1537,
+ "eb": 0.020950903456633505,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.01714565643370346,
+ "pool_holdout": 0.01616326530612245,
+ "lift_select": 0.5568204365079366,
+ "lift_holdout": 0.5698625815737071,
+ "eb_lift": 0.22193650255641528,
+ "p_value": 0.0008683425416905227,
+ "held_up": true,
+ "z_path": [
+ 0.96,
+ 1.92,
+ 1.988,
+ 2.839,
+ 3.036,
+ 3.079,
+ 3.457,
+ 3.674,
+ 3.682,
+ 3.342
+ ],
+ "first_cross": 2,
+ "final_z": 3.342
+ },
+ {
+ "id": "54f36ac739383600191f0000",
+ "variation": "headline",
+ "week": 201509,
+ "impressions": 12249,
+ "pooled_ctr": 0.0032655726998122294,
+ "arms": [
+ {
+ "headline": "Think you're not capable of changing the world? Maybe you should watch this.",
+ "impressions": 3089,
+ "clicks": 10,
+ "ctr": 0.0032372936225315,
+ "ctr_select": 0.0045336787564766836,
+ "ctr_holdout": 0.001941747572815534,
+ "n_select": 1544,
+ "n_holdout": 1545,
+ "eb": 0.004360775512356986,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A solid reality check about your place in the world.",
+ "impressions": 3042,
+ "clicks": 8,
+ "ctr": 0.0026298487836949,
+ "ctr_select": 0.0019723865877712033,
+ "ctr_holdout": 0.003287310979618672,
+ "n_select": 1521,
+ "n_holdout": 1521,
+ "eb": 0.0033448835450087928,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Listen to this 2-minute reality check about what you have to offer.",
+ "impressions": 3098,
+ "clicks": 14,
+ "ctr": 0.0045190445448676,
+ "ctr_select": 0.006455777921239509,
+ "ctr_holdout": 0.0025823111684958036,
+ "n_select": 1549,
+ "n_holdout": 1549,
+ "eb": 0.005130992464892195,
+ "labelled_winner": false
+ },
+ {
+ "headline": "\"People think that if they can't tackle all of the world problems ... there's no purpose.\" NOT TRUE!",
+ "impressions": 3020,
+ "clicks": 8,
+ "ctr": 0.0026490066225165,
+ "ctr_select": 0.003973509933774834,
+ "ctr_holdout": 0.0013245033112582781,
+ "n_select": 1510,
+ "n_holdout": 1510,
+ "eb": 0.004138256991223857,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.004245591116917048,
+ "pool_holdout": 0.002285714285714286,
+ "lift_select": 0.5205839996027213,
+ "lift_holdout": 0.12976113621691399,
+ "eb_lift": 0.20854607134614622,
+ "p_value": 0.12165613302743672,
+ "held_up": false,
+ "z_path": [
+ 0.999,
+ 0.575,
+ -0.003,
+ 0.444,
+ 0.813,
+ 1.262,
+ 0.827,
+ 0.465,
+ 0.442,
+ 0.811
+ ],
+ "first_cross": null,
+ "final_z": 0.811
+ },
+ {
+ "id": "54cab7bb383837000c320000",
+ "variation": "other",
+ "week": 201504,
+ "impressions": 12211,
+ "pooled_ctr": 0.0258783064450086,
+ "arms": [
+ {
+ "headline": "After Seeing His Mom Harassed, This Little Kid Decided To Send A Big Message Out To The World",
+ "impressions": 3070,
+ "clicks": 91,
+ "ctr": 0.0296416938110749,
+ "ctr_select": 0.028013029315960912,
+ "ctr_holdout": 0.031270358306188926,
+ "n_select": 1535,
+ "n_holdout": 1535,
+ "eb": 0.02820538871856726,
+ "labelled_winner": false
+ },
+ {
+ "headline": "After Seeing His Mom Harassed, This Little Kid Decided To Send A Big Message Out To The World",
+ "impressions": 3040,
+ "clicks": 70,
+ "ctr": 0.0230263157894736,
+ "ctr_select": 0.024342105263157894,
+ "ctr_holdout": 0.021710526315789475,
+ "n_select": 1520,
+ "n_holdout": 1520,
+ "eb": 0.026752194746961014,
+ "labelled_winner": false
+ },
+ {
+ "headline": "After Seeing His Mom Harassed, This Little Kid Decided To Send A Big Message Out To The World",
+ "impressions": 3114,
+ "clicks": 74,
+ "ctr": 0.0237636480411046,
+ "ctr_select": 0.028259473346178548,
+ "ctr_holdout": 0.019267822736030827,
+ "n_select": 1557,
+ "n_holdout": 1557,
+ "eb": 0.028303327055211602,
+ "labelled_winner": false
+ },
+ {
+ "headline": "After Seeing His Mom Harassed, This Little Kid Decided To Send A Big Message Out To The World",
+ "impressions": 2987,
+ "clicks": 81,
+ "ctr": 0.0271175092065617,
+ "ctr_select": 0.03279785809906292,
+ "ctr_holdout": 0.021433355659745478,
+ "n_select": 1494,
+ "n_holdout": 1493,
+ "eb": 0.03008287783024594,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.028332787422207666,
+ "pool_holdout": 0.023423423423423424,
+ "lift_select": 0.1575937662016078,
+ "lift_holdout": -0.08496058529548156,
+ "eb_lift": 0.061769086887177416,
+ "p_value": 0.23137121693022744,
+ "held_up": false,
+ "z_path": [
+ 1.006,
+ -0.083,
+ 0.452,
+ 0.296,
+ 0.063,
+ 0.631,
+ 0.411,
+ 0.977,
+ 0.943,
+ 0.832
+ ],
+ "first_cross": null,
+ "final_z": 0.832
+ },
+ {
+ "id": "54d508b4376437000c1a0100",
+ "variation": "image",
+ "week": 201505,
+ "impressions": 12210,
+ "pooled_ctr": 0.01891891891891892,
+ "arms": [
+ {
+ "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer",
+ "impressions": 2049,
+ "clicks": 54,
+ "ctr": 0.0263543191800878,
+ "ctr_select": 0.02734375,
+ "ctr_holdout": 0.025365853658536587,
+ "n_select": 1024,
+ "n_holdout": 1025,
+ "eb": 0.021218635290760048,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer",
+ "impressions": 2091,
+ "clicks": 50,
+ "ctr": 0.0239120038259206,
+ "ctr_select": 0.02676864244741874,
+ "ctr_holdout": 0.021052631578947368,
+ "n_select": 1046,
+ "n_holdout": 1045,
+ "eb": 0.021079856477267504,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer",
+ "impressions": 2029,
+ "clicks": 23,
+ "ctr": 0.0113356333169048,
+ "ctr_select": 0.007889546351084813,
+ "ctr_holdout": 0.014778325123152709,
+ "n_select": 1014,
+ "n_holdout": 1015,
+ "eb": 0.015279376870174978,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer",
+ "impressions": 2046,
+ "clicks": 30,
+ "ctr": 0.0146627565982404,
+ "ctr_select": 0.011730205278592375,
+ "ctr_holdout": 0.017595307917888565,
+ "n_select": 1023,
+ "n_holdout": 1023,
+ "eb": 0.016435568166026274,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer",
+ "impressions": 1981,
+ "clicks": 33,
+ "ctr": 0.01665825340737,
+ "ctr_select": 0.01818181818181818,
+ "ctr_holdout": 0.015136226034308779,
+ "n_select": 990,
+ "n_holdout": 991,
+ "eb": 0.018413491886296374,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer",
+ "impressions": 2014,
+ "clicks": 41,
+ "ctr": 0.0203574975173783,
+ "ctr_select": 0.018867924528301886,
+ "ctr_holdout": 0.021847070506454815,
+ "n_select": 1007,
+ "n_holdout": 1007,
+ "eb": 0.018620118382586672,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.018512450851900394,
+ "pool_holdout": 0.019325253848673438,
+ "lift_select": 0.47704646017699104,
+ "lift_holdout": 0.3125754443985118,
+ "eb_lift": 0.14618185676813567,
+ "p_value": 0.021554080511398552,
+ "held_up": true,
+ "z_path": [
+ 0.679,
+ 0.916,
+ 0.428,
+ 0.225,
+ 0.636,
+ 0.962,
+ 0.553,
+ 0.535,
+ 0.724,
+ 0.502
+ ],
+ "first_cross": null,
+ "final_z": 0.502
+ },
+ {
+ "id": "5536963931363319a04f0100",
+ "variation": "image",
+ "week": 201516,
+ "impressions": 12161,
+ "pooled_ctr": 0.0037003535893429815,
+ "arms": [
+ {
+ "headline": "Most workers have the same rights, right? Umm \u2026 no.",
+ "impressions": 2048,
+ "clicks": 7,
+ "ctr": 0.00341796875,
+ "ctr_select": 0.0029296875,
+ "ctr_holdout": 0.00390625,
+ "n_select": 1024,
+ "n_holdout": 1024,
+ "eb": 0.0034069589945267416,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Most workers have the same rights, right? Umm \u2026 no.",
+ "impressions": 2019,
+ "clicks": 1,
+ "ctr": 0.0004952947003467,
+ "ctr_select": 0.0009900990099009901,
+ "ctr_holdout": 0.0,
+ "n_select": 1010,
+ "n_holdout": 1009,
+ "eb": 0.0028202763245961638,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Most workers have the same rights, right? Umm \u2026 no.",
+ "impressions": 2008,
+ "clicks": 16,
+ "ctr": 0.0079681274900398,
+ "ctr_select": 0.00796812749003984,
+ "ctr_holdout": 0.00796812749003984,
+ "n_select": 1004,
+ "n_holdout": 1004,
+ "eb": 0.004932726728544162,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Most workers have the same rights, right? Umm \u2026 no.",
+ "impressions": 1936,
+ "clicks": 6,
+ "ctr": 0.003099173553719,
+ "ctr_select": 0.004132231404958678,
+ "ctr_holdout": 0.002066115702479339,
+ "n_select": 968,
+ "n_holdout": 968,
+ "eb": 0.003769375081074814,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Most workers have the same rights, right? Umm \u2026 no.",
+ "impressions": 1998,
+ "clicks": 11,
+ "ctr": 0.0055055055055055,
+ "ctr_select": 0.004004004004004004,
+ "ctr_holdout": 0.007007007007007007,
+ "n_select": 999,
+ "n_holdout": 999,
+ "eb": 0.003734144065794147,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Most workers have the same rights, right? Umm \u2026 no.",
+ "impressions": 2152,
+ "clicks": 4,
+ "ctr": 0.0018587360594795,
+ "ctr_select": 0.0018587360594795538,
+ "ctr_holdout": 0.0018587360594795538,
+ "n_select": 1076,
+ "n_holdout": 1076,
+ "eb": 0.003060091957398493,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.003617826015457984,
+ "pool_holdout": 0.0037828947368421053,
+ "lift_select": 1.2024628757696485,
+ "lift_holdout": 1.106357179975749,
+ "eb_lift": 0.3634505107398658,
+ "p_value": 0.011982555804691151,
+ "held_up": true,
+ "z_path": [
+ -1.041,
+ -1.038,
+ -0.426,
+ 0.244,
+ 0.518,
+ 1.226,
+ 1.224,
+ 1.813,
+ 1.991,
+ 2.052
+ ],
+ "first_cross": 8,
+ "final_z": 2.052
+ },
+ {
+ "id": "5318d184bc91b20a60000013",
+ "variation": "headline",
+ "week": 201409,
+ "impressions": 12111,
+ "pooled_ctr": 0.04062422591032945,
+ "arms": [
+ {
+ "headline": "Never Before Have I Laughed So Hard At One Guy's Non-Attempt To Save The Planet",
+ "impressions": 4058,
+ "clicks": 184,
+ "ctr": 0.0453425332676195,
+ "ctr_select": 0.040413997042878264,
+ "ctr_holdout": 0.05027106949236077,
+ "n_select": 2029,
+ "n_holdout": 2029,
+ "eb": 0.03938308971403499,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Never Before Have I Laughed So Hard At One Dude's Non-Attempt To Save The Planet",
+ "impressions": 4092,
+ "clicks": 147,
+ "ctr": 0.0359237536656891,
+ "ctr_select": 0.03470185728250244,
+ "ctr_holdout": 0.03714565004887586,
+ "n_select": 2046,
+ "n_holdout": 2046,
+ "eb": 0.03670886401944899,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Never Before Have I Laughed So Hard At One Man's Non-Attempt To Save The Planet",
+ "impressions": 3961,
+ "clicks": 161,
+ "ctr": 0.0406463014390305,
+ "ctr_select": 0.04040404040404041,
+ "ctr_holdout": 0.0408884401817264,
+ "n_select": 1980,
+ "n_holdout": 1981,
+ "eb": 0.03936674873129519,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.03848059454995871,
+ "pool_holdout": 0.042767503302509904,
+ "lift_select": 0.050243571221578964,
+ "lift_holdout": 0.17545018087157094,
+ "eb_lift": 0.0234532541565744,
+ "p_value": 0.5787280405430407,
+ "held_up": true,
+ "z_path": [
+ 0.272,
+ 0.153,
+ 0.801,
+ -0.407,
+ 0.304,
+ 0.592,
+ -0.134,
+ -0.031,
+ 0.41,
+ 1.036
+ ],
+ "first_cross": null,
+ "final_z": 1.036
+ },
+ {
+ "id": "53149f4bd167d1a490000017",
+ "variation": "headline",
+ "week": 201409,
+ "impressions": 12098,
+ "pooled_ctr": 0.06968093899818152,
+ "arms": [
+ {
+ "headline": "Oscar Winner Lupita Nyong\u2019o's Speech On Beauty That Left An Entire Audience Speechless",
+ "impressions": 2056,
+ "clicks": 167,
+ "ctr": 0.0812256809338521,
+ "ctr_select": 0.08268482490272373,
+ "ctr_holdout": 0.07976653696498054,
+ "n_select": 1028,
+ "n_holdout": 1028,
+ "eb": 0.07384829861406061,
+ "labelled_winner": true
+ },
+ {
+ "headline": "An Oscar Winner's Speech On Beauty That Left An Entire Audience Speechless",
+ "impressions": 1973,
+ "clicks": 142,
+ "ctr": 0.0719716168271667,
+ "ctr_select": 0.08417849898580122,
+ "ctr_holdout": 0.05977710233029382,
+ "n_select": 986,
+ "n_holdout": 987,
+ "eb": 0.07418175180508511,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Lupita Nyong\u2019o's Speech On Beauty That Left An Entire Audience Speechless",
+ "impressions": 2067,
+ "clicks": 144,
+ "ctr": 0.06966618287373,
+ "ctr_select": 0.07059961315280464,
+ "ctr_holdout": 0.06873184898354308,
+ "n_select": 1034,
+ "n_holdout": 1033,
+ "eb": 0.07013582485054663,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Listen To (Or Read) Oscar Winner Lupita Nyong\u2019o's Speech On Beauty That Left An Audience Speechless",
+ "impressions": 1994,
+ "clicks": 129,
+ "ctr": 0.0646940822467402,
+ "ctr_select": 0.058174523570712136,
+ "ctr_holdout": 0.0712136409227683,
+ "n_select": 997,
+ "n_holdout": 997,
+ "eb": 0.06639340982935578,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Speech On Beauty That Left An Entire Audience Speechless, By Oscar Winner Lupita Nyong\u2019o",
+ "impressions": 1968,
+ "clicks": 123,
+ "ctr": 0.0625,
+ "ctr_select": 0.05589430894308943,
+ "ctr_holdout": 0.06910569105691057,
+ "n_select": 984,
+ "n_holdout": 984,
+ "eb": 0.06574620167664721,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Oscar Winner Lupita Nyong\u2019o Delivers A Speech On Beauty And It Left An Entire Audience Speechless",
+ "impressions": 2040,
+ "clicks": 138,
+ "ctr": 0.0676470588235294,
+ "ctr_select": 0.06764705882352941,
+ "ctr_holdout": 0.06764705882352941,
+ "n_select": 1020,
+ "n_holdout": 1020,
+ "eb": 0.0692315792539781,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.06992891387006117,
+ "pool_holdout": 0.06943296412630187,
+ "lift_select": 0.20377243585132754,
+ "lift_holdout": -0.13906740000964934,
+ "eb_lift": 0.06081658786988142,
+ "p_value": 0.055142689705152174,
+ "held_up": false,
+ "z_path": [
+ 0.341,
+ 0.291,
+ 0.066,
+ -0.626,
+ -0.65,
+ -0.617,
+ -0.462,
+ -0.462,
+ -0.394,
+ -1.103
+ ],
+ "first_cross": null,
+ "final_z": -1.103
+ },
+ {
+ "id": "52eabc0217d1b029dc000009",
+ "variation": "image",
+ "week": 201404,
+ "impressions": 12086,
+ "pooled_ctr": 0.031689558166473605,
+ "arms": [
+ {
+ "headline": "Here Are Some Secrets About An Industry That Nobody Wants To Talk About",
+ "impressions": 3038,
+ "clicks": 103,
+ "ctr": 0.0339038841342988,
+ "ctr_select": 0.032916392363396975,
+ "ctr_holdout": 0.03489137590520079,
+ "n_select": 1519,
+ "n_holdout": 1519,
+ "eb": 0.03132557840743455,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here Are Some Secrets About An Industry That Nobody Wants To Talk About",
+ "impressions": 2952,
+ "clicks": 104,
+ "ctr": 0.035230352303523,
+ "ctr_select": 0.034552845528455285,
+ "ctr_holdout": 0.03590785907859079,
+ "n_select": 1476,
+ "n_holdout": 1476,
+ "eb": 0.03194423599542653,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here Are Some Secrets About An Industry That Nobody Wants To Talk About",
+ "impressions": 3071,
+ "clicks": 89,
+ "ctr": 0.0289807880169325,
+ "ctr_select": 0.03125,
+ "ctr_holdout": 0.026710097719869708,
+ "n_select": 1536,
+ "n_holdout": 1535,
+ "eb": 0.030668408326376826,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here Are Some Secrets About An Industry That Nobody Wants To Talk About",
+ "impressions": 3025,
+ "clicks": 87,
+ "ctr": 0.0287603305785124,
+ "ctr_select": 0.022486772486772486,
+ "ctr_holdout": 0.03502974223397224,
+ "n_select": 1512,
+ "n_holdout": 1513,
+ "eb": 0.027204960497552954,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.030282972033758068,
+ "pool_holdout": 0.03309614429918915,
+ "lift_select": 0.14099915589319822,
+ "lift_holdout": 0.08495596205962053,
+ "eb_lift": 0.05485802251564209,
+ "p_value": 0.27083056617654155,
+ "held_up": true,
+ "z_path": [
+ -0.421,
+ 0.249,
+ 0.8,
+ -0.453,
+ -0.659,
+ -0.856,
+ -0.874,
+ -0.209,
+ 0.35,
+ 0.281
+ ],
+ "first_cross": null,
+ "final_z": 0.281
+ },
+ {
+ "id": "52278c3138274a5b8100063c",
+ "variation": "headline",
+ "week": 201335,
+ "impressions": 11725,
+ "pooled_ctr": 0.04247334754797441,
+ "arms": [
+ {
+ "headline": "There's No Good Way To Say This, But I Think This Is A Sex Tape You Should Share. Trust Me.",
+ "impressions": 2209,
+ "clicks": 124,
+ "ctr": 0.0561339972838388,
+ "ctr_select": 0.057065217391304345,
+ "ctr_holdout": 0.05520361990950226,
+ "n_select": 1104,
+ "n_holdout": 1105,
+ "eb": 0.047761571528432065,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Alyssa Milano Made A Sex Tape And It's Really Important That You Watch It. I'm Not Joking.",
+ "impressions": 2338,
+ "clicks": 161,
+ "ctr": 0.0688622754491018,
+ "ctr_select": 0.07613344739093242,
+ "ctr_holdout": 0.06159110350727117,
+ "n_select": 1169,
+ "n_holdout": 1169,
+ "eb": 0.05432809945547935,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Celebrity Sex Tape About Syria. Trust Me.",
+ "impressions": 2177,
+ "clicks": 55,
+ "ctr": 0.0252641249425815,
+ "ctr_select": 0.027573529411764705,
+ "ctr_holdout": 0.02295684113865932,
+ "n_select": 1088,
+ "n_holdout": 1089,
+ "eb": 0.03829631135232144,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Sharing Celebrity Sex Tapes Is Not A Good Idea, But I Have A Really Important Reason To Share This",
+ "impressions": 2307,
+ "clicks": 78,
+ "ctr": 0.0338101430429128,
+ "ctr_select": 0.030329289428076257,
+ "ctr_holdout": 0.0372940156114484,
+ "n_select": 1154,
+ "n_holdout": 1153,
+ "eb": 0.039008482647491086,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This Is A Totally Safe For Work Celebrity Sex Tape About A Really Important Issue",
+ "impressions": 2694,
+ "clicks": 80,
+ "ctr": 0.0296956198960653,
+ "ctr_select": 0.02746844840386043,
+ "ctr_holdout": 0.03192279138827023,
+ "n_select": 1347,
+ "n_holdout": 1347,
+ "eb": 0.03749986240195908,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.04332992152848857,
+ "pool_holdout": 0.04161691966570016,
+ "lift_select": 0.7570640496285268,
+ "lift_holdout": 0.47995344206201174,
+ "eb_lift": 0.2538240905827558,
+ "p_value": 7.426059028263499e-10,
+ "held_up": true,
+ "z_path": [
+ 2.529,
+ 1.987,
+ 2.524,
+ 2.216,
+ 1.885,
+ 1.733,
+ 1.775,
+ 1.796,
+ 1.52,
+ 1.77
+ ],
+ "first_cross": 0,
+ "final_z": 1.77
+ },
+ {
+ "id": "5143606c220cb800020005dd",
+ "variation": "headline",
+ "week": 201310,
+ "impressions": 11468,
+ "pooled_ctr": 0.0177886292291594,
+ "arms": [
+ {
+ "headline": "Finally, An Answer To The Question: Do Feminist D**k Jokes Exist?",
+ "impressions": 2806,
+ "clicks": 32,
+ "ctr": 0.0114041339985744,
+ "ctr_select": 0.011404133998574484,
+ "ctr_holdout": 0.011404133998574484,
+ "n_select": 1403,
+ "n_holdout": 1403,
+ "eb": 0.015487850094401573,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Nothing Makes Me Happier Than Smart Comedy And Feminist D**k Jokes",
+ "impressions": 2868,
+ "clicks": 40,
+ "ctr": 0.0139470013947001,
+ "ctr_select": 0.014644351464435146,
+ "ctr_holdout": 0.013249651324965132,
+ "n_select": 1434,
+ "n_holdout": 1434,
+ "eb": 0.016692605115439255,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Didn't Think This Kind Of Joke Was Possible, But Here We Are",
+ "impressions": 2895,
+ "clicks": 55,
+ "ctr": 0.0189982728842832,
+ "ctr_select": 0.020027624309392266,
+ "ctr_holdout": 0.01796821008984105,
+ "n_select": 1448,
+ "n_holdout": 1447,
+ "eb": 0.018754986150121472,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Joke About Male Genitalia That Even Hillary Clinton Would Laugh At",
+ "impressions": 2899,
+ "clicks": 77,
+ "ctr": 0.0265608830631252,
+ "ctr_select": 0.025517241379310347,
+ "ctr_holdout": 0.027605244996549344,
+ "n_select": 1450,
+ "n_holdout": 1449,
+ "eb": 0.02086834307307899,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.017959895379250218,
+ "pool_holdout": 0.017617303331589045,
+ "lift_select": 0.42079009039169746,
+ "lift_holdout": 0.5669393026259149,
+ "eb_lift": 0.161941238098136,
+ "p_value": 0.012181290252280666,
+ "held_up": true,
+ "z_path": [
+ 0.0,
+ 0.202,
+ -0.162,
+ 0.281,
+ 0.906,
+ 1.299,
+ 0.971,
+ 1.887,
+ 2.107,
+ 1.929
+ ],
+ "first_cross": 8,
+ "final_z": 1.929
+ },
+ {
+ "id": "54b004a736633300245c0000",
+ "variation": "headline",
+ "week": 201501,
+ "impressions": 11448,
+ "pooled_ctr": 0.0159853249475891,
+ "arms": [
+ {
+ "headline": "That Song That's Been Stuck In Your Head For The Last 6 Months? She Just Made It 10 Times Better.",
+ "impressions": 2883,
+ "clicks": 51,
+ "ctr": 0.0176899063475546,
+ "ctr_select": 0.018030513176144243,
+ "ctr_holdout": 0.01734906315058987,
+ "n_select": 1442,
+ "n_holdout": 1441,
+ "eb": 0.016823588034115163,
+ "labelled_winner": false
+ },
+ {
+ "headline": "She Just Rewrote \"Rude\" In The Most Hilariously Perfect Way Possible",
+ "impressions": 2838,
+ "clicks": 27,
+ "ctr": 0.0095137420718816,
+ "ctr_select": 0.009161381254404511,
+ "ctr_holdout": 0.009866102889358703,
+ "n_select": 1419,
+ "n_holdout": 1419,
+ "eb": 0.013448132358486669,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's One Of The Most Popular Songs In The World Right Now. Thank God Someone Finally Fixed It.",
+ "impressions": 2873,
+ "clicks": 62,
+ "ctr": 0.0215802297250261,
+ "ctr_select": 0.022284122562674095,
+ "ctr_holdout": 0.020876826722338204,
+ "n_select": 1436,
+ "n_holdout": 1437,
+ "eb": 0.01844890312559755,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her Boyfriend And Father Are Fighting Over Her, So She Wrote Them This Hilarious Song",
+ "impressions": 2854,
+ "clicks": 43,
+ "ctr": 0.0150665732305536,
+ "ctr_select": 0.014716187806587245,
+ "ctr_holdout": 0.015416958654519973,
+ "n_select": 1427,
+ "n_holdout": 1427,
+ "eb": 0.015555756408183666,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.01607267645003494,
+ "pool_holdout": 0.015897973445143255,
+ "lift_select": 0.3864599733559404,
+ "lift_holdout": 0.3131753424029,
+ "eb_lift": 0.14784262490130828,
+ "p_value": 0.03057557866492494,
+ "held_up": true,
+ "z_path": [
+ 0.287,
+ 0.421,
+ 0.339,
+ 0.304,
+ 0.661,
+ 1.316,
+ 1.355,
+ 1.07,
+ 0.818,
+ 1.064
+ ],
+ "first_cross": null,
+ "final_z": 1.064
+ },
+ {
+ "id": "51e59879cde6d2e201002399",
+ "variation": "image",
+ "week": 201328,
+ "impressions": 11426,
+ "pooled_ctr": 0.021529844214948364,
+ "arms": [
+ {
+ "headline": "How Can Our Kids Accept Diversity If These Books Are What We Use To Raise Them?",
+ "impressions": 2891,
+ "clicks": 71,
+ "ctr": 0.024558976132826,
+ "ctr_select": 0.022130013831258646,
+ "ctr_holdout": 0.02698961937716263,
+ "n_select": 1446,
+ "n_holdout": 1445,
+ "eb": 0.02165033609357659,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How Can Our Kids Accept Diversity If These Books Are What We Use To Raise Them?",
+ "impressions": 2682,
+ "clicks": 32,
+ "ctr": 0.01193139448173,
+ "ctr_select": 0.013422818791946308,
+ "ctr_holdout": 0.010439970171513796,
+ "n_select": 1341,
+ "n_holdout": 1341,
+ "eb": 0.018445175225729755,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How Can Our Kids Accept Diversity If These Books Are What We Use To Raise Them?",
+ "impressions": 2739,
+ "clicks": 69,
+ "ctr": 0.0251916757940854,
+ "ctr_select": 0.02408759124087591,
+ "ctr_holdout": 0.026296566837107377,
+ "n_select": 1370,
+ "n_holdout": 1369,
+ "eb": 0.02236770116929509,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How Can Our Kids Accept Diversity If These Books Are What We Use To Raise Them?",
+ "impressions": 3114,
+ "clicks": 74,
+ "ctr": 0.0237636480411046,
+ "ctr_select": 0.025048169556840076,
+ "ctr_holdout": 0.0224791265253693,
+ "n_select": 1557,
+ "n_holdout": 1557,
+ "eb": 0.02283670288758872,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.021351067553377668,
+ "pool_holdout": 0.021708683473389355,
+ "lift_select": 0.1731577118670835,
+ "lift_holdout": 0.035490086394430964,
+ "eb_lift": 0.06958131393181932,
+ "p_value": 0.23672511803976526,
+ "held_up": false,
+ "z_path": [
+ -1.33,
+ -0.335,
+ -0.739,
+ -0.896,
+ -0.282,
+ 0.113,
+ -0.439,
+ -0.294,
+ 0.439,
+ -0.353
+ ],
+ "first_cross": null,
+ "final_z": -0.353
+ },
+ {
+ "id": "54a2cb0334303400154b0000",
+ "variation": "image",
+ "week": 201452,
+ "impressions": 11411,
+ "pooled_ctr": 0.007887126456927527,
+ "arms": [
+ {
+ "headline": "When He Mentioned Beyonc\u00e9, I Almost Rolled My Eyes, But Then I Saw Where He Was Going",
+ "impressions": 2843,
+ "clicks": 15,
+ "ctr": 0.0052761167780513,
+ "ctr_select": 0.004922644163150493,
+ "ctr_holdout": 0.005629838142153413,
+ "n_select": 1422,
+ "n_holdout": 1421,
+ "eb": 0.006975804494400704,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When He Mentioned Beyonc\u00e9, I Almost Rolled My Eyes, But Then I Saw Where He Was Going",
+ "impressions": 2819,
+ "clicks": 17,
+ "ctr": 0.0060305072720822,
+ "ctr_select": 0.0035460992907801418,
+ "ctr_holdout": 0.008516678495386799,
+ "n_select": 1410,
+ "n_holdout": 1409,
+ "eb": 0.006461736429362888,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When He Mentioned Beyonc\u00e9, I Almost Rolled My Eyes, But Then I Saw Where He Was Going",
+ "impressions": 2898,
+ "clicks": 26,
+ "ctr": 0.0089717046238785,
+ "ctr_select": 0.00966183574879227,
+ "ctr_holdout": 0.008281573498964804,
+ "n_select": 1449,
+ "n_holdout": 1449,
+ "eb": 0.008784193118279602,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When He Mentioned Beyonc\u00e9, I Almost Rolled My Eyes, But Then I Saw Where He Was Going",
+ "impressions": 2851,
+ "clicks": 32,
+ "ctr": 0.0112241318835496,
+ "ctr_select": 0.014726507713884993,
+ "ctr_holdout": 0.0077192982456140355,
+ "n_select": 1426,
+ "n_holdout": 1425,
+ "eb": 0.01070796929091377,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.008235500262835115,
+ "pool_holdout": 0.007538569424964937,
+ "lift_select": 0.7881740324072692,
+ "lift_holdout": 0.023973888208894456,
+ "eb_lift": 0.3002208668775508,
+ "p_value": 0.0017390935993718255,
+ "held_up": false,
+ "z_path": [
+ 0.357,
+ 0.029,
+ 0.035,
+ 0.451,
+ 0.56,
+ 0.663,
+ 0.354,
+ 0.483,
+ 0.618,
+ 0.854
+ ],
+ "first_cross": null,
+ "final_z": 0.854
+ },
+ {
+ "id": "5230d6a46b6514d4750081c0",
+ "variation": "other",
+ "week": 201336,
+ "impressions": 11404,
+ "pooled_ctr": 0.03560154331813399,
+ "arms": [
+ {
+ "headline": "Why We're Going To Invade Syria Is Explained Perfectly By \u2014 A Dead Comedian?",
+ "impressions": 2955,
+ "clicks": 112,
+ "ctr": 0.037901861252115,
+ "ctr_select": 0.04736129905277402,
+ "ctr_holdout": 0.02843601895734597,
+ "n_select": 1478,
+ "n_holdout": 1477,
+ "eb": 0.04210822511825934,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why We're Going To Invade Syria Is Explained Perfectly By \u2014 A Dead Comedian?",
+ "impressions": 3016,
+ "clicks": 90,
+ "ctr": 0.029840848806366,
+ "ctr_select": 0.03050397877984085,
+ "ctr_holdout": 0.029177718832891247,
+ "n_select": 1508,
+ "n_holdout": 1508,
+ "eb": 0.035504659258116814,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why We're Going To Invade Syria Is Explained Perfectly By \u2014 A Dead Comedian?",
+ "impressions": 2756,
+ "clicks": 100,
+ "ctr": 0.0362844702467343,
+ "ctr_select": 0.034107402031930335,
+ "ctr_holdout": 0.038461538461538464,
+ "n_select": 1378,
+ "n_holdout": 1378,
+ "eb": 0.03702415737521486,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why We're Going To Invade Syria Is Explained Perfectly By \u2014 A Dead Comedian?",
+ "impressions": 2677,
+ "clicks": 104,
+ "ctr": 0.038849458348898,
+ "ctr_select": 0.043348281016442454,
+ "ctr_holdout": 0.03435399551904406,
+ "n_select": 1338,
+ "n_holdout": 1339,
+ "eb": 0.04043827071374037,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.03875833041038232,
+ "pool_holdout": 0.032444756225885656,
+ "lift_select": 0.22196437646568978,
+ "lift_holdout": -0.12355578327142314,
+ "eb_lift": 0.08643031504214838,
+ "p_value": 0.046497459022196576,
+ "held_up": false,
+ "z_path": [
+ -1.544,
+ -2.149,
+ -0.786,
+ 0.217,
+ 0.073,
+ -0.139,
+ -0.051,
+ -0.179,
+ 0.316,
+ -0.185
+ ],
+ "first_cross": 1,
+ "final_z": -0.185
+ },
+ {
+ "id": "54c6f5a83564300016020000",
+ "variation": "headline",
+ "week": 201504,
+ "impressions": 11398,
+ "pooled_ctr": 0.008159326197578523,
+ "arms": [
+ {
+ "headline": "5 Minutes Will Change How You Think About Love",
+ "impressions": 2965,
+ "clicks": 21,
+ "ctr": 0.0070826306913996,
+ "ctr_select": 0.004048582995951417,
+ "ctr_holdout": 0.010114632501685773,
+ "n_select": 1482,
+ "n_holdout": 1483,
+ "eb": 0.006288431608325599,
+ "labelled_winner": false
+ },
+ {
+ "headline": "True Fact: If You're In Love, You Don't Exactly Know What Love Is",
+ "impressions": 2786,
+ "clicks": 28,
+ "ctr": 0.0100502512562814,
+ "ctr_select": 0.01148600143575018,
+ "ctr_holdout": 0.008614501076812634,
+ "n_select": 1393,
+ "n_holdout": 1393,
+ "eb": 0.009134168785987194,
+ "labelled_winner": false
+ },
+ {
+ "headline": "True Fact: If You're In Love, You Don't Exactly Know What Love Is",
+ "impressions": 2815,
+ "clicks": 30,
+ "ctr": 0.0106571936056838,
+ "ctr_select": 0.01065340909090909,
+ "ctr_holdout": 0.010660980810234541,
+ "n_select": 1408,
+ "n_holdout": 1407,
+ "eb": 0.008828987683834019,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Problem With Examining Love Is That You Only Care If You're In It (Or Falling Out Of It)",
+ "impressions": 2832,
+ "clicks": 14,
+ "ctr": 0.0049435028248587,
+ "ctr_select": 0.0049435028248587575,
+ "ctr_holdout": 0.0049435028248587575,
+ "n_select": 1416,
+ "n_holdout": 1416,
+ "eb": 0.006667422249263987,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.007720652746095806,
+ "pool_holdout": 0.008597999649061239,
+ "lift_select": 0.48769823141682433,
+ "lift_holdout": 0.0019192170766368921,
+ "eb_lift": 0.18308245253047772,
+ "p_value": 0.06472729924384903,
+ "held_up": false,
+ "z_path": [
+ 0.352,
+ 0.281,
+ -0.195,
+ -0.367,
+ -0.338,
+ -0.48,
+ -0.277,
+ 0.182,
+ 0.178,
+ -0.224
+ ],
+ "first_cross": null,
+ "final_z": -0.224
+ },
+ {
+ "id": "54a310426565330015080000",
+ "variation": "headline",
+ "week": 201452,
+ "impressions": 11372,
+ "pooled_ctr": 0.009145269081955681,
+ "arms": [
+ {
+ "headline": "They Took Utah's Biggest Secret And Made Sure Everyone Had A Peek. (And Nope: Not THAT Utah Secret.)",
+ "impressions": 2840,
+ "clicks": 30,
+ "ctr": 0.0105633802816901,
+ "ctr_select": 0.01267605633802817,
+ "ctr_holdout": 0.008450704225352112,
+ "n_select": 1420,
+ "n_holdout": 1420,
+ "eb": 0.010159495974712603,
+ "labelled_winner": true
+ },
+ {
+ "headline": "It's 1 Of Utah's Big Secrets \u2014 So, They Made Sure Everyone Had A Peek. (No, Not THAT Utah Secret.)",
+ "impressions": 2869,
+ "clicks": 31,
+ "ctr": 0.0108051585918438,
+ "ctr_select": 0.008368200836820083,
+ "ctr_holdout": 0.0132404181184669,
+ "n_select": 1434,
+ "n_holdout": 1435,
+ "eb": 0.008522308379331723,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Took 1 Of Utah's Dirtiest Secrets And Made Sure Everyone Had A Peek. (Not THAT Utah Secret.)",
+ "impressions": 2871,
+ "clicks": 22,
+ "ctr": 0.0076628352490421,
+ "ctr_select": 0.006267409470752089,
+ "ctr_holdout": 0.009059233449477353,
+ "n_select": 1436,
+ "n_holdout": 1435,
+ "eb": 0.007718555692205654,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Took 1 Of Our Government's Dirtiest Secrets And Made Sure Everyone Had A Peek \u2014 With A Blimp",
+ "impressions": 2792,
+ "clicks": 21,
+ "ctr": 0.0075214899713467,
+ "ctr_select": 0.0071633237822349575,
+ "ctr_holdout": 0.007879656160458453,
+ "n_select": 1396,
+ "n_holdout": 1396,
+ "eb": 0.008070965158936105,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.008617657404150545,
+ "pool_holdout": 0.009672880759760817,
+ "lift_select": 0.470939925265881,
+ "lift_holdout": -0.12635083226632537,
+ "eb_lift": 0.1789162063717522,
+ "p_value": 0.056108674822637136,
+ "held_up": false,
+ "z_path": [
+ -1.728,
+ 0.015,
+ 0.297,
+ 0.021,
+ -0.384,
+ 0.762,
+ 0.701,
+ 0.188,
+ 0.178,
+ -0.089
+ ],
+ "first_cross": null,
+ "final_z": -0.089
+ },
+ {
+ "id": "5148f04dcb47730002005288",
+ "variation": "headline",
+ "week": 201311,
+ "impressions": 11361,
+ "pooled_ctr": 0.005633306927207112,
+ "arms": [
+ {
+ "headline": "Fate Says These Kids Won't Amount To Anything. Fortunately, They Aren't Listening.",
+ "impressions": 3677,
+ "clicks": 20,
+ "ctr": 0.0054392167527875,
+ "ctr_select": 0.00544069640914037,
+ "ctr_holdout": 0.005437737901033171,
+ "n_select": 1838,
+ "n_holdout": 1839,
+ "eb": 0.005646584255024341,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Meet The Cutest People Ever To Spit In The Face Of Fate",
+ "impressions": 3942,
+ "clicks": 20,
+ "ctr": 0.0050735667174023,
+ "ctr_select": 0.0050735667174023336,
+ "ctr_holdout": 0.0050735667174023336,
+ "n_select": 1971,
+ "n_holdout": 1971,
+ "eb": 0.005471473823457493,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Tiny And Terrifying: Why Fate Doesn't Stand A Chance Against These Kids",
+ "impressions": 3742,
+ "clicks": 24,
+ "ctr": 0.0064136825227151,
+ "ctr_select": 0.00694815606627472,
+ "ctr_holdout": 0.005879208979155532,
+ "n_select": 1871,
+ "n_holdout": 1871,
+ "eb": 0.006318311583657129,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.0058098591549295775,
+ "pool_holdout": 0.00545678577715191,
+ "lift_select": 0.1959250441345577,
+ "lift_holdout": 0.0774124584058895,
+ "eb_lift": 0.08751544833856029,
+ "p_value": 0.4288726505073468,
+ "held_up": true,
+ "z_path": [
+ -0.47,
+ -0.027,
+ 0.307,
+ 0.466,
+ -0.258,
+ 0.168,
+ -0.243,
+ 0.66,
+ 0.427,
+ 0.547
+ ],
+ "first_cross": null,
+ "final_z": 0.547
+ },
+ {
+ "id": "5220d3fd762f11fc44008b62",
+ "variation": "other",
+ "week": 201334,
+ "impressions": 11361,
+ "pooled_ctr": 0.029398820526362116,
+ "arms": [
+ {
+ "headline": "The Warning Label That We Never Knew We Needed Until We Saw It",
+ "impressions": 2548,
+ "clicks": 79,
+ "ctr": 0.0310047095761381,
+ "ctr_select": 0.027472527472527472,
+ "ctr_holdout": 0.03453689167974882,
+ "n_select": 1274,
+ "n_holdout": 1274,
+ "eb": 0.029285260726749616,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Warning Label That We Never Knew We Needed Until We Saw It",
+ "impressions": 2955,
+ "clicks": 103,
+ "ctr": 0.0348561759729272,
+ "ctr_select": 0.036535859269282815,
+ "ctr_holdout": 0.03317535545023697,
+ "n_select": 1478,
+ "n_holdout": 1477,
+ "eb": 0.03271698997005576,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Warning Label That We Never Knew We Needed Until We Saw It",
+ "impressions": 3081,
+ "clicks": 79,
+ "ctr": 0.0256410256410256,
+ "ctr_select": 0.025324675324675326,
+ "ctr_holdout": 0.025957170668397145,
+ "n_select": 1540,
+ "n_holdout": 1541,
+ "eb": 0.028302840733404177,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Warning Label That We Never Knew We Needed Until We Saw It",
+ "impressions": 2777,
+ "clicks": 73,
+ "ctr": 0.026287360460929,
+ "ctr_select": 0.03170028818443804,
+ "ctr_holdout": 0.02087832973362131,
+ "n_select": 1388,
+ "n_holdout": 1389,
+ "eb": 0.030813037876534544,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.03028169014084507,
+ "pool_holdout": 0.02851610631930998,
+ "lift_select": 0.20653302703213017,
+ "lift_holdout": 0.16339008835059388,
+ "eb_lift": 0.08042152924370183,
+ "p_value": 0.10282204850119653,
+ "held_up": false,
+ "z_path": [
+ 1.302,
+ 1.255,
+ 0.052,
+ -0.511,
+ -0.099,
+ 0.25,
+ 0.389,
+ 0.739,
+ 1.335,
+ 1.879
+ ],
+ "first_cross": null,
+ "final_z": 1.879
+ },
+ {
+ "id": "549a190b63356600181f0000",
+ "variation": "headline",
+ "week": 201451,
+ "impressions": 11358,
+ "pooled_ctr": 0.004138052474027118,
+ "arms": [
+ {
+ "headline": "This 47 Seconds Is Just Between The President Of The United States And Your Daughters",
+ "impressions": 2881,
+ "clicks": 7,
+ "ctr": 0.0024297119055883,
+ "ctr_select": 0.001388888888888889,
+ "ctr_holdout": 0.0034698126301179735,
+ "n_select": 1440,
+ "n_holdout": 1441,
+ "eb": 0.002487534793471008,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What? Girls Don't Like Tools? The Leader Of The Free World Would Like To Know The Answer.",
+ "impressions": 2816,
+ "clicks": 7,
+ "ctr": 0.0024857954545454,
+ "ctr_select": 0.0007102272727272727,
+ "ctr_holdout": 0.004261363636363636,
+ "n_select": 1408,
+ "n_holdout": 1408,
+ "eb": 0.0022404941155694967,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Obama Separated Boy Toys From Girl Toys. He's Got A Wonderful Way Of Doing It.",
+ "impressions": 2869,
+ "clicks": 21,
+ "ctr": 0.0073196235622168,
+ "ctr_select": 0.0041841004184100415,
+ "ctr_holdout": 0.010452961672473868,
+ "n_select": 1434,
+ "n_holdout": 1435,
+ "eb": 0.003557697030636737,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Obama Walked Into This Miltary Event And Shook Things Up. He's Got Daughters.",
+ "impressions": 2792,
+ "clicks": 12,
+ "ctr": 0.0042979942693409,
+ "ctr_select": 0.0064469914040114614,
+ "ctr_holdout": 0.002148997134670487,
+ "n_select": 1396,
+ "n_holdout": 1396,
+ "eb": 0.00440192096497903,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.0031701303275801338,
+ "pool_holdout": 0.0051056338028169015,
+ "lift_select": 1.0336676217765044,
+ "lift_holdout": -0.57909297500247,
+ "eb_lift": 0.38856151328616284,
+ "p_value": 0.012141966477187772,
+ "held_up": false,
+ "z_path": [
+ -0.422,
+ 0.035,
+ 0.041,
+ -0.726,
+ -0.632,
+ -0.563,
+ -0.686,
+ -0.858,
+ -1.184,
+ -1.493
+ ],
+ "first_cross": null,
+ "final_z": -1.493
+ },
+ {
+ "id": "54a5d6446135350018020000",
+ "variation": "headline",
+ "week": 201500,
+ "impressions": 11348,
+ "pooled_ctr": 0.006432851603806839,
+ "arms": [
+ {
+ "headline": "Christopher Columbus Called Native People \u2018Indians.\u2019 I Believe I\u2019ll Call These Guys Heroes.",
+ "impressions": 2832,
+ "clicks": 18,
+ "ctr": 0.0063559322033898,
+ "ctr_select": 0.007768361581920904,
+ "ctr_holdout": 0.0049435028248587575,
+ "n_select": 1416,
+ "n_holdout": 1416,
+ "eb": 0.006666490792782979,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some People Call Native Americans \u2018Indians.\u2019 I Believe I\u2019ll Call These Guys Heroes.",
+ "impressions": 2837,
+ "clicks": 18,
+ "ctr": 0.0063447303489601,
+ "ctr_select": 0.005641748942172073,
+ "ctr_holdout": 0.007047216349541931,
+ "n_select": 1418,
+ "n_holdout": 1419,
+ "eb": 0.005859858880932215,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Folks Call Native Americans \u2018Indians.\u2019 I Believe I\u2019ll Call These Guys Straight-Up Heroes.",
+ "impressions": 2890,
+ "clicks": 16,
+ "ctr": 0.0055363321799307,
+ "ctr_select": 0.004152249134948097,
+ "ctr_holdout": 0.006920415224913495,
+ "n_select": 1445,
+ "n_holdout": 1445,
+ "eb": 0.005286276744071447,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Some Call Native Americans \u2018Indians.\u2019 I Believe I\u2019ll Call These Guys Simply 'Heroes.'",
+ "impressions": 2789,
+ "clicks": 21,
+ "ctr": 0.007529580494801,
+ "ctr_select": 0.006456241032998565,
+ "ctr_holdout": 0.008602150537634409,
+ "n_select": 1394,
+ "n_holdout": 1395,
+ "eb": 0.006167167477674416,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.005993301604089547,
+ "pool_holdout": 0.006872246696035243,
+ "lift_select": 0.2961739780658026,
+ "lift_holdout": -0.28065696074170654,
+ "eb_lift": 0.11232359610170106,
+ "p_value": 0.3177879886874009,
+ "held_up": false,
+ "z_path": [
+ -0.594,
+ -1.031,
+ -1.188,
+ -0.808,
+ -0.933,
+ -0.857,
+ -0.826,
+ -0.774,
+ -0.397,
+ -0.53
+ ],
+ "first_cross": null,
+ "final_z": -0.53
+ },
+ {
+ "id": "54ae476463316300244d0000",
+ "variation": "other",
+ "week": 201501,
+ "impressions": 11314,
+ "pooled_ctr": 0.025720346473395793,
+ "arms": [
+ {
+ "headline": "Girls Said She'd Steal Their Boyfriends, So She Told Them The Truth. It Only Made Things Worse.",
+ "impressions": 2825,
+ "clicks": 58,
+ "ctr": 0.0205309734513274,
+ "ctr_select": 0.018413597733711047,
+ "ctr_holdout": 0.02264685067232838,
+ "n_select": 1412,
+ "n_holdout": 1413,
+ "eb": 0.023231633366076098,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Girls Said She'd Steal Their Boyfriends, So She Told Them The Truth. It Only Made Things Worse.",
+ "impressions": 2845,
+ "clicks": 75,
+ "ctr": 0.0263620386643233,
+ "ctr_select": 0.03164556962025317,
+ "ctr_holdout": 0.02108222066057625,
+ "n_select": 1422,
+ "n_holdout": 1423,
+ "eb": 0.028250136236798156,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Girls Said She'd Steal Their Boyfriends, So She Told Them The Truth. It Only Made Things Worse.",
+ "impressions": 2785,
+ "clicks": 88,
+ "ctr": 0.0315978456014362,
+ "ctr_select": 0.03089080459770115,
+ "ctr_holdout": 0.03230437903804738,
+ "n_select": 1392,
+ "n_holdout": 1393,
+ "eb": 0.0279394651243093,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Girls Said She'd Steal Their Boyfriends, So She Told Them The Truth. It Only Made Things Worse.",
+ "impressions": 2859,
+ "clicks": 70,
+ "ctr": 0.024484085344526,
+ "ctr_select": 0.023776223776223775,
+ "ctr_holdout": 0.025192442267319804,
+ "n_select": 1430,
+ "n_holdout": 1429,
+ "eb": 0.02525469690987138,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.02616690240452617,
+ "pool_holdout": 0.02527394839165783,
+ "lift_select": 0.20937393089291834,
+ "lift_holdout": -0.16585171680041666,
+ "eb_lift": 0.07961331456304288,
+ "p_value": 0.13469497063318195,
+ "held_up": false,
+ "z_path": [
+ -1.214,
+ -1.137,
+ -2.079,
+ -1.855,
+ -1.524,
+ -0.755,
+ -1.558,
+ -1.491,
+ -0.803,
+ -1.171
+ ],
+ "first_cross": 2,
+ "final_z": -1.171
+ },
+ {
+ "id": "522e6e2a36430f3b45001e97",
+ "variation": "headline",
+ "week": 201336,
+ "impressions": 11313,
+ "pooled_ctr": 0.016352868381508,
+ "arms": [
+ {
+ "headline": "20 Years Ago We Thought Of It As Sci-fi, And Now 162 Countries Can Stare Me In The Face",
+ "impressions": 3100,
+ "clicks": 65,
+ "ctr": 0.0209677419354838,
+ "ctr_select": 0.02064516129032258,
+ "ctr_holdout": 0.02129032258064516,
+ "n_select": 1550,
+ "n_holdout": 1550,
+ "eb": 0.017911562602866898,
+ "labelled_winner": true
+ },
+ {
+ "headline": "20 Years Ago It Was Only In Sci-Fi Films, But Now 162 Countries Can Stare Each Other In The Face",
+ "impressions": 2955,
+ "clicks": 35,
+ "ctr": 0.0118443316412859,
+ "ctr_select": 0.011502029769959404,
+ "ctr_holdout": 0.012186865267433988,
+ "n_select": 1478,
+ "n_holdout": 1477,
+ "eb": 0.014299481620797484,
+ "labelled_winner": false
+ },
+ {
+ "headline": "20 Years Ago Only Sci-fi Movies Could Do What Now 162 Countries Can Do Just By Logging On",
+ "impressions": 2811,
+ "clicks": 38,
+ "ctr": 0.0135183208822483,
+ "ctr_select": 0.013513513513513514,
+ "ctr_holdout": 0.013523131672597865,
+ "n_select": 1406,
+ "n_holdout": 1405,
+ "eb": 0.015113071476098335,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Videochatting? No Biggie. But Watch when A Comedian Asks Everyone About Their Message To The World.",
+ "impressions": 2447,
+ "clicks": 47,
+ "ctr": 0.0192071924805884,
+ "ctr_select": 0.018790849673202614,
+ "ctr_holdout": 0.019623875715453803,
+ "n_select": 1224,
+ "n_holdout": 1223,
+ "eb": 0.01701910061004081,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.016083421703782255,
+ "pool_holdout": 0.016622458001768346,
+ "lift_select": 0.2836299184686282,
+ "lift_holdout": 0.28081674673987655,
+ "eb_lift": 0.11366616711011979,
+ "p_value": 0.09383632488081853,
+ "held_up": true,
+ "z_path": [
+ 0.543,
+ 1.209,
+ 0.17,
+ 0.092,
+ 0.503,
+ 0.054,
+ 0.2,
+ 0.402,
+ 0.605,
+ 0.463
+ ],
+ "first_cross": null,
+ "final_z": 0.463
+ },
+ {
+ "id": "52deecd74ded95426b000009",
+ "variation": "image",
+ "week": 201403,
+ "impressions": 11304,
+ "pooled_ctr": 0.021142958244869073,
+ "arms": [
+ {
+ "headline": "There Are People In Our Lives Who Are So Good At What They Do, You Never See Them. Here\u2019s One Story.",
+ "impressions": 2873,
+ "clicks": 58,
+ "ctr": 0.0201879568395405,
+ "ctr_select": 0.02158774373259053,
+ "ctr_holdout": 0.018789144050104383,
+ "n_select": 1436,
+ "n_holdout": 1437,
+ "eb": 0.021588580703635483,
+ "labelled_winner": false
+ },
+ {
+ "headline": "There Are People In Our Lives Who Are So Good At What They Do, You Never See Them. Here\u2019s One Story.",
+ "impressions": 2806,
+ "clicks": 56,
+ "ctr": 0.0199572344975053,
+ "ctr_select": 0.023521026372059873,
+ "ctr_holdout": 0.01639344262295082,
+ "n_select": 1403,
+ "n_holdout": 1403,
+ "eb": 0.022317589985911122,
+ "labelled_winner": false
+ },
+ {
+ "headline": "There Are People In Our Lives Who Are So Good At What They Do, You Never See Them. Here\u2019s One Story.",
+ "impressions": 2824,
+ "clicks": 52,
+ "ctr": 0.018413597733711,
+ "ctr_select": 0.0169971671388102,
+ "ctr_holdout": 0.019830028328611898,
+ "n_select": 1412,
+ "n_holdout": 1412,
+ "eb": 0.01985067201064436,
+ "labelled_winner": false
+ },
+ {
+ "headline": "There Are People In Our Lives Who Are So Good At What They Do, You Never See Them. Here\u2019s One Story.",
+ "impressions": 2801,
+ "clicks": 73,
+ "ctr": 0.0260621206711888,
+ "ctr_select": 0.024285714285714285,
+ "ctr_holdout": 0.027837259100642397,
+ "n_select": 1400,
+ "n_holdout": 1401,
+ "eb": 0.022604582865570964,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.021589099274464698,
+ "pool_holdout": 0.020696975057491598,
+ "lift_select": 0.1249063231850116,
+ "lift_holdout": 0.3449916726147988,
+ "eb_lift": 0.04703686699460241,
+ "p_value": 0.42346297961794266,
+ "held_up": true,
+ "z_path": [
+ 0.305,
+ 1.809,
+ 1.435,
+ 0.157,
+ 0.517,
+ 1.092,
+ 1.205,
+ 1.237,
+ 1.258,
+ 1.525
+ ],
+ "first_cross": null,
+ "final_z": 1.525
+ },
+ {
+ "id": "53b8d9e555a05d3c30000018",
+ "variation": "image",
+ "week": 201427,
+ "impressions": 11283,
+ "pooled_ctr": 0.01701675086413188,
+ "arms": [
+ {
+ "headline": "What Every Dog Owner Should Know About Dog Poop",
+ "impressions": 2821,
+ "clicks": 47,
+ "ctr": 0.0166607585962424,
+ "ctr_select": 0.02056737588652482,
+ "ctr_holdout": 0.012756909992912827,
+ "n_select": 1410,
+ "n_holdout": 1411,
+ "eb": 0.019573129870720005,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Every Dog Owner Should Know About Dog Poop",
+ "impressions": 2827,
+ "clicks": 52,
+ "ctr": 0.0183940573045631,
+ "ctr_select": 0.02050919377652051,
+ "ctr_holdout": 0.01627742392073602,
+ "n_select": 1414,
+ "n_holdout": 1413,
+ "eb": 0.0195521494683188,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Every Dog Owner Should Know About Dog Poop",
+ "impressions": 2773,
+ "clicks": 44,
+ "ctr": 0.0158672917417958,
+ "ctr_select": 0.01948051948051948,
+ "ctr_holdout": 0.012256669069935111,
+ "n_select": 1386,
+ "n_holdout": 1387,
+ "eb": 0.019159963083055873,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What Every Dog Owner Should Know About Dog Poop",
+ "impressions": 2862,
+ "clicks": 49,
+ "ctr": 0.017120894479385,
+ "ctr_select": 0.015373864430468204,
+ "ctr_holdout": 0.018867924528301886,
+ "n_select": 1431,
+ "n_holdout": 1431,
+ "eb": 0.01759616711959618,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.01896826803758199,
+ "pool_holdout": 0.015065579581708614,
+ "lift_select": 0.08430436799893948,
+ "lift_holdout": -0.1532413390586569,
+ "eb_lift": 0.031888089726463065,
+ "p_value": 0.6112689254942019,
+ "held_up": false,
+ "z_path": [
+ -1.902,
+ -0.535,
+ 0.006,
+ -0.328,
+ -0.131,
+ 0.012,
+ -0.24,
+ 0.365,
+ -0.212,
+ -0.496
+ ],
+ "first_cross": null,
+ "final_z": -0.496
+ },
+ {
+ "id": "519109d5f11e26001000aa4a",
+ "variation": "headline",
+ "week": 201319,
+ "impressions": 11250,
+ "pooled_ctr": 0.010666666666666666,
+ "arms": [
+ {
+ "headline": "It Doesn't Take A Rocket Scientist To Understand How Much This Epic Song Rocks. Especially IN SPACE.",
+ "impressions": 2874,
+ "clicks": 29,
+ "ctr": 0.0100904662491301,
+ "ctr_select": 0.010438413361169102,
+ "ctr_holdout": 0.009742519137091163,
+ "n_select": 1437,
+ "n_holdout": 1437,
+ "eb": 0.010250093731769381,
+ "labelled_winner": false
+ },
+ {
+ "headline": "OMG. There's A Riff In The Space Time Continuum.",
+ "impressions": 2810,
+ "clicks": 22,
+ "ctr": 0.0078291814946619,
+ "ctr_select": 0.006405693950177936,
+ "ctr_holdout": 0.009252669039145907,
+ "n_select": 1405,
+ "n_holdout": 1405,
+ "eb": 0.008726468272328328,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Well, That's Never Happened Before. IN SPACE.",
+ "impressions": 2786,
+ "clicks": 32,
+ "ctr": 0.0114860014357501,
+ "ctr_select": 0.010050251256281407,
+ "ctr_holdout": 0.012921751615218953,
+ "n_select": 1393,
+ "n_holdout": 1393,
+ "eb": 0.010102144230219707,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Final Act Of The Departing Commander Of The International Space Station Truly Rocked",
+ "impressions": 2780,
+ "clicks": 37,
+ "ctr": 0.0133093525179856,
+ "ctr_select": 0.013669064748201438,
+ "ctr_holdout": 0.012949640287769784,
+ "n_select": 1390,
+ "n_holdout": 1390,
+ "eb": 0.01145886390733993,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.010133333333333333,
+ "pool_holdout": 0.0112,
+ "lift_select": 0.34892086330935257,
+ "lift_holdout": 0.15621788283658788,
+ "eb_lift": 0.13080893822433537,
+ "p_value": 0.12929214730272892,
+ "held_up": true,
+ "z_path": [
+ -1.308,
+ -0.519,
+ 0.529,
+ 1.404,
+ 2.027,
+ 1.21,
+ 1.413,
+ 1.075,
+ 1.275,
+ 1.127
+ ],
+ "first_cross": 4,
+ "final_z": 1.127
+ },
+ {
+ "id": "51eeb293145e76e55b00389c",
+ "variation": "image",
+ "week": 201329,
+ "impressions": 11213,
+ "pooled_ctr": 0.015517702666547757,
+ "arms": [
+ {
+ "headline": "If You Know Nothing About Keystone XL, Here's An Adorable Animation You Should Watch",
+ "impressions": 2775,
+ "clicks": 40,
+ "ctr": 0.0144144144144144,
+ "ctr_select": 0.01440922190201729,
+ "ctr_holdout": 0.014419610670511895,
+ "n_select": 1388,
+ "n_holdout": 1387,
+ "eb": 0.014657111676282985,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Know Nothing About Keystone XL, Here's An Adorable Animation You Should Watch",
+ "impressions": 2824,
+ "clicks": 47,
+ "ctr": 0.0166430594900849,
+ "ctr_select": 0.014164305949008499,
+ "ctr_holdout": 0.019121813031161474,
+ "n_select": 1412,
+ "n_holdout": 1412,
+ "eb": 0.014562795539135002,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Know Nothing About Keystone XL, Here's An Adorable Animation You Should Watch",
+ "impressions": 2921,
+ "clicks": 40,
+ "ctr": 0.0136939404313591,
+ "ctr_select": 0.013013698630136987,
+ "ctr_holdout": 0.014373716632443531,
+ "n_select": 1460,
+ "n_holdout": 1461,
+ "eb": 0.01411304695713643,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Know Nothing About Keystone XL, Here's An Adorable Animation You Should Watch",
+ "impressions": 2693,
+ "clicks": 47,
+ "ctr": 0.0174526550315633,
+ "ctr_select": 0.017830609212481426,
+ "ctr_holdout": 0.017074981440237565,
+ "n_select": 1346,
+ "n_holdout": 1347,
+ "eb": 0.01591693141647762,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.014805565465572601,
+ "pool_holdout": 0.016229712858926344,
+ "lift_select": 0.2043180150020587,
+ "lift_holdout": 0.052081548740791295,
+ "eb_lift": 0.0750640665153437,
+ "p_value": 0.29181419278839826,
+ "held_up": false,
+ "z_path": [
+ 0.963,
+ 1.013,
+ 0.895,
+ 0.442,
+ 1.372,
+ 0.964,
+ 0.639,
+ 0.368,
+ 0.248,
+ 0.898
+ ],
+ "first_cross": null,
+ "final_z": 0.898
+ },
+ {
+ "id": "51436090220cb80002000a00",
+ "variation": "headline",
+ "week": 201307,
+ "impressions": 11143,
+ "pooled_ctr": 0.017051063447904514,
+ "arms": [
+ {
+ "headline": "A Congressperson Thinks Legalized Bribery Is A Good Idea. A Human Person Corrects Them.",
+ "impressions": 2729,
+ "clicks": 52,
+ "ctr": 0.0190545987541223,
+ "ctr_select": 0.018328445747800588,
+ "ctr_holdout": 0.01978021978021978,
+ "n_select": 1364,
+ "n_holdout": 1365,
+ "eb": 0.017188141736879197,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Congressperson Endorses Legalized Bribery Of Congress. Honest Shellacking Ensues.",
+ "impressions": 2761,
+ "clicks": 38,
+ "ctr": 0.0137631293009779,
+ "ctr_select": 0.010144927536231883,
+ "ctr_holdout": 0.017378711078928313,
+ "n_select": 1380,
+ "n_holdout": 1381,
+ "eb": 0.014138944325836213,
+ "labelled_winner": false
+ },
+ {
+ "headline": "2 Members Of Congress Debate Whether Accepting Legalized Bribes Is Cool. Righteous Honesty Prevails.",
+ "impressions": 2820,
+ "clicks": 26,
+ "ctr": 0.0092198581560283,
+ "ctr_select": 0.011347517730496455,
+ "ctr_holdout": 0.0070921985815602835,
+ "n_select": 1410,
+ "n_holdout": 1410,
+ "eb": 0.014561680232845488,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch A Real Person Correct An Elected Official On How Democracy Should Actually Function",
+ "impressions": 2833,
+ "clicks": 74,
+ "ctr": 0.0261207200847158,
+ "ctr_select": 0.026129943502824857,
+ "ctr_holdout": 0.02611150317572336,
+ "n_select": 1416,
+ "n_holdout": 1417,
+ "eb": 0.02016273087297812,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.01651705565529623,
+ "pool_holdout": 0.017584783778934146,
+ "lift_select": 0.5819976664210267,
+ "lift_holdout": 0.484891910186799,
+ "eb_lift": 0.22072185828791446,
+ "p_value": 0.0010145035597310008,
+ "held_up": true,
+ "z_path": [
+ 0.164,
+ 0.604,
+ 1.226,
+ 1.419,
+ 1.589,
+ 1.716,
+ 2.384,
+ 2.313,
+ 1.846,
+ 1.771
+ ],
+ "first_cross": 6,
+ "final_z": 1.771
+ },
+ {
+ "id": "5143606c220cb800020005de",
+ "variation": "both",
+ "week": 201310,
+ "impressions": 11134,
+ "pooled_ctr": 0.02811208909646129,
+ "arms": [
+ {
+ "headline": "An 'Arab' Filmmaker Drives His Car Into Cardboard Twin Towers. Trust Me, You'll Be On His Side.",
+ "impressions": 2216,
+ "clicks": 42,
+ "ctr": 0.0189530685920577,
+ "ctr_select": 0.018050541516245487,
+ "ctr_holdout": 0.019855595667870037,
+ "n_select": 1108,
+ "n_holdout": 1108,
+ "eb": 0.023220260531002783,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Racists Spray-Paint A Comedian's Car. They Shouldn't Have Done That.",
+ "impressions": 2224,
+ "clicks": 73,
+ "ctr": 0.0328237410071942,
+ "ctr_select": 0.0341726618705036,
+ "ctr_holdout": 0.03147482014388489,
+ "n_select": 1112,
+ "n_holdout": 1112,
+ "eb": 0.02844145507933867,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Racists Spray Paint 'Go Home Arab' On A Car \u2014 Filmmaker Responds Perfectly",
+ "impressions": 2225,
+ "clicks": 70,
+ "ctr": 0.0314606741573033,
+ "ctr_select": 0.026079136690647483,
+ "ctr_holdout": 0.036837376460017966,
+ "n_select": 1112,
+ "n_holdout": 1113,
+ "eb": 0.025817317101277416,
+ "labelled_winner": false
+ },
+ {
+ "headline": "An 'Arab' Filmmaker Drives His Car Into Cardboard Twin Towers. Trust Me, You'll Be On His Side.",
+ "impressions": 2240,
+ "clicks": 58,
+ "ctr": 0.0258928571428571,
+ "ctr_select": 0.020535714285714286,
+ "ctr_holdout": 0.03125,
+ "n_select": 1120,
+ "n_holdout": 1120,
+ "eb": 0.024011882455746036,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Idiot Racists Accidentally Make A Filmmaker Famous When They Spray Paint 'Go Home Arab' On His Car",
+ "impressions": 2229,
+ "clicks": 70,
+ "ctr": 0.0314042171377299,
+ "ctr_select": 0.0296229802513465,
+ "ctr_holdout": 0.033183856502242155,
+ "n_select": 1114,
+ "n_holdout": 1115,
+ "eb": 0.026967874552904225,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.025691699604743084,
+ "pool_holdout": 0.0305316091954023,
+ "lift_select": 0.3301051466519094,
+ "lift_holdout": 0.030892932712653254,
+ "eb_lift": 0.10702894385733575,
+ "p_value": 0.045689773370043846,
+ "held_up": false,
+ "z_path": [
+ 0.481,
+ 0.371,
+ 0.146,
+ 0.368,
+ 0.336,
+ 0.112,
+ 0.01,
+ 0.201,
+ 0.369,
+ 0.269
+ ],
+ "first_cross": null,
+ "final_z": 0.269
+ },
+ {
+ "id": "51ef1b498faddd30ea00f6e3",
+ "variation": "image",
+ "week": 201329,
+ "impressions": 11117,
+ "pooled_ctr": 0.028694791760367006,
+ "arms": [
+ {
+ "headline": "Meet 4 Kenyan Women Who Want You To Keep Your 'Poverty Porn' To Yourself",
+ "impressions": 3066,
+ "clicks": 87,
+ "ctr": 0.0283757338551859,
+ "ctr_select": 0.028049575994781473,
+ "ctr_holdout": 0.028701891715590344,
+ "n_select": 1533,
+ "n_holdout": 1533,
+ "eb": 0.027191230694499902,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Meet 4 Kenyan Women Who Want You To Keep Your 'Poverty Porn' To Yourself",
+ "impressions": 2687,
+ "clicks": 84,
+ "ctr": 0.0312616300707108,
+ "ctr_select": 0.027529761904761904,
+ "ctr_holdout": 0.034996276991809384,
+ "n_select": 1344,
+ "n_holdout": 1343,
+ "eb": 0.026956132805984957,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Meet 4 Kenyan Women Who Want You To Keep Your 'Poverty Porn' To Yourself",
+ "impressions": 2530,
+ "clicks": 59,
+ "ctr": 0.0233201581027668,
+ "ctr_select": 0.02608695652173913,
+ "ctr_holdout": 0.020553359683794466,
+ "n_select": 1265,
+ "n_holdout": 1265,
+ "eb": 0.026434049822006773,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Meet 4 Kenyan Women Who Want You To Keep Your 'Poverty Porn' To Yourself",
+ "impressions": 2834,
+ "clicks": 89,
+ "ctr": 0.0314043754410726,
+ "ctr_select": 0.024700070571630206,
+ "ctr_holdout": 0.038108680310515175,
+ "n_select": 1417,
+ "n_holdout": 1417,
+ "eb": 0.025893718118468613,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.026623493434070875,
+ "pool_holdout": 0.03076646275638719,
+ "lift_select": 0.053564817263447306,
+ "lift_holdout": -0.06710459558332671,
+ "eb_lift": 0.02132467182922282,
+ "p_value": 0.6835879884360658,
+ "held_up": false,
+ "z_path": [
+ -0.522,
+ -1.27,
+ -1.509,
+ -1.048,
+ -1.39,
+ -0.589,
+ -0.65,
+ -0.456,
+ -0.746,
+ -0.643
+ ],
+ "first_cross": null,
+ "final_z": -0.643
+ },
+ {
+ "id": "52253bf7661ea3d61f002f61",
+ "variation": "other",
+ "week": 201335,
+ "impressions": 11100,
+ "pooled_ctr": 0.011531531531531532,
+ "arms": [
+ {
+ "headline": "I Don't Know About You, But Most Folks I Know Are Working Harder While Falling More Behind",
+ "impressions": 2884,
+ "clicks": 32,
+ "ctr": 0.0110957004160887,
+ "ctr_select": 0.011095700416088766,
+ "ctr_holdout": 0.011095700416088766,
+ "n_select": 1442,
+ "n_holdout": 1442,
+ "eb": 0.01191974026433826,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Don't Know About You, But Most Folks I Know Are Working Harder While Falling More Behind",
+ "impressions": 2862,
+ "clicks": 40,
+ "ctr": 0.0139762403913347,
+ "ctr_select": 0.014675052410901468,
+ "ctr_holdout": 0.013277428371767994,
+ "n_select": 1431,
+ "n_holdout": 1431,
+ "eb": 0.013288513514996643,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Don't Know About You, But Most Folks I Know Are Working Harder While Falling More Behind",
+ "impressions": 2766,
+ "clicks": 22,
+ "ctr": 0.0079537237888647,
+ "ctr_select": 0.008676789587852495,
+ "ctr_holdout": 0.0072306579898770785,
+ "n_select": 1383,
+ "n_holdout": 1383,
+ "eb": 0.011028898702144262,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I Don't Know About You, But Most Folks I Know Are Working Harder While Falling More Behind",
+ "impressions": 2588,
+ "clicks": 34,
+ "ctr": 0.0131375579598145,
+ "ctr_select": 0.015455950540958269,
+ "ctr_holdout": 0.010819165378670788,
+ "n_select": 1294,
+ "n_holdout": 1294,
+ "eb": 0.013515699292616328,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.012432432432432432,
+ "pool_holdout": 0.010630630630630631,
+ "lift_select": 0.24319602177273048,
+ "lift_holdout": 0.017735048332591097,
+ "eb_lift": 0.08713233440609591,
+ "p_value": 0.2623336593653335,
+ "held_up": false,
+ "z_path": [
+ 1.087,
+ 0.732,
+ 0.455,
+ -0.704,
+ -0.232,
+ -0.727,
+ -0.794,
+ -0.584,
+ -0.782,
+ -0.267
+ ],
+ "first_cross": null,
+ "final_z": -0.267
+ },
+ {
+ "id": "51708b0defe57b0002008c03",
+ "variation": "other",
+ "week": 201315,
+ "impressions": 11086,
+ "pooled_ctr": 0.00847916290817247,
+ "arms": [
+ {
+ "headline": "These Talented Artists Said Everything By Saying Almost Nothing At All",
+ "impressions": 5581,
+ "clicks": 52,
+ "ctr": 0.0093173266439706,
+ "ctr_select": 0.010752688172043012,
+ "ctr_holdout": 0.007882479398065209,
+ "n_select": 2790,
+ "n_holdout": 2791,
+ "eb": 0.009967367573197555,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These Talented Artists Said Everything By Saying Almost Nothing At All",
+ "impressions": 5505,
+ "clicks": 42,
+ "ctr": 0.0076294277929155,
+ "ctr_select": 0.007267441860465116,
+ "ctr_holdout": 0.007991282237559027,
+ "n_select": 2752,
+ "n_holdout": 2753,
+ "eb": 0.008069573938791343,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.009022013713460845,
+ "pool_holdout": 0.007936507936507936,
+ "lift_select": 0.19182795698924737,
+ "lift_holdout": -0.006807595843783654,
+ "eb_lift": 0.10478302181321686,
+ "p_value": 0.17007457502193757,
+ "held_up": false,
+ "z_path": [
+ -0.327,
+ 0.177,
+ -0.217,
+ 0.429,
+ -0.186,
+ 0.327,
+ 0.188,
+ 0.85,
+ 0.905,
+ 0.969
+ ],
+ "first_cross": null,
+ "final_z": 0.969
+ },
+ {
+ "id": "5175a64c162a4e0002000e8f",
+ "variation": "both",
+ "week": 201316,
+ "impressions": 11075,
+ "pooled_ctr": 0.020225733634311512,
+ "arms": [
+ {
+ "headline": "Voices In Your Head? It May Not Be As Bad As Actually Going To A Doctor.",
+ "impressions": 2772,
+ "clicks": 33,
+ "ctr": 0.0119047619047619,
+ "ctr_select": 0.010822510822510822,
+ "ctr_holdout": 0.012987012987012988,
+ "n_select": 1386,
+ "n_holdout": 1386,
+ "eb": 0.015123748090861687,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Woman With Voices In Her Head Does The Only Logical Thing And Becomes A Psychologist",
+ "impressions": 2787,
+ "clicks": 56,
+ "ctr": 0.0200932902762827,
+ "ctr_select": 0.01721664275466284,
+ "ctr_holdout": 0.02297200287150036,
+ "n_select": 1394,
+ "n_holdout": 1393,
+ "eb": 0.01751591758359473,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Having Voices In Your Head Is Tough, But It's Nothing Compared To A Doctor Mistreating It",
+ "impressions": 2786,
+ "clicks": 42,
+ "ctr": 0.0150753768844221,
+ "ctr_select": 0.014357501794687724,
+ "ctr_holdout": 0.015793251974156496,
+ "n_select": 1393,
+ "n_holdout": 1393,
+ "eb": 0.016442673574134512,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Woman With Schizophrenia Is Given The Correct Answers To Test Questions By One Of The Voices",
+ "impressions": 2730,
+ "clicks": 93,
+ "ctr": 0.034065934065934,
+ "ctr_select": 0.02857142857142857,
+ "ctr_holdout": 0.03956043956043956,
+ "n_select": 1365,
+ "n_holdout": 1365,
+ "eb": 0.021726950596271725,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.01769591910436981,
+ "pool_holdout": 0.022756005056890013,
+ "lift_select": 0.6145772594752186,
+ "lift_holdout": 0.7384615384615383,
+ "eb_lift": 0.22779441226686536,
+ "p_value": 0.00044672514843723947,
+ "held_up": true,
+ "z_path": [
+ 1.875,
+ 2.882,
+ 3.626,
+ 3.438,
+ 2.923,
+ 3.069,
+ 3.091,
+ 3.234,
+ 3.276,
+ 3.201
+ ],
+ "first_cross": 1,
+ "final_z": 3.201
+ },
+ {
+ "id": "51f94ff94dcfeddbb6001229",
+ "variation": "image",
+ "week": 201330,
+ "impressions": 11035,
+ "pooled_ctr": 0.033801540552786585,
+ "arms": [
+ {
+ "headline": "What World Do We Live In When Even An Act Of Kindness Faces This Kind of Discrimination?",
+ "impressions": 2954,
+ "clicks": 95,
+ "ctr": 0.0321597833446174,
+ "ctr_select": 0.03182125930941097,
+ "ctr_holdout": 0.03249830737982397,
+ "n_select": 1477,
+ "n_holdout": 1477,
+ "eb": 0.033309400522441475,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What World Do We Live In When Even An Act Of Kindness Faces This Kind of Discrimination?",
+ "impressions": 2839,
+ "clicks": 106,
+ "ctr": 0.0373370905248326,
+ "ctr_select": 0.036619718309859155,
+ "ctr_holdout": 0.03805496828752643,
+ "n_select": 1420,
+ "n_holdout": 1419,
+ "eb": 0.035155091740654745,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What World Do We Live In When Even An Act Of Kindness Faces This Kind of Discrimination?",
+ "impressions": 2645,
+ "clicks": 87,
+ "ctr": 0.0328922495274102,
+ "ctr_select": 0.03252647503782148,
+ "ctr_holdout": 0.03325774754346183,
+ "n_select": 1322,
+ "n_holdout": 1323,
+ "eb": 0.03362892076158049,
+ "labelled_winner": false
+ },
+ {
+ "headline": "What World Do We Live In When Even An Act Of Kindness Faces This Kind of Discrimination?",
+ "impressions": 2597,
+ "clicks": 85,
+ "ctr": 0.03273007316134,
+ "ctr_select": 0.0362095531587057,
+ "ctr_holdout": 0.029253271747498075,
+ "n_select": 1298,
+ "n_holdout": 1299,
+ "eb": 0.03495842739929436,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.03425774877650897,
+ "pool_holdout": 0.03334541500543675,
+ "lift_select": 0.06894701542588866,
+ "lift_holdout": 0.14123540766614573,
+ "eb_lift": 0.026193868429588596,
+ "p_value": 0.5701427590385715,
+ "held_up": true,
+ "z_path": [
+ -0.167,
+ 0.465,
+ 1.408,
+ 0.863,
+ 0.116,
+ 0.888,
+ 0.392,
+ 0.533,
+ 1.069,
+ 0.921
+ ],
+ "first_cross": null,
+ "final_z": 0.921
+ },
+ {
+ "id": "51f2bb3ffc2f41984e000314",
+ "variation": "headline",
+ "week": 201329,
+ "impressions": 11008,
+ "pooled_ctr": 0.024164244186046513,
+ "arms": [
+ {
+ "headline": "He\u2019s Caught Lots Of Criminals In The Act; Now He Has To Hide His Face So They Won't Catch Him",
+ "impressions": 2665,
+ "clicks": 83,
+ "ctr": 0.0311444652908067,
+ "ctr_select": 0.03153153153153153,
+ "ctr_holdout": 0.03075768942235559,
+ "n_select": 1332,
+ "n_holdout": 1333,
+ "eb": 0.027202006338853023,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Bravest Man You\u2019ll Ever See Wearing A Mop On His Head",
+ "impressions": 3152,
+ "clicks": 58,
+ "ctr": 0.0184010152284263,
+ "ctr_select": 0.01649746192893401,
+ "ctr_holdout": 0.02030456852791878,
+ "n_select": 1576,
+ "n_holdout": 1576,
+ "eb": 0.021388178131353013,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Just Imagine How Much Justice We Could Get Done If We Had More Reporters Like Him",
+ "impressions": 2361,
+ "clicks": 61,
+ "ctr": 0.0258365099534095,
+ "ctr_select": 0.02711864406779661,
+ "ctr_holdout": 0.024555461473327687,
+ "n_select": 1180,
+ "n_holdout": 1181,
+ "eb": 0.025525104693315356,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Naming, Shaming, And Jailing: This Is What Justice Looks Like, People",
+ "impressions": 2830,
+ "clicks": 64,
+ "ctr": 0.0226148409893992,
+ "ctr_select": 0.025441696113074206,
+ "ctr_holdout": 0.019787985865724382,
+ "n_select": 1415,
+ "n_holdout": 1415,
+ "eb": 0.02498972793337895,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.02471379247683082,
+ "pool_holdout": 0.023614895549500452,
+ "lift_select": 0.2758677795442499,
+ "lift_holdout": 0.30246984823128864,
+ "eb_lift": 0.10068118296108963,
+ "p_value": 0.06563235505527663,
+ "held_up": true,
+ "z_path": [
+ -0.209,
+ -1.297,
+ -0.871,
+ -0.323,
+ -0.049,
+ 0.561,
+ 0.836,
+ 0.931,
+ 0.903,
+ 1.126
+ ],
+ "first_cross": null,
+ "final_z": 1.126
+ },
+ {
+ "id": "54f8ec713831360024600100",
+ "variation": "image",
+ "week": 201509,
+ "impressions": 10993,
+ "pooled_ctr": 0.030292004002547075,
+ "arms": [
+ {
+ "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.",
+ "impressions": 1786,
+ "clicks": 72,
+ "ctr": 0.0403135498320268,
+ "ctr_select": 0.043673012318029114,
+ "ctr_holdout": 0.036954087346024636,
+ "n_select": 893,
+ "n_holdout": 893,
+ "eb": 0.034606737781897844,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.",
+ "impressions": 1869,
+ "clicks": 23,
+ "ctr": 0.0123060460139111,
+ "ctr_select": 0.011777301927194861,
+ "ctr_holdout": 0.012834224598930482,
+ "n_select": 934,
+ "n_holdout": 935,
+ "eb": 0.0255595018696753,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.",
+ "impressions": 1804,
+ "clicks": 52,
+ "ctr": 0.0288248337028824,
+ "ctr_select": 0.028824833702882482,
+ "ctr_holdout": 0.028824833702882482,
+ "n_select": 902,
+ "n_holdout": 902,
+ "eb": 0.03047235565624745,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.",
+ "impressions": 1829,
+ "clicks": 57,
+ "ctr": 0.0311645708037178,
+ "ctr_select": 0.03282275711159737,
+ "ctr_holdout": 0.029508196721311476,
+ "n_select": 914,
+ "n_holdout": 915,
+ "eb": 0.03159694466929576,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.",
+ "impressions": 1867,
+ "clicks": 63,
+ "ctr": 0.0337439742903053,
+ "ctr_select": 0.034261241970021415,
+ "ctr_holdout": 0.03322615219721329,
+ "n_select": 934,
+ "n_holdout": 933,
+ "eb": 0.032017666775371115,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.",
+ "impressions": 1838,
+ "clicks": 66,
+ "ctr": 0.0359085963003264,
+ "ctr_select": 0.035908596300326445,
+ "ctr_holdout": 0.035908596300326445,
+ "n_select": 919,
+ "n_holdout": 919,
+ "eb": 0.032475004820722464,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.03111353711790393,
+ "pool_holdout": 0.029470620338366383,
+ "lift_select": 0.403665939765427,
+ "lift_holdout": 0.25392974161171233,
+ "eb_lift": 0.11227269502520798,
+ "p_value": 0.01817422469029952,
+ "held_up": true,
+ "z_path": [
+ 1.591,
+ 1.231,
+ 1.378,
+ 0.935,
+ 1.25,
+ 0.585,
+ 0.664,
+ 0.448,
+ 0.255,
+ 0.693
+ ],
+ "first_cross": null,
+ "final_z": 0.693
+ },
+ {
+ "id": "517568f4c270aa000200b91a",
+ "variation": "image",
+ "week": 201316,
+ "impressions": 10951,
+ "pooled_ctr": 0.03972239978084193,
+ "arms": [
+ {
+ "headline": "The One Where The Weather Lady Starts Telling The Truth. A Little Too Much Truth, To Be Honest.",
+ "impressions": 2761,
+ "clicks": 87,
+ "ctr": 0.0315103223469757,
+ "ctr_select": 0.035507246376811595,
+ "ctr_holdout": 0.027516292541636494,
+ "n_select": 1380,
+ "n_holdout": 1381,
+ "eb": 0.03855686912472771,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The One Where The Weather Lady Starts Telling The Truth. A Little Too Much Truth, To Be Honest.",
+ "impressions": 2808,
+ "clicks": 133,
+ "ctr": 0.0473646723646723,
+ "ctr_select": 0.046296296296296294,
+ "ctr_holdout": 0.04843304843304843,
+ "n_select": 1404,
+ "n_holdout": 1404,
+ "eb": 0.04260734187597669,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The One Where The Weather Lady Starts Telling The Truth. A Little Too Much Truth, To Be Honest.",
+ "impressions": 2645,
+ "clicks": 103,
+ "ctr": 0.0389413988657845,
+ "ctr_select": 0.036308623298033284,
+ "ctr_holdout": 0.04157218442932729,
+ "n_select": 1322,
+ "n_holdout": 1323,
+ "eb": 0.03889653975387367,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The One Where The Weather Lady Starts Telling The Truth. A Little Too Much Truth, To Be Honest.",
+ "impressions": 2737,
+ "clicks": 112,
+ "ctr": 0.0409207161125319,
+ "ctr_select": 0.04312865497076023,
+ "ctr_holdout": 0.03871439006574142,
+ "n_select": 1368,
+ "n_holdout": 1369,
+ "eb": 0.04139559418249401,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.040372670807453416,
+ "pool_holdout": 0.039072484937009315,
+ "lift_select": 0.14672364672364657,
+ "lift_holdout": 0.23956918816731876,
+ "eb_lift": 0.055351083389576505,
+ "p_value": 0.190951409805865,
+ "held_up": true,
+ "z_path": [
+ 0.854,
+ 0.693,
+ 0.9,
+ 1.447,
+ 1.452,
+ 2.09,
+ 1.835,
+ 1.406,
+ 1.556,
+ 1.167
+ ],
+ "first_cross": 5,
+ "final_z": 1.167
+ },
+ {
+ "id": "51f1823e9a72d2ca6300081c",
+ "variation": "headline",
+ "week": 201329,
+ "impressions": 10912,
+ "pooled_ctr": 0.08165322580645161,
+ "arms": [
+ {
+ "headline": "The Most Anti-Gay Airline In The Entire World Has A New Safety Video",
+ "impressions": 2861,
+ "clicks": 264,
+ "ctr": 0.0922754281719678,
+ "ctr_select": 0.09300699300699301,
+ "ctr_holdout": 0.09154437456324249,
+ "n_select": 1430,
+ "n_holdout": 1431,
+ "eb": 0.08457758962906371,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Anti-Gay Airline In The History Of The World Has A New Safety Video",
+ "impressions": 2625,
+ "clicks": 211,
+ "ctr": 0.0803809523809523,
+ "ctr_select": 0.07850609756097561,
+ "ctr_holdout": 0.08225437928408226,
+ "n_select": 1312,
+ "n_holdout": 1313,
+ "eb": 0.0790620216526423,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Anti-Gay Airline In The Entire World Has A New Jerktacular Safety Video",
+ "impressions": 2812,
+ "clicks": 201,
+ "ctr": 0.071479374110953,
+ "ctr_select": 0.07610241820768136,
+ "ctr_holdout": 0.06685633001422475,
+ "n_select": 1406,
+ "n_holdout": 1406,
+ "eb": 0.07814040279215385,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Anti-Gay Airline In The Entire World Has A New Scumtacular Safety Video",
+ "impressions": 2614,
+ "clicks": 215,
+ "ctr": 0.0822494261667941,
+ "ctr_select": 0.06885998469778118,
+ "ctr_holdout": 0.09563886763580719,
+ "n_select": 1307,
+ "n_holdout": 1307,
+ "eb": 0.07558457582988534,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.07937671860678276,
+ "pool_holdout": 0.08392889866226864,
+ "lift_select": 0.17171627448763727,
+ "lift_holdout": 0.0907372314227386,
+ "eb_lift": 0.06552136588116064,
+ "p_value": 0.026437325468948777,
+ "held_up": false,
+ "z_path": [
+ 1.232,
+ 0.936,
+ 1.694,
+ 1.488,
+ 2.102,
+ 1.632,
+ 2.241,
+ 2.252,
+ 2.098,
+ 1.565
+ ],
+ "first_cross": 4,
+ "final_z": 1.565
+ },
+ {
+ "id": "51d47483c6d06f695c000505",
+ "variation": "image",
+ "week": 201326,
+ "impressions": 10900,
+ "pooled_ctr": 0.034128440366972476,
+ "arms": [
+ {
+ "headline": "The Next Time Someone Talks About Starving Kids In Africa, Show Them This",
+ "impressions": 2563,
+ "clicks": 60,
+ "ctr": 0.0234100663285212,
+ "ctr_select": 0.0249609984399376,
+ "ctr_holdout": 0.02185792349726776,
+ "n_select": 1282,
+ "n_holdout": 1281,
+ "eb": 0.030154702701359407,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Next Time Someone Talks About Starving Kids In Africa, Show Them This",
+ "impressions": 2645,
+ "clicks": 90,
+ "ctr": 0.0340264650283553,
+ "ctr_select": 0.03328290468986384,
+ "ctr_holdout": 0.03476946334089191,
+ "n_select": 1322,
+ "n_holdout": 1323,
+ "eb": 0.03312028192465662,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Next Time Someone Talks About Starving Kids In Africa, Show Them This",
+ "impressions": 2923,
+ "clicks": 118,
+ "ctr": 0.040369483407458,
+ "ctr_select": 0.03967168262653899,
+ "ctr_holdout": 0.04106776180698152,
+ "n_select": 1462,
+ "n_holdout": 1461,
+ "eb": 0.035597506629444046,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Next Time Someone Talks About Starving Kids In Africa, Show Them This",
+ "impressions": 2769,
+ "clicks": 104,
+ "ctr": 0.0375586854460093,
+ "ctr_select": 0.033236994219653176,
+ "ctr_holdout": 0.04187725631768953,
+ "n_select": 1384,
+ "n_holdout": 1385,
+ "eb": 0.03310584058352231,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.03302752293577982,
+ "pool_holdout": 0.03522935779816514,
+ "lift_select": 0.20117039063687492,
+ "lift_holdout": 0.16572553045858984,
+ "eb_lift": 0.0778133951692781,
+ "p_value": 0.09654473783038803,
+ "held_up": false,
+ "z_path": [
+ -0.217,
+ 1.007,
+ 1.098,
+ 0.95,
+ 0.94,
+ 1.25,
+ 1.045,
+ 0.734,
+ 0.995,
+ 1.246
+ ],
+ "first_cross": null,
+ "final_z": 1.246
+ },
+ {
+ "id": "516e6376eb1fc000020183d9",
+ "variation": "both",
+ "week": 201315,
+ "impressions": 10871,
+ "pooled_ctr": 0.00901481004507405,
+ "arms": [
+ {
+ "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Unless It's Friday.",
+ "impressions": 2204,
+ "clicks": 17,
+ "ctr": 0.0077132486388384,
+ "ctr_select": 0.007259528130671506,
+ "ctr_holdout": 0.008166969147005444,
+ "n_select": 1102,
+ "n_holdout": 1102,
+ "eb": 0.008448607234753339,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Except On Friday.",
+ "impressions": 2126,
+ "clicks": 21,
+ "ctr": 0.0098777046095954,
+ "ctr_select": 0.011288805268109126,
+ "ctr_holdout": 0.008466603951081843,
+ "n_select": 1063,
+ "n_holdout": 1063,
+ "eb": 0.00972925876305196,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Unless It's Friday.",
+ "impressions": 2210,
+ "clicks": 18,
+ "ctr": 0.0081447963800904,
+ "ctr_select": 0.009954751131221719,
+ "ctr_holdout": 0.006334841628959276,
+ "n_select": 1105,
+ "n_holdout": 1105,
+ "eb": 0.009317703612814482,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Unless It's Friday.",
+ "impressions": 2192,
+ "clicks": 20,
+ "ctr": 0.0091240875912408,
+ "ctr_select": 0.010036496350364963,
+ "ctr_holdout": 0.008211678832116789,
+ "n_select": 1096,
+ "n_holdout": 1096,
+ "eb": 0.009342269154280408,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Unless It's Friday.",
+ "impressions": 2139,
+ "clicks": 22,
+ "ctr": 0.0102851799906498,
+ "ctr_select": 0.0065420560747663555,
+ "ctr_holdout": 0.01403180542563143,
+ "n_select": 1070,
+ "n_holdout": 1069,
+ "eb": 0.008233226574463081,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.009013980868285504,
+ "pool_holdout": 0.009015639374425023,
+ "lift_select": 0.25236623341716746,
+ "lift_holdout": -0.060898112772860835,
+ "eb_lift": 0.07935205379490728,
+ "p_value": 0.3816129727632338,
+ "held_up": false,
+ "z_path": [
+ -1.381,
+ -0.585,
+ -0.832,
+ -0.832,
+ -0.319,
+ -0.099,
+ -0.439,
+ -0.08,
+ -0.072,
+ 0.255
+ ],
+ "first_cross": null,
+ "final_z": 0.255
+ },
+ {
+ "id": "515b3b374c9a93000200b70d",
+ "variation": "image",
+ "week": 201313,
+ "impressions": 10868,
+ "pooled_ctr": 0.03422892896577107,
+ "arms": [
+ {
+ "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It",
+ "impressions": 2161,
+ "clicks": 77,
+ "ctr": 0.0356316520129569,
+ "ctr_select": 0.03611111111111111,
+ "ctr_holdout": 0.03515263644773358,
+ "n_select": 1080,
+ "n_holdout": 1081,
+ "eb": 0.03645911208577955,
+ "labelled_winner": false
+ },
+ {
+ "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It",
+ "impressions": 2172,
+ "clicks": 65,
+ "ctr": 0.0299263351749539,
+ "ctr_select": 0.03314917127071823,
+ "ctr_holdout": 0.026703499079189688,
+ "n_select": 1086,
+ "n_holdout": 1086,
+ "eb": 0.035513448001647664,
+ "labelled_winner": false
+ },
+ {
+ "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It",
+ "impressions": 2200,
+ "clicks": 84,
+ "ctr": 0.0381818181818181,
+ "ctr_select": 0.04454545454545455,
+ "ctr_holdout": 0.031818181818181815,
+ "n_select": 1100,
+ "n_holdout": 1100,
+ "eb": 0.03917170359020483,
+ "labelled_winner": false
+ },
+ {
+ "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It",
+ "impressions": 2255,
+ "clicks": 72,
+ "ctr": 0.0319290465631929,
+ "ctr_select": 0.03280141843971631,
+ "ctr_holdout": 0.031055900621118012,
+ "n_select": 1128,
+ "n_holdout": 1127,
+ "eb": 0.03537078754266193,
+ "labelled_winner": false
+ },
+ {
+ "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It",
+ "impressions": 2080,
+ "clicks": 74,
+ "ctr": 0.035576923076923,
+ "ctr_select": 0.03653846153846154,
+ "ctr_holdout": 0.03461538461538462,
+ "n_select": 1040,
+ "n_holdout": 1040,
+ "eb": 0.03659562362545704,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.03662127346337873,
+ "pool_holdout": 0.03183658446816342,
+ "lift_select": 0.21638190954773884,
+ "lift_holdout": -0.0005780346820811744,
+ "eb_lift": 0.0696434035636837,
+ "p_value": 0.11717217104502756,
+ "held_up": false,
+ "z_path": [
+ -0.114,
+ 0.959,
+ 1.694,
+ 1.387,
+ 1.419,
+ 0.552,
+ 0.854,
+ -0.051,
+ 0.179,
+ 0.452
+ ],
+ "first_cross": null,
+ "final_z": 0.452
+ },
+ {
+ "id": "5194806d83c7ab001000a221",
+ "variation": "image",
+ "week": 201319,
+ "impressions": 10849,
+ "pooled_ctr": 0.017052262881371554,
+ "arms": [
+ {
+ "headline": "They Were Happily Married For 16 Years. Paid Taxes. Never Had A Parking Ticket. So We Deported Him.",
+ "impressions": 2766,
+ "clicks": 45,
+ "ctr": 0.0162689804772234,
+ "ctr_select": 0.018076644974692697,
+ "ctr_holdout": 0.014461315979754157,
+ "n_select": 1383,
+ "n_holdout": 1383,
+ "eb": 0.017722715188389155,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Were Happily Married For 16 Years. Paid Taxes. Never Had A Parking Ticket. So We Deported Him.",
+ "impressions": 2728,
+ "clicks": 52,
+ "ctr": 0.0190615835777126,
+ "ctr_select": 0.017595307917888565,
+ "ctr_holdout": 0.020527859237536656,
+ "n_select": 1364,
+ "n_holdout": 1364,
+ "eb": 0.01754256231744982,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Were Happily Married For 16 Years. Paid Taxes. Never Had A Parking Ticket. So We Deported Him.",
+ "impressions": 2704,
+ "clicks": 45,
+ "ctr": 0.0166420118343195,
+ "ctr_select": 0.017011834319526627,
+ "ctr_holdout": 0.016272189349112426,
+ "n_select": 1352,
+ "n_holdout": 1352,
+ "eb": 0.017327424909484686,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They Were Happily Married For 16 Years. Paid Taxes. Never Had A Parking Ticket. So We Deported Him.",
+ "impressions": 2651,
+ "clicks": 43,
+ "ctr": 0.0162202942285929,
+ "ctr_select": 0.017345399698340876,
+ "ctr_holdout": 0.01509433962264151,
+ "n_select": 1326,
+ "n_holdout": 1325,
+ "eb": 0.017451066628465278,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.017511520737327188,
+ "pool_holdout": 0.016592920353982302,
+ "lift_select": 0.03227156829166189,
+ "lift_holdout": -0.12846469028681617,
+ "eb_lift": 0.012060314705380826,
+ "p_value": 0.8527406540647168,
+ "held_up": false,
+ "z_path": [
+ -0.023,
+ 0.187,
+ -0.602,
+ -0.7,
+ -0.197,
+ -0.694,
+ -0.298,
+ -0.402,
+ -0.807,
+ -0.786
+ ],
+ "first_cross": null,
+ "final_z": -0.786
+ },
+ {
+ "id": "520af3a05ac98b06e20036e6",
+ "variation": "headline",
+ "week": 201332,
+ "impressions": 10804,
+ "pooled_ctr": 0.023417252869307664,
+ "arms": [
+ {
+ "headline": "If You Don\u2019t Say, 'Holy S***' Once Over The Next Three Minutes, I\u2019ve Failed",
+ "impressions": 2536,
+ "clicks": 73,
+ "ctr": 0.0287854889589905,
+ "ctr_select": 0.033123028391167195,
+ "ctr_holdout": 0.02444794952681388,
+ "n_select": 1268,
+ "n_holdout": 1268,
+ "eb": 0.025949435360090842,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Don\u2019t Say, 'Holy Crap' Once Over The Next Three Minutes, I\u2019ve Failed",
+ "impressions": 2686,
+ "clicks": 72,
+ "ctr": 0.0268056589724497,
+ "ctr_select": 0.021593447505584513,
+ "ctr_holdout": 0.03201787043931496,
+ "n_select": 1343,
+ "n_holdout": 1343,
+ "eb": 0.02186655038625483,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Don\u2019t Say, 'Holy S***' At Least Once , I\u2019ve Failed",
+ "impressions": 2855,
+ "clicks": 61,
+ "ctr": 0.0213660245183887,
+ "ctr_select": 0.021708683473389355,
+ "ctr_holdout": 0.02102312543798178,
+ "n_select": 1428,
+ "n_holdout": 1427,
+ "eb": 0.02190428520318719,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You Don\u2019t Say, 'Holy Crap' At Least Once, I\u2019ve Failed",
+ "impressions": 2727,
+ "clicks": 47,
+ "ctr": 0.0172350568390172,
+ "ctr_select": 0.012463343108504398,
+ "ctr_holdout": 0.022010271460014674,
+ "n_select": 1364,
+ "n_holdout": 1363,
+ "eb": 0.018482460067696372,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.022024801036461226,
+ "pool_holdout": 0.0248102203295686,
+ "lift_select": 0.5038968268695492,
+ "lift_holdout": -0.014601676161777877,
+ "eb_lift": 0.17819159034093124,
+ "p_value": 0.0020837292088948705,
+ "held_up": false,
+ "z_path": [
+ 0.75,
+ 0.185,
+ 0.858,
+ 1.753,
+ 1.565,
+ 1.846,
+ 2.013,
+ 1.874,
+ 1.756,
+ 1.746
+ ],
+ "first_cross": 6,
+ "final_z": 1.746
+ },
+ {
+ "id": "521d6395188487ad41003a3b",
+ "variation": "image",
+ "week": 201334,
+ "impressions": 10791,
+ "pooled_ctr": 0.02075803910666296,
+ "arms": [
+ {
+ "headline": "If 287,000 Women Had Been Given A Few More Choices About Their Sex Lives They\u2019d Still Be Alive",
+ "impressions": 2709,
+ "clicks": 66,
+ "ctr": 0.0243632336655592,
+ "ctr_select": 0.024372230428360415,
+ "ctr_holdout": 0.024354243542435424,
+ "n_select": 1354,
+ "n_holdout": 1355,
+ "eb": 0.021273023355833864,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If 287,000 Women Had Been Given A Few More Choices About Their Sex Lives They\u2019d Still Be Alive",
+ "impressions": 2614,
+ "clicks": 52,
+ "ctr": 0.0198928844682478,
+ "ctr_select": 0.019127773527161437,
+ "ctr_holdout": 0.020657995409334353,
+ "n_select": 1307,
+ "n_holdout": 1307,
+ "eb": 0.01934178150542352,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If 287,000 Women Had Been Given A Few More Choices About Their Sex Lives They\u2019d Still Be Alive",
+ "impressions": 2807,
+ "clicks": 71,
+ "ctr": 0.0252939080869255,
+ "ctr_select": 0.022792022792022793,
+ "ctr_holdout": 0.027797576621525304,
+ "n_select": 1404,
+ "n_holdout": 1403,
+ "eb": 0.020718531472943405,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If 287,000 Women Had Been Given A Few More Choices About Their Sex Lives They\u2019d Still Be Alive",
+ "impressions": 2661,
+ "clicks": 35,
+ "ctr": 0.0131529500187899,
+ "ctr_select": 0.011278195488721804,
+ "ctr_holdout": 0.015026296018031555,
+ "n_select": 1330,
+ "n_holdout": 1331,
+ "eb": 0.016478369616506512,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.019462465245597776,
+ "pool_holdout": 0.022053372868791696,
+ "lift_select": 0.2522684110571851,
+ "lift_holdout": 0.10433191726875246,
+ "eb_lift": 0.09302820004498757,
+ "p_value": 0.13076553528697094,
+ "held_up": false,
+ "z_path": [
+ -0.776,
+ -0.489,
+ -1.2,
+ -1.625,
+ -0.8,
+ -0.833,
+ -0.647,
+ -0.656,
+ 0.021,
+ -0.222
+ ],
+ "first_cross": null,
+ "final_z": -0.222
+ },
+ {
+ "id": "51436069220cb800020005c4",
+ "variation": "headline",
+ "week": 201306,
+ "impressions": 10599,
+ "pooled_ctr": 0.011133125766581754,
+ "arms": [
+ {
+ "headline": "Shooting Victim Gabby Giffords Can Finally Speak Again \u2014 And It's About Gun Violence",
+ "impressions": 2654,
+ "clicks": 34,
+ "ctr": 0.0128108515448379,
+ "ctr_select": 0.012057272042200452,
+ "ctr_holdout": 0.013564431047475508,
+ "n_select": 1327,
+ "n_holdout": 1327,
+ "eb": 0.011110250041202481,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Optimism About Gun Violence From The Woman Who Knows Far Too Much About Gun Violence",
+ "impressions": 2512,
+ "clicks": 21,
+ "ctr": 0.0083598726114649,
+ "ctr_select": 0.007165605095541401,
+ "ctr_holdout": 0.009554140127388535,
+ "n_select": 1256,
+ "n_holdout": 1256,
+ "eb": 0.009372226078229805,
+ "labelled_winner": false
+ },
+ {
+ "headline": "2 Years Ago Gabby Giffords Was Shot In The Head. Now She's Got Something Important To Say.",
+ "impressions": 2753,
+ "clicks": 39,
+ "ctr": 0.0141663639665819,
+ "ctr_select": 0.015261627906976744,
+ "ctr_holdout": 0.013071895424836602,
+ "n_select": 1376,
+ "n_holdout": 1377,
+ "eb": 0.01231652031675617,
+ "labelled_winner": false
+ },
+ {
+ "headline": "90% Of Americans Agree On Gun Control Reform. So Why Hasn't Anything Happened Yet?",
+ "impressions": 2680,
+ "clicks": 24,
+ "ctr": 0.008955223880597,
+ "ctr_select": 0.007462686567164179,
+ "ctr_holdout": 0.010447761194029851,
+ "n_select": 1340,
+ "n_holdout": 1340,
+ "eb": 0.009430386436632078,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.010568031704095112,
+ "pool_holdout": 0.01169811320754717,
+ "lift_select": 0.44413154069767424,
+ "lift_holdout": 0.11743622180054825,
+ "eb_lift": 0.16545073497305274,
+ "p_value": 0.047832974893710224,
+ "held_up": false,
+ "z_path": [
+ -0.43,
+ -0.064,
+ 0.395,
+ 0.463,
+ 0.717,
+ -0.273,
+ -0.691,
+ -0.272,
+ -0.273,
+ 0.432
+ ],
+ "first_cross": null,
+ "final_z": 0.432
+ },
+ {
+ "id": "5349aebefe9c5af4c3000048",
+ "variation": "headline",
+ "week": 201414,
+ "impressions": 10598,
+ "pooled_ctr": 0.02349499905642574,
+ "arms": [
+ {
+ "headline": "The Most Honest And Evil Company Ad I Have Ever Laid My Eyes On. Also The Rudest.",
+ "impressions": 2063,
+ "clicks": 43,
+ "ctr": 0.0208434318952981,
+ "ctr_select": 0.02422480620155039,
+ "ctr_holdout": 0.01745877788554801,
+ "n_select": 1032,
+ "n_holdout": 1031,
+ "eb": 0.022482223237236587,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest And Awful Company Ad I Have Ever Laid My Eyes On. Did I Mention Awful?",
+ "impressions": 2141,
+ "clicks": 55,
+ "ctr": 0.0256889304063521,
+ "ctr_select": 0.02336448598130841,
+ "ctr_holdout": 0.028011204481792718,
+ "n_select": 1070,
+ "n_holdout": 1071,
+ "eb": 0.02223003884609498,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest And Awful Company Ad I Have Ever Laid My Eyes On. Also The Rudest.",
+ "impressions": 2129,
+ "clicks": 48,
+ "ctr": 0.0225457961484264,
+ "ctr_select": 0.016917293233082706,
+ "ctr_holdout": 0.028169014084507043,
+ "n_select": 1064,
+ "n_holdout": 1065,
+ "eb": 0.02019951442709867,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest And Disturbing Company Ad I Have Ever Laid My Eyes On. Also The Rudest.",
+ "impressions": 2156,
+ "clicks": 49,
+ "ctr": 0.0227272727272727,
+ "ctr_select": 0.017625231910946195,
+ "ctr_holdout": 0.027829313543599257,
+ "n_select": 1078,
+ "n_holdout": 1078,
+ "eb": 0.020410724864659658,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Honest And Gross Company Ad I Have Ever Laid My Eyes On. Also The Rudest.",
+ "impressions": 2109,
+ "clicks": 54,
+ "ctr": 0.0256045519203413,
+ "ctr_select": 0.026565464895635674,
+ "ctr_holdout": 0.024644549763033177,
+ "n_select": 1054,
+ "n_holdout": 1055,
+ "eb": 0.023225288395580204,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.021706304265760666,
+ "pool_holdout": 0.025283018867924528,
+ "lift_select": 0.22385941753980676,
+ "lift_holdout": -0.025252882506896723,
+ "eb_lift": 0.06997893843290348,
+ "p_value": 0.22645326693522005,
+ "held_up": false,
+ "z_path": [
+ -1.169,
+ -1.553,
+ -1.52,
+ -0.692,
+ -0.37,
+ 0.575,
+ 0.657,
+ 0.588,
+ 0.541,
+ 1.02
+ ],
+ "first_cross": null,
+ "final_z": 1.02
+ },
+ {
+ "id": "520b42103715ea76550065fb",
+ "variation": "image",
+ "week": 201332,
+ "impressions": 10570,
+ "pooled_ctr": 0.025733207190160833,
+ "arms": [
+ {
+ "headline": "Watch The Daily Show Supersize The Hilarity Of Our Minimum Wage Dispute",
+ "impressions": 2533,
+ "clicks": 76,
+ "ctr": 0.0300039478878799,
+ "ctr_select": 0.030015797788309637,
+ "ctr_holdout": 0.02999210734017364,
+ "n_select": 1266,
+ "n_holdout": 1267,
+ "eb": 0.028592189524651028,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch The Daily Show Supersize The Hilarity Of Our Minimum Wage Dispute",
+ "impressions": 2559,
+ "clicks": 56,
+ "ctr": 0.0218835482610394,
+ "ctr_select": 0.0265625,
+ "ctr_holdout": 0.017200938232994525,
+ "n_select": 1280,
+ "n_holdout": 1279,
+ "eb": 0.02736910437003197,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch The Daily Show Supersize The Hilarity Of Our Minimum Wage Dispute",
+ "impressions": 2772,
+ "clicks": 69,
+ "ctr": 0.0248917748917748,
+ "ctr_select": 0.026695526695526696,
+ "ctr_holdout": 0.023088023088023088,
+ "n_select": 1386,
+ "n_holdout": 1386,
+ "eb": 0.027395800646202912,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch The Daily Show Supersize The Hilarity Of Our Minimum Wage Dispute",
+ "impressions": 2706,
+ "clicks": 71,
+ "ctr": 0.0262379896526238,
+ "ctr_select": 0.028085735402808575,
+ "ctr_holdout": 0.024390243902439025,
+ "n_select": 1353,
+ "n_holdout": 1353,
+ "eb": 0.02791451985833549,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.02781456953642384,
+ "pool_holdout": 0.023651844843897825,
+ "lift_select": 0.07913939667494163,
+ "lift_holdout": 0.2680662983425415,
+ "eb_lift": 0.027957290052929906,
+ "p_value": 0.5849431591592602,
+ "held_up": true,
+ "z_path": [
+ 0.103,
+ -0.444,
+ 0.528,
+ 0.521,
+ -0.087,
+ -0.248,
+ 0.238,
+ 0.274,
+ 0.653,
+ 0.825
+ ],
+ "first_cross": null,
+ "final_z": 0.825
+ },
+ {
+ "id": "51d3a5338505f3a3a1002c75",
+ "variation": "other",
+ "week": 201326,
+ "impressions": 10530,
+ "pooled_ctr": 0.02678062678062678,
+ "arms": [
+ {
+ "headline": "If A 14-Year Old In Africa Can Build These With Scraps, Imagine What We Could Do In America?",
+ "impressions": 2686,
+ "clicks": 60,
+ "ctr": 0.0223380491437081,
+ "ctr_select": 0.014892032762472078,
+ "ctr_holdout": 0.029784065524944156,
+ "n_select": 1343,
+ "n_holdout": 1343,
+ "eb": 0.02037476429104088,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If A 14-Year Old In Africa Can Build These With Scraps, Imagine What We Could Do In America?",
+ "impressions": 2620,
+ "clicks": 62,
+ "ctr": 0.0236641221374045,
+ "ctr_select": 0.022900763358778626,
+ "ctr_holdout": 0.024427480916030534,
+ "n_select": 1310,
+ "n_holdout": 1310,
+ "eb": 0.023316676239208734,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If A 14-Year Old In Africa Can Build These With Scraps, Imagine What We Could Do In America?",
+ "impressions": 2577,
+ "clicks": 88,
+ "ctr": 0.0341482343810632,
+ "ctr_select": 0.031055900621118012,
+ "ctr_holdout": 0.037238169123351435,
+ "n_select": 1288,
+ "n_holdout": 1289,
+ "eb": 0.02623233022311065,
+ "labelled_winner": true
+ },
+ {
+ "headline": "If A 14-Year Old In Africa Can Build These With Scraps, Imagine What We Could Do In America?",
+ "impressions": 2647,
+ "clicks": 72,
+ "ctr": 0.0272006044578768,
+ "ctr_select": 0.0256797583081571,
+ "ctr_holdout": 0.02872260015117158,
+ "n_select": 1324,
+ "n_holdout": 1323,
+ "eb": 0.024325427330211254,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.02355175688509022,
+ "pool_holdout": 0.030009496676163344,
+ "lift_select": 0.3186235223402123,
+ "lift_holdout": 0.24087949642053985,
+ "eb_lift": 0.11381627923127069,
+ "p_value": 0.04101713464094579,
+ "held_up": true,
+ "z_path": [
+ 1.363,
+ 2.446,
+ 1.426,
+ 1.748,
+ 1.939,
+ 1.513,
+ 1.212,
+ 1.325,
+ 1.273,
+ 1.457
+ ],
+ "first_cross": 1,
+ "final_z": 1.457
+ },
+ {
+ "id": "53208267c532cf5ce700003d",
+ "variation": "image",
+ "week": 201410,
+ "impressions": 10511,
+ "pooled_ctr": 0.01141661116925126,
+ "arms": [
+ {
+ "headline": "HIV-Positive, Homeless, And Gay In New York At The Age Of 16? Give This Story A Listen.",
+ "impressions": 2665,
+ "clicks": 22,
+ "ctr": 0.0082551594746716,
+ "ctr_select": 0.010510510510510511,
+ "ctr_holdout": 0.006001500375093774,
+ "n_select": 1332,
+ "n_holdout": 1333,
+ "eb": 0.010725386208783449,
+ "labelled_winner": false
+ },
+ {
+ "headline": "HIV-Positive, Homeless, And Gay In New York At The Age Of 16? Give This Story A Listen.",
+ "impressions": 2592,
+ "clicks": 29,
+ "ctr": 0.0111882716049382,
+ "ctr_select": 0.005401234567901234,
+ "ctr_holdout": 0.016975308641975308,
+ "n_select": 1296,
+ "n_holdout": 1296,
+ "eb": 0.008895159488932967,
+ "labelled_winner": false
+ },
+ {
+ "headline": "HIV-Positive, Homeless, And Gay In New York At The Age Of 16? Give This Story A Listen.",
+ "impressions": 2657,
+ "clicks": 24,
+ "ctr": 0.0090327436958976,
+ "ctr_select": 0.00677710843373494,
+ "ctr_holdout": 0.011286681715575621,
+ "n_select": 1328,
+ "n_holdout": 1329,
+ "eb": 0.009365674335562653,
+ "labelled_winner": false
+ },
+ {
+ "headline": "HIV-Positive, Homeless, And Gay In New York At The Age Of 16? Give This Story A Listen.",
+ "impressions": 2597,
+ "clicks": 45,
+ "ctr": 0.0173276857912976,
+ "ctr_select": 0.020801232665639446,
+ "ctr_holdout": 0.013856812933025405,
+ "n_select": 1298,
+ "n_holdout": 1299,
+ "eb": 0.014421675064450069,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.010848877046060145,
+ "pool_holdout": 0.011984021304926764,
+ "lift_select": 0.9173627443029762,
+ "lift_holdout": 0.15627405696689767,
+ "eb_lift": 0.329324224361766,
+ "p_value": 6.637202282107459e-05,
+ "held_up": false,
+ "z_path": [
+ 0.329,
+ 0.557,
+ 0.531,
+ 0.451,
+ 1.087,
+ 1.301,
+ 1.515,
+ 2.132,
+ 2.767,
+ 2.935
+ ],
+ "first_cross": 7,
+ "final_z": 2.935
+ },
+ {
+ "id": "53e4faa3220776c2b3000002",
+ "variation": "headline",
+ "week": 201431,
+ "impressions": 10328,
+ "pooled_ctr": 0.013555383423702556,
+ "arms": [
+ {
+ "headline": "A Politician Decides To Be Sexist To A Woman Who Wants Him To Learn Facts. Then He Gets Xenophobic.",
+ "impressions": 2506,
+ "clicks": 30,
+ "ctr": 0.0119712689545091,
+ "ctr_select": 0.009577015163607342,
+ "ctr_holdout": 0.014365522745411013,
+ "n_select": 1253,
+ "n_holdout": 1253,
+ "eb": 0.0126621063974509,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Politician Decides To Be Awful To A Woman Who Wants Him To Learn Facts. Then He Gets Xenophobic.",
+ "impressions": 2586,
+ "clicks": 36,
+ "ctr": 0.0139211136890951,
+ "ctr_select": 0.014694508894044857,
+ "ctr_holdout": 0.013147718484145398,
+ "n_select": 1293,
+ "n_holdout": 1293,
+ "eb": 0.014460516642586579,
+ "labelled_winner": true
+ },
+ {
+ "headline": "A Politician Decides To Be Rude To A Woman Who Wants Him To Learn Facts. Then He Gets Xenophobic.",
+ "impressions": 2639,
+ "clicks": 38,
+ "ctr": 0.0143993937097385,
+ "ctr_select": 0.013636363636363636,
+ "ctr_holdout": 0.015163002274450341,
+ "n_select": 1320,
+ "n_holdout": 1319,
+ "eb": 0.014078287534439819,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Politician Decides To Be Hostile To A Woman Who Wants Him To Learn Facts. Then He Gets Xenophobic.",
+ "impressions": 2597,
+ "clicks": 36,
+ "ctr": 0.0138621486330381,
+ "ctr_select": 0.01926040061633282,
+ "ctr_holdout": 0.008468052347959968,
+ "n_select": 1298,
+ "n_holdout": 1299,
+ "eb": 0.0160999505502359,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.014329976762199844,
+ "pool_holdout": 0.012780790085205267,
+ "lift_select": 0.3440636321992254,
+ "lift_holdout": -0.33743905568385946,
+ "eb_lift": 0.12351546812727299,
+ "p_value": 0.08409416499354987,
+ "held_up": false,
+ "z_path": [
+ -0.645,
+ -0.007,
+ 1.09,
+ 0.345,
+ 0.944,
+ 1.338,
+ 1.101,
+ 0.234,
+ 0.346,
+ -0.018
+ ],
+ "first_cross": null,
+ "final_z": -0.018
+ },
+ {
+ "id": "538e322d13f17fb43e000064",
+ "variation": "headline",
+ "week": 201424,
+ "impressions": 10282,
+ "pooled_ctr": 0.008947675549503988,
+ "arms": [
+ {
+ "headline": "There's A Lot To Lose For Girls Like Christine By That Awkward Thing Called Puberty",
+ "impressions": 2614,
+ "clicks": 24,
+ "ctr": 0.0091813312930374,
+ "ctr_select": 0.009946442234123947,
+ "ctr_holdout": 0.008416220351951033,
+ "n_select": 1307,
+ "n_holdout": 1307,
+ "eb": 0.008934683937558223,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why Are Girls Dropping Out Of School Because Of Something Half The World Goes Through?",
+ "impressions": 2588,
+ "clicks": 27,
+ "ctr": 0.0104327666151468,
+ "ctr_select": 0.010046367851622875,
+ "ctr_holdout": 0.010819165378670788,
+ "n_select": 1294,
+ "n_holdout": 1294,
+ "eb": 0.008966843575680544,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Puberty Is So Awkward, And Here It's Making Girls Drop Out Of School",
+ "impressions": 2530,
+ "clicks": 19,
+ "ctr": 0.0075098814229249,
+ "ctr_select": 0.0071146245059288534,
+ "ctr_holdout": 0.007905138339920948,
+ "n_select": 1265,
+ "n_holdout": 1265,
+ "eb": 0.007922948208380177,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Girl's Body Starts To Change, So... She Never Goes To School Again?",
+ "impressions": 2550,
+ "clicks": 22,
+ "ctr": 0.0086274509803921,
+ "ctr_select": 0.006274509803921568,
+ "ctr_holdout": 0.010980392156862745,
+ "n_select": 1275,
+ "n_holdout": 1275,
+ "eb": 0.007622552914701078,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.008364131491927641,
+ "pool_holdout": 0.009531219607080334,
+ "lift_select": 0.2011250494230976,
+ "lift_holdout": 0.1351291675866637,
+ "eb_lift": 0.07205913540868991,
+ "p_value": 0.4424145827960426,
+ "held_up": false,
+ "z_path": [
+ -0.569,
+ -0.89,
+ -0.515,
+ 0.267,
+ 0.882,
+ 0.607,
+ 0.539,
+ 0.682,
+ 0.798,
+ 0.458
+ ],
+ "first_cross": null,
+ "final_z": 0.458
+ },
+ {
+ "id": "537ccb78db41d8f8ac0000a1",
+ "variation": "headline",
+ "week": 201420,
+ "impressions": 10275,
+ "pooled_ctr": 0.020632603406326035,
+ "arms": [
+ {
+ "headline": "Thought I Knew Where This Bullying Video Was Going But It Opened My Eyes To Something Bigger",
+ "impressions": 2557,
+ "clicks": 61,
+ "ctr": 0.0238560813453265,
+ "ctr_select": 0.024256651017214397,
+ "ctr_holdout": 0.023455824863174355,
+ "n_select": 1278,
+ "n_holdout": 1279,
+ "eb": 0.022677572252107932,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Sticks and Stones May Break Adult Bones But Kids Are So Resilient. Isn't That How The Rhyme Goes?",
+ "impressions": 2516,
+ "clicks": 20,
+ "ctr": 0.0079491255961844,
+ "ctr_select": 0.00794912559618442,
+ "ctr_holdout": 0.00794912559618442,
+ "n_select": 1258,
+ "n_holdout": 1258,
+ "eb": 0.016931435343406364,
+ "labelled_winner": false
+ },
+ {
+ "headline": "We Tell Kids It Builds Character But As Adults We\u2019d Call The Police. That Sounds Fair?",
+ "impressions": 2557,
+ "clicks": 50,
+ "ctr": 0.0195541650371529,
+ "ctr_select": 0.02112676056338028,
+ "ctr_holdout": 0.017982799061767005,
+ "n_select": 1278,
+ "n_holdout": 1279,
+ "eb": 0.02156513168324566,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Shocking Type Of Bullying Is The Kind That Goes Unnoticed. But We\u2019d Notice\u2026.Right?",
+ "impressions": 2645,
+ "clicks": 81,
+ "ctr": 0.0306238185255198,
+ "ctr_select": 0.03328290468986384,
+ "ctr_holdout": 0.027966742252456538,
+ "n_select": 1322,
+ "n_holdout": 1323,
+ "eb": 0.02597515018162272,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.021806853582554516,
+ "pool_holdout": 0.01945903872348706,
+ "lift_select": 0.5262589150637562,
+ "lift_holdout": 0.43721088435374145,
+ "eb_lift": 0.19114617261441325,
+ "p_value": 0.0009154802431240878,
+ "held_up": true,
+ "z_path": [
+ -1.061,
+ -1.182,
+ 0.232,
+ 0.463,
+ 0.771,
+ 0.964,
+ 1.235,
+ 1.481,
+ 1.311,
+ 1.498
+ ],
+ "first_cross": null,
+ "final_z": 1.498
+ },
+ {
+ "id": "53e55f16220776c2b3000038",
+ "variation": "headline",
+ "week": 201431,
+ "impressions": 10238,
+ "pooled_ctr": 0.015921078335612425,
+ "arms": [
+ {
+ "headline": "Hey Big Government Agency: Someone Fixed Your Really Clueless Poster About Rape",
+ "impressions": 2556,
+ "clicks": 34,
+ "ctr": 0.0133020344287949,
+ "ctr_select": 0.013302034428794992,
+ "ctr_holdout": 0.013302034428794992,
+ "n_select": 1278,
+ "n_holdout": 1278,
+ "eb": 0.014295786110415888,
+ "labelled_winner": false
+ },
+ {
+ "headline": "1 Poster Missed The Mark About Sexual Assault, So Someone Corrected The Problem Like A Boss",
+ "impressions": 2535,
+ "clicks": 65,
+ "ctr": 0.0256410256410256,
+ "ctr_select": 0.01971608832807571,
+ "ctr_holdout": 0.03157063930544594,
+ "n_select": 1268,
+ "n_holdout": 1267,
+ "eb": 0.016566741035193896,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Boom! That's The Sound Of Someone Giving A Government Agency The Smackdown On One Huge Mistake.",
+ "impressions": 2560,
+ "clicks": 25,
+ "ctr": 0.009765625,
+ "ctr_select": 0.01171875,
+ "ctr_holdout": 0.0078125,
+ "n_select": 1280,
+ "n_holdout": 1280,
+ "eb": 0.013731927849170697,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Misinformation About Rape Cleared Up Using 30 Characters",
+ "impressions": 2587,
+ "clicks": 39,
+ "ctr": 0.0150753768844221,
+ "ctr_select": 0.014683153013910355,
+ "ctr_holdout": 0.015467904098994586,
+ "n_select": 1294,
+ "n_holdout": 1293,
+ "eb": 0.014786211269742106,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.01484375,
+ "pool_holdout": 0.016998827667057445,
+ "lift_select": 0.32824173999667927,
+ "lift_holdout": 0.8572245053479575,
+ "eb_lift": 0.11607518552885199,
+ "p_value": 0.09810505308966028,
+ "held_up": true,
+ "z_path": [
+ 0.618,
+ 1.701,
+ 2.42,
+ 2.685,
+ 2.052,
+ 1.887,
+ 2.482,
+ 2.563,
+ 2.37,
+ 2.681
+ ],
+ "first_cross": 2,
+ "final_z": 2.681
+ },
+ {
+ "id": "55357665346161001c390000",
+ "variation": "both",
+ "week": 201516,
+ "impressions": 10227,
+ "pooled_ctr": 0.013689253935660506,
+ "arms": [
+ {
+ "headline": "The man who found the Titanic made a discovery during his latest adventure: a beautiful whale.
",
+ "impressions": 2038,
+ "clicks": 42,
+ "ctr": 0.0206084396467124,
+ "ctr_select": 0.018645731108930325,
+ "ctr_holdout": 0.022571148184494603,
+ "n_select": 1019,
+ "n_holdout": 1019,
+ "eb": 0.015473641256922497,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The man who found the Titanic made a discovery during his latest adventure: a beautiful whale.
",
+ "impressions": 1979,
+ "clicks": 37,
+ "ctr": 0.0186963112683173,
+ "ctr_select": 0.01717171717171717,
+ "ctr_holdout": 0.020222446916076844,
+ "n_select": 990,
+ "n_holdout": 989,
+ "eb": 0.01500465510943511,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A whale interrupts a deep sea expedition on the site of the largest oil spill in U.S. history.",
+ "impressions": 2019,
+ "clicks": 21,
+ "ctr": 0.0104011887072808,
+ "ctr_select": 0.011881188118811881,
+ "ctr_holdout": 0.008919722497522299,
+ "n_select": 1010,
+ "n_holdout": 1009,
+ "eb": 0.013411934383989474,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A whale interrupts a deep sea expedition on the site of the largest oil spill in U.S. history.",
+ "impressions": 2077,
+ "clicks": 28,
+ "ctr": 0.0134809821858449,
+ "ctr_select": 0.016377649325626204,
+ "ctr_holdout": 0.010587102983638113,
+ "n_select": 1038,
+ "n_holdout": 1039,
+ "eb": 0.01479002807546293,
+ "labelled_winner": false
+ },
+ {
+ "headline": "At the site of the largest oil spill in U.S. history, a whale interrupts a deep sea expedition",
+ "impressions": 2114,
+ "clicks": 12,
+ "ctr": 0.0056764427625354,
+ "ctr_select": 0.006622516556291391,
+ "ctr_holdout": 0.004730368968779565,
+ "n_select": 1057,
+ "n_holdout": 1057,
+ "eb": 0.011743529860432934,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.014078998826750098,
+ "pool_holdout": 0.013299432818306279,
+ "lift_select": 0.32436484570930113,
+ "lift_holdout": 0.6971511862841309,
+ "eb_lift": 0.09905835260974505,
+ "p_value": 0.16674598729037737,
+ "held_up": true,
+ "z_path": [
+ -2.19,
+ -0.733,
+ -1.397,
+ -0.75,
+ -0.251,
+ -0.25,
+ -0.669,
+ -0.247,
+ 0.128,
+ 0.436
+ ],
+ "first_cross": 0,
+ "final_z": 0.436
+ },
+ {
+ "id": "54ecbc63326439000c5a0000",
+ "variation": "both",
+ "week": 201508,
+ "impressions": 10193,
+ "pooled_ctr": 0.009320121652114197,
+ "arms": [
+ {
+ "headline": "When people say his TV show\u2019s racist, he\u2019s like, \u2018I don\u2019t know if you know any of us.\u2019",
+ "impressions": 2466,
+ "clicks": 28,
+ "ctr": 0.0113544201135442,
+ "ctr_select": 0.0113544201135442,
+ "ctr_holdout": 0.0113544201135442,
+ "n_select": 1233,
+ "n_holdout": 1233,
+ "eb": 0.010346107619141311,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When people say his TV show is racist, he\u2019s like, \u2018I don\u2019t know if you know any of us.\u2019",
+ "impressions": 2544,
+ "clicks": 22,
+ "ctr": 0.0086477987421383,
+ "ctr_select": 0.008647798742138365,
+ "ctr_holdout": 0.008647798742138365,
+ "n_select": 1272,
+ "n_holdout": 1272,
+ "eb": 0.009397978144965052,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When people say his TV show is racist, he\u2019s like, \u2018I don\u2019t know if you know any of us.\u2019",
+ "impressions": 2563,
+ "clicks": 17,
+ "ctr": 0.0066328521264143,
+ "ctr_select": 0.00546021840873635,
+ "ctr_holdout": 0.0078064012490242,
+ "n_select": 1282,
+ "n_holdout": 1281,
+ "eb": 0.008260666037845846,
+ "labelled_winner": false
+ },
+ {
+ "headline": "When people say his TV show is racist, he\u2019s like, \u2018I don\u2019t know if you know any of us.\u2019",
+ "impressions": 2620,
+ "clicks": 28,
+ "ctr": 0.0106870229007633,
+ "ctr_select": 0.013740458015267175,
+ "ctr_holdout": 0.007633587786259542,
+ "n_select": 1310,
+ "n_holdout": 1310,
+ "eb": 0.011229133156035975,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.009809691975671964,
+ "pool_holdout": 0.008830455259026688,
+ "lift_select": 0.40070229007633573,
+ "lift_holdout": -0.1355385920271417,
+ "eb_lift": 0.1446978339263072,
+ "p_value": 0.09399426860419993,
+ "held_up": false,
+ "z_path": [
+ 0.528,
+ 1.059,
+ 0.664,
+ 0.317,
+ 0.027,
+ -0.005,
+ -0.017,
+ -0.2,
+ -0.499,
+ -0.228
+ ],
+ "first_cross": null,
+ "final_z": -0.228
+ },
+ {
+ "id": "53ceacf3d7c9144ec1000011",
+ "variation": "headline",
+ "week": 201429,
+ "impressions": 10187,
+ "pooled_ctr": 0.007165995877098263,
+ "arms": [
+ {
+ "headline": "A Simple Message That Anyone Who Really Cares About The 'Middle Class' Can Appreciate",
+ "impressions": 2588,
+ "clicks": 20,
+ "ctr": 0.0077279752704791,
+ "ctr_select": 0.008500772797527048,
+ "ctr_holdout": 0.006955177743431221,
+ "n_select": 1294,
+ "n_holdout": 1294,
+ "eb": 0.008210658959453781,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Graphic That Even The Opposite Of A 'Progressive' Can Appreciate",
+ "impressions": 2519,
+ "clicks": 20,
+ "ctr": 0.0079396585946804,
+ "ctr_select": 0.007142857142857143,
+ "ctr_holdout": 0.008737092930897538,
+ "n_select": 1260,
+ "n_holdout": 1259,
+ "eb": 0.007729668742435282,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Any Politician Who Disagrees With The Premise Of This Graphic Has Some Questionable Priorities",
+ "impressions": 2574,
+ "clicks": 17,
+ "ctr": 0.0066045066045066,
+ "ctr_select": 0.009324009324009324,
+ "ctr_holdout": 0.003885003885003885,
+ "n_select": 1287,
+ "n_holdout": 1287,
+ "eb": 0.00850401909652067,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Something To Think Of Next Time A Bus Or Train Gets You Where You Need To Go",
+ "impressions": 2506,
+ "clicks": 16,
+ "ctr": 0.0063846767757382,
+ "ctr_select": 0.007182761372705506,
+ "ctr_holdout": 0.00558659217877095,
+ "n_select": 1253,
+ "n_holdout": 1253,
+ "eb": 0.007744821994172841,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.008048684727129957,
+ "pool_holdout": 0.006283133712939329,
+ "lift_select": 0.15845130479276803,
+ "lift_holdout": -0.3816773504273505,
+ "eb_lift": 0.056572518967714425,
+ "p_value": 0.5536509287153362,
+ "held_up": false,
+ "z_path": [
+ -0.445,
+ -1.414,
+ -0.901,
+ -1.602,
+ -1.141,
+ -1.334,
+ -0.99,
+ -0.743,
+ -0.672,
+ -0.478
+ ],
+ "first_cross": null,
+ "final_z": -0.478
+ },
+ {
+ "id": "55386c3a653733000c440000",
+ "variation": "headline",
+ "week": 201516,
+ "impressions": 10176,
+ "pooled_ctr": 0.0013757861635220125,
+ "arms": [
+ {
+ "headline": "4 families are racing the clock to get a promising treatment approved before a son dies.",
+ "impressions": 2037,
+ "clicks": 1,
+ "ctr": 0.0004909180166912,
+ "ctr_select": 0.0009823182711198428,
+ "ctr_holdout": 0.0,
+ "n_select": 1018,
+ "n_holdout": 1019,
+ "eb": 0.0012557062432375877,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A new drug may be life-saving. That's why these 4 families are racing to get it approved.",
+ "impressions": 1991,
+ "clicks": 4,
+ "ctr": 0.0020090406830738,
+ "ctr_select": 0.001004016064257028,
+ "ctr_holdout": 0.003015075376884422,
+ "n_select": 996,
+ "n_holdout": 995,
+ "eb": 0.001264043013762127,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Their sons' disease has 4 families racing the clock to get a promising experimental drug approved.",
+ "impressions": 1997,
+ "clicks": 3,
+ "ctr": 0.0015022533800701,
+ "ctr_select": 0.002004008016032064,
+ "ctr_holdout": 0.001001001001001001,
+ "n_select": 998,
+ "n_holdout": 999,
+ "eb": 0.0015648762026799121,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Getting a promising new drug cleared takes time, and 4 families wonder how much they have left.",
+ "impressions": 2068,
+ "clicks": 2,
+ "ctr": 0.0009671179883945,
+ "ctr_select": 0.0009671179883945841,
+ "ctr_holdout": 0.0009671179883945841,
+ "n_select": 1034,
+ "n_holdout": 1034,
+ "eb": 0.001249711878094209,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Duchenne muscular dystrophy is merciless, so they're desperate to get this promising drug approved.",
+ "impressions": 2083,
+ "clicks": 4,
+ "ctr": 0.0019203072491598,
+ "ctr_select": 0.0019193857965451055,
+ "ctr_holdout": 0.0019212295869356388,
+ "n_select": 1042,
+ "n_holdout": 1041,
+ "eb": 0.0015443819323701332,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.0013757861635220125,
+ "pool_holdout": 0.0013757861635220125,
+ "lift_select": 0.45662754079587753,
+ "lift_holdout": -0.2724152724152724,
+ "eb_lift": 0.1374414456050561,
+ "p_value": 0.5503794650473408,
+ "held_up": false,
+ "z_path": [
+ 0.0,
+ 1.023,
+ 0.031,
+ 0.03,
+ 0.042,
+ 0.042,
+ -0.402,
+ -0.766,
+ -0.766,
+ -0.323
+ ],
+ "first_cross": null,
+ "final_z": -0.323
+ },
+ {
+ "id": "5519b9fa373836000ca10000",
+ "variation": "headline",
+ "week": 201513,
+ "impressions": 10172,
+ "pooled_ctr": 0.011207235548564688,
+ "arms": [
+ {
+ "headline": "Justin Timberlake gave the best advice to 'weird' and 'different' kids.",
+ "impressions": 2042,
+ "clicks": 22,
+ "ctr": 0.0107737512242899,
+ "ctr_select": 0.011753183153770812,
+ "ctr_holdout": 0.009794319294809012,
+ "n_select": 1021,
+ "n_holdout": 1021,
+ "eb": 0.011512238230895517,
+ "labelled_winner": false
+ },
+ {
+ "headline": "My favorite 90s boy band star gave the best advice to 'weird' and 'different' kids.",
+ "impressions": 1993,
+ "clicks": 23,
+ "ctr": 0.0115403913697942,
+ "ctr_select": 0.013052208835341365,
+ "ctr_holdout": 0.010030090270812437,
+ "n_select": 996,
+ "n_holdout": 997,
+ "eb": 0.011900869322371183,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Justin Timberlake to kids who are weird: 'Your critics don't count.'",
+ "impressions": 2011,
+ "clicks": 15,
+ "ctr": 0.0074589756340129,
+ "ctr_select": 0.005964214711729622,
+ "ctr_holdout": 0.008955223880597015,
+ "n_select": 1006,
+ "n_holdout": 1005,
+ "eb": 0.009758975165361108,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This fantastic acceptance speech is just what kids who don't fit in need to hear.",
+ "impressions": 2029,
+ "clicks": 24,
+ "ctr": 0.011828486939379,
+ "ctr_select": 0.013806706114398421,
+ "ctr_holdout": 0.009852216748768473,
+ "n_select": 1014,
+ "n_holdout": 1015,
+ "eb": 0.01213672035744689,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This is what an inspiring acceptance speech looks like.",
+ "impressions": 2097,
+ "clicks": 30,
+ "ctr": 0.0143061516452074,
+ "ctr_select": 0.012404580152671756,
+ "ctr_holdout": 0.016205910390848427,
+ "n_select": 1048,
+ "n_holdout": 1049,
+ "eb": 0.011717000995779872,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.011406096361848575,
+ "pool_holdout": 0.01100845291920582,
+ "lift_select": 0.21046725158130974,
+ "lift_holdout": -0.10503166783954965,
+ "eb_lift": 0.06405556926926614,
+ "p_value": 0.4210741118025991,
+ "held_up": false,
+ "z_path": [
+ -1.009,
+ -1.363,
+ -0.021,
+ 0.507,
+ 1.058,
+ 0.714,
+ 0.853,
+ 0.427,
+ 0.546,
+ 0.085
+ ],
+ "first_cross": null,
+ "final_z": 0.085
+ },
+ {
+ "id": "552589eb313437000c270100",
+ "variation": "other",
+ "week": 201514,
+ "impressions": 10168,
+ "pooled_ctr": 0.012293469708890637,
+ "arms": [
+ {
+ "headline": "Listen to some boys talk about their plans to be 'manly.'",
+ "impressions": 2051,
+ "clicks": 23,
+ "ctr": 0.0112140419307654,
+ "ctr_select": 0.014619883040935672,
+ "ctr_holdout": 0.007804878048780488,
+ "n_select": 1026,
+ "n_holdout": 1025,
+ "eb": 0.013073801126785543,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Listen to some boys talk about their plans to be 'manly.'",
+ "impressions": 2067,
+ "clicks": 36,
+ "ctr": 0.0174165457184325,
+ "ctr_select": 0.01160541586073501,
+ "ctr_holdout": 0.023233301064859633,
+ "n_select": 1034,
+ "n_holdout": 1033,
+ "eb": 0.012147527135929573,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Listen to some boys talk about their plans to be 'manly.'",
+ "impressions": 2045,
+ "clicks": 22,
+ "ctr": 0.0107579462102689,
+ "ctr_select": 0.012720156555772993,
+ "ctr_holdout": 0.008797653958944282,
+ "n_select": 1022,
+ "n_holdout": 1023,
+ "eb": 0.012490603194592392,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Listen to some boys talk about their plans to be 'manly.'",
+ "impressions": 1987,
+ "clicks": 20,
+ "ctr": 0.0100654252642174,
+ "ctr_select": 0.009054325955734407,
+ "ctr_holdout": 0.011077542799597181,
+ "n_select": 994,
+ "n_holdout": 993,
+ "eb": 0.011388370009044241,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Listen to some boys talk about their plans to be 'manly.'",
+ "impressions": 2018,
+ "clicks": 24,
+ "ctr": 0.0118929633300297,
+ "ctr_select": 0.013875123885034688,
+ "ctr_holdout": 0.009910802775024777,
+ "n_select": 1009,
+ "n_holdout": 1009,
+ "eb": 0.01284001211140214,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.012389380530973451,
+ "pool_holdout": 0.012197521148927798,
+ "lift_select": 0.180033416875522,
+ "lift_holdout": -0.360125885129819,
+ "eb_lift": 0.05524251951911885,
+ "p_value": 0.4697244963614622,
+ "held_up": false,
+ "z_path": [
+ 0.985,
+ 1.65,
+ 0.699,
+ 1.061,
+ 0.692,
+ 0.13,
+ -0.544,
+ -0.685,
+ -0.206,
+ -0.203
+ ],
+ "first_cross": null,
+ "final_z": -0.203
+ },
+ {
+ "id": "54fdb1f362306300264e0000",
+ "variation": "headline",
+ "week": 201510,
+ "impressions": 10161,
+ "pooled_ctr": 0.006298592658202933,
+ "arms": [
+ {
+ "headline": "The way President Obama uses the word 'we' here makes me want to stand up and cheer.",
+ "impressions": 2003,
+ "clicks": 14,
+ "ctr": 0.0069895157264103,
+ "ctr_select": 0.00499001996007984,
+ "ctr_holdout": 0.008991008991008992,
+ "n_select": 1002,
+ "n_holdout": 1001,
+ "eb": 0.00480393075371331,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Was President Obama's speech on the 50th anniversary of the march from Selma one of his best? Yes.",
+ "impressions": 2014,
+ "clicks": 6,
+ "ctr": 0.0029791459781529,
+ "ctr_select": 0.0029791459781529296,
+ "ctr_holdout": 0.0029791459781529296,
+ "n_select": 1007,
+ "n_holdout": 1007,
+ "eb": 0.004195147690234376,
+ "labelled_winner": false
+ },
+ {
+ "headline": "President Obama just said something kind of revolutionary about what it means to be an American.",
+ "impressions": 1984,
+ "clicks": 11,
+ "ctr": 0.0055443548387096,
+ "ctr_select": 0.0030241935483870967,
+ "ctr_holdout": 0.008064516129032258,
+ "n_select": 992,
+ "n_holdout": 992,
+ "eb": 0.004214160669959203,
+ "labelled_winner": false
+ },
+ {
+ "headline": "There's something powerful about hearing the first black president say these words in Selma",
+ "impressions": 2079,
+ "clicks": 24,
+ "ctr": 0.0115440115440115,
+ "ctr_select": 0.009615384615384616,
+ "ctr_holdout": 0.013474494706448507,
+ "n_select": 1040,
+ "n_holdout": 1039,
+ "eb": 0.006238678921992437,
+ "labelled_winner": true
+ },
+ {
+ "headline": "President Obama's powerful speech at Selma is just what America needed to hear.",
+ "impressions": 2081,
+ "clicks": 9,
+ "ctr": 0.004324843825084,
+ "ctr_select": 0.0028846153846153848,
+ "ctr_holdout": 0.005763688760806916,
+ "n_select": 1040,
+ "n_holdout": 1041,
+ "eb": 0.004153917095357814,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.004723479629994096,
+ "pool_holdout": 0.007874015748031496,
+ "lift_select": 1.035657051282051,
+ "lift_holdout": 0.7112608277189605,
+ "eb_lift": 0.3207803167768155,
+ "p_value": 0.009879649717886754,
+ "held_up": true,
+ "z_path": [
+ -0.042,
+ 0.662,
+ 0.521,
+ 0.771,
+ 0.656,
+ 1.199,
+ 1.087,
+ 1.52,
+ 1.314,
+ 1.515
+ ],
+ "first_cross": null,
+ "final_z": 1.515
+ },
+ {
+ "id": "54ecb3693264390024330000",
+ "variation": "headline",
+ "week": 201508,
+ "impressions": 10153,
+ "pooled_ctr": 0.014773958435930266,
+ "arms": [
+ {
+ "headline": "They held up racist signs at a basketball game, so he went on TV and gave them a piece of his mind.",
+ "impressions": 2491,
+ "clicks": 37,
+ "ctr": 0.0148534725010036,
+ "ctr_select": 0.016051364365971106,
+ "ctr_holdout": 0.013654618473895583,
+ "n_select": 1246,
+ "n_holdout": 1245,
+ "eb": 0.015221533064889072,
+ "labelled_winner": false
+ },
+ {
+ "headline": "They claimed the racist signs were an accident. He doesn't buy it for 1 second.",
+ "impressions": 2604,
+ "clicks": 35,
+ "ctr": 0.0134408602150537,
+ "ctr_select": 0.01152073732718894,
+ "ctr_holdout": 0.015360983102918587,
+ "n_select": 1302,
+ "n_holdout": 1302,
+ "eb": 0.013604711349600203,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A white TV news anchor from Texas smacks down a group of racist sports fans so perfectly",
+ "impressions": 2601,
+ "clicks": 42,
+ "ctr": 0.0161476355247981,
+ "ctr_select": 0.01769230769230769,
+ "ctr_holdout": 0.014604150653343582,
+ "n_select": 1300,
+ "n_holdout": 1301,
+ "eb": 0.015823583703809506,
+ "labelled_winner": false
+ },
+ {
+ "headline": "His story about his dad explains exactly what these racist kids did and why it's so sad.",
+ "impressions": 2457,
+ "clicks": 36,
+ "ctr": 0.0146520146520146,
+ "ctr_select": 0.013843648208469055,
+ "ctr_holdout": 0.015459723352318959,
+ "n_select": 1228,
+ "n_holdout": 1229,
+ "eb": 0.01445271049965201,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.014775413711583925,
+ "pool_holdout": 0.014772503446917471,
+ "lift_select": 0.19741538461538455,
+ "lift_holdout": -0.011396361772995078,
+ "eb_lift": 0.0709401450738274,
+ "p_value": 0.3121851920454233,
+ "held_up": false,
+ "z_path": [
+ 1.085,
+ 0.417,
+ 0.526,
+ 0.444,
+ 0.181,
+ -0.002,
+ 0.376,
+ 0.33,
+ 0.668,
+ 0.374
+ ],
+ "first_cross": null,
+ "final_z": 0.374
+ },
+ {
+ "id": "54664b956c94ae538d00000b",
+ "variation": "headline",
+ "week": 201445,
+ "impressions": 10146,
+ "pooled_ctr": 0.0035481963335304554,
+ "arms": [
+ {
+ "headline": "One Moment On A Motorcyle And All Of His Dreams Came Crashing Down \u2026 Good Thing He Had More",
+ "impressions": 2102,
+ "clicks": 5,
+ "ctr": 0.0023786869647954,
+ "ctr_select": 0.0019029495718363464,
+ "ctr_holdout": 0.0028544243577545195,
+ "n_select": 1051,
+ "n_holdout": 1051,
+ "eb": 0.0024924081156793157,
+ "labelled_winner": false
+ },
+ {
+ "headline": "He Took A Traumatic Turn That Threw His Life Way Off Course, But He Still Survived",
+ "impressions": 2010,
+ "clicks": 8,
+ "ctr": 0.0039800995024875,
+ "ctr_select": 0.0029850746268656717,
+ "ctr_holdout": 0.004975124378109453,
+ "n_select": 1005,
+ "n_holdout": 1005,
+ "eb": 0.002827873713939021,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Sometimes A Hero\u2019s Story Skips The Hollywood Ending And Sometimes That's OK.",
+ "impressions": 2035,
+ "clicks": 4,
+ "ctr": 0.0019656019656019,
+ "ctr_select": 0.0019646365422396855,
+ "ctr_holdout": 0.0019665683382497543,
+ "n_select": 1018,
+ "n_holdout": 1017,
+ "eb": 0.0025170654662849032,
+ "labelled_winner": false
+ },
+ {
+ "headline": "After Living Through What He Did, Most Would Give Up. But He's Not Like Most.",
+ "impressions": 2001,
+ "clicks": 14,
+ "ctr": 0.0069965017491254,
+ "ctr_select": 0.006,
+ "ctr_holdout": 0.007992007992007992,
+ "n_select": 1000,
+ "n_holdout": 1001,
+ "eb": 0.003736377030669639,
+ "labelled_winner": false
+ },
+ {
+ "headline": "His Story Makes Me Want To Swim, And I Haven't Used My Sea Legs For Years",
+ "impressions": 1998,
+ "clicks": 5,
+ "ctr": 0.0025025025025025,
+ "ctr_select": 0.001001001001001001,
+ "ctr_holdout": 0.004004004004004004,
+ "n_select": 999,
+ "n_holdout": 999,
+ "eb": 0.0022299799852022626,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.0027597082594125765,
+ "pool_holdout": 0.004336684407648335,
+ "lift_select": 1.174142857142857,
+ "lift_holdout": 0.8428843883389336,
+ "eb_lift": 0.3539029054705056,
+ "p_value": 0.029268122693604133,
+ "held_up": true,
+ "z_path": [
+ -1.001,
+ -0.576,
+ -0.998,
+ -0.444,
+ 0.006,
+ 0.006,
+ 0.913,
+ 1.615,
+ 1.225,
+ 1.293
+ ],
+ "first_cross": null,
+ "final_z": 1.293
+ },
+ {
+ "id": "53663b77d5854b2e3f000037",
+ "variation": "image",
+ "week": 201418,
+ "impressions": 10068,
+ "pooled_ctr": 0.03675009932459277,
+ "arms": [
+ {
+ "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day",
+ "impressions": 2051,
+ "clicks": 77,
+ "ctr": 0.0375426621160409,
+ "ctr_select": 0.03898635477582846,
+ "ctr_holdout": 0.03609756097560975,
+ "n_select": 1026,
+ "n_holdout": 1025,
+ "eb": 0.038813230120448954,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day",
+ "impressions": 2028,
+ "clicks": 70,
+ "ctr": 0.034516765285996,
+ "ctr_select": 0.03944773175542406,
+ "ctr_holdout": 0.029585798816568046,
+ "n_select": 1014,
+ "n_holdout": 1014,
+ "eb": 0.03895302634529779,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day",
+ "impressions": 1917,
+ "clicks": 73,
+ "ctr": 0.0380803338549817,
+ "ctr_select": 0.03966597077244259,
+ "ctr_holdout": 0.0364963503649635,
+ "n_select": 958,
+ "n_holdout": 959,
+ "eb": 0.03900839454214944,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day",
+ "impressions": 2083,
+ "clicks": 72,
+ "ctr": 0.0345655304848775,
+ "ctr_select": 0.0345489443378119,
+ "ctr_holdout": 0.0345821325648415,
+ "n_select": 1042,
+ "n_holdout": 1041,
+ "eb": 0.03743780527923385,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day",
+ "impressions": 1989,
+ "clicks": 78,
+ "ctr": 0.0392156862745098,
+ "ctr_select": 0.04124748490945674,
+ "ctr_holdout": 0.0371859296482412,
+ "n_select": 994,
+ "n_holdout": 995,
+ "eb": 0.0394902313336687,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 4,
+ "pool_select": 0.03873659117997616,
+ "pool_holdout": 0.034763607469209375,
+ "lift_select": 0.06481968735489874,
+ "lift_holdout": 0.06967982770997838,
+ "eb_lift": 0.019455510429170486,
+ "p_value": 0.6469978497298443,
+ "held_up": true,
+ "z_path": [
+ -0.057,
+ -0.891,
+ -0.129,
+ 0.974,
+ 0.608,
+ 0.625,
+ 0.724,
+ 0.883,
+ 0.21,
+ 0.184
+ ],
+ "first_cross": null,
+ "final_z": 0.184
+ },
+ {
+ "id": "530f12bdfa892222b800001a",
+ "variation": "headline",
+ "week": 201408,
+ "impressions": 10059,
+ "pooled_ctr": 0.016104980614375188,
+ "arms": [
+ {
+ "headline": "A Late Night Host Got Left Off The List Of Late Night Hosts, So He Decided Call Them Out For It",
+ "impressions": 2092,
+ "clicks": 32,
+ "ctr": 0.0152963671128107,
+ "ctr_select": 0.01338432122370937,
+ "ctr_holdout": 0.017208413001912046,
+ "n_select": 1046,
+ "n_holdout": 1046,
+ "eb": 0.014300964695494172,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Why A New List Of All White Late Night Hosts Rubbed Arsenio \u2013 A Late Night Host \u2013 The Wrong Way.",
+ "impressions": 2044,
+ "clicks": 30,
+ "ctr": 0.0146771037181996,
+ "ctr_select": 0.019569471624266144,
+ "ctr_holdout": 0.009784735812133072,
+ "n_select": 1022,
+ "n_holdout": 1022,
+ "eb": 0.016200305208825663,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If You're Going To Leave Someone Off The List Of Late Night Hosts, Next Time Don't Let It Be Arsenio",
+ "impressions": 1994,
+ "clicks": 23,
+ "ctr": 0.0115346038114343,
+ "ctr_select": 0.00802407221664995,
+ "ctr_holdout": 0.015045135406218655,
+ "n_select": 997,
+ "n_holdout": 997,
+ "eb": 0.012702250965879526,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Late Night Host Has Some Choice Words For An Article That Conveniently Left Him Off The List",
+ "impressions": 2000,
+ "clicks": 37,
+ "ctr": 0.0185,
+ "ctr_select": 0.018,
+ "ctr_holdout": 0.019,
+ "n_select": 1000,
+ "n_holdout": 1000,
+ "eb": 0.01570490344780625,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How A Late Night Host Used His Monologue To Shut Down A Racially Biased Article About Late Night",
+ "impressions": 1929,
+ "clicks": 40,
+ "ctr": 0.0207361327112493,
+ "ctr_select": 0.014522821576763486,
+ "ctr_holdout": 0.02694300518134715,
+ "n_select": 964,
+ "n_holdout": 965,
+ "eb": 0.014658303856677286,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.014714655000994234,
+ "pool_holdout": 0.01749502982107356,
+ "lift_select": 0.32993071349235725,
+ "lift_holdout": -0.44071339619284833,
+ "eb_lift": 0.10096398507005744,
+ "p_value": 0.14873319729001686,
+ "held_up": false,
+ "z_path": [
+ 0.974,
+ 0.539,
+ -0.061,
+ -0.066,
+ -1.056,
+ -0.538,
+ -0.937,
+ -0.906,
+ -0.614,
+ -0.952
+ ],
+ "first_cross": null,
+ "final_z": -0.952
+ },
+ {
+ "id": "54db5928366238000c800000",
+ "variation": "headline",
+ "week": 201506,
+ "impressions": 9755,
+ "pooled_ctr": 0.009841107124551511,
+ "arms": [
+ {
+ "headline": "Ever Been In A Forest And Thought You Heard Whispering? Maybe It Wasn't Your Head.",
+ "impressions": 2516,
+ "clicks": 35,
+ "ctr": 0.0139109697933227,
+ "ctr_select": 0.014308426073131956,
+ "ctr_holdout": 0.013513513513513514,
+ "n_select": 1258,
+ "n_holdout": 1258,
+ "eb": 0.011810778009823174,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Ever Feel Like Trees Know Something You Don't? Maybe It's Not Just In Your Head.",
+ "impressions": 2444,
+ "clicks": 13,
+ "ctr": 0.0053191489361702,
+ "ctr_select": 0.0057283142389525366,
+ "ctr_holdout": 0.004909983633387889,
+ "n_select": 1222,
+ "n_holdout": 1222,
+ "eb": 0.008823287869690064,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Ever Been In A Forest And Felt Like Something Was Watching You? Maybe It Wasn't Your Imagination.",
+ "impressions": 2363,
+ "clicks": 20,
+ "ctr": 0.0084638171815488,
+ "ctr_select": 0.009306260575296108,
+ "ctr_holdout": 0.007620660457239628,
+ "n_select": 1182,
+ "n_holdout": 1181,
+ "eb": 0.01006709001552215,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Ever Heard Trees Talking? Maybe That Wasn't Just Your Imagination.",
+ "impressions": 2432,
+ "clicks": 28,
+ "ctr": 0.0115131578947368,
+ "ctr_select": 0.012335526315789474,
+ "ctr_holdout": 0.01069078947368421,
+ "n_select": 1216,
+ "n_holdout": 1216,
+ "eb": 0.011102186745691073,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.010455104551045511,
+ "pool_holdout": 0.009226983801517326,
+ "lift_select": 0.36855887028897416,
+ "lift_holdout": 0.46456456456456463,
+ "eb_lift": 0.12966617905720468,
+ "p_value": 0.11881451760004201,
+ "held_up": true,
+ "z_path": [
+ -0.032,
+ -0.599,
+ -0.767,
+ -0.088,
+ 0.748,
+ 0.218,
+ 0.508,
+ 0.885,
+ 0.537,
+ 0.752
+ ],
+ "first_cross": null,
+ "final_z": 0.752
+ },
+ {
+ "id": "5227cdeb6886ffa1810001ca",
+ "variation": "headline",
+ "week": 201335,
+ "impressions": 9644,
+ "pooled_ctr": 0.012339278307756118,
+ "arms": [
+ {
+ "headline": "All They Wanted To Do Was Play The Game. There's Nothing Fun About What's Happening Here, Though.",
+ "impressions": 2325,
+ "clicks": 28,
+ "ctr": 0.0120430107526881,
+ "ctr_select": 0.00946643717728055,
+ "ctr_holdout": 0.014617368873602751,
+ "n_select": 1162,
+ "n_holdout": 1163,
+ "eb": 0.010482085088237829,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Important Thing To Happen In Football Has Nothing To Do With Touchdowns",
+ "impressions": 2239,
+ "clicks": 26,
+ "ctr": 0.0116123269316659,
+ "ctr_select": 0.009821428571428571,
+ "ctr_holdout": 0.013404825737265416,
+ "n_select": 1120,
+ "n_holdout": 1119,
+ "eb": 0.010610149744159296,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's Time We All Pay Attention To The Most Important Thing In Football \u2014 And It's Not The Score",
+ "impressions": 2423,
+ "clicks": 24,
+ "ctr": 0.0099050763516302,
+ "ctr_select": 0.007425742574257425,
+ "ctr_holdout": 0.012386457473162676,
+ "n_select": 1212,
+ "n_holdout": 1211,
+ "eb": 0.009766980151319866,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Sports Commentator Proves That A 'Maybe' Does Not A Job Make",
+ "impressions": 2657,
+ "clicks": 41,
+ "ctr": 0.0154309371471584,
+ "ctr_select": 0.016566265060240965,
+ "ctr_holdout": 0.014296463506395787,
+ "n_select": 1328,
+ "n_holdout": 1329,
+ "eb": 0.013022058126480104,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.010991289921194525,
+ "pool_holdout": 0.013687266694317711,
+ "lift_select": 0.5072175494430553,
+ "lift_holdout": 0.044508288300613374,
+ "eb_lift": 0.18476159029975592,
+ "p_value": 0.022072021193859894,
+ "held_up": false,
+ "z_path": [
+ -1.893,
+ 1.449,
+ 2.172,
+ 1.385,
+ 1.151,
+ 1.31,
+ 1.675,
+ 1.566,
+ 1.527,
+ 1.146
+ ],
+ "first_cross": 2,
+ "final_z": 1.146
+ },
+ {
+ "id": "531d5e30365e2adfdb000020",
+ "variation": "image",
+ "week": 201410,
+ "impressions": 9243,
+ "pooled_ctr": 0.013199177756139782,
+ "arms": [
+ {
+ "headline": "Have Our Beauty Standards Gotten So Bad That They Affect Even Our Fruits And Vegetables?",
+ "impressions": 2225,
+ "clicks": 37,
+ "ctr": 0.016629213483146,
+ "ctr_select": 0.017985611510791366,
+ "ctr_holdout": 0.015274034141958671,
+ "n_select": 1112,
+ "n_holdout": 1113,
+ "eb": 0.015042518979300038,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Have Our Beauty Standards Gotten So Bad That They Affect Even Our Fruits And Vegetables?",
+ "impressions": 2319,
+ "clicks": 22,
+ "ctr": 0.0094868477792151,
+ "ctr_select": 0.010344827586206896,
+ "ctr_holdout": 0.008628127696289905,
+ "n_select": 1160,
+ "n_holdout": 1159,
+ "eb": 0.01253452614847292,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Have Our Beauty Standards Gotten So Bad That They Affect Even Our Fruits And Vegetables?",
+ "impressions": 2276,
+ "clicks": 31,
+ "ctr": 0.0136203866432337,
+ "ctr_select": 0.014059753954305799,
+ "ctr_holdout": 0.013181019332161687,
+ "n_select": 1138,
+ "n_holdout": 1138,
+ "eb": 0.013771833326566804,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Have Our Beauty Standards Gotten So Bad That They Affect Even Our Fruits And Vegetables?",
+ "impressions": 2423,
+ "clicks": 32,
+ "ctr": 0.0132067684688402,
+ "ctr_select": 0.012376237623762377,
+ "ctr_holdout": 0.014037985136251032,
+ "n_select": 1212,
+ "n_holdout": 1211,
+ "eb": 0.013199796898768741,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.013630463003028993,
+ "pool_holdout": 0.012767799177667172,
+ "lift_select": 0.3195158159186935,
+ "lift_holdout": 0.1962934198303561,
+ "eb_lift": 0.10359559876705982,
+ "p_value": 0.15063820243661535,
+ "held_up": true,
+ "z_path": [
+ 1.027,
+ 1.437,
+ 0.29,
+ 0.44,
+ -0.104,
+ -0.083,
+ 0.788,
+ 1.163,
+ 0.734,
+ 0.827
+ ],
+ "first_cross": null,
+ "final_z": 0.827
+ },
+ {
+ "id": "52b04a2aa3623369cb0046c7",
+ "variation": "headline",
+ "week": 201350,
+ "impressions": 9152,
+ "pooled_ctr": 0.00590034965034965,
+ "arms": [
+ {
+ "headline": "The Most Important You Can Ask Today: What Will Your Last 10 Years Will Look Like?",
+ "impressions": 2202,
+ "clicks": 8,
+ "ctr": 0.0036330608537693,
+ "ctr_select": 0.0027247956403269754,
+ "ctr_holdout": 0.004541326067211626,
+ "n_select": 1101,
+ "n_holdout": 1101,
+ "eb": 0.004433973254697844,
+ "labelled_winner": false
+ },
+ {
+ "headline": "We Like To Imagine The Next 10 Years. Maybe We Should Ask What The Last 10 Will Look Like.",
+ "impressions": 2381,
+ "clicks": 9,
+ "ctr": 0.0037799244015119,
+ "ctr_select": 0.0033613445378151263,
+ "ctr_holdout": 0.0041981528127623844,
+ "n_select": 1190,
+ "n_holdout": 1191,
+ "eb": 0.004606558360743928,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Scary Ad Shows Two Perspectives On Living Your Last 10 Years. Which Describes Where You're Headed?",
+ "impressions": 2248,
+ "clicks": 23,
+ "ctr": 0.0102313167259786,
+ "ctr_select": 0.008896797153024912,
+ "ctr_holdout": 0.011565836298932384,
+ "n_select": 1124,
+ "n_holdout": 1124,
+ "eb": 0.006438222089757918,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Which Side Of This Ad Describes What Your Last 10 Years Will Look Like?",
+ "impressions": 2321,
+ "clicks": 14,
+ "ctr": 0.0060318828091339,
+ "ctr_select": 0.00603448275862069,
+ "ctr_holdout": 0.006029285099052541,
+ "n_select": 1160,
+ "n_holdout": 1161,
+ "eb": 0.005508936030499569,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.005245901639344262,
+ "pool_holdout": 0.006554511688879178,
+ "lift_select": 0.6959519572953738,
+ "lift_holdout": 0.7645610913404506,
+ "eb_lift": 0.22728608586010313,
+ "p_value": 0.051067510214896146,
+ "held_up": true,
+ "z_path": [
+ 0.61,
+ 0.86,
+ 0.422,
+ 0.685,
+ 1.875,
+ 2.141,
+ 1.959,
+ 2.286,
+ 1.315,
+ 1.583
+ ],
+ "first_cross": 5,
+ "final_z": 1.583
+ },
+ {
+ "id": "54f8e0603737324288250100",
+ "variation": "other",
+ "week": 201509,
+ "impressions": 9137,
+ "pooled_ctr": 0.011929517347050455,
+ "arms": [
+ {
+ "headline": "It's a pop song like no other. But not every woman is allowed to hear it.",
+ "impressions": 1796,
+ "clicks": 32,
+ "ctr": 0.0178173719376391,
+ "ctr_select": 0.01670378619153675,
+ "ctr_holdout": 0.01893095768374165,
+ "n_select": 898,
+ "n_holdout": 898,
+ "eb": 0.012867448501116123,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's a pop song like no other. But not every woman is allowed to hear it.",
+ "impressions": 1812,
+ "clicks": 15,
+ "ctr": 0.0082781456953642,
+ "ctr_select": 0.009933774834437087,
+ "ctr_holdout": 0.006622516556291391,
+ "n_select": 906,
+ "n_holdout": 906,
+ "eb": 0.010974299655541747,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's a pop song like no other. But not every woman is allowed to hear it.",
+ "impressions": 1868,
+ "clicks": 26,
+ "ctr": 0.0139186295503211,
+ "ctr_select": 0.010706638115631691,
+ "ctr_holdout": 0.017130620985010708,
+ "n_select": 934,
+ "n_holdout": 934,
+ "eb": 0.011187332868517726,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's a pop song like no other. But not every woman is allowed to hear it.",
+ "impressions": 1835,
+ "clicks": 20,
+ "ctr": 0.0108991825613079,
+ "ctr_select": 0.011982570806100218,
+ "ctr_holdout": 0.009814612868047983,
+ "n_select": 918,
+ "n_holdout": 917,
+ "eb": 0.011551704770196836,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's a pop song like no other. But not every woman is allowed to hear it.",
+ "impressions": 1826,
+ "clicks": 16,
+ "ctr": 0.008762322015334,
+ "ctr_select": 0.007667031763417305,
+ "ctr_holdout": 0.009857612267250822,
+ "n_select": 913,
+ "n_holdout": 913,
+ "eb": 0.010331460155389886,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.011381046180783541,
+ "pool_holdout": 0.012478108581436076,
+ "lift_select": 0.46768459825252706,
+ "lift_holdout": 0.5171335912163484,
+ "eb_lift": 0.13060331156922245,
+ "p_value": 0.09342684866125113,
+ "held_up": true,
+ "z_path": [
+ 0.411,
+ 0.948,
+ 1.655,
+ 2.029,
+ 2.344,
+ 2.487,
+ 2.276,
+ 1.979,
+ 2.111,
+ 1.754
+ ],
+ "first_cross": 3,
+ "final_z": 1.754
+ },
+ {
+ "id": "54d3a1753139330019030000",
+ "variation": "both",
+ "week": 201505,
+ "impressions": 9136,
+ "pooled_ctr": 0.011711908931698774,
+ "arms": [
+ {
+ "headline": "Watch Men Learn What Feminism Means And Then Realize Something Interesting",
+ "impressions": 1566,
+ "clicks": 18,
+ "ctr": 0.0114942528735632,
+ "ctr_select": 0.010217113665389528,
+ "ctr_holdout": 0.01277139208173691,
+ "n_select": 783,
+ "n_holdout": 783,
+ "eb": 0.01109085095490348,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch Men Learn What Feminism Means And Then Realize Something Obvious",
+ "impressions": 1453,
+ "clicks": 22,
+ "ctr": 0.0151410874053682,
+ "ctr_select": 0.01790633608815427,
+ "ctr_holdout": 0.012379642365887207,
+ "n_select": 726,
+ "n_holdout": 727,
+ "eb": 0.012941290469923076,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Watch Men Learn What Feminism Means And Then Realize Something Obvious",
+ "impressions": 1598,
+ "clicks": 16,
+ "ctr": 0.0100125156445556,
+ "ctr_select": 0.008760951188986232,
+ "ctr_holdout": 0.011264080100125156,
+ "n_select": 799,
+ "n_holdout": 799,
+ "eb": 0.010713062113358878,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch Men Learn What Feminism Means And Then Realize Something",
+ "impressions": 1481,
+ "clicks": 17,
+ "ctr": 0.0114787305874409,
+ "ctr_select": 0.008108108108108109,
+ "ctr_holdout": 0.014844804318488529,
+ "n_select": 740,
+ "n_holdout": 741,
+ "eb": 0.010592733234950068,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch Men Learn What Feminism Means And Then Realize Something Interesting",
+ "impressions": 1522,
+ "clicks": 17,
+ "ctr": 0.0111695137976346,
+ "ctr_select": 0.011826544021024968,
+ "ctr_holdout": 0.010512483574244415,
+ "n_select": 761,
+ "n_holdout": 761,
+ "eb": 0.011494917492236901,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Watch Men Learn What Feminism Means And Then Realize Something Interesting",
+ "impressions": 1516,
+ "clicks": 17,
+ "ctr": 0.0112137203166226,
+ "ctr_select": 0.011873350923482849,
+ "ctr_holdout": 0.010554089709762533,
+ "n_select": 758,
+ "n_holdout": 758,
+ "eb": 0.011506129501929776,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.011386030216772498,
+ "pool_holdout": 0.012037644998905668,
+ "lift_select": 0.572658402203857,
+ "lift_holdout": 0.028410653995248225,
+ "eb_lift": 0.1365937226180518,
+ "p_value": 0.07097528747784294,
+ "held_up": false,
+ "z_path": [
+ 0.029,
+ 0.87,
+ 0.07,
+ 1.092,
+ 0.787,
+ 1.298,
+ 0.664,
+ 0.298,
+ 0.627,
+ 0.94
+ ],
+ "first_cross": null,
+ "final_z": 0.94
+ },
+ {
+ "id": "52d815dc392cef894b0021c6",
+ "variation": "headline",
+ "week": 201402,
+ "impressions": 8932,
+ "pooled_ctr": 0.03884908195253023,
+ "arms": [
+ {
+ "headline": "To The Men That Mistreated These Girls (And Their Mothers): You. Just. Got. Burned.",
+ "impressions": 4433,
+ "clicks": 170,
+ "ctr": 0.0383487480261673,
+ "ctr_select": 0.037454873646209384,
+ "ctr_holdout": 0.03924221921515562,
+ "n_select": 2216,
+ "n_holdout": 2217,
+ "eb": 0.03811038071424032,
+ "labelled_winner": false
+ },
+ {
+ "headline": "To The Men That Mistreated These Girls (And Their Mothers): You. Just. Got. Told.",
+ "impressions": 4499,
+ "clicks": 177,
+ "ctr": 0.0393420760168926,
+ "ctr_select": 0.04,
+ "ctr_holdout": 0.03868385949310805,
+ "n_select": 2250,
+ "n_holdout": 2249,
+ "eb": 0.0393592039561831,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.0387371249440215,
+ "pool_holdout": 0.03896103896103896,
+ "lift_select": 0.0326011560693642,
+ "lift_holdout": -0.007114273010226713,
+ "eb_lift": 0.016058987678113867,
+ "p_value": 0.6594314470492292,
+ "held_up": false,
+ "z_path": [
+ -1.152,
+ 0.3,
+ 0.116,
+ -0.171,
+ 0.284,
+ -0.178,
+ -0.054,
+ -0.065,
+ -0.017,
+ 0.243
+ ],
+ "first_cross": null,
+ "final_z": 0.243
+ },
+ {
+ "id": "550975176133340018390000",
+ "variation": "headline",
+ "week": 201511,
+ "impressions": 8894,
+ "pooled_ctr": 0.008882392624241061,
+ "arms": [
+ {
+ "headline": "Her name was 'Essence.' This is a little story about making it felt all around her.",
+ "impressions": 2197,
+ "clicks": 27,
+ "ctr": 0.0122894856622667,
+ "ctr_select": 0.012750455373406194,
+ "ctr_holdout": 0.011828935395814377,
+ "n_select": 1098,
+ "n_holdout": 1099,
+ "eb": 0.010049516325297993,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her name was 'Essence.' This is a story about making that felt all around her after she was gone.",
+ "impressions": 2246,
+ "clicks": 16,
+ "ctr": 0.0071237756010685,
+ "ctr_select": 0.007123775601068566,
+ "ctr_holdout": 0.007123775601068566,
+ "n_select": 1123,
+ "n_holdout": 1123,
+ "eb": 0.008232664499078892,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her name was 'Essence.' This is a story about her impact on the world, through her family.",
+ "impressions": 2251,
+ "clicks": 17,
+ "ctr": 0.0075521990226565,
+ "ctr_select": 0.004440497335701598,
+ "ctr_holdout": 0.010666666666666666,
+ "n_select": 1126,
+ "n_holdout": 1125,
+ "eb": 0.007354335954321053,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Her name was 'Essence,' and how she's being remembered is remarkable and beautiful.",
+ "impressions": 2200,
+ "clicks": 19,
+ "ctr": 0.0086363636363636,
+ "ctr_select": 0.01090909090909091,
+ "ctr_holdout": 0.006363636363636364,
+ "n_select": 1100,
+ "n_holdout": 1100,
+ "eb": 0.009458446193521423,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.008769957274567124,
+ "pool_holdout": 0.008994827973915,
+ "lift_select": 0.4538788473214703,
+ "lift_holdout": 0.31508189262966324,
+ "eb_lift": 0.1459025409897481,
+ "p_value": 0.10306896837806057,
+ "held_up": true,
+ "z_path": [
+ 0.45,
+ -0.379,
+ -0.503,
+ -0.231,
+ 0.0,
+ -0.201,
+ 0.0,
+ 0.177,
+ 0.655,
+ 1.19
+ ],
+ "first_cross": null,
+ "final_z": 1.19
+ },
+ {
+ "id": "54997b38666230001ea80000",
+ "variation": "image",
+ "week": 201451,
+ "impressions": 8528,
+ "pooled_ctr": 0.014657598499061914,
+ "arms": [
+ {
+ "headline": "Your Doctor Asks You This Question All The Time. Why Can't You Ever Answer It?",
+ "impressions": 2890,
+ "clicks": 39,
+ "ctr": 0.0134948096885813,
+ "ctr_select": 0.010380622837370242,
+ "ctr_holdout": 0.01660899653979239,
+ "n_select": 1445,
+ "n_holdout": 1445,
+ "eb": 0.012942870025825295,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Your Doctor Asks You This Question All The Time. Why Can't You Ever Answer It?",
+ "impressions": 2772,
+ "clicks": 41,
+ "ctr": 0.0147907647907647,
+ "ctr_select": 0.016594516594516596,
+ "ctr_holdout": 0.012987012987012988,
+ "n_select": 1386,
+ "n_holdout": 1386,
+ "eb": 0.015309053777959929,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Your Doctor Asks You This Question All The Time. Why Can't You Ever Answer It?",
+ "impressions": 2866,
+ "clicks": 45,
+ "ctr": 0.0157013258897418,
+ "ctr_select": 0.01674808094905792,
+ "ctr_holdout": 0.01465457083042568,
+ "n_select": 1433,
+ "n_holdout": 1433,
+ "eb": 0.01538383305758285,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.014540337711069419,
+ "pool_holdout": 0.01477485928705441,
+ "lift_select": 0.15183576075456418,
+ "lift_holdout": -0.008141428239125448,
+ "eb_lift": 0.05801071221827847,
+ "p_value": 0.3915290449871476,
+ "held_up": false,
+ "z_path": [
+ 1.099,
+ 1.337,
+ 1.461,
+ 0.703,
+ -0.103,
+ -0.265,
+ -0.666,
+ -0.134,
+ 0.445,
+ 0.279
+ ],
+ "first_cross": null,
+ "final_z": 0.279
+ },
+ {
+ "id": "53062f508fa4ee627e000018",
+ "variation": "other",
+ "week": 201407,
+ "impressions": 8481,
+ "pooled_ctr": 0.010729866760995166,
+ "arms": [
+ {
+ "headline": "Meet The Woman Who Altered The Course Of Her Life To Work With Malala...And To Change The World",
+ "impressions": 2085,
+ "clicks": 22,
+ "ctr": 0.0105515587529976,
+ "ctr_select": 0.01055662188099808,
+ "ctr_holdout": 0.010546500479386385,
+ "n_select": 1042,
+ "n_holdout": 1043,
+ "eb": 0.010109166145989425,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Meet The Woman Who Altered The Course Of Her Life To Work With Malala...And To Change The World",
+ "impressions": 2161,
+ "clicks": 22,
+ "ctr": 0.0101804720037019,
+ "ctr_select": 0.012037037037037037,
+ "ctr_holdout": 0.008325624421831638,
+ "n_select": 1080,
+ "n_holdout": 1081,
+ "eb": 0.010584738673895167,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Meet The Woman Who Altered The Course Of Her Life To Work With Malala...And To Change The World",
+ "impressions": 2154,
+ "clicks": 22,
+ "ctr": 0.0102135561745589,
+ "ctr_select": 0.005571030640668524,
+ "ctr_holdout": 0.014856081708449397,
+ "n_select": 1077,
+ "n_holdout": 1077,
+ "eb": 0.0085320534599906,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Meet The Woman Who Altered The Course Of Her Life To Work With Malala...And To Change The World",
+ "impressions": 2081,
+ "clicks": 25,
+ "ctr": 0.012013455069678,
+ "ctr_select": 0.011538461538461539,
+ "ctr_holdout": 0.012487992315081652,
+ "n_select": 1040,
+ "n_holdout": 1041,
+ "eb": 0.010413010750845877,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.009907997169143666,
+ "pool_holdout": 0.01155115511551155,
+ "lift_select": 0.21488095238095228,
+ "lift_holdout": -0.27923880005286106,
+ "eb_lift": 0.06830255330099066,
+ "p_value": 0.413175899839451,
+ "held_up": false,
+ "z_path": [
+ 0.967,
+ -0.054,
+ -0.646,
+ 0.148,
+ -0.306,
+ -0.698,
+ -0.471,
+ -0.455,
+ -0.438,
+ -0.57
+ ],
+ "first_cross": null,
+ "final_z": -0.57
+ },
+ {
+ "id": "5342bdf93ce9c40360000024",
+ "variation": "headline",
+ "week": 201417,
+ "impressions": 8465,
+ "pooled_ctr": 0.010277613703484938,
+ "arms": [
+ {
+ "headline": "Scientists Studying The 'Big Bang' Have Discovered Some Bananas-Crazy Information",
+ "impressions": 2080,
+ "clicks": 26,
+ "ctr": 0.0125,
+ "ctr_select": 0.010576923076923078,
+ "ctr_holdout": 0.014423076923076924,
+ "n_select": 1040,
+ "n_holdout": 1040,
+ "eb": 0.01045101629195058,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Either Scientists Are Just Messing With Me, Or The 'Big Bang' Is Super Crazy To Think About",
+ "impressions": 2144,
+ "clicks": 14,
+ "ctr": 0.0065298507462686,
+ "ctr_select": 0.0065298507462686565,
+ "ctr_holdout": 0.0065298507462686565,
+ "n_select": 1072,
+ "n_holdout": 1072,
+ "eb": 0.00917230860205343,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Yeah, But How Much Do We Really Know About The 'Big Bang' Theory? Oh, A Lot? Oh. Crazy.",
+ "impressions": 2099,
+ "clicks": 25,
+ "ctr": 0.0119104335397808,
+ "ctr_select": 0.01619047619047619,
+ "ctr_holdout": 0.0076263107721639654,
+ "n_select": 1050,
+ "n_holdout": 1049,
+ "eb": 0.012201615767956973,
+ "labelled_winner": false
+ },
+ {
+ "headline": "The Most Bad-Assed Scientists Take On The Most Bad-Assed Problem",
+ "impressions": 2142,
+ "clicks": 22,
+ "ctr": 0.0102707749766573,
+ "ctr_select": 0.008403361344537815,
+ "ctr_holdout": 0.012138188608776844,
+ "n_select": 1071,
+ "n_holdout": 1071,
+ "eb": 0.009765212568038488,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.010394519253484526,
+ "pool_holdout": 0.010160680529300567,
+ "lift_select": 0.5575974025974024,
+ "lift_holdout": -0.24942913516749066,
+ "eb_lift": 0.17385089876731508,
+ "p_value": 0.032723083563681477,
+ "held_up": false,
+ "z_path": [
+ 0.571,
+ 0.991,
+ 0.563,
+ 0.224,
+ -0.68,
+ -0.651,
+ -0.605,
+ -0.027,
+ -0.502,
+ -0.174
+ ],
+ "first_cross": null,
+ "final_z": -0.174
+ },
+ {
+ "id": "5527349366386200182e0000",
+ "variation": "headline",
+ "week": 201514,
+ "impressions": 8148,
+ "pooled_ctr": 0.005031909671084929,
+ "arms": [
+ {
+ "headline": "Considering the perspective of people who don't believe in climate change can help you reach them.",
+ "impressions": 2050,
+ "clicks": 5,
+ "ctr": 0.0024390243902439,
+ "ctr_select": 0.001951219512195122,
+ "ctr_holdout": 0.002926829268292683,
+ "n_select": 1025,
+ "n_holdout": 1025,
+ "eb": 0.0036625238927699266,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These five steps can help you through tough discussions. Let's use climate change as an example.",
+ "impressions": 2085,
+ "clicks": 17,
+ "ctr": 0.0081534772182254,
+ "ctr_select": 0.008637236084452975,
+ "ctr_holdout": 0.007670182166826462,
+ "n_select": 1042,
+ "n_holdout": 1043,
+ "eb": 0.00572751238392299,
+ "labelled_winner": false
+ },
+ {
+ "headline": "If facts don't change peoples minds, what can? A step-by-step guide to talking about tough topics.",
+ "impressions": 2025,
+ "clicks": 12,
+ "ctr": 0.0059259259259259,
+ "ctr_select": 0.005928853754940711,
+ "ctr_holdout": 0.005923000987166831,
+ "n_select": 1012,
+ "n_holdout": 1013,
+ "eb": 0.004878133587599783,
+ "labelled_winner": false
+ },
+ {
+ "headline": "How to help folks who don't believe in climate change discover why it's good for them if they do.",
+ "impressions": 1988,
+ "clicks": 7,
+ "ctr": 0.0035211267605633,
+ "ctr_select": 0.001006036217303823,
+ "ctr_holdout": 0.006036217303822937,
+ "n_select": 994,
+ "n_holdout": 994,
+ "eb": 0.0033948479638698705,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.00441934691873312,
+ "pool_holdout": 0.005644171779141104,
+ "lift_select": 0.954414587332054,
+ "lift_holdout": 0.3589561882529493,
+ "eb_lift": 0.2960087744287967,
+ "p_value": 0.017338648457058986,
+ "held_up": true,
+ "z_path": [
+ -0.605,
+ -0.029,
+ 1.099,
+ 2.069,
+ 1.555,
+ 1.447,
+ 1.643,
+ 0.787,
+ 0.976,
+ 0.853
+ ],
+ "first_cross": 3,
+ "final_z": 0.853
+ },
+ {
+ "id": "552e7acc346134000cf40000",
+ "variation": "headline",
+ "week": 201515,
+ "impressions": 8119,
+ "pooled_ctr": 0.007390072669047913,
+ "arms": [
+ {
+ "headline": "Why are people out standing in the street in Germany, Finland, Brazil, Japan, and the U.S. today?",
+ "impressions": 2031,
+ "clicks": 15,
+ "ctr": 0.0073855243722304,
+ "ctr_select": 0.005905511811023622,
+ "ctr_holdout": 0.008866995073891626,
+ "n_select": 1016,
+ "n_holdout": 1015,
+ "eb": 0.006595697029212951,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A massive, global protest is going down today. You should know why.",
+ "impressions": 2045,
+ "clicks": 10,
+ "ctr": 0.0048899755501222,
+ "ctr_select": 0.0019569471624266144,
+ "ctr_holdout": 0.007820136852394917,
+ "n_select": 1022,
+ "n_holdout": 1023,
+ "eb": 0.005386134206634738,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Imagine working 60 hours a week and not being able to pay your bills. Now image how pissed you'd be.",
+ "impressions": 2066,
+ "clicks": 14,
+ "ctr": 0.0067763794772507,
+ "ctr_select": 0.007744433688286544,
+ "ctr_holdout": 0.005808325266214908,
+ "n_select": 1033,
+ "n_holdout": 1033,
+ "eb": 0.007159123908305138,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These photos from around the world should scare the pants off of McDonald's and Walmart.",
+ "impressions": 1977,
+ "clicks": 21,
+ "ctr": 0.0106221547799696,
+ "ctr_select": 0.012145748987854251,
+ "ctr_holdout": 0.00910010111223458,
+ "n_select": 988,
+ "n_holdout": 989,
+ "eb": 0.0084666120581788,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 3,
+ "pool_select": 0.006898250800689825,
+ "pool_holdout": 0.007881773399014778,
+ "lift_select": 0.7606998264893001,
+ "lift_holdout": 0.15457532861476242,
+ "eb_lift": 0.2273563694338483,
+ "p_value": 0.02196068790332485,
+ "held_up": true,
+ "z_path": [
+ -0.541,
+ -0.399,
+ 0.063,
+ 0.705,
+ 1.156,
+ 0.822,
+ 1.338,
+ 1.052,
+ 0.664,
+ 1.319
+ ],
+ "first_cross": null,
+ "final_z": 1.319
+ },
+ {
+ "id": "5318a8133d3a7922dd00006e",
+ "variation": "image",
+ "week": 201409,
+ "impressions": 8108,
+ "pooled_ctr": 0.051060680809077455,
+ "arms": [
+ {
+ "headline": "Watch A Powerful Twist Happen As This Girl Grows Up A Second-A-Day",
+ "impressions": 4027,
+ "clicks": 210,
+ "ctr": 0.0521480009932952,
+ "ctr_select": 0.05114200595829196,
+ "ctr_holdout": 0.053154495777446596,
+ "n_select": 2014,
+ "n_holdout": 2013,
+ "eb": 0.05109849250825092,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Watch A Powerful Twist Happen As This Girl Grows Up A Second-A-Day",
+ "impressions": 4081,
+ "clicks": 204,
+ "ctr": 0.0499877481009556,
+ "ctr_select": 0.050980392156862744,
+ "ctr_holdout": 0.04899559039686428,
+ "n_select": 2040,
+ "n_holdout": 2041,
+ "eb": 0.051023094711642604,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.051060680809077455,
+ "pool_holdout": 0.051060680809077455,
+ "lift_select": 0.0015927157242299206,
+ "lift_holdout": 0.04100640522593468,
+ "eb_lift": 0.0007405247751171018,
+ "p_value": 0.9813525526912157,
+ "held_up": true,
+ "z_path": [
+ -2.452,
+ -2.509,
+ -1.906,
+ -1.731,
+ -1.302,
+ -0.669,
+ -0.426,
+ -0.162,
+ 0.183,
+ 0.442
+ ],
+ "first_cross": 0,
+ "final_z": 0.442
+ },
+ {
+ "id": "54f77c1f3135360026660100",
+ "variation": "image",
+ "week": 201509,
+ "impressions": 7653,
+ "pooled_ctr": 0.007578727296485038,
+ "arms": [
+ {
+ "headline": "EXCLUSIVE: A massively viral Chinese documentary, summarized in English for the first time.",
+ "impressions": 2547,
+ "clicks": 21,
+ "ctr": 0.0082449941107184,
+ "ctr_select": 0.00706436420722135,
+ "ctr_holdout": 0.009426551453260016,
+ "n_select": 1274,
+ "n_holdout": 1273,
+ "eb": 0.007900078399988042,
+ "labelled_winner": false
+ },
+ {
+ "headline": "EXCLUSIVE: A massively viral Chinese documentary, summarized in English for the first time.",
+ "impressions": 2543,
+ "clicks": 16,
+ "ctr": 0.0062917813605977,
+ "ctr_select": 0.009433962264150943,
+ "ctr_holdout": 0.003147128245476003,
+ "n_select": 1272,
+ "n_holdout": 1271,
+ "eb": 0.008740204896686274,
+ "labelled_winner": false
+ },
+ {
+ "headline": "EXCLUSIVE: A massively viral Chinese documentary, summarized in English for the first time.",
+ "impressions": 2563,
+ "clicks": 21,
+ "ctr": 0.0081935232149824,
+ "ctr_select": 0.00858034321372855,
+ "ctr_holdout": 0.0078064012490242,
+ "n_select": 1282,
+ "n_holdout": 1281,
+ "eb": 0.008438123405908042,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.008359456635318705,
+ "pool_holdout": 0.0067973856209150325,
+ "lift_select": 0.12853773584905648,
+ "lift_holdout": -0.5370090177328573,
+ "eb_lift": 0.04554701076609535,
+ "p_value": 0.6064830776223172,
+ "held_up": false,
+ "z_path": [
+ 0.008,
+ -0.29,
+ 0.015,
+ -0.227,
+ -0.639,
+ -0.984,
+ -0.738,
+ -0.738,
+ -0.851,
+ -0.801
+ ],
+ "first_cross": null,
+ "final_z": -0.801
+ },
+ {
+ "id": "54e2b61c633065000c140000",
+ "variation": "headline",
+ "week": 201507,
+ "impressions": 7624,
+ "pooled_ctr": 0.002229800629590766,
+ "arms": [
+ {
+ "headline": "For once, an education reform idea that all my teacher friends will love.",
+ "impressions": 1546,
+ "clicks": 2,
+ "ctr": 0.001293661060802,
+ "ctr_select": 0.0,
+ "ctr_holdout": 0.00258732212160414,
+ "n_select": 773,
+ "n_holdout": 773,
+ "eb": 0.0019677109947059137,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Stop thinking of teachers as teachers. Think of them as designers.",
+ "impressions": 1477,
+ "clicks": 1,
+ "ctr": 0.0006770480704129,
+ "ctr_select": 0.0013550135501355014,
+ "ctr_holdout": 0.0,
+ "n_select": 738,
+ "n_holdout": 739,
+ "eb": 0.0023175066839829315,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Here's the simplest (and best) education idea I've heard in a long time.",
+ "impressions": 1542,
+ "clicks": 7,
+ "ctr": 0.0045395590142671,
+ "ctr_select": 0.005188067444876783,
+ "ctr_holdout": 0.0038910505836575876,
+ "n_select": 771,
+ "n_holdout": 771,
+ "eb": 0.003264029315525094,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Forget charter schools. Forget merit pay. Here's how to dramatically improve teachers.",
+ "impressions": 1529,
+ "clicks": 4,
+ "ctr": 0.0026160889470241,
+ "ctr_select": 0.003926701570680628,
+ "ctr_holdout": 0.00130718954248366,
+ "n_select": 764,
+ "n_holdout": 765,
+ "eb": 0.0029469470159465484,
+ "labelled_winner": false
+ },
+ {
+ "headline": "I'm not a teacher because I love creative freedom. What if we tried giving teachers some of that?",
+ "impressions": 1530,
+ "clicks": 3,
+ "ctr": 0.0019607843137254,
+ "ctr_select": 0.00261437908496732,
+ "ctr_holdout": 0.00130718954248366,
+ "n_select": 765,
+ "n_holdout": 765,
+ "eb": 0.0026215998547545073,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.0026239832065074785,
+ "pool_holdout": 0.0018358248098610018,
+ "lift_select": 0.977172503242542,
+ "lift_holdout": 1.1195108393551974,
+ "eb_lift": 0.24392157214661325,
+ "p_value": 0.1191782144187907,
+ "held_up": true,
+ "z_path": [
+ 0.0,
+ -0.009,
+ 0.569,
+ 0.57,
+ 0.438,
+ 0.438,
+ 1.125,
+ 1.405,
+ 0.621,
+ 0.892
+ ],
+ "first_cross": null,
+ "final_z": 0.892
+ },
+ {
+ "id": "531f270a65114a89d8000037",
+ "variation": "image",
+ "week": 201410,
+ "impressions": 7262,
+ "pooled_ctr": 0.03552740291930598,
+ "arms": [
+ {
+ "headline": "Ellen, Katy Perry, And A Hockey Player Walk Into An Ad And Shatter A Ridiculous Argument",
+ "impressions": 2371,
+ "clicks": 69,
+ "ctr": 0.0291016448755799,
+ "ctr_select": 0.03204047217537943,
+ "ctr_holdout": 0.026160337552742614,
+ "n_select": 1186,
+ "n_holdout": 1185,
+ "eb": 0.03653325471312591,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Ellen, Katy Perry, And A Hockey Player Walk Into An Ad And Shatter A Ridiculous Argument",
+ "impressions": 2473,
+ "clicks": 105,
+ "ctr": 0.0424585523655479,
+ "ctr_select": 0.04611650485436893,
+ "ctr_holdout": 0.038803556992724336,
+ "n_select": 1236,
+ "n_holdout": 1237,
+ "eb": 0.041365780913075044,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Ellen, Katy Perry, And A Hockey Player Walk Into An Ad And Shatter A Ridiculous Argument",
+ "impressions": 2418,
+ "clicks": 84,
+ "ctr": 0.0347394540942928,
+ "ctr_select": 0.0380479735318445,
+ "ctr_holdout": 0.03143093465674111,
+ "n_select": 1209,
+ "n_holdout": 1209,
+ "eb": 0.03856340749094694,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.03883227760947398,
+ "pool_holdout": 0.03222252822913798,
+ "lift_select": 0.18758176685257855,
+ "lift_holdout": 0.20423688410753904,
+ "eb_lift": 0.06524220209486153,
+ "p_value": 0.10265026803004539,
+ "held_up": true,
+ "z_path": [
+ 1.788,
+ 2.022,
+ 1.511,
+ 0.939,
+ 1.107,
+ 0.482,
+ 0.765,
+ 0.609,
+ 1.029,
+ 1.4
+ ],
+ "first_cross": 1,
+ "final_z": 1.4
+ },
+ {
+ "id": "54fde77b3233330019640000",
+ "variation": "other",
+ "week": 201510,
+ "impressions": 6920,
+ "pooled_ctr": 0.014884393063583815,
+ "arms": [
+ {
+ "headline": "Coming out was easy. The hard part? Admitting she still liked men.",
+ "impressions": 1730,
+ "clicks": 33,
+ "ctr": 0.0190751445086705,
+ "ctr_select": 0.020809248554913295,
+ "ctr_holdout": 0.017341040462427744,
+ "n_select": 865,
+ "n_holdout": 865,
+ "eb": 0.017438361121974746,
+ "labelled_winner": true
+ },
+ {
+ "headline": "Coming out was easy. The hard part? Admitting she still liked men.",
+ "impressions": 1807,
+ "clicks": 30,
+ "ctr": 0.0166021029330381,
+ "ctr_select": 0.018805309734513276,
+ "ctr_holdout": 0.014396456256921373,
+ "n_select": 904,
+ "n_holdout": 903,
+ "eb": 0.016916867103047916,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Coming out was easy. The hard part? Admitting she still liked men.",
+ "impressions": 1679,
+ "clicks": 18,
+ "ctr": 0.0107206670637284,
+ "ctr_select": 0.008333333333333333,
+ "ctr_holdout": 0.013110846245530394,
+ "n_select": 840,
+ "n_holdout": 839,
+ "eb": 0.014092872744019156,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Coming out was easy. The hard part? Admitting she still liked men.",
+ "impressions": 1704,
+ "clicks": 22,
+ "ctr": 0.0129107981220657,
+ "ctr_select": 0.01643192488262911,
+ "ctr_holdout": 0.009389671361502348,
+ "n_select": 852,
+ "n_holdout": 852,
+ "eb": 0.01624793173184783,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.016180294712510836,
+ "pool_holdout": 0.013587742122000578,
+ "lift_select": 0.2860858794384804,
+ "lift_holdout": 0.2762267863731398,
+ "eb_lift": 0.07775299719918904,
+ "p_value": 0.21279563128668189,
+ "held_up": true,
+ "z_path": [
+ -0.23,
+ 0.746,
+ 0.663,
+ 1.306,
+ 0.763,
+ 0.305,
+ 0.447,
+ 0.442,
+ 0.836,
+ 0.556
+ ],
+ "first_cross": null,
+ "final_z": 0.556
+ },
+ {
+ "id": "53fb5762842e02dc3100001f",
+ "variation": "headline",
+ "week": 201434,
+ "impressions": 6790,
+ "pooled_ctr": 0.01752577319587629,
+ "arms": [
+ {
+ "headline": "He Didn't Want To Perform In Their Town. And Sure Enough, Within 10 Seconds, The Heckling Started",
+ "impressions": 3425,
+ "clicks": 63,
+ "ctr": 0.0183941605839416,
+ "ctr_select": 0.018107476635514017,
+ "ctr_holdout": 0.018680677174547577,
+ "n_select": 1712,
+ "n_holdout": 1713,
+ "eb": 0.017860608422402773,
+ "labelled_winner": true
+ },
+ {
+ "headline": "He Didn't Want To Perform In Their Town. And Sure Enough, Within 10 Seconds, They Started Heckling",
+ "impressions": 3365,
+ "clicks": 56,
+ "ctr": 0.0166419019316493,
+ "ctr_select": 0.017241379310344827,
+ "ctr_holdout": 0.016042780748663103,
+ "n_select": 1682,
+ "n_holdout": 1683,
+ "eb": 0.01749453532002019,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.01767825574543312,
+ "pool_holdout": 0.017373380447585393,
+ "lift_select": 0.024279595015576083,
+ "lift_holdout": 0.0752471133010777,
+ "eb_lift": 0.01031508309391671,
+ "p_value": 0.8481821328783327,
+ "held_up": true,
+ "z_path": [
+ 2.055,
+ 0.891,
+ 0.712,
+ 0.36,
+ 1.093,
+ 0.862,
+ 0.353,
+ 0.431,
+ 0.296,
+ 0.55
+ ],
+ "first_cross": 0,
+ "final_z": 0.55
+ },
+ {
+ "id": "53cd4f31ddae0f8434000038",
+ "variation": "image",
+ "week": 201429,
+ "impressions": 6621,
+ "pooled_ctr": 0.01616070080048331,
+ "arms": [
+ {
+ "headline": "Which Cities Will Be Completely Underwater In Less Than 100 Years?",
+ "impressions": 3307,
+ "clicks": 48,
+ "ctr": 0.0145146658602963,
+ "ctr_select": 0.013301088270858524,
+ "ctr_holdout": 0.01572897761645493,
+ "n_select": 1654,
+ "n_holdout": 1653,
+ "eb": 0.014175289817459198,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Which Cities Will Be Completely Underwater In Less Than 100 Years?",
+ "impressions": 3314,
+ "clicks": 59,
+ "ctr": 0.0178032589016294,
+ "ctr_select": 0.016294508147254073,
+ "ctr_holdout": 0.01931200965600483,
+ "n_select": 1657,
+ "n_holdout": 1657,
+ "eb": 0.01542254797366253,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.014799154334038054,
+ "pool_holdout": 0.017522658610271902,
+ "lift_select": 0.10104319337873946,
+ "lift_holdout": 0.10211641312717212,
+ "eb_lift": 0.0421235987917683,
+ "p_value": 0.47569672424122555,
+ "held_up": true,
+ "z_path": [
+ 1.27,
+ 0.894,
+ 1.867,
+ 1.989,
+ 1.912,
+ 1.897,
+ 1.418,
+ 1.692,
+ 1.335,
+ 1.061
+ ],
+ "first_cross": 3,
+ "final_z": 1.061
+ },
+ {
+ "id": "54451b14ecd9a21c0b00002b",
+ "variation": "image",
+ "week": 201442,
+ "impressions": 6274,
+ "pooled_ctr": 0.02693656359579216,
+ "arms": [
+ {
+ "headline": "These Congressmen Think They\u2019re Smarter Than Scientists. Jon Stewart Disproves That Real Quick.",
+ "impressions": 3175,
+ "clicks": 82,
+ "ctr": 0.0258267716535433,
+ "ctr_select": 0.02707808564231738,
+ "ctr_holdout": 0.024574669187145556,
+ "n_select": 1588,
+ "n_holdout": 1587,
+ "eb": 0.027083563522875723,
+ "labelled_winner": false
+ },
+ {
+ "headline": "These Congressmen Think They\u2019re Smarter Than Scientists. Jon Stewart Disproves That Real Quick.",
+ "impressions": 3099,
+ "clicks": 87,
+ "ctr": 0.0280735721200387,
+ "ctr_select": 0.027096774193548386,
+ "ctr_holdout": 0.029051000645577793,
+ "n_select": 1550,
+ "n_holdout": 1549,
+ "eb": 0.027091106877113132,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.02708731676226896,
+ "pool_holdout": 0.026785714285714284,
+ "lift_select": 0.0003491461100568749,
+ "lift_holdout": 0.08457069076823776,
+ "eb_lift": 0.00013992212212965427,
+ "p_value": 0.9974274747438153,
+ "held_up": true,
+ "z_path": [
+ 2.057,
+ 1.247,
+ 1.37,
+ 0.836,
+ 1.119,
+ 0.735,
+ 0.045,
+ 0.49,
+ 0.476,
+ 0.55
+ ],
+ "first_cross": 0,
+ "final_z": 0.55
+ },
+ {
+ "id": "5441cdf0275a490d8a000029",
+ "variation": "headline",
+ "week": 201441,
+ "impressions": 6174,
+ "pooled_ctr": 0.025267249757045675,
+ "arms": [
+ {
+ "headline": "Since When Does Being Popular Mean Having To Put Up With This Kinda Bullying",
+ "impressions": 3085,
+ "clicks": 84,
+ "ctr": 0.0272285251215559,
+ "ctr_select": 0.029831387808041506,
+ "ctr_holdout": 0.02462734931950745,
+ "n_select": 1542,
+ "n_holdout": 1543,
+ "eb": 0.028263104759697795,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Since When Does Being Popular Mean Having To Put Up With This Kind Of Bullying",
+ "impressions": 3089,
+ "clicks": 72,
+ "ctr": 0.0233085140822272,
+ "ctr_select": 0.02461139896373057,
+ "ctr_holdout": 0.022006472491909384,
+ "n_select": 1544,
+ "n_holdout": 1545,
+ "eb": 0.02617683938479512,
+ "labelled_winner": true
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.027219701879455604,
+ "pool_holdout": 0.023316062176165803,
+ "lift_select": 0.09594836637638204,
+ "lift_holdout": 0.056239648592208535,
+ "eb_lift": 0.0383326343860404,
+ "p_value": 0.37291741586952387,
+ "held_up": true,
+ "z_path": [
+ -0.304,
+ 0.694,
+ 1.598,
+ 1.307,
+ 1.321,
+ 1.578,
+ 1.369,
+ 0.74,
+ 0.787,
+ 0.981
+ ],
+ "first_cross": null,
+ "final_z": 0.981
+ },
+ {
+ "id": "54e238c0396166001a8b0000",
+ "variation": "other",
+ "week": 201507,
+ "impressions": 6158,
+ "pooled_ctr": 0.03004222150048717,
+ "arms": [
+ {
+ "headline": "5 men watch something horrifying that's being done to girls. Here's how they reacted.",
+ "impressions": 1586,
+ "clicks": 46,
+ "ctr": 0.0290037831021437,
+ "ctr_select": 0.025220680958385876,
+ "ctr_holdout": 0.03278688524590164,
+ "n_select": 793,
+ "n_holdout": 793,
+ "eb": 0.028215193845817697,
+ "labelled_winner": false
+ },
+ {
+ "headline": "5 men watch something horrifying that's being done to girls. Here's how they reacted.",
+ "impressions": 1542,
+ "clicks": 49,
+ "ctr": 0.0317769130998702,
+ "ctr_select": 0.03372243839169909,
+ "ctr_holdout": 0.029831387808041506,
+ "n_select": 771,
+ "n_holdout": 771,
+ "eb": 0.030358729697407993,
+ "labelled_winner": false
+ },
+ {
+ "headline": "5 men watch something horrifying that's being done to girls. Here's how they reacted.",
+ "impressions": 1493,
+ "clicks": 39,
+ "ctr": 0.0261219022103148,
+ "ctr_select": 0.032171581769437,
+ "ctr_holdout": 0.020080321285140562,
+ "n_select": 746,
+ "n_holdout": 747,
+ "eb": 0.029953653258016598,
+ "labelled_winner": false
+ },
+ {
+ "headline": "5 men watch something horrifying that's being done to girls. Here's how they reacted.",
+ "impressions": 1537,
+ "clicks": 51,
+ "ctr": 0.033181522446324,
+ "ctr_select": 0.026041666666666668,
+ "ctr_holdout": 0.04031209362808843,
+ "n_select": 768,
+ "n_holdout": 769,
+ "eb": 0.028443790394690936,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.029239766081871343,
+ "pool_holdout": 0.030844155844155844,
+ "lift_select": 0.15330739299610885,
+ "lift_holdout": -0.032835005802443806,
+ "eb_lift": 0.03826855565135334,
+ "p_value": 0.3934602692724115,
+ "held_up": false,
+ "z_path": [
+ 1.642,
+ 0.887,
+ -0.466,
+ -0.265,
+ 0.812,
+ 1.508,
+ 1.161,
+ 0.834,
+ 1.234,
+ 0.928
+ ],
+ "first_cross": null,
+ "final_z": 0.928
+ },
+ {
+ "id": "5526f68d646532002c810100",
+ "variation": "other",
+ "week": 201514,
+ "impressions": 6127,
+ "pooled_ctr": 0.010935204831075566,
+ "arms": [
+ {
+ "headline": "A Utah town's solution to homelessness is so good, every state will be mad that they weren't first.",
+ "impressions": 2033,
+ "clicks": 25,
+ "ctr": 0.0122970978848991,
+ "ctr_select": 0.010826771653543307,
+ "ctr_holdout": 0.01376597836774828,
+ "n_select": 1016,
+ "n_holdout": 1017,
+ "eb": 0.013059689106881671,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Utah town's solution to homelessness is so good, every state will be mad that they weren't first.",
+ "impressions": 2078,
+ "clicks": 17,
+ "ctr": 0.0081809432146294,
+ "ctr_select": 0.015399422521655439,
+ "ctr_holdout": 0.0009624639076034649,
+ "n_select": 1039,
+ "n_holdout": 1039,
+ "eb": 0.01445976369329842,
+ "labelled_winner": false
+ },
+ {
+ "headline": "A Utah town's solution to homelessness is so good, every state will be mad that they weren't first.",
+ "impressions": 2016,
+ "clicks": 25,
+ "ctr": 0.0124007936507936,
+ "ctr_select": 0.015873015873015872,
+ "ctr_holdout": 0.008928571428571428,
+ "n_select": 1008,
+ "n_holdout": 1008,
+ "eb": 0.014594548245083638,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 2,
+ "pool_select": 0.014038524322559582,
+ "pool_holdout": 0.007832898172323759,
+ "lift_select": 0.13067552602436328,
+ "lift_holdout": 0.13988095238095233,
+ "eb_lift": 0.039607006388167054,
+ "p_value": 0.5455792075042505,
+ "held_up": false,
+ "z_path": [
+ 0.029,
+ 0.752,
+ 0.051,
+ 0.561,
+ 0.791,
+ 0.482,
+ 0.817,
+ 1.098,
+ 0.9,
+ 1.34
+ ],
+ "first_cross": null,
+ "final_z": 1.34
+ },
+ {
+ "id": "54fa0d6161316400266b0100",
+ "variation": "headline",
+ "week": 201509,
+ "impressions": 6106,
+ "pooled_ctr": 0.025548640681297086,
+ "arms": [
+ {
+ "headline": "This billboard lets you determine how the women on it look. You just have to see it.",
+ "impressions": 1446,
+ "clicks": 43,
+ "ctr": 0.0297372060857538,
+ "ctr_select": 0.027662517289073305,
+ "ctr_holdout": 0.0318118948824343,
+ "n_select": 723,
+ "n_holdout": 723,
+ "eb": 0.02380428615235581,
+ "labelled_winner": false
+ },
+ {
+ "headline": "This billboard takes on domestic violence in a way you really have to see to believe",
+ "impressions": 1514,
+ "clicks": 46,
+ "ctr": 0.0303830911492734,
+ "ctr_select": 0.022457067371202115,
+ "ctr_holdout": 0.03830911492734478,
+ "n_select": 757,
+ "n_holdout": 757,
+ "eb": 0.02256535273674273,
+ "labelled_winner": false
+ },
+ {
+ "headline": "People can control how the women on this billboard look. They just have to look up.",
+ "impressions": 1630,
+ "clicks": 33,
+ "ctr": 0.0202453987730061,
+ "ctr_select": 0.01717791411042945,
+ "ctr_holdout": 0.023312883435582823,
+ "n_select": 815,
+ "n_holdout": 815,
+ "eb": 0.021189927826654173,
+ "labelled_winner": false
+ },
+ {
+ "headline": "It's hard to ignore the women on this billboard, but that's sort of the point",
+ "impressions": 1516,
+ "clicks": 34,
+ "ctr": 0.0224274406332453,
+ "ctr_select": 0.023746701846965697,
+ "ctr_holdout": 0.021108179419525065,
+ "n_select": 758,
+ "n_holdout": 758,
+ "eb": 0.022883145556820774,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 0,
+ "pool_select": 0.02260072060268588,
+ "pool_holdout": 0.028496560759908286,
+ "lift_select": 0.22396616352957688,
+ "lift_holdout": 0.11634155259852785,
+ "eb_lift": 0.05325341482814916,
+ "p_value": 0.2945264791676031,
+ "held_up": false,
+ "z_path": [
+ -0.649,
+ -0.452,
+ -0.346,
+ 0.509,
+ -0.021,
+ 0.613,
+ 1.109,
+ 0.952,
+ 0.803,
+ 1.25
+ ],
+ "first_cross": null,
+ "final_z": 1.25
+ },
+ {
+ "id": "54920116623736001e000000",
+ "variation": "other",
+ "week": 201450,
+ "impressions": 5724,
+ "pooled_ctr": 0.009783368273934312,
+ "arms": [
+ {
+ "headline": "Native American Rap. It's Seriously Good. For My Ears, Your Ears, And For America.",
+ "impressions": 2875,
+ "clicks": 27,
+ "ctr": 0.009391304347826,
+ "ctr_select": 0.009040333796940195,
+ "ctr_holdout": 0.009742519137091163,
+ "n_select": 1438,
+ "n_holdout": 1437,
+ "eb": 0.009930119018841973,
+ "labelled_winner": false
+ },
+ {
+ "headline": "Native American Rap. It's Seriously Good. For My Ears, Your Ears, And For America.",
+ "impressions": 2849,
+ "clicks": 29,
+ "ctr": 0.0101790101790101,
+ "ctr_select": 0.011938202247191011,
+ "ctr_holdout": 0.008421052631578947,
+ "n_select": 1424,
+ "n_holdout": 1425,
+ "eb": 0.011036307170048719,
+ "labelled_winner": false
+ }
+ ],
+ "winner": 1,
+ "pool_select": 0.010482180293501049,
+ "pool_holdout": 0.009084556254367574,
+ "lift_select": 0.13890449438202235,
+ "lift_holdout": -0.07303643724696351,
+ "eb_lift": 0.05286370402264762,
+ "p_value": 0.4465984444995771,
+ "held_up": false,
+ "z_path": [
+ 0.461,
+ 0.724,
+ 0.925,
+ 2.028,
+ 1.256,
+ 0.761,
+ 0.537,
+ 0.624,
+ 0.173,
+ 0.303
+ ],
+ "first_cross": 3,
+ "final_z": 0.303
+ }
+ ]
+}
\ No newline at end of file