{ "meta": { "source": "Upworthy Research Archive (Matias et al.), CC BY 4.0", "alpha": 0.05, "looks": 10, "obf_bounds": [ 6.198, 4.383, 3.582, 3.119, 2.816, 2.599, 2.437, 2.31, 2.207, 2.122 ], "pocock_bounds": [ 2.655, 2.623, 2.59, 2.562, 2.54, 2.521, 2.506, 2.493, 2.482, 2.472 ], "z_fixed": 1.96 }, "dataset": { "filters": { "rows_raw": 150817, "tests_raw": 32487, "rows_kept": 150024, "tests_kept": 32363, "dropped_bad_counts": 0, "dropped_single_arm": 0, "dropped_small_arm": 409, "dropped_too_many_arms": 384 }, "tests": 32363, "arms": 150024, "impressions": 536874152, "clicks": 8160148, "pooled_ctr": 0.015199368361470306, "median_arm_impressions": 3120.0, "median_test_impressions": 14351.0, "arms_per_test": { "2": 837, "3": 1166, "4": 16918, "5": 6156, "6": 5674, "7": 889, "8": 517, "9": 112, "10": 54, "11": 31, "12": 9 }, "variation_types": { "headline": 15672, "image": 9343, "other": 5289, "both": 2059 }, "weeks_covered": 121 }, "curse": { "config": { "replicates": 20, "selection_fraction": 0.5, "kappa": 2317.6977046341626 }, "vs_test_average": { "n_decisions": 647242, "median_in_sample": 0.31868131868131866, "median_out_of_sample": 0.14206439625286016, "mean_in_sample": 0.3783102340137593, "mean_out_of_sample": 0.1826517713416575, "shrinkage_median": 0.554211722103094, "shrinkage_mean": 0.5171905094827163 }, "vs_test_average_per_test": { "n_tests": 32354, "median_in_sample": 0.32796999656388864, "median_out_of_sample": 0.12094147264307453, "mean_in_sample": 0.37807799590129176, "mean_out_of_sample": 0.1826698304748117, "shrinkage_median": 0.6312422663348258, "shrinkage_mean": 0.5168461733951242 }, "vs_runner_up": { "n_tests": 31915, "median_in_sample": 0.1839019416355784, "median_out_of_sample": 0.08402156695679681, "mean_in_sample": 0.24954621195953383, "mean_out_of_sample": 0.23182197507525645, "shrinkage_median": 0.5431175646677258, "shrinkage_mean": 0.07102587029913132 }, "significant_winners_only": { "n_decisions": 254265, "median_in_sample": 0.516036467571201, "median_out_of_sample": 0.2724248103230493, "mean_in_sample": 0.5738378899261888, "mean_out_of_sample": 0.3141318330156038, "shrinkage_median": 0.47208225107567414, "shrinkage_mean": 0.452577394190527, "share_of_decisions_calling_a_significant_winner": 0.3928344096653586 }, "replication_rate": 0.4353953588975064, "by_arm_count": { "2": { "n_tests": 837, "median_in_sample": 0.11092271666361864, "median_out_of_sample": 0.010025987512021212, "replication_rate": 0.6079450418160095 }, "3": { "n_tests": 1166, "median_in_sample": 0.21263491821073582, "median_out_of_sample": 0.07485807867954185, "replication_rate": 0.5227701543739279 }, "4": { "n_tests": 16918, "median_in_sample": 0.2900668703837036, "median_out_of_sample": 0.1121344691893478, "replication_rate": 0.46564605745359966 }, "5": { "n_tests": 6156, "median_in_sample": 0.36963398940845227, "median_out_of_sample": 0.12748433326520914, "replication_rate": 0.3992608836907082 }, "6": { "n_tests": 5674, "median_in_sample": 0.42449651059914206, "median_out_of_sample": 0.15421368957962794, "replication_rate": 0.36602044413112444 }, "7": { "n_tests": 889, "median_in_sample": 0.45016965378795915, "median_out_of_sample": 0.18826688119268736, "replication_rate": 0.3787401574803149 }, "8": { "n_tests": 517, "median_in_sample": 0.4867977961371479, "median_out_of_sample": 0.1805987566388986, "replication_rate": 0.325531914893617 }, "9": { "n_tests": 112, "median_in_sample": 0.5414530560736706, "median_out_of_sample": 0.21368699748534725, "replication_rate": 0.28125 }, "10": { "n_tests": 54, "median_in_sample": 0.5724740546312979, "median_out_of_sample": 0.20635668625340123, "replication_rate": 0.24259259259259258 } }, "by_traffic_quartile": { "0": { "n_tests": 8097, "median_arm_impressions": 2328.4, "median_in_sample": 0.34772269165223346, "median_out_of_sample": 0.1115969694241283, "replication_rate": 0.41384463381499315 }, "1": { "n_tests": 8086, "median_arm_impressions": 3009.875, "median_in_sample": 0.3459243269824711, "median_out_of_sample": 0.12285629813929196, "replication_rate": 0.42260697501855055 }, "2": { "n_tests": 8091, "median_arm_impressions": 3565.3333333333335, "median_in_sample": 0.3237282969614249, "median_out_of_sample": 0.12243480212000284, "replication_rate": 0.43411197626992953 }, "3": { "n_tests": 8089, "median_arm_impressions": 5152.833333333333, "median_in_sample": 0.2936309405044912, "median_out_of_sample": 0.12586099934658254, "replication_rate": 0.47103473853381134 } }, "by_variation_type": { "both": { "n_tests": 2059, "median_in_sample": 0.3544615687415575, "median_out_of_sample": 0.15396077938951375 }, "headline": { "n_tests": 15672, "median_in_sample": 0.3807955229577384, "median_out_of_sample": 0.1710960371768363 }, "image": { "n_tests": 9343, "median_in_sample": 0.3096719983063988, "median_out_of_sample": 0.10780890071112145 }, "other": { "n_tests": 5289, "median_in_sample": 0.23199220153270478, "median_out_of_sample": 0.03363531542064699 } }, "estimators": { "raw": { "bias": 0.0024386243267708043, "relative_bias": 0.12967131621106626, "rmse": 0.0030924567126945223, "mae": 0.002552567502497645 }, "eb": { "bias": -0.0005040170360299849, "relative_bias": -0.026800582499458978, "rmse": 0.00292609761262605, "mae": 0.001946116177398383 }, "pool": { "bias": -0.0025313968087278546, "relative_bias": -0.13460439660047907, "rmse": 0.004708668441440616, "mae": 0.002998323774879991 } }, "selection_fraction_sensitivity": { "0.3": { "median_in_sample": 0.37186440677966104, "median_out_of_sample": 0.12236164463568211, "shrinkage_median": 0.6709509100498977 }, "0.5": { "median_in_sample": 0.31835916404474596, "median_out_of_sample": 0.14139019815782683, "shrinkage_median": 0.5558783470798592 }, "0.7": { "median_in_sample": 0.2917112426035502, "median_out_of_sample": 0.15568690632691196, "shrinkage_median": 0.46629788781059067 } } }, "peeking": { "tau_fitted": 0.0035931873430655294, "kappa": 2317.6977046341626, "aa_nulls": { "config": { "n_arms": 150024, "replicates": 20, "alpha": 0.05, "tau": 0.0035931873430655294, "median_impressions_per_arm": 3120.0 }, "by_looks": { "1": { "naive": 0.049325774542739835, "fixed": 0.049325774542739835, "pocock": 0.04932310830267157, "obrien_fleming": 0.04932310830267157, "msprt": 0.0003159494480883058, "stop_look_share": [ 1.0 ], "median_false_lift_at_stop": 0.7058823529411764 }, "2": { "naive": 0.08149262784621127, "fixed": 0.04950674558737268, "pocock": 0.046423238948434906, "obrien_fleming": 0.04848957500133313, "msprt": 0.00031861568815656164, "stop_look_share": [ 0.589251463088456, 0.410748536911544 ], "median_false_lift_at_stop": 0.856126301598723 }, "5": { "naive": 0.13586759451821043, "fixed": 0.049445088785794275, "pocock": 0.04313776462432677, "obrien_fleming": 0.04826827707566788, "msprt": 0.00045459393163760456, "stop_look_share": [ 0.3160635615255551, 0.2418242295200997, 0.18213840674274165, 0.14366592423246366, 0.11630787797913988 ], "median_false_lift_at_stop": 1.0000000000000002 }, "10": { "naive": 0.1787110728950035, "fixed": 0.04966072095131445, "pocock": 0.03914107076201141, "obrien_fleming": 0.04880919052951528, "msprt": 0.0005572441742654508, "stop_look_share": [ 0.18706722439898624, 0.1803069268339988, 0.1351089759967476, 0.10821884342031894, 0.0903119807392129, 0.07592233770157343, 0.06600288315035462, 0.058297076381105484, 0.05204589915687434, 0.04671785222082769 ], "median_false_lift_at_stop": 1.1428571428571428 }, "20": { "naive": 0.215034261184877, "fixed": 0.04950674558737268, "pocock": 0.03601623740201568, "obrien_fleming": 0.04730576441102757, "msprt": 0.0006462299365434863, "stop_look_share": [ 0.09008285725799202, 0.12529486706571236, 0.10784307647480029, 0.0886445569322046, 0.07246677805228098, 0.06223903683474735, 0.054517471939194616, 0.04839694609163585, 0.04383406229948265, 0.039751645210986876, 0.036416276352048804, 0.03357842301528504, 0.030591779989646718, 0.028524223271327295, 0.026715498615945917, 0.025117559353136826, 0.023260788027389703, 0.022321553116369037, 0.020658518364677327, 0.019744081735135754 ], "median_false_lift_at_stop": 1.2727272727272727 } } }, "declarations": { "config": { "n_pairs": 32363, "looks": 10, "alpha": 0.05, "tau": 0.0035931873430655294 }, "rules": { "fixed_horizon": { "declaration_rate": 0.17096684485369093, "n_declared": 5533, "mean_stop_fraction": 1.0, "sign_replication_rate": 0.871294287780188, "median_declared_lift": 0.7454761194531032, "median_holdout_lift": 0.43277883447732646, "lift_shrinkage": 0.41945982817689453 }, "naive_peeking": { "declaration_rate": 0.30435991718938293, "n_declared": 9850, "mean_stop_fraction": 0.44750253807106594, "sign_replication_rate": 0.7745964057264697, "median_declared_lift": 1.0513098969995522, "median_holdout_lift": 0.3023096337401675, "lift_shrinkage": 0.7124447942486208 }, "pocock": { "declaration_rate": 0.13123010845718877, "n_declared": 4247, "mean_stop_fraction": 0.5864610313162233, "sign_replication_rate": 0.8707018370230806, "median_declared_lift": 1.0308263441408942, "median_holdout_lift": 0.4457428473778206, "lift_shrinkage": 0.5675868686211069 }, "obrien_fleming": { "declaration_rate": 0.16416895837839507, "n_declared": 5313, "mean_stop_fraction": 0.7526068134763788, "sign_replication_rate": 0.8697289156626506, "median_declared_lift": 0.8693797760173938, "median_holdout_lift": 0.43127506704294133, "lift_shrinkage": 0.5039278817611779 }, "msprt": { "declaration_rate": 0.01554244044124463, "n_declared": 503, "mean_stop_fraction": 0.8240556660039762, "sign_replication_rate": 0.9681908548707754, "median_declared_lift": 1.168246463701647, "median_holdout_lift": 0.7381621178342287, "lift_shrinkage": 0.36814521526962274 } } }, "tau_sweep": { "looks": 10, "reference_fixed_horizon": 0.17096684485369093, "rows": [ { "tau": 0.001, "false_positive_rate": 0.0, "declaration_rate": 6.179896795723511e-05, "sign_replication_rate": 1.0 }, { "tau": 0.002, "false_positive_rate": 3.9993601023836184e-05, "declaration_rate": 0.0026882551061397273, "sign_replication_rate": 0.9770114942528736 }, { "tau": 0.005, "false_positive_rate": 0.00157308164027089, "declaration_rate": 0.03000339894323765, "sign_replication_rate": 0.9618949536560247 }, { "tau": 0.01, "false_positive_rate": 0.004439289713645817, "declaration_rate": 0.05166393721224855, "sign_replication_rate": 0.9342105263157895 }, { "tau": 0.02, "false_positive_rate": 0.005725750546579214, "declaration_rate": 0.05323981089515805, "sign_replication_rate": 0.926291352292513 } ] } }, "multiplicity": { "config": { "n_source_arms": 9435, "min_impressions": 6000, "replicates": 20, "alpha": 0.05, "median_impressions": 6980.0 }, "by_arms": { "2": { "false_positive_rate": 0.05049814520402755, "mean_z_of_best_arm": 0.7999737674187941, "median_apparent_lift": 0.07704830053667266, "p90_apparent_lift": 0.22580645161290325, "impressions_per_arm": 3490.0 }, "3": { "false_positive_rate": 0.08264440911499735, "mean_z_of_best_arm": 1.0481762339852339, "median_apparent_lift": 0.15416639979504243, "p90_apparent_lift": 0.3770491803278688, "impressions_per_arm": 2326.6666666666665 }, "4": { "false_positive_rate": 0.11354531001589827, "mean_z_of_best_arm": 1.2122554958989449, "median_apparent_lift": 0.2248299319727891, "p90_apparent_lift": 0.5140050740702207, "impressions_per_arm": 1745.0 }, "5": { "false_positive_rate": 0.14760466348701645, "mean_z_of_best_arm": 1.3407993799508509, "median_apparent_lift": 0.2908414771253136, "p90_apparent_lift": 0.6659731853906612, "impressions_per_arm": 1396.0 }, "6": { "false_positive_rate": 0.18210386857445682, "mean_z_of_best_arm": 1.441989063728598, "median_apparent_lift": 0.3528476316386132, "p90_apparent_lift": 0.7634700939199208, "impressions_per_arm": 1163.3333333333333 }, "8": { "false_positive_rate": 0.24803921568627452, "mean_z_of_best_arm": 1.5983881465015841, "median_apparent_lift": 0.46626016260162606, "p90_apparent_lift": 0.9993447777785404, "impressions_per_arm": 872.5 }, "10": { "false_positive_rate": 0.31072602013778483, "mean_z_of_best_arm": 1.7271592612405282, "median_apparent_lift": 0.573984181404664, "p90_apparent_lift": 1.1926331414677687, "impressions_per_arm": 698.0 } } }, "power": { "config": { "alpha": 0.05, "power_target": 0.8, "n_tests": 32363 }, "median_baseline_ctr": 0.01303538175046555, "median_arm_impressions": 3089.0, "median_mde_relative": 0.6841496794023025, "median_mde_relative_textbook_approx": 0.5920487129962937, "median_true_lift": 0.12094147264307445, "share_adequately_powered": 0.04554583938448228, "median_realised_power": 0.08750987660410837, "impressions_needed_per_arm": 86101.29373885738, "traffic_shortfall_factor": 27.873516911251983, "mde_percentiles": { "0.1": 0.4075025264945719, "0.25": 0.5184872621134372, "0.5": 0.6841496794023025, "0.75": 0.9158746575643475, "0.9": 1.2112357920260604 }, "true_lift_percentiles": { "0.1": 0.0, "0.25": 0.002579204641405175, "0.5": 0.12094147264307445, "0.75": 0.29955706894309253, "0.9": 0.5213373762361565 }, "by_traffic_decile": [ { "bucket": 0, "arm_impressions": 2025.0, "mde_rel": 0.9808284054349048, "true_lift": 0.1050711503936414, "powered": 0.15584199568516222 }, { "bucket": 1, "arm_impressions": 2527.5, "mde_rel": 0.7468712650440182, "true_lift": 0.1211473309573299, "powered": 0.18858134726721915 }, { "bucket": 2, "arm_impressions": 2750.5, "mde_rel": 0.5995502549171345, "true_lift": 0.1118253342529459, "powered": 0.20996959931462145 }, { "bucket": 3, "arm_impressions": 2866.0, "mde_rel": 0.7447778080360874, "true_lift": 0.124143202607286, "powered": 0.1884605661314108 }, { "bucket": 4, "arm_impressions": 3060.5, "mde_rel": 0.7817054107527598, "true_lift": 0.1232680405448523, "powered": 0.1856882725721644 }, { "bucket": 5, "arm_impressions": 3277.5, "mde_rel": 0.6736432191043422, "true_lift": 0.1182253975230108, "powered": 0.197932508546423 }, { "bucket": 6, "arm_impressions": 3597.0, "mde_rel": 0.6734410197359542, "true_lift": 0.1188595265068763, "powered": 0.20313257032144985 }, { "bucket": 7, "arm_impressions": 4097.0, "mde_rel": 0.6467085432336616, "true_lift": 0.1302693541561495, "powered": 0.21891303590705605 }, { "bucket": 8, "arm_impressions": 5017.0, "mde_rel": 0.6035956176663714, "true_lift": 0.12436662287529635, "powered": 0.2222169108018201 }, { "bucket": 9, "arm_impressions": 6359.5, "mde_rel": 0.5113104259135894, "true_lift": 0.1284647786925055, "powered": 0.2593972109001994 } ] }, "tests": [ { "id": "53f5f61b73269e19c2000049", "variation": "headline", "week": 201433, "impressions": 70362, "pooled_ctr": 0.009365850885421108, "arms": [ { "headline": "Yo Doctors \u2014 There's Somethin' We've Really, Really Gotta Talk About (Your Patients Can Come Too)", "impressions": 11641, "clicks": 77, "ctr": 0.0066145520144317, "ctr_select": 0.006872852233676976, "ctr_holdout": 0.00635629616904312, "n_select": 5820, "n_holdout": 5821, "eb": 0.007514070900554288, "labelled_winner": false }, { "headline": "The Biggest Problem With The Culture Of Medicine? What Happens After The Mess Ups.", "impressions": 11747, "clicks": 89, "ctr": 0.007576402485741, "ctr_select": 0.008341845420497106, "ctr_holdout": 0.006810829218457347, "n_select": 5874, "n_holdout": 5873, "eb": 0.008563211198603638, "labelled_winner": false }, { "headline": "'Do You Remember' Can Be The Most Gut-Wrenching Words Doctors Ever Hear. Here's Why.", "impressions": 11752, "clicks": 150, "ctr": 0.0127637848876786, "ctr_select": 0.012423417290673928, "ctr_holdout": 0.013104152484683457, "n_select": 5876, "n_holdout": 5876, "eb": 0.011490201483348783, "labelled_winner": false }, { "headline": "As A Young Man, He Let A Woman Die. Now He's Here To Tell You Why You Should Care.", "impressions": 11661, "clicks": 108, "ctr": 0.0092616413686647, "ctr_select": 0.008919382504288165, "ctr_holdout": 0.009603841536614645, "n_select": 5830, "n_holdout": 5831, "eb": 0.008977657268542874, "labelled_winner": false }, { "headline": "3 Of The Scariest Words Doctors Ever Have To Hear", "impressions": 11782, "clicks": 159, "ctr": 0.0134951621116958, "ctr_select": 0.01222203361059243, "ctr_holdout": 0.014768290612799185, "n_select": 5891, "n_holdout": 5891, "eb": 0.011347383089441014, "labelled_winner": false }, { "headline": "Yo Doctors \u2014 There's Something This Guy Really, Really Wants To Talk To You About", "impressions": 11779, "clicks": 76, "ctr": 0.0064521606248408, "ctr_select": 0.005942275042444821, "ctr_holdout": 0.0069621327899473595, "n_select": 5890, "n_holdout": 5889, "eb": 0.0068408023224585356, "labelled_winner": false } ], "winner": 2, "pool_select": 0.00912424319945425, "pool_holdout": 0.009607458571387966, "lift_select": 0.36158331371713226, "lift_holdout": 0.36395617918239265, "eb_lift": 0.25930460556290824, "p_value": 0.0035658625066109315, "held_up": false, "z_path": [ 0.32, 0.714, 1.03, 0.368, -0.376, -0.2, -0.117, -0.299, -0.7, -0.493 ], "first_cross": null, "final_z": -0.493 }, { "id": "53f548f1ba722b505e000029", "variation": "both", "week": 201433, "impressions": 61132, "pooled_ctr": 0.0062324151017470394, "arms": [ { "headline": "HIV-Positive People Are Living Longer Than Ever. And There's A Big Problem With That.", "impressions": 10280, "clicks": 92, "ctr": 0.0089494163424124, "ctr_select": 0.0071984435797665365, "ctr_holdout": 0.010700389105058366, "n_select": 5140, "n_holdout": 5140, "eb": 0.0068219658592151055, "labelled_winner": true }, { "headline": "HIV-Positive People Are Living Longer Than Ever. And There's A Big Problem With That.", "impressions": 10114, "clicks": 87, "ctr": 0.0086019379078505, "ctr_select": 0.008107573660272888, "ctr_holdout": 0.009096302155428119, "n_select": 5057, "n_holdout": 5057, "eb": 0.007441140142582069, "labelled_winner": false }, { "headline": "You Can Live For Years With HIV. Here's The Problem With That.", "impressions": 10078, "clicks": 63, "ctr": 0.0062512403254614, "ctr_select": 0.006350466362373487, "ctr_holdout": 0.0061520142885493154, "n_select": 5039, "n_holdout": 5039, "eb": 0.006235971759511419, "labelled_winner": false }, { "headline": "You Can Live For Years With HIV. Here's The Problem With That.", "impressions": 10063, "clicks": 67, "ctr": 0.0066580542581735, "ctr_select": 0.007551669316375199, "ctr_holdout": 0.005764261578215067, "n_select": 5032, "n_holdout": 5031, "eb": 0.007058271130886368, "labelled_winner": false }, { "headline": "Living With HIV Is Better Than Dying From It. But For How Long?", "impressions": 10285, "clicks": 32, "ctr": 0.0031113271754982, "ctr_select": 0.0031116297160637884, "ctr_holdout": 0.003111024693758507, "n_select": 5142, "n_holdout": 5143, "eb": 0.004005009359937099, "labelled_winner": false }, { "headline": "Living With HIV Is Better Than Dying From It. But For How Long?", "impressions": 10312, "clicks": 40, "ctr": 0.0038789759503491, "ctr_select": 0.0036850271528316523, "ctr_holdout": 0.004072924747866563, "n_select": 5156, "n_holdout": 5156, "eb": 0.004398914757948509, "labelled_winner": false } ], "winner": 1, "pool_select": 0.005987044428449911, "pool_holdout": 0.006477785775044167, "lift_select": 0.35418631967159087, "lift_holdout": 0.40423016001422174, "eb_lift": 0.24287371365116694, "p_value": 0.0323757607505977, "held_up": false, "z_path": [ -0.009, 0.173, 1.031, 0.633, 1.307, 1.231, 1.549, 1.82, 2.225, 1.586 ], "first_cross": 8, "final_z": 1.586 }, { "id": "5420dc081afbe86848000074", "variation": "both", "week": 201438, "impressions": 48797, "pooled_ctr": 0.011086747136094432, "arms": [ { "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.", "impressions": 6916, "clicks": 88, "ctr": 0.0127241179872758, "ctr_select": 0.012434933487565067, "ctr_holdout": 0.013013302486986697, "n_select": 3458, "n_holdout": 3458, "eb": 0.011671804855457222, "labelled_winner": false }, { "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.", "impressions": 6942, "clicks": 100, "ctr": 0.0144050705848458, "ctr_select": 0.012100259291270527, "ctr_holdout": 0.016709881878421203, "n_select": 3471, "n_holdout": 3471, "eb": 0.011472842407962525, "labelled_winner": false }, { "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.", "impressions": 6867, "clicks": 72, "ctr": 0.0104849279161205, "ctr_select": 0.011357018054746652, "ctr_holdout": 0.009612583745994756, "n_select": 3434, "n_holdout": 3433, "eb": 0.011025060733200604, "labelled_winner": false }, { "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.", "impressions": 7120, "clicks": 88, "ctr": 0.0123595505617977, "ctr_select": 0.013202247191011237, "ctr_holdout": 0.011516853932584269, "n_select": 3560, "n_holdout": 3560, "eb": 0.012149794035221992, "labelled_winner": false }, { "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.", "impressions": 7058, "clicks": 48, "ctr": 0.0068007934258996, "ctr_select": 0.006517427033153868, "ctr_holdout": 0.007084159818645509, "n_select": 3529, "n_holdout": 3529, "eb": 0.008109332636613346, "labelled_winner": false }, { "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16, And Set On Making An Impact.", "impressions": 7014, "clicks": 67, "ctr": 0.0095523239235814, "ctr_select": 0.007698887938408896, "ctr_holdout": 0.01140575990875392, "n_select": 3507, "n_holdout": 3507, "eb": 0.008826692666247351, "labelled_winner": false }, { "headline": "An 11-Year-Old Saw Something He Could Never Forget. Now He's 16 And He's Set On Making An Impact.", "impressions": 6880, "clicks": 78, "ctr": 0.0113372093023255, "ctr_select": 0.010465116279069767, "ctr_holdout": 0.012209302325581395, "n_select": 3440, "n_holdout": 3440, "eb": 0.010492530106951961, "labelled_winner": false } ], "winner": 3, "pool_select": 0.010533218574531744, "pool_holdout": 0.011640298385113533, "lift_select": 0.2533915533598565, "lift_holdout": -0.01060492166482041, "eb_lift": 0.153474025935336, "p_value": 0.09143196774647187, "held_up": false, "z_path": [ -0.054, -0.422, 0.292, 0.101, 0.583, 0.683, 0.365, -0.426, -0.106, -0.194 ], "first_cross": null, "final_z": -0.194 }, { "id": "53f65f602f4a3a4260000025", "variation": "image", "week": 201433, "impressions": 44996, "pooled_ctr": 0.004778202506889501, "arms": [ { "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.", "impressions": 8962, "clicks": 47, "ctr": 0.0052443650970765, "ctr_select": 0.005802276277616604, "ctr_holdout": 0.0046864539165364875, "n_select": 4481, "n_holdout": 4481, "eb": 0.005627421162244576, "labelled_winner": false }, { "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.", "impressions": 9087, "clicks": 40, "ctr": 0.0044018928139099, "ctr_select": 0.005061619718309859, "ctr_holdout": 0.0037420206911732336, "n_select": 4544, "n_holdout": 4543, "eb": 0.005138543966305752, "labelled_winner": false }, { "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.", "impressions": 8990, "clicks": 50, "ctr": 0.0055617352614015, "ctr_select": 0.006674082313681869, "ctr_holdout": 0.004449388209121246, "n_select": 4495, "n_holdout": 4495, "eb": 0.0062029958132468165, "labelled_winner": false }, { "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.", "impressions": 9004, "clicks": 35, "ctr": 0.0038871612616614, "ctr_select": 0.0046645935139937806, "ctr_holdout": 0.003109729009329187, "n_select": 4502, "n_holdout": 4502, "eb": 0.004876922230168862, "labelled_winner": false }, { "headline": "These Students Have No Classes, No Tests, And No Teachers. Because They Made It That Way.", "impressions": 8953, "clicks": 43, "ctr": 0.0048028593767452, "ctr_select": 0.0042448614834673815, "ctr_holdout": 0.005360732633459906, "n_select": 4476, "n_holdout": 4477, "eb": 0.004601196093467512, "labelled_winner": false } ], "winner": 2, "pool_select": 0.005289359054138146, "pool_holdout": 0.004267045959640857, "lift_select": 0.2617941503631487, "lift_holdout": 0.04273266592510194, "eb_lift": 0.1727310908103099, "p_value": 0.15248894768407653, "held_up": false, "z_path": [ -0.338, -0.825, -1.04, -1.355, -0.801, -0.391, -0.37, 0.096, 0.297, 0.29 ], "first_cross": null, "final_z": 0.29 }, { "id": "527d0b432a1baa15cf011f0c", "variation": "headline", "week": 201344, "impressions": 38694, "pooled_ctr": 0.015583811443634672, "arms": [ { "headline": "The Most Hilarious Parody Video I've Seen Yet: What Does The Farmer Say?", "impressions": 9566, "clicks": 208, "ctr": 0.0217436755174576, "ctr_select": 0.022370896926615095, "ctr_holdout": 0.02111645410830023, "n_select": 4783, "n_holdout": 4783, "eb": 0.020045901744286353, "labelled_winner": false }, { "headline": "You Will No Longer Wonder What The Farmer Says After You See This", "impressions": 9379, "clicks": 129, "ctr": 0.0137541315705299, "ctr_select": 0.011940298507462687, "ctr_holdout": 0.015568351460865857, "n_select": 4690, "n_holdout": 4689, "eb": 0.013034222130126042, "labelled_winner": false }, { "headline": "You Will Never Look At Farming The Same Way Agains After Watching This", "impressions": 10265, "clicks": 174, "ctr": 0.0169508037018996, "ctr_select": 0.0161730319563523, "ctr_holdout": 0.017728423923631404, "n_select": 5132, "n_holdout": 5133, "eb": 0.015885193358834076, "labelled_winner": false }, { "headline": "Forget What The Fox Says, These Farmers Are Actually Using Real Words Like 'Work' And 'Corn'", "impressions": 9484, "clicks": 92, "ctr": 0.0097005482918599, "ctr_select": 0.01033319274567693, "ctr_holdout": 0.009067903838043019, "n_select": 4742, "n_holdout": 4742, "eb": 0.011946671377672906, "labelled_winner": false } ], "winner": 0, "pool_select": 0.015247842042693957, "pool_holdout": 0.015919780844575387, "lift_select": 0.46715167064143137, "lift_holdout": 0.3264286936145602, "eb_lift": 0.31467139337867156, "p_value": 3.5942110764181427e-06, "held_up": true, "z_path": [ 0.723, 1.917, 1.424, 1.426, 0.931, 1.869, 2.064, 1.838, 2.469, 2.454 ], "first_cross": 6, "final_z": 2.454 }, { "id": "52af71ac99f9bdaee6000ee8", "variation": "headline", "week": 201350, "impressions": 37837, "pooled_ctr": 0.00827232603007638, "arms": [ { "headline": "What Are Humans Actually \"Wired\" For? It's Possible The Answer Won't Surprise You.", "impressions": 9179, "clicks": 84, "ctr": 0.0091513236736027, "ctr_select": 0.009368191721132898, "ctr_holdout": 0.00893440836783613, "n_select": 4590, "n_holdout": 4589, "eb": 0.00895623670245161, "labelled_winner": false }, { "headline": "Watch This Total Genius Prove In Just 10 Minutes That Being Nicer Is Better", "impressions": 10169, "clicks": 78, "ctr": 0.0076703707345855, "ctr_select": 0.00885129819040126, "ctr_holdout": 0.006489675516224189, "n_select": 5084, "n_holdout": 5085, "eb": 0.008628692802692883, "labelled_winner": false }, { "headline": "What Are Humans Actually \"Wired\" For? (Hint: It's Not Narcissism, Violence, Or Materialism)", "impressions": 9029, "clicks": 80, "ctr": 0.0088603389079632, "ctr_select": 0.007532122286220647, "ctr_holdout": 0.010188261351052049, "n_select": 4514, "n_holdout": 4515, "eb": 0.0077384828775172295, "labelled_winner": false }, { "headline": "The History (And Science) Behind Being Nice To Your Extended Family", "impressions": 9460, "clicks": 71, "ctr": 0.0075052854122621, "ctr_select": 0.006765327695560253, "ctr_holdout": 0.00824524312896406, "n_select": 4730, "n_holdout": 4730, "eb": 0.007217530865184245, "labelled_winner": false } ], "winner": 0, "pool_select": 0.00814039539063326, "pool_holdout": 0.008404249696072731, "lift_select": 0.15082760376878013, "lift_holdout": 0.06308221327730634, "eb_lift": 0.100221337253114, "p_value": 0.28745298465617475, "held_up": false, "z_path": [ -0.8, 0.074, 0.961, 1.288, 1.13, 0.817, 0.833, 1.204, 1.451, 1.129 ], "first_cross": null, "final_z": 1.129 }, { "id": "54220c5ea369ffe8c6000017", "variation": "headline", "week": 201438, "impressions": 35866, "pooled_ctr": 0.0012546701611554118, "arms": [ { "headline": "He Visited One Of The World's Oldest Cultures, And Came Away With These", "impressions": 7025, "clicks": 11, "ctr": 0.0015658362989323, "ctr_select": 0.0011389521640091116, "ctr_holdout": 0.0019925989183034445, "n_select": 3512, "n_holdout": 3513, "eb": 0.0010187044302131009, "labelled_winner": false }, { "headline": "For Him, A Journey Through The Desert Mountains Of Ethiopia Was A Journey Through The Old Testament", "impressions": 7325, "clicks": 6, "ctr": 0.0008191126279863, "ctr_select": 0.0005461496450027307, "ctr_holdout": 0.001092001092001092, "n_select": 3662, "n_holdout": 3663, "eb": 0.0006586852836158442, "labelled_winner": false }, { "headline": "Where He Went, The Mountains Have Names Like Golgotha and Sina, And The River Is Called Jordanus", "impressions": 7288, "clicks": 10, "ctr": 0.0013721185510428, "ctr_select": 0.0013721185510428102, "ctr_holdout": 0.0013721185510428102, "n_select": 3644, "n_holdout": 3644, "eb": 0.0011638863998622954, "labelled_winner": false }, { "headline": "He Couldn't Find A Map For Such A Remote Area, But Farmers And Villagers Helped Guide Him On His Way", "impressions": 7128, "clicks": 6, "ctr": 0.0008417508417508, "ctr_select": 0.0002805836139169473, "ctr_holdout": 0.0014029180695847362, "n_select": 3564, "n_holdout": 3564, "eb": 0.0004996412644938463, "labelled_winner": false }, { "headline": "We Live In A World Where Everyone Is Connected \u2014 But Not Here.", "impressions": 7100, "clicks": 12, "ctr": 0.0016901408450704, "ctr_select": 0.0008450704225352113, "ctr_holdout": 0.0025352112676056337, "n_select": 3550, "n_holdout": 3550, "eb": 0.0008416825690617075, "labelled_winner": false } ], "winner": 2, "pool_select": 0.0008364934195850993, "pool_holdout": 0.0016728002676480427, "lift_select": 0.6403219904866448, "lift_holdout": -0.17974753018660805, "eb_lift": 0.39138739482204543, "p_value": 0.21022864171444344, "held_up": false, "z_path": [ 1.967, 1.59, 0.947, 0.576, 0.767, 0.467, -0.074, 0.167, 0.149, -0.303 ], "first_cross": 0, "final_z": -0.303 }, { "id": "5432e069a1243c5276000050", "variation": "headline", "week": 201440, "impressions": 35834, "pooled_ctr": 0.011720712172796785, "arms": [ { "headline": "Here Is The Most Thoughtful And Well-Reasoned Debate On Religion You\u2019ll Ever See", "impressions": 7176, "clicks": 106, "ctr": 0.0147714604236343, "ctr_select": 0.013656633221850614, "ctr_holdout": 0.01588628762541806, "n_select": 3588, "n_holdout": 3588, "eb": 0.01309374568466064, "labelled_winner": true }, { "headline": "Here Is The Most Thoughtful And Well-Reasoned And Informative Debate On Religion I've Ever Seen", "impressions": 7098, "clicks": 97, "ctr": 0.013665821358129, "ctr_select": 0.014652014652014652, "ctr_holdout": 0.01267962806424345, "n_select": 3549, "n_holdout": 3549, "eb": 0.013692149805419863, "labelled_winner": false }, { "headline": "Screw All The Shouting You See On TV. Watch This Thoughtful Debate On Religion With Actual Experts.", "impressions": 7215, "clicks": 73, "ctr": 0.0101178101178101, "ctr_select": 0.0102549889135255, "ctr_holdout": 0.0099805932908234, "n_select": 3608, "n_holdout": 3607, "eb": 0.011024474600497188, "labelled_winner": false }, { "headline": "This Is What A Debate On Religion Should Look Like. No Shouting, Just Thoughtfulness And Experts.", "impressions": 7175, "clicks": 74, "ctr": 0.0103135888501742, "ctr_select": 0.010869565217391304, "ctr_holdout": 0.009757457485363814, "n_select": 3588, "n_holdout": 3587, "eb": 0.011400465652370287, "labelled_winner": false }, { "headline": "You've Seen Lots Of Shout-Matches On Religion. Maybe You've Been In Some. This Debate Is Different.", "impressions": 7170, "clicks": 70, "ctr": 0.00976290097629, "ctr_select": 0.011715481171548118, "ctr_holdout": 0.007810320781032078, "n_select": 3585, "n_holdout": 3585, "eb": 0.011914502038576373, "labelled_winner": false } ], "winner": 1, "pool_select": 0.012222346244000447, "pool_holdout": 0.011219022103148023, "lift_select": 0.1987890344054728, "lift_holdout": 0.13019013133823698, "eb_lift": 0.12025543476490008, "p_value": 0.14128018213105395, "held_up": false, "z_path": [ -2.122, -1.875, -2.337, -2.029, -1.775, -1.071, -1.235, -0.01, -0.227, -0.558 ], "first_cross": 0, "final_z": -0.558 }, { "id": "53f7ab00029b3462d3000024", "variation": "image", "week": 201433, "impressions": 35820, "pooled_ctr": 0.0049692908989391405, "arms": [ { "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.", "impressions": 5239, "clicks": 35, "ctr": 0.0066806642489024, "ctr_select": 0.004961832061068702, "ctr_holdout": 0.008400152730049638, "n_select": 2620, "n_holdout": 2619, "eb": 0.004834292299664501, "labelled_winner": false }, { "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.", "impressions": 5062, "clicks": 14, "ctr": 0.0027657052548399, "ctr_select": 0.0031608060055314103, "ctr_holdout": 0.002370604504148558, "n_select": 2531, "n_holdout": 2531, "eb": 0.0038918231535755824, "labelled_winner": false }, { "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.", "impressions": 5078, "clicks": 30, "ctr": 0.0059078377313903, "ctr_select": 0.004332414336352895, "ctr_holdout": 0.007483261126427727, "n_select": 2539, "n_holdout": 2539, "eb": 0.004503116175156597, "labelled_winner": false }, { "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.", "impressions": 5086, "clicks": 23, "ctr": 0.0045222178529296, "ctr_select": 0.005112072355485647, "ctr_holdout": 0.003932363350373574, "n_select": 2543, "n_holdout": 2543, "eb": 0.004910874002476275, "labelled_winner": false }, { "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.", "impressions": 5068, "clicks": 36, "ctr": 0.0071033938437253, "ctr_select": 0.006314127861089187, "ctr_holdout": 0.007892659826361484, "n_select": 2534, "n_holdout": 2534, "eb": 0.005538324031589709, "labelled_winner": false }, { "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.", "impressions": 5174, "clicks": 27, "ctr": 0.0052183996907614, "ctr_select": 0.005798221878623889, "ctr_holdout": 0.004638577502899111, "n_select": 2587, "n_holdout": 2587, "eb": 0.005274590922727144, "labelled_winner": false }, { "headline": "A Deadly Disease You've Probably Never Heard Of Is Killing Children. It Doesn't Have To.", "impressions": 5113, "clicks": 13, "ctr": 0.0025425386270291, "ctr_select": 0.003129890453834116, "ctr_holdout": 0.0019554165037152915, "n_select": 2556, "n_holdout": 2557, "eb": 0.0038718597531482483, "labelled_winner": false } ], "winner": 4, "pool_select": 0.004690117252931323, "pool_holdout": 0.005248464544946957, "lift_select": 0.3462622618108018, "lift_holdout": 0.5038035903205764, "eb_lift": 0.18084980244966298, "p_value": 0.19657893949011507, "held_up": false, "z_path": [ 0.015, 0.407, 0.338, 1.045, 2.098, 2.254, 2.401, 1.972, 1.15, 1.22 ], "first_cross": 4, "final_z": 1.22 }, { "id": "53fd0b577861f02488000064", "variation": "headline", "week": 201434, "impressions": 33250, "pooled_ctr": 0.013353383458646617, "arms": [ { "headline": "Leave It To Matt Damon To Put A New Unexpected Twist On The Ice Bucket Challenge", "impressions": 5493, "clicks": 83, "ctr": 0.0151101401784088, "ctr_select": 0.01529497450837582, "ctr_holdout": 0.014925373134328358, "n_select": 2746, "n_holdout": 2747, "eb": 0.014268994368627795, "labelled_winner": false }, { "headline": "Matt Damon Did The Ice Bucket Challenge? Not Surprising. How He Did It? Pretty Impressive.", "impressions": 5620, "clicks": 67, "ctr": 0.0119217081850533, "ctr_select": 0.012455516014234875, "ctr_holdout": 0.011387900355871887, "n_select": 2810, "n_holdout": 2810, "eb": 0.012725764620033098, "labelled_winner": false }, { "headline": "What Matt Damon Did With The Toilet That Disgusted His Wife But Made Him Feel Great", "impressions": 5544, "clicks": 64, "ctr": 0.0115440115440115, "ctr_select": 0.011904761904761904, "ctr_holdout": 0.011183261183261184, "n_select": 2772, "n_holdout": 2772, "eb": 0.012427825325316628, "labelled_winner": false }, { "headline": "The Very Good Reason Matt Damon Risked Disgusting His Wife To Help Strangers", "impressions": 5468, "clicks": 81, "ctr": 0.0148134601316752, "ctr_select": 0.012801755669348939, "ctr_holdout": 0.016825164594001463, "n_select": 2734, "n_holdout": 2734, "eb": 0.012917216715481189, "labelled_winner": false }, { "headline": "The Ice Bucket Challenge Posed A Problem For Matt Damon. Here's How He Fixed It.", "impressions": 5577, "clicks": 101, "ctr": 0.0181100950331719, "ctr_select": 0.01721664275466284, "ctr_holdout": 0.0190032269630692, "n_select": 2788, "n_holdout": 2789, "eb": 0.015326773843432132, "labelled_winner": false }, { "headline": "A Newer, More Nervy Twist To The Ice Bucket Challenge. First Up ... Matt Damon.", "impressions": 5548, "clicks": 48, "ctr": 0.0086517664023071, "ctr_select": 0.008651766402307137, "ctr_holdout": 0.008651766402307137, "n_select": 2774, "n_holdout": 2774, "eb": 0.010655360388437767, "labelled_winner": false } ], "winner": 4, "pool_select": 0.013053416746871993, "pool_holdout": 0.013653314086370745, "lift_select": 0.31893764586873297, "lift_holdout": 0.3918398743964251, "eb_lift": 0.17415801093647798, "p_value": 0.03376206928772575, "held_up": true, "z_path": [ -0.808, -0.568, -0.456, -0.066, 0.123, 0.287, 0.424, 0.789, 0.828, 1.234 ], "first_cross": null, "final_z": 1.234 }, { "id": "53f4ef85ba722bdd33000008", "variation": "headline", "week": 201433, "impressions": 33006, "pooled_ctr": 0.013452099618251227, "arms": [ { "headline": "A Boy Called Her \u2018Whore\u2019 And She Thought It Was Love. Harry Potter\u2019s Penis Taught Her Otherwise.", "impressions": 5485, "clicks": 80, "ctr": 0.0145852324521422, "ctr_select": 0.016046681254558718, "ctr_holdout": 0.013124316441851986, "n_select": 2742, "n_holdout": 2743, "eb": 0.015219018567841035, "labelled_winner": false }, { "headline": "Here\u2019s Why This Girl\u2019s Harry Potter Erotica Is Better Than Whatever Porn *You\u2019re* Watching", "impressions": 5436, "clicks": 68, "ctr": 0.0125091979396615, "ctr_select": 0.012877115526122149, "ctr_holdout": 0.012141280353200883, "n_select": 2718, "n_holdout": 2718, "eb": 0.013504312074155844, "labelled_winner": false }, { "headline": "In 3 Minutes She Covers Everything From Harry Potter Erotica To Rape Culture And It. Is. Awesome.", "impressions": 5611, "clicks": 71, "ctr": 0.0126537159151666, "ctr_select": 0.011404133998574484, "ctr_holdout": 0.013903743315508022, "n_select": 2806, "n_holdout": 2805, "eb": 0.012686859581059402, "labelled_winner": false }, { "headline": "A Harry Potter Fan Issues A Smackdown To Anyone Who Thinks Their \u2018Normal\u2019 Porn Is Better Than Hers", "impressions": 5389, "clicks": 81, "ctr": 0.0150306179254036, "ctr_select": 0.016332590942835932, "ctr_holdout": 0.013729128014842301, "n_select": 2694, "n_holdout": 2695, "eb": 0.015364780130949857, "labelled_winner": false }, { "headline": "A 3-Minute Poem That\u2019ll Convince You To Throw Out Your Porn And Pick Up Her Harry Potter Erotica", "impressions": 5527, "clicks": 70, "ctr": 0.0126650986068391, "ctr_select": 0.015195369030390739, "ctr_holdout": 0.010133912414042706, "n_select": 2764, "n_holdout": 2763, "eb": 0.014759562192393258, "labelled_winner": false }, { "headline": "In 3 Mins She Argues That Her Harry Potter Erotica Is Better Than Your Porn. Holy $#*% She\u2019s Right.", "impressions": 5558, "clicks": 74, "ctr": 0.0133141417776178, "ctr_select": 0.013673983447283196, "ctr_holdout": 0.012954300107952501, "n_select": 2779, "n_holdout": 2779, "eb": 0.013931301683819716, "labelled_winner": false } ], "winner": 3, "pool_select": 0.014239835181482155, "pool_holdout": 0.0126643640550203, "lift_select": 0.14696488650902717, "lift_holdout": 0.08407559631072958, "eb_lift": 0.07899985745134241, "p_value": 0.31621896596095755, "held_up": false, "z_path": [ 1.261, 1.281, 1.936, 2.007, 1.556, 1.543, 1.238, 0.711, 0.439, 0.192 ], "first_cross": 3, "final_z": 0.192 }, { "id": "5282632083121c4e7e00961f", "variation": "headline", "week": 201345, "impressions": 32340, "pooled_ctr": 0.014625850340136054, "arms": [ { "headline": "Some Call It The Miracle Of Life. They Call It An Inescapable Curse.", "impressions": 7467, "clicks": 139, "ctr": 0.0186152403910539, "ctr_select": 0.019550080342795928, "ctr_holdout": 0.017680150013394052, "n_select": 3734, "n_holdout": 3733, "eb": 0.017959878851117935, "labelled_winner": true }, { "headline": "Some Call It The Miracle Of Life. They Call It An Inescapable Curse. But It Doesn't Have To Be.", "impressions": 8892, "clicks": 130, "ctr": 0.0146198830409356, "ctr_select": 0.014619883040935672, "ctr_holdout": 0.014619883040935672, "n_select": 4446, "n_holdout": 4446, "eb": 0.014886495821631944, "labelled_winner": false }, { "headline": "MUST WATCH: It's Called The Miracle Of Life But For Some It Is More Like An Inescapable Curse.", "impressions": 7702, "clicks": 101, "ctr": 0.0131134770189561, "ctr_select": 0.013243313425084394, "ctr_holdout": 0.012983640612827837, "n_select": 3851, "n_holdout": 3851, "eb": 0.014052845798181174, "labelled_winner": false }, { "headline": "The Miracle Of Life Has Become An Inescapable Curse For Them. But It Doesn't Have To Be.", "impressions": 8279, "clicks": 103, "ctr": 0.0124411160768208, "ctr_select": 0.014492753623188406, "ctr_holdout": 0.01038898284609809, "n_select": 4140, "n_holdout": 4139, "eb": 0.014817627271426896, "labelled_winner": false } ], "winner": 0, "pool_select": 0.015397934574237833, "pool_holdout": 0.013853670604242687, "lift_select": 0.2696560209773211, "lift_holdout": 0.27620689985075186, "eb_lift": 0.1663823329374623, "p_value": 0.018789544921063144, "held_up": true, "z_path": [ 0.085, 0.246, -0.033, -0.849, -0.233, 0.145, 0.354, 1.098, 1.506, 2.001 ], "first_cross": 9, "final_z": 2.001 }, { "id": "52729dad576e674866000cdd", "variation": "headline", "week": 201343, "impressions": 32308, "pooled_ctr": 0.025071189798192398, "arms": [ { "headline": "The Disappointing Difference Between Men And Women, According To The World\u2019s Biggest Search Engine", "impressions": 8098, "clicks": 218, "ctr": 0.0269202272165966, "ctr_select": 0.027167201778216844, "ctr_holdout": 0.026673252654976536, "n_select": 4049, "n_holdout": 4049, "eb": 0.026021081735781063, "labelled_winner": false }, { "headline": "The Frustrating Between Men And Women, According To The World\u2019s Biggest Search Engine", "impressions": 8261, "clicks": 159, "ctr": 0.0192470645200338, "ctr_select": 0.016707021791767553, "ctr_holdout": 0.02178649237472767, "n_select": 4130, "n_holdout": 4131, "eb": 0.019335329767351247, "labelled_winner": false }, { "headline": "The Upsetting Difference Between Men And Women, According To The World\u2019s Biggest Search Engine", "impressions": 8283, "clicks": 224, "ctr": 0.0270433417843776, "ctr_select": 0.02583293095123129, "ctr_holdout": 0.02825404491668679, "n_select": 4142, "n_holdout": 4141, "eb": 0.025182039283757437, "labelled_winner": false }, { "headline": "The Difference Between Men And Women, According To The World\u2019s Biggest Search Engine", "impressions": 7666, "clicks": 209, "ctr": 0.0272632402817636, "ctr_select": 0.026611009653013305, "ctr_holdout": 0.027915470910513957, "n_select": 3833, "n_holdout": 3833, "eb": 0.02563422377927988, "labelled_winner": false } ], "winner": 0, "pool_select": 0.02401881886839173, "pool_holdout": 0.026123560727993066, "lift_select": 0.1310798389827703, "lift_holdout": 0.021041998550926477, "eb_lift": 0.08336225350465787, "p_value": 0.13064836721119705, "held_up": false, "z_path": [ -0.042, -0.054, -0.585, -0.974, -0.957, -0.189, 0.348, 0.53, 0.333, -0.133 ], "first_cross": null, "final_z": -0.133 }, { "id": "5393c9cb73ead54d2c000016", "variation": "image", "week": 201423, "impressions": 30752, "pooled_ctr": 0.018080124869927158, "arms": [ { "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.", "impressions": 5072, "clicks": 104, "ctr": 0.0205047318611987, "ctr_select": 0.02168769716088328, "ctr_holdout": 0.019321766561514197, "n_select": 2536, "n_holdout": 2536, "eb": 0.02012031726422287, "labelled_winner": false }, { "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.", "impressions": 5019, "clicks": 78, "ctr": 0.0155409444112372, "ctr_select": 0.015537848605577689, "ctr_holdout": 0.015544041450777202, "n_select": 2510, "n_holdout": 2509, "eb": 0.016914467872229298, "labelled_winner": false }, { "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.", "impressions": 5133, "clicks": 109, "ctr": 0.0212351451392947, "ctr_select": 0.021823850350740453, "ctr_holdout": 0.020646669263731984, "n_select": 2566, "n_holdout": 2567, "eb": 0.02020148332036455, "labelled_winner": false }, { "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.", "impressions": 5183, "clicks": 86, "ctr": 0.0165927069264904, "ctr_select": 0.018132716049382717, "ctr_holdout": 0.015052103434967195, "n_select": 2592, "n_holdout": 2591, "eb": 0.01826139675305129, "labelled_winner": false }, { "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.", "impressions": 5140, "clicks": 91, "ctr": 0.017704280155642, "ctr_select": 0.017509727626459144, "ctr_holdout": 0.0178988326848249, "n_select": 2570, "n_holdout": 2570, "eb": 0.017934402456755608, "labelled_winner": false }, { "headline": "So You're Not Sure If You Should Use The Word 'Gay?' Here's A Handy Flow Chart.", "impressions": 5205, "clicks": 88, "ctr": 0.0169068203650336, "ctr_select": 0.015757109915449655, "ctr_holdout": 0.018056089127929314, "n_select": 2602, "n_holdout": 2603, "eb": 0.017004690683142417, "labelled_winner": false } ], "winner": 2, "pool_select": 0.018405306971904265, "pool_holdout": 0.017754942767950053, "lift_select": 0.18573683036390531, "lift_holdout": 0.16286881538147613, "eb_lift": 0.09759013262871141, "p_value": 0.1580991264810938, "held_up": true, "z_path": [ 0.367, 0.244, -0.405, -0.705, -0.848, -0.426, 0.256, -0.004, -0.157, 0.258 ], "first_cross": null, "final_z": 0.258 }, { "id": "53fa81f12a06a92e4900000c", "variation": "headline", "week": 201434, "impressions": 30652, "pooled_ctr": 0.013506459611118361, "arms": [ { "headline": "Does Your Best Friend Feel Like Family? That's Because They Kinda Are.", "impressions": 5055, "clicks": 41, "ctr": 0.0081107814045499, "ctr_select": 0.010680379746835443, "ctr_holdout": 0.00554016620498615, "n_select": 2528, "n_holdout": 2527, "eb": 0.012468111852896136, "labelled_winner": false }, { "headline": "A Scientific Reason Kissing Your Friend Is Kinda Like Kissing Your Cousin", "impressions": 5183, "clicks": 58, "ctr": 0.0111904302527493, "ctr_select": 0.011188271604938271, "ctr_holdout": 0.011192589733693555, "n_select": 2592, "n_holdout": 2591, "eb": 0.0127129417617274, "labelled_winner": false }, { "headline": "A Bizarre Scientific Reason You Might Wanna Keep That Friend 'In The Friend Zone'", "impressions": 5063, "clicks": 61, "ctr": 0.0120481927710843, "ctr_select": 0.014612954186413903, "ctr_holdout": 0.009482418016594232, "n_select": 2532, "n_holdout": 2531, "eb": 0.014519812424476196, "labelled_winner": false }, { "headline": "A Bizarre Reason You Were Drawn To Your Best Friend ... Long Before You Met", "impressions": 5188, "clicks": 82, "ctr": 0.0158057054741711, "ctr_select": 0.015420200462606014, "ctr_holdout": 0.016191210485736313, "n_select": 2594, "n_holdout": 2594, "eb": 0.014947316671673932, "labelled_winner": false }, { "headline": "The Scientific Reason Someone Chooses To Be Your Friend Might Freak You Out ... Just A Bit", "impressions": 5031, "clicks": 111, "ctr": 0.0220632081097197, "ctr_select": 0.023052464228934817, "ctr_holdout": 0.02107355864811133, "n_select": 2516, "n_holdout": 2515, "eb": 0.018912374453838473, "labelled_winner": false }, { "headline": "Best Friends Share Almost Everything, But Things Just Got Weird", "impressions": 5132, "clicks": 61, "ctr": 0.0118862042088854, "ctr_select": 0.011691348402182385, "ctr_holdout": 0.012081060015588464, "n_select": 2566, "n_holdout": 2566, "eb": 0.012985386242585084, "labelled_winner": false } ], "winner": 4, "pool_select": 0.014418058455114822, "pool_holdout": 0.012594622813886713, "lift_select": 0.5988605054348999, "lift_holdout": 0.6732187187754304, "eb_lift": 0.3117143693594395, "p_value": 7.069069302376094e-05, "held_up": true, "z_path": [ 2.063, 1.643, 1.994, 2.638, 2.176, 2.633, 3.033, 3.106, 2.349, 2.323 ], "first_cross": 0, "final_z": 2.323 }, { "id": "5391cd299c9fb52c18000018", "variation": "headline", "week": 201422, "impressions": 28169, "pooled_ctr": 0.004792502396251198, "arms": [ { "headline": "It\u2019s Not A Revolution If You Can\u2019t Dance To It", "impressions": 5816, "clicks": 24, "ctr": 0.0041265474552957, "ctr_select": 0.002407152682255846, "ctr_holdout": 0.005845942228335626, "n_select": 2908, "n_holdout": 2908, "eb": 0.003417940235709462, "labelled_winner": false }, { "headline": "We Spent An Afternoon In The Virtual Record Stacks And Made These Playlists For You", "impressions": 5599, "clicks": 26, "ctr": 0.0046436863725665, "ctr_select": 0.005, "ctr_holdout": 0.004287245444801715, "n_select": 2800, "n_holdout": 2799, "eb": 0.0048578724026257965, "labelled_winner": false }, { "headline": "Bob Your Head To Our 3 Songza Playlists", "impressions": 5804, "clicks": 26, "ctr": 0.0044796691936595, "ctr_select": 0.00585802894555479, "ctr_holdout": 0.0031013094417643005, "n_select": 2902, "n_holdout": 2902, "eb": 0.005337688889451963, "labelled_winner": false }, { "headline": "If You've Ever Wondered What We Listened To In Our Offices, The Wait Is Over.", "impressions": 5457, "clicks": 26, "ctr": 0.0047645226314824, "ctr_select": 0.004398826979472141, "ctr_holdout": 0.005130084279956028, "n_select": 2728, "n_holdout": 2729, "eb": 0.0045308149204672235, "labelled_winner": false }, { "headline": "We Tore Through Some Virtual Record Stacks And Made 3 Playlists That'll Get You Through The Summer", "impressions": 5493, "clicks": 33, "ctr": 0.00600764609503, "ctr_select": 0.005826656955571741, "ctr_holdout": 0.006188569348380051, "n_select": 2746, "n_holdout": 2747, "eb": 0.005304645737390898, "labelled_winner": false } ], "winner": 2, "pool_select": 0.004686168702073275, "pool_holdout": 0.004898828541001065, "lift_select": 0.25006787377566164, "lift_holdout": -0.3669283552572439, "eb_lift": 0.13903045937941583, "p_value": 0.29955775065097434, "held_up": false, "z_path": [ -0.626, -0.087, -0.365, -0.351, -0.136, 0.026, -0.672, -1.393, -1.034, -1.126 ], "first_cross": null, "final_z": -1.126 }, { "id": "541f8fa29d99f1ec4d000032", "variation": "headline", "week": 201438, "impressions": 27970, "pooled_ctr": 0.0039685377189846265, "arms": [ { "headline": "Quit Your Job, Sell Everything, And Live Like A Boss By Calling The Entire Planet Your Home", "impressions": 6973, "clicks": 38, "ctr": 0.0054495912806539, "ctr_select": 0.005737234652897304, "ctr_holdout": 0.005162030398623459, "n_select": 3486, "n_holdout": 3487, "eb": 0.005159530469016437, "labelled_winner": false }, { "headline": "Here's A Guy Who's Been Earth Hopping For 10 Years And He's Ready To Teach You How You Can Do It Too", "impressions": 6757, "clicks": 31, "ctr": 0.0045878348379458, "ctr_select": 0.004440497335701598, "ctr_holdout": 0.004735128736312519, "n_select": 3378, "n_holdout": 3379, "eb": 0.004379508259317443, "labelled_winner": false }, { "headline": "A World Traveler Teaches Us How To Have A Vacation That Never Has To End", "impressions": 7097, "clicks": 34, "ctr": 0.0047907566577427, "ctr_select": 0.005355129650507328, "ctr_holdout": 0.00422654268808115, "n_select": 3548, "n_holdout": 3549, "eb": 0.004934511902506225, "labelled_winner": false }, { "headline": "One Guy Explains How To Be A Continent Bounding Nomad In Search Of Unforgettable Adventure", "impressions": 7143, "clicks": 8, "ctr": 0.0011199776004479, "ctr_select": 0.0016797312430011197, "ctr_holdout": 0.0005600672080649678, "n_select": 3572, "n_holdout": 3571, "eb": 0.00270716018709675, "labelled_winner": false } ], "winner": 0, "pool_select": 0.004290617848970252, "pool_holdout": 0.0036465036465036463, "lift_select": 0.33715815643526503, "lift_holdout": 0.4156109246107391, "eb_lift": 0.2025145679787641, "p_value": 0.1315082638851518, "held_up": true, "z_path": [ 1.019, 0.865, 1.647, 1.782, 1.5, 0.948, 0.492, 0.478, 0.857, 0.548 ], "first_cross": null, "final_z": 0.548 }, { "id": "5420bab11afbe8fce400007e", "variation": "image", "week": 201438, "impressions": 27932, "pooled_ctr": 0.01202921380495489, "arms": [ { "headline": "The Messages These Women Read Would Be Funny If They Were From A Movie. Except They're Not.", "impressions": 6955, "clicks": 132, "ctr": 0.018979151689432, "ctr_select": 0.019838987924094307, "ctr_holdout": 0.0181190681622088, "n_select": 3478, "n_holdout": 3477, "eb": 0.016744495492083924, "labelled_winner": false }, { "headline": "The Messages These Women Read Would Be Funny If They Were From A Movie. Except They're Not.", "impressions": 7038, "clicks": 62, "ctr": 0.0088093208297811, "ctr_select": 0.007956805910770106, "ctr_holdout": 0.00966183574879227, "n_select": 3519, "n_holdout": 3519, "eb": 0.009602353406829517, "labelled_winner": false }, { "headline": "The Messages These Women Read Would Be Funny If They Were From A Movie. Except They're Not.", "impressions": 6877, "clicks": 76, "ctr": 0.0110513305220299, "ctr_select": 0.011052937754508435, "ctr_holdout": 0.011049723756906077, "n_select": 3438, "n_holdout": 3439, "eb": 0.011474896264192496, "labelled_winner": false }, { "headline": "The Messages These Women Read Would Be Funny If They Were From A Movie. Except They're Not.", "impressions": 7062, "clicks": 66, "ctr": 0.0093457943925233, "ctr_select": 0.009629000283205891, "ctr_holdout": 0.009062588501840838, "n_select": 3531, "n_holdout": 3531, "eb": 0.010608521284929166, "labelled_winner": false } ], "winner": 0, "pool_select": 0.012100816268079621, "pool_holdout": 0.01195761134183016, "lift_select": 0.6394751795733793, "lift_holdout": 0.5152748859485514, "eb_lift": 0.3837492546890182, "p_value": 1.4611751204554249e-06, "held_up": true, "z_path": [ 0.206, -0.034, 0.372, 0.8, 1.006, 2.033, 1.864, 1.952, 2.777, 3.831 ], "first_cross": 5, "final_z": 3.831 }, { "id": "53e271645fba149a24000001", "variation": "image", "week": 201431, "impressions": 27615, "pooled_ctr": 0.009053050878145934, "arms": [ { "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something", "impressions": 4595, "clicks": 43, "ctr": 0.0093579978237214, "ctr_select": 0.009138381201044387, "ctr_holdout": 0.00957771005659556, "n_select": 2298, "n_holdout": 2297, "eb": 0.009059003931114687, "labelled_winner": false }, { "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something", "impressions": 4643, "clicks": 57, "ctr": 0.0122765453370665, "ctr_select": 0.014211886304909561, "ctr_holdout": 0.01034037052994399, "n_select": 2322, "n_holdout": 2321, "eb": 0.011598519359864435, "labelled_winner": false }, { "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something", "impressions": 4585, "clicks": 28, "ctr": 0.0061068702290076, "ctr_select": 0.005235602094240838, "ctr_holdout": 0.006977758395115569, "n_select": 2292, "n_holdout": 2293, "eb": 0.007118389480969683, "labelled_winner": false }, { "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something", "impressions": 4619, "clicks": 34, "ctr": 0.0073609006278415, "ctr_select": 0.008225108225108226, "ctr_holdout": 0.0064963187527068, "n_select": 2310, "n_holdout": 2309, "eb": 0.008603332843294554, "labelled_winner": false }, { "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something", "impressions": 4561, "clicks": 40, "ctr": 0.0087700065775049, "ctr_select": 0.007894736842105263, "ctr_holdout": 0.009644892590968873, "n_select": 2280, "n_holdout": 2281, "eb": 0.008441969469196823, "labelled_winner": false }, { "headline": "The People You See Riding A Bike On The Side Of The Road Would Like You To Know Something", "impressions": 4612, "clicks": 48, "ctr": 0.01040763226366, "ctr_select": 0.009106678230702514, "ctr_holdout": 0.011708586296617519, "n_select": 2306, "n_holdout": 2306, "eb": 0.00904332988923771, "labelled_winner": false } ], "winner": 1, "pool_select": 0.008980301274623407, "pool_holdout": 0.009125805750706163, "lift_select": 0.5825623072434776, "lift_holdout": 0.1330912373566402, "eb_lift": 0.2915512525887751, "p_value": 0.003390270502562463, "held_up": false, "z_path": [ -0.019, 0.832, 0.512, 0.294, -0.311, -0.289, 0.197, 0.397, 0.688, 1.355 ], "first_cross": null, "final_z": 1.355 }, { "id": "544d103de8a6ed45d400000c", "variation": "image", "week": 201443, "impressions": 27386, "pooled_ctr": 0.008215876725334113, "arms": [ { "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised", "impressions": 3795, "clicks": 37, "ctr": 0.0097496706192358, "ctr_select": 0.007903055848261328, "ctr_holdout": 0.011597258829731154, "n_select": 1898, "n_holdout": 1897, "eb": 0.007533299221811757, "labelled_winner": true }, { "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised", "impressions": 3925, "clicks": 29, "ctr": 0.0073885350318471, "ctr_select": 0.009683995922528032, "ctr_holdout": 0.005094243504839531, "n_select": 1962, "n_holdout": 1963, "eb": 0.008355289253022327, "labelled_winner": false }, { "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised", "impressions": 3945, "clicks": 38, "ctr": 0.0096324461343472, "ctr_select": 0.006592292089249493, "ctr_holdout": 0.012671059300557527, "n_select": 1972, "n_holdout": 1973, "eb": 0.006937111723645805, "labelled_winner": false }, { "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised", "impressions": 4045, "clicks": 28, "ctr": 0.0069221260815822, "ctr_select": 0.006429277942631058, "ctr_holdout": 0.007414730598121601, "n_select": 2022, "n_holdout": 2023, "eb": 0.006857185514543291, "labelled_winner": false }, { "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised", "impressions": 3922, "clicks": 26, "ctr": 0.0066292707802141, "ctr_select": 0.006119326874043855, "ctr_holdout": 0.0071392146863844975, "n_select": 1961, "n_holdout": 1961, "eb": 0.006721230202023124, "labelled_winner": false }, { "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised", "impressions": 3792, "clicks": 31, "ctr": 0.008175105485232, "ctr_select": 0.008438818565400843, "ctr_holdout": 0.007911392405063292, "n_select": 1896, "n_holdout": 1896, "eb": 0.007774196094249299, "labelled_winner": false }, { "headline": "\u200bA 13-Year-Old's List Of 'Important' Things Sure Has Me Surprised", "impressions": 3962, "clicks": 36, "ctr": 0.0090863200403836, "ctr_select": 0.005552751135790005, "ctr_holdout": 0.012619888944977285, "n_select": 1981, "n_holdout": 1981, "eb": 0.006457330602193736, "labelled_winner": false } ], "winner": 1, "pool_select": 0.007230499561787906, "pool_holdout": 0.009201109975171607, "lift_select": 0.33932598152781623, "lift_holdout": -0.4463446781327576, "eb_lift": 0.15556182275133024, "p_value": 0.16579781078707745, "held_up": false, "z_path": [ 0.285, -0.359, -0.085, 0.092, -0.455, -0.613, -0.27, 0.033, -0.401, -0.393 ], "first_cross": null, "final_z": -0.393 }, { "id": "52a65dc3476fb047c10009c0", "variation": "headline", "week": 201349, "impressions": 27366, "pooled_ctr": 0.03749177811883359, "arms": [ { "headline": "This Ad Totally Nails 5 Ridiculous Double Standards Women Face In Less Than 60 Seconds", "impressions": 6717, "clicks": 251, "ctr": 0.0373678725621557, "ctr_select": 0.03871351995235259, "ctr_holdout": 0.03602262578148258, "n_select": 3358, "n_holdout": 3359, "eb": 0.03785648863959225, "labelled_winner": false }, { "headline": "This Ad Totally Nails 5 Unfortunate Double Standards Women Face In Less Than 60 Seconds", "impressions": 6990, "clicks": 263, "ctr": 0.0376251788268955, "ctr_select": 0.03547925608011445, "ctr_holdout": 0.03977110157367668, "n_select": 3495, "n_holdout": 3495, "eb": 0.035932022666640334, "labelled_winner": false }, { "headline": "NAILED IT: This Ad Calls Out 5 Ridiculous Double Standards Women Face In Less Than 60 Seconds", "impressions": 7423, "clicks": 279, "ctr": 0.0375858817189815, "ctr_select": 0.03529094827586207, "ctr_holdout": 0.039881433575855565, "n_select": 3712, "n_holdout": 3711, "eb": 0.03579980228715958, "labelled_winner": true }, { "headline": "5 Ridiculous Double Standards Women Face In Less Than 60 Seconds", "impressions": 6236, "clicks": 233, "ctr": 0.0373636946760744, "ctr_select": 0.03720333547145606, "ctr_holdout": 0.03752405388069275, "n_select": 3118, "n_holdout": 3118, "eb": 0.03695238340903314, "labelled_winner": false } ], "winner": 0, "pool_select": 0.036614777461083096, "pool_holdout": 0.03836877877658408, "lift_select": 0.05731954792024041, "lift_holdout": -0.06114745034661673, "eb_lift": 0.03391284242622894, "p_value": 0.4560013598018976, "held_up": false, "z_path": [ -0.702, -0.464, -0.27, -0.358, -0.841, -0.2, 0.392, 0.448, 0.555, 0.001 ], "first_cross": null, "final_z": 0.001 }, { "id": "53fb622a2a06a94b42000029", "variation": "headline", "week": 201434, "impressions": 26961, "pooled_ctr": 0.006565038388783799, "arms": [ { "headline": "Something That No Parent Should Ever Have To Teach Their Child", "impressions": 4564, "clicks": 29, "ctr": 0.0063540753724802, "ctr_select": 0.006573181419807186, "ctr_holdout": 0.006134969325153374, "n_select": 2282, "n_holdout": 2282, "eb": 0.006625270787771982, "labelled_winner": false }, { "headline": "Here\u2019s Something I Sincerely Never Want To Have To Teach My Child", "impressions": 4490, "clicks": 25, "ctr": 0.0055679287305122, "ctr_select": 0.004008908685968819, "ctr_holdout": 0.0071269487750556795, "n_select": 2245, "n_holdout": 2245, "eb": 0.005363985172683472, "labelled_winner": false }, { "headline": "Mary Engelbreit: No One Should Have To Teach Their Children This In The USA", "impressions": 4558, "clicks": 36, "ctr": 0.0078982009653356, "ctr_select": 0.009653356735410267, "ctr_holdout": 0.00614304519526108, "n_select": 2279, "n_holdout": 2279, "eb": 0.008152427077663794, "labelled_winner": false }, { "headline": "I Have No Words After Understanding What It Means To Have To Teach Kids These 4 Words.", "impressions": 4553, "clicks": 41, "ctr": 0.0090050516143202, "ctr_select": 0.007908611599297012, "ctr_holdout": 0.010101010101010102, "n_select": 2276, "n_holdout": 2277, "eb": 0.007286992960230153, "labelled_winner": false }, { "headline": "4 Words That Leave Me With No Words", "impressions": 4366, "clicks": 27, "ctr": 0.0061841502519468, "ctr_select": 0.007787448465414567, "ctr_holdout": 0.004580852038479157, "n_select": 2183, "n_holdout": 2183, "eb": 0.0072153796958318965, "labelled_winner": false }, { "headline": "No One Should Have To Teach Their Children This In The USA", "impressions": 4430, "clicks": 19, "ctr": 0.0042889390519187, "ctr_select": 0.004063205417607223, "ctr_holdout": 0.004514672686230248, "n_select": 2215, "n_holdout": 2215, "eb": 0.0053994871111904185, "labelled_winner": false } ], "winner": 2, "pool_select": 0.0066765578635014835, "pool_holdout": 0.006453527186410504, "lift_select": 0.4458583199258934, "lift_holdout": -0.048110433594084956, "eb_lift": 0.221052411187866, "p_value": 0.055578146416599694, "held_up": false, "z_path": [ -0.578, 0.302, -0.449, -0.603, -0.672, -0.623, -0.288, -0.783, -0.126, -0.577 ], "first_cross": null, "final_z": -0.577 }, { "id": "534eb20dfcf40f8485000075", "variation": "headline", "week": 201415, "impressions": 26798, "pooled_ctr": 0.021792671094857826, "arms": [ { "headline": "The Most Honest And Awful Corporate Ad I Have Ever Laid My Eyes On. No, They Aren't Drunk. I Swear.", "impressions": 3786, "clicks": 138, "ctr": 0.0364500792393026, "ctr_select": 0.036978341257263604, "ctr_holdout": 0.03592181722134179, "n_select": 1893, "n_holdout": 1893, "eb": 0.02853752070455732, "labelled_winner": false }, { "headline": "The Most Honest Corporate Ad I Have Ever Laid My Eyes On. No, They Aren't Drunk. I Swear.", "impressions": 3871, "clicks": 97, "ctr": 0.025058124515629, "ctr_select": 0.027892561983471075, "ctr_holdout": 0.022222222222222223, "n_select": 1936, "n_holdout": 1935, "eb": 0.02448760588067879, "labelled_winner": false }, { "headline": "The Most Honest Coal Company Ad I Have Ever Laid My Eyes On. No, They Aren't Drunk. I Swear.", "impressions": 3766, "clicks": 62, "ctr": 0.0164630908125331, "ctr_select": 0.014338821030270845, "ctr_holdout": 0.01858736059479554, "n_select": 1883, "n_holdout": 1883, "eb": 0.018369061130369884, "labelled_winner": false }, { "headline": "The Most Honest Energy Company Ad I Have Ever Laid My Eyes On. No, They Aren't Drunk. I Swear.", "impressions": 3833, "clicks": 74, "ctr": 0.0193060266110096, "ctr_select": 0.019311064718162838, "ctr_holdout": 0.019300991131977047, "n_select": 1916, "n_holdout": 1917, "eb": 0.020587882982580876, "labelled_winner": false }, { "headline": "The Most Honest Energy Company Ad I Have Ever Laid My Eyes On", "impressions": 3866, "clicks": 45, "ctr": 0.011639937920331, "ctr_select": 0.012933264355923435, "ctr_holdout": 0.010346611484738748, "n_select": 1933, "n_holdout": 1933, "eb": 0.01768247900684302, "labelled_winner": false }, { "headline": "The Most Honest, In-Your-Face, Company Ad I Have Ever Laid My Eyes On. And No, They Aren't Drunk.", "impressions": 3876, "clicks": 93, "ctr": 0.0239938080495356, "ctr_select": 0.021671826625386997, "ctr_holdout": 0.02631578947368421, "n_select": 1938, "n_holdout": 1938, "eb": 0.021656348576232366, "labelled_winner": false }, { "headline": "An Extremely Honest And Rude Energy Company Ad That Doesn't Mix Words About Not Caring About You", "impressions": 3800, "clicks": 75, "ctr": 0.0197368421052631, "ctr_select": 0.018421052631578946, "ctr_holdout": 0.021052631578947368, "n_select": 1900, "n_holdout": 1900, "eb": 0.02019179156274224, "labelled_winner": false } ], "winner": 0, "pool_select": 0.021643406224345102, "pool_holdout": 0.02194193596537055, "lift_select": 0.7085268776071552, "lift_holdout": 0.6371307107100632, "eb_lift": 0.3185318617943569, "p_value": 7.502304424942291e-07, "held_up": true, "z_path": [ 3.374, 1.994, 2.56, 3.039, 2.588, 3.049, 3.243, 3.396, 3.372, 2.89 ], "first_cross": 0, "final_z": 2.89 }, { "id": "5277d8c0ae0d4e881d000140", "variation": "headline", "week": 201344, "impressions": 26603, "pooled_ctr": 0.03213923241739654, "arms": [ { "headline": "Something Truly Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It", "impressions": 5278, "clicks": 163, "ctr": 0.030882910193255, "ctr_select": 0.028798787419477074, "ctr_holdout": 0.03296703296703297, "n_select": 2639, "n_holdout": 2639, "eb": 0.030518361180697322, "labelled_winner": false }, { "headline": "Something Really Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It", "impressions": 5428, "clicks": 180, "ctr": 0.0331613854089904, "ctr_select": 0.03500368459837878, "ctr_holdout": 0.03131908621960206, "n_select": 2714, "n_holdout": 2714, "eb": 0.03383953107054521, "labelled_winner": false }, { "headline": "Something Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It", "impressions": 5339, "clicks": 168, "ctr": 0.0314665667728039, "ctr_select": 0.03258426966292135, "ctr_holdout": 0.030348445110528286, "n_select": 2670, "n_holdout": 2669, "eb": 0.03253410700147092, "labelled_winner": false }, { "headline": "Something Very Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It", "impressions": 5590, "clicks": 191, "ctr": 0.0341681574239713, "ctr_select": 0.03398926654740608, "ctr_holdout": 0.03434704830053667, "n_select": 2795, "n_holdout": 2795, "eb": 0.033303414488837335, "labelled_winner": false }, { "headline": "Something Way Insane Just Happened In Congress, And You Probably Haven't Heard A Word About It", "impressions": 4968, "clicks": 153, "ctr": 0.0307971014492753, "ctr_select": 0.03180354267310789, "ctr_holdout": 0.029790660225442835, "n_select": 2484, "n_holdout": 2484, "eb": 0.03212828051725769, "labelled_winner": false } ], "winner": 1, "pool_select": 0.03247631935047361, "pool_holdout": 0.03180212014134275, "lift_select": 0.07782178825841313, "lift_holdout": -0.015188733316957248, "eb_lift": 0.04197556088053789, "p_value": 0.4050986674098218, "held_up": false, "z_path": [ 1.186, 1.803, 2.255, 0.51, -0.074, -0.101, -0.133, -0.097, -0.396, -0.293 ], "first_cross": 2, "final_z": -0.293 }, { "id": "52318ead117f64b609005c31", "variation": "image", "week": 201336, "impressions": 26332, "pooled_ctr": 0.028938174084763787, "arms": [ { "headline": "5 Reasons Why my Girlfriend Thinks She's Not Beautiful Enough, No Matter What Anyone Tells Her", "impressions": 9151, "clicks": 233, "ctr": 0.0254616981750628, "ctr_select": 0.024912587412587412, "ctr_holdout": 0.026010928961748635, "n_select": 4576, "n_holdout": 4575, "eb": 0.02608725987833294, "labelled_winner": false }, { "headline": "5 Reasons Why my Girlfriend Thinks She's Not Beautiful Enough, No Matter What Anyone Tells Her", "impressions": 8488, "clicks": 285, "ctr": 0.0335768143261074, "ctr_select": 0.033930254476908575, "ctr_holdout": 0.03322337417530632, "n_select": 4244, "n_holdout": 4244, "eb": 0.03197917566291753, "labelled_winner": false }, { "headline": "5 Reasons Why my Girlfriend Thinks She's Not Beautiful Enough, No Matter What Anyone Tells Her", "impressions": 8693, "clicks": 244, "ctr": 0.0280685609110778, "ctr_select": 0.026691210308329497, "ctr_holdout": 0.029445594662985967, "n_select": 4346, "n_holdout": 4347, "eb": 0.027287805000068135, "labelled_winner": false } ], "winner": 1, "pool_select": 0.02840650159501747, "pool_holdout": 0.0294698465745101, "lift_select": 0.19445382471384565, "lift_holdout": 0.12736841338165727, "eb_lift": 0.12576959031543367, "p_value": 0.008506467301024528, "held_up": true, "z_path": [ 2.756, 2.552, 2.231, 2.421, 2.251, 2.629, 2.375, 2.086, 2.233, 2.09 ], "first_cross": 0, "final_z": 2.09 }, { "id": "527f702108d1715c87007d77", "variation": "headline", "week": 201345, "impressions": 26268, "pooled_ctr": 0.0063194761687224, "arms": [ { "headline": "How Can We Do This To 2 Million Of Our Fellow Americans?", "impressions": 6363, "clicks": 46, "ctr": 0.0072292943580072, "ctr_select": 0.009428032683846637, "ctr_holdout": 0.005029864822382898, "n_select": 3182, "n_holdout": 3181, "eb": 0.008310317238059441, "labelled_winner": false }, { "headline": "Congress Is About To Let 2 Million Of These People Fall Off The Edge", "impressions": 6574, "clicks": 41, "ctr": 0.0062366899908731, "ctr_select": 0.0060845756008518406, "ctr_holdout": 0.006388804380894433, "n_select": 3287, "n_holdout": 3287, "eb": 0.006370411843874417, "labelled_winner": false }, { "headline": "Congress Is About To Go Home For The Holidays, While These People May Not HAVE A Home...", "impressions": 6487, "clicks": 26, "ctr": 0.0040080160320641, "ctr_select": 0.0052404438964241675, "ctr_holdout": 0.0027752081406105457, "n_select": 3244, "n_holdout": 3243, "eb": 0.0058802607361574406, "labelled_winner": false }, { "headline": "What Happened To An America Where We Give A Damn About Our Fellow Americans?", "impressions": 6844, "clicks": 53, "ctr": 0.0077440093512565, "ctr_select": 0.006428988895382817, "ctr_holdout": 0.009059029807130333, "n_select": 3422, "n_holdout": 3422, "eb": 0.006569027600268092, "labelled_winner": false } ], "winner": 0, "pool_select": 0.006775789874381424, "pool_holdout": 0.00586309297190284, "lift_select": 0.3914293180036581, "lift_holdout": -0.14211409464474534, "eb_lift": 0.22647210024618825, "p_value": 0.03616509041168524, "held_up": false, "z_path": [ -0.147, -0.232, -0.315, 0.096, -0.014, -0.235, -0.288, 0.11, 0.123, -0.343 ], "first_cross": null, "final_z": -0.343 }, { "id": "5419f47d3088cf4a200000c0", "variation": "headline", "week": 201437, "impressions": 25835, "pooled_ctr": 0.004296497000193536, "arms": [ { "headline": "Morgan Freeman Is Speaking. Clouds Are Parting. I'm Over Here Feeling Inspired About Our Planet.", "impressions": 4431, "clicks": 20, "ctr": 0.0045136538027533, "ctr_select": 0.004963898916967509, "ctr_holdout": 0.004063205417607223, "n_select": 2216, "n_holdout": 2215, "eb": 0.004404817864750659, "labelled_winner": false }, { "headline": "The Rallying Cry The Environmental Movement Has Been Waiting For", "impressions": 4320, "clicks": 9, "ctr": 0.0020833333333333, "ctr_select": 0.001851851851851852, "ctr_holdout": 0.0023148148148148147, "n_select": 2160, "n_holdout": 2160, "eb": 0.0028966030085792274, "labelled_winner": false }, { "headline": "62 Seconds That Will Motivate You To Participate In The Future Of Our Planet, Via Morgan Freeman", "impressions": 4360, "clicks": 26, "ctr": 0.0059633027522935, "ctr_select": 0.005963302752293578, "ctr_holdout": 0.005963302752293578, "n_select": 2180, "n_holdout": 2180, "eb": 0.00488474639371937, "labelled_winner": false }, { "headline": "Morgan Freeman Has Something To Say About The Environment. And Yes It's An Inspiring Voiceover.", "impressions": 4263, "clicks": 24, "ctr": 0.0056298381421534, "ctr_select": 0.005628517823639775, "ctr_holdout": 0.005631159080244017, "n_select": 2132, "n_holdout": 2131, "eb": 0.00471270500486183, "labelled_winner": false }, { "headline": "You Know That 62 Second Speech You Gave In 2nd Grade About Trees? This Is The PHD Version Of That", "impressions": 4255, "clicks": 17, "ctr": 0.0039952996474735, "ctr_select": 0.0018796992481203006, "ctr_holdout": 0.0061118946873530795, "n_select": 2128, "n_holdout": 2127, "eb": 0.0029174526709793303, "labelled_winner": false }, { "headline": "Well, Well, Well. Pessimists Beware Because I've Got An Inspiring Video On The FUTURE OF OUR PLANET!", "impressions": 4206, "clicks": 15, "ctr": 0.0035663338088445, "ctr_select": 0.0028530670470756064, "ctr_holdout": 0.0042796005706134095, "n_select": 2103, "n_holdout": 2103, "eb": 0.0033863687686807066, "labelled_winner": false } ], "winner": 2, "pool_select": 0.0038702685966406068, "pool_holdout": 0.004722824403840198, "lift_select": 0.5407981651376148, "lift_holdout": 0.26265603850203045, "eb_lift": 0.2621207732092108, "p_value": 0.08429894800707505, "held_up": false, "z_path": [ -0.015, -1.164, -0.316, -0.544, -0.706, -0.256, -0.059, -0.409, -0.229, 0.204 ], "first_cross": null, "final_z": 0.204 }, { "id": "5388de3a5f5ab749ca00000a", "variation": "image", "week": 201421, "impressions": 25592, "pooled_ctr": 0.003594873397936855, "arms": [ { "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley", "impressions": 5071, "clicks": 10, "ctr": 0.0019719976336028, "ctr_select": 0.002365930599369085, "ctr_holdout": 0.0015779092702169625, "n_select": 2536, "n_holdout": 2535, "eb": 0.0031765171984118497, "labelled_winner": false }, { "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley", "impressions": 5138, "clicks": 35, "ctr": 0.0068119891008174, "ctr_select": 0.009342156481121059, "ctr_holdout": 0.004281821720513819, "n_select": 2569, "n_holdout": 2569, "eb": 0.006838535193812306, "labelled_winner": true }, { "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley", "impressions": 5179, "clicks": 17, "ctr": 0.0032824869665958, "ctr_select": 0.0015444015444015444, "ctr_holdout": 0.005021243723445346, "n_select": 2590, "n_holdout": 2589, "eb": 0.0027340425271084926, "labelled_winner": false }, { "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley", "impressions": 5023, "clicks": 20, "ctr": 0.0039816842524387, "ctr_select": 0.0051751592356687895, "ctr_holdout": 0.0027877339705296694, "n_select": 2512, "n_holdout": 2511, "eb": 0.004641668196573108, "labelled_winner": false }, { "headline": "I Found A Life-Saving Use For Drones, And It's In Tornado Alley", "impressions": 5181, "clicks": 10, "ctr": 0.0019301293186643, "ctr_select": 0.0019305019305019305, "ctr_holdout": 0.0019297568506368198, "n_select": 2590, "n_holdout": 2591, "eb": 0.002937804058519797, "labelled_winner": false } ], "winner": 1, "pool_select": 0.004063452371649606, "pool_holdout": 0.0031262211801484957, "lift_select": 1.2990687786328112, "lift_holdout": 0.36964772284935754, "eb_lift": 0.6829372091387707, "p_value": 2.546077688100374e-06, "held_up": false, "z_path": [ -2.026, -0.407, -0.301, 0.168, 1.122, 1.654, 2.016, 2.165, 2.025, 1.944 ], "first_cross": 0, "final_z": 1.944 }, { "id": "529520d47a0573ac2e00093e", "variation": "headline", "week": 201347, "impressions": 25571, "pooled_ctr": 0.023190332798873725, "arms": [ { "headline": "Wondering What Happens To Divorced, Dark-Skinned, Single Mothers In This Country?", "impressions": 6376, "clicks": 171, "ctr": 0.026819322459222, "ctr_select": 0.027603513174404015, "ctr_holdout": 0.026035131744040152, "n_select": 3188, "n_holdout": 3188, "eb": 0.02576180550900934, "labelled_winner": false }, { "headline": "I Thought This Ad Was Trying To Sell Me Jewelry, And Then I Realized It Had A Powerful Message", "impressions": 5851, "clicks": 138, "ctr": 0.0235857118441292, "ctr_select": 0.024265208475734792, "ctr_holdout": 0.022905982905982905, "n_select": 2926, "n_holdout": 2925, "eb": 0.023807000420306253, "labelled_winner": false }, { "headline": "Marriage Gets A Shiny Reboot In This Fantastic Ad From India", "impressions": 6612, "clicks": 146, "ctr": 0.0220810647307924, "ctr_select": 0.02026618269812462, "ctr_holdout": 0.023895946763460376, "n_select": 3306, "n_holdout": 3306, "eb": 0.021487057058314128, "labelled_winner": false }, { "headline": "Every Marriage Isn't The Same, Especially For Women In This Country", "impressions": 6732, "clicks": 138, "ctr": 0.0204991087344028, "ctr_select": 0.02109328579916815, "ctr_holdout": 0.01990493166963755, "n_select": 3366, "n_holdout": 3366, "eb": 0.021963995966287883, "labelled_winner": false } ], "winner": 0, "pool_select": 0.02322853120600657, "pool_holdout": 0.02315213140398905, "lift_select": 0.18834518332636274, "lift_holdout": 0.12452418698497758, "eb_lift": 0.10905873817573553, "p_value": 0.05838428814795746, "held_up": true, "z_path": [ 0.671, 0.92, 0.736, 1.086, 0.741, 0.868, 1.108, 1.351, 1.015, 1.138 ], "first_cross": null, "final_z": 1.138 }, { "id": "540903f9844567e9bc000036", "variation": "headline", "week": 201435, "impressions": 24691, "pooled_ctr": 0.00486007047102183, "arms": [ { "headline": "The Outrageous Truth About What Happens To Thousands Of Rape Kits", "impressions": 6062, "clicks": 21, "ctr": 0.0034642032332563, "ctr_select": 0.0026393929396238865, "ctr_holdout": 0.004289013526888816, "n_select": 3031, "n_holdout": 3031, "eb": 0.0037421456332816377, "labelled_winner": false }, { "headline": "Thousands Of Rape Victims Nationwide May Have Subjected Themselves To This Exam For Nothing", "impressions": 6168, "clicks": 31, "ctr": 0.0050259403372243, "ctr_select": 0.005512321660181583, "ctr_holdout": 0.004539559014267186, "n_select": 3084, "n_holdout": 3084, "eb": 0.005371571558743007, "labelled_winner": false }, { "headline": "Could This Be One Reason Why There Are So Many Unsolved Crimes In The U.S.?", "impressions": 6224, "clicks": 28, "ctr": 0.0044987146529562, "ctr_select": 0.005784061696658098, "ctr_holdout": 0.003213367609254499, "n_select": 3112, "n_holdout": 3112, "eb": 0.005528043620830382, "labelled_winner": false }, { "headline": "Unbelievable. Just When You Thought The News About Rape In This Country Couldn't Get Worse.", "impressions": 6237, "clicks": 40, "ctr": 0.006413339746673, "ctr_select": 0.006735086593970494, "ctr_holdout": 0.006091696056428343, "n_select": 3118, "n_holdout": 3119, "eb": 0.006073848759284948, "labelled_winner": false } ], "winner": 3, "pool_select": 0.005184285135682463, "pool_holdout": 0.004535882067066256, "lift_select": 0.2991350625400897, "lift_holdout": 0.3430014198690057, "eb_lift": 0.17158848333394805, "p_value": 0.16309374620995776, "held_up": true, "z_path": [ -0.003, 0.24, -0.197, -0.371, -0.694, -0.167, 0.431, 0.681, 0.782, 1.451 ], "first_cross": null, "final_z": 1.451 }, { "id": "541c94937bc1ef0682000055", "variation": "image", "week": 201437, "impressions": 24644, "pooled_ctr": 0.004463561110209382, "arms": [ { "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected", "impressions": 4182, "clicks": 23, "ctr": 0.0054997608799617, "ctr_select": 0.0052606408417025345, "ctr_holdout": 0.005738880918220947, "n_select": 2091, "n_holdout": 2091, "eb": 0.004926936558748172, "labelled_winner": false }, { "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected", "impressions": 4126, "clicks": 11, "ctr": 0.0026660203587009, "ctr_select": 0.004362578768783325, "ctr_holdout": 0.0009694619486185168, "n_select": 2063, "n_holdout": 2063, "eb": 0.004501879660988422, "labelled_winner": false }, { "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected", "impressions": 4059, "clicks": 20, "ctr": 0.0049273220004927, "ctr_select": 0.006896551724137931, "ctr_holdout": 0.002957121734844751, "n_select": 2030, "n_holdout": 2029, "eb": 0.005686083894719949, "labelled_winner": false }, { "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected", "impressions": 4178, "clicks": 20, "ctr": 0.0047869794159885, "ctr_select": 0.005265677357587362, "ctr_holdout": 0.00430828147438966, "n_select": 2089, "n_holdout": 2089, "eb": 0.004929172671542369, "labelled_winner": false }, { "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected", "impressions": 4002, "clicks": 18, "ctr": 0.0044977511244377, "ctr_select": 0.004497751124437781, "ctr_holdout": 0.004497751124437781, "n_select": 2001, "n_holdout": 2001, "eb": 0.004566509454058166, "labelled_winner": false }, { "headline": "This 16-Year-Old Kid Is Really Into Apps, But Definitely Not The Kind I Expected", "impressions": 4097, "clicks": 18, "ctr": 0.0043934586282645, "ctr_select": 0.00146484375, "ctr_holdout": 0.007320644216691069, "n_select": 2048, "n_holdout": 2049, "eb": 0.0031429967958766466, "labelled_winner": false } ], "winner": 2, "pool_select": 0.004625872423307905, "pool_holdout": 0.004301249797110858, "lift_select": 0.49086509376890497, "lift_holdout": -0.3124970940234524, "eb_lift": 0.22919167983752997, "p_value": 0.09900465270977789, "held_up": false, "z_path": [ 1.761, 1.672, 1.458, 1.313, 0.954, 0.559, 0.065, -0.108, 0.082, 0.092 ], "first_cross": null, "final_z": 0.092 }, { "id": "53dbf3892866d1c05b000019", "variation": "other", "week": 201430, "impressions": 24600, "pooled_ctr": 0.017195121951219514, "arms": [ { "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.", "impressions": 4985, "clicks": 74, "ctr": 0.0148445336008024, "ctr_select": 0.01565008025682183, "ctr_holdout": 0.014039310068190935, "n_select": 2492, "n_holdout": 2493, "eb": 0.016649951646199795, "labelled_winner": false }, { "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.", "impressions": 4830, "clicks": 74, "ctr": 0.0153209109730848, "ctr_select": 0.015320910973084885, "ctr_holdout": 0.015320910973084885, "n_select": 2415, "n_holdout": 2415, "eb": 0.016498250910583486, "labelled_winner": false }, { "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.", "impressions": 4966, "clicks": 88, "ctr": 0.017720499395892, "ctr_select": 0.019331453886427707, "ctr_holdout": 0.016109544905356425, "n_select": 2483, "n_holdout": 2483, "eb": 0.0185558932671403, "labelled_winner": false }, { "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.", "impressions": 4882, "clicks": 110, "ctr": 0.0225317492830807, "ctr_select": 0.02498975829578042, "ctr_holdout": 0.02007374027038099, "n_select": 2441, "n_holdout": 2441, "eb": 0.021451506389150794, "labelled_winner": true }, { "headline": "It Was Fourth Of July. She Was Wearing Something That Made Her Happy. Then It Got Worse From There.", "impressions": 4937, "clicks": 77, "ctr": 0.0155965161028964, "ctr_select": 0.013371150729335495, "ctr_holdout": 0.017820980153908466, "n_select": 2468, "n_holdout": 2469, "eb": 0.015479714513363734, "labelled_winner": false } ], "winner": 3, "pool_select": 0.017725018294170256, "pool_holdout": 0.016665311763271278, "lift_select": 0.4098579691734101, "lift_holdout": 0.2045223369071052, "eb_lift": 0.2102388856888331, "p_value": 0.0023789532901485836, "held_up": true, "z_path": [ 0.488, 0.842, 0.455, 0.646, 1.408, 1.784, 1.122, 0.74, 1.088, 1.701 ], "first_cross": null, "final_z": 1.701 }, { "id": "54482589b8fbabf5e4000003", "variation": "other", "week": 201442, "impressions": 24558, "pooled_ctr": 0.012134538643211987, "arms": [ { "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'", "impressions": 3069, "clicks": 39, "ctr": 0.0127077223851417, "ctr_select": 0.013689700130378096, "ctr_holdout": 0.011726384364820847, "n_select": 1534, "n_holdout": 1535, "eb": 0.012312301579783124, "labelled_winner": false }, { "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'", "impressions": 3095, "clicks": 34, "ctr": 0.0109854604200323, "ctr_select": 0.009689922480620155, "ctr_holdout": 0.012281835811247576, "n_select": 1548, "n_holdout": 1547, "eb": 0.010715598295220138, "labelled_winner": false }, { "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'", "impressions": 3152, "clicks": 49, "ctr": 0.0155456852791878, "ctr_select": 0.015228426395939087, "ctr_holdout": 0.015862944162436547, "n_select": 1576, "n_holdout": 1576, "eb": 0.012949968785096484, "labelled_winner": true }, { "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'", "impressions": 3059, "clicks": 44, "ctr": 0.0143837855508336, "ctr_select": 0.01568627450980392, "ctr_holdout": 0.013080444735120994, "n_select": 1530, "n_holdout": 1529, "eb": 0.013104788266730132, "labelled_winner": false }, { "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'", "impressions": 3084, "clicks": 46, "ctr": 0.0149156939040207, "ctr_select": 0.011673151750972763, "ctr_holdout": 0.018158236057068743, "n_select": 1542, "n_holdout": 1542, "eb": 0.011509518913949466, "labelled_winner": false }, { "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'", "impressions": 3080, "clicks": 37, "ctr": 0.012012987012987, "ctr_select": 0.00909090909090909, "ctr_holdout": 0.014935064935064935, "n_select": 1540, "n_holdout": 1540, "eb": 0.010478598072901022, "labelled_winner": false }, { "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'", "impressions": 2940, "clicks": 23, "ctr": 0.0078231292517006, "ctr_select": 0.007482993197278911, "ctr_holdout": 0.00816326530612245, "n_select": 1470, "n_holdout": 1470, "eb": 0.009880213958951296, "labelled_winner": false }, { "headline": "In 2 Minutes, She'll Take You From 'Have No Clue' To 'OMG, I Can't Believe We Do This'", "impressions": 3079, "clicks": 26, "ctr": 0.0084443000974342, "ctr_select": 0.008441558441558441, "ctr_holdout": 0.008447043534762833, "n_select": 1540, "n_holdout": 1539, "eb": 0.0102193761025536, "labelled_winner": false } ], "winner": 3, "pool_select": 0.011400651465798045, "pool_holdout": 0.012868545365694739, "lift_select": 0.37591036414565826, "lift_holdout": 0.016466458593769273, "eb_lift": 0.14947714225318598, "p_value": 0.09148048238766168, "held_up": false, "z_path": [ -1.224, -1.445, -1.465, -1.337, -1.539, -0.901, -0.54, -0.697, -0.627, -0.377 ], "first_cross": null, "final_z": -0.377 }, { "id": "542d3b2952a3f66b70000008", "variation": "headline", "week": 201439, "impressions": 24451, "pooled_ctr": 0.004907774733139749, "arms": [ { "headline": "The Perfect Response When Your Mom Tells You To Fix Your Hair", "impressions": 4099, "clicks": 27, "ctr": 0.0065869724323005, "ctr_select": 0.006829268292682927, "ctr_holdout": 0.00634455832113226, "n_select": 2050, "n_holdout": 2049, "eb": 0.005679321480686004, "labelled_winner": false }, { "headline": "For Moms, Grandmothers, BFFs And Anyone In Between Who Has Something To Say About Your Hair", "impressions": 4068, "clicks": 19, "ctr": 0.0046705998033431, "ctr_select": 0.004424778761061947, "ctr_holdout": 0.004916420845624385, "n_select": 2034, "n_holdout": 2034, "eb": 0.004551225921318163, "labelled_winner": false }, { "headline": "Why \u2018Nappy\u2019 Hair Is Not A Bad Thing", "impressions": 4130, "clicks": 12, "ctr": 0.0029055690072639, "ctr_select": 0.003389830508474576, "ctr_holdout": 0.002421307506053269, "n_select": 2065, "n_holdout": 2065, "eb": 0.004062693937627633, "labelled_winner": false }, { "headline": "Something That Should Be Played In Beauty Salons Across The Nation", "impressions": 4102, "clicks": 23, "ctr": 0.0056070209653827, "ctr_select": 0.0048756704046806435, "ctr_holdout": 0.006338371526084837, "n_select": 2051, "n_holdout": 2051, "eb": 0.004762416811994553, "labelled_winner": false }, { "headline": "You Don\u2019t Have To Have Curly Hair To Understand Why What She Says Hits Home", "impressions": 4049, "clicks": 20, "ctr": 0.004939491232403, "ctr_select": 0.004940711462450593, "ctr_holdout": 0.0049382716049382715, "n_select": 2024, "n_holdout": 2025, "eb": 0.004792033165474578, "labelled_winner": false }, { "headline": "Next Time You Feel Like You\u2019re Having A 'Bad Hair Day,' Listen To This", "impressions": 4003, "clicks": 19, "ctr": 0.0047464401698725, "ctr_select": 0.0034965034965034965, "ctr_holdout": 0.005997001499250375, "n_select": 2002, "n_holdout": 2001, "eb": 0.004121945703739868, "labelled_winner": false } ], "winner": 0, "pool_select": 0.004662195321446098, "pool_holdout": 0.0051533742331288344, "lift_select": 0.4648181429182714, "lift_holdout": 0.23114643612447416, "eb_lift": 0.21816463899766836, "p_value": 0.11438484724503767, "held_up": true, "z_path": [ 1.123, 1.138, 1.585, 1.865, 1.608, 1.935, 2.015, 1.3, 1.407, 0.982 ], "first_cross": 6, "final_z": 0.982 }, { "id": "542c3500d9a91cd011000037", "variation": "image", "week": 201439, "impressions": 23766, "pooled_ctr": 0.010982075233526888, "arms": [ { "headline": "CNN Asks An Expert A Dumb Question. Here's His Epic Response.", "impressions": 5909, "clicks": 69, "ctr": 0.0116771027246573, "ctr_select": 0.010832769126607989, "ctr_holdout": 0.012521150592216581, "n_select": 2954, "n_holdout": 2955, "eb": 0.01069531075049896, "labelled_winner": false }, { "headline": "CNN Asks An Expert A Dumb Question. Here's His Epic Response.", "impressions": 5825, "clicks": 67, "ctr": 0.0115021459227467, "ctr_select": 0.01201923076923077, "ctr_holdout": 0.010985238585650533, "n_select": 2912, "n_holdout": 2913, "eb": 0.011354852323707776, "labelled_winner": false }, { "headline": "CNN Asks An Expert A Dumb Question. Here's His Epic Response.", "impressions": 6079, "clicks": 63, "ctr": 0.0103635466359598, "ctr_select": 0.008881578947368421, "ctr_holdout": 0.011846001974333662, "n_select": 3040, "n_holdout": 3039, "eb": 0.00959039646624911, "labelled_winner": false }, { "headline": "CNN Asks An Expert A Dumb Question. Here's His Epic Response.", "impressions": 5953, "clicks": 62, "ctr": 0.0104149168486477, "ctr_select": 0.010416666666666666, "ctr_holdout": 0.010413167618407793, "n_select": 2976, "n_holdout": 2977, "eb": 0.010461958393519908, "labelled_winner": false } ], "winner": 1, "pool_select": 0.010520114458845313, "pool_holdout": 0.01144395826321104, "lift_select": 0.14250000000000007, "lift_holdout": -0.04008400476565488, "eb_lift": 0.07934684248236623, "p_value": 0.3614691120953939, "held_up": false, "z_path": [ -0.886, 0.032, 0.355, 1.109, 0.293, -0.043, 0.172, -0.214, -0.1, -0.089 ], "first_cross": null, "final_z": -0.089 }, { "id": "53c05eac5d80f0f0b4000029", "variation": "headline", "week": 201427, "impressions": 23313, "pooled_ctr": 0.01282546218847853, "arms": [ { "headline": "A Straight Kid Stands Up To Some Bullying Politicians For His Two Gay Moms And Wins", "impressions": 3910, "clicks": 57, "ctr": 0.0145780051150895, "ctr_select": 0.01687979539641944, "ctr_holdout": 0.012276214833759591, "n_select": 1955, "n_holdout": 1955, "eb": 0.014051501564002556, "labelled_winner": false }, { "headline": "A Straight Kid Stands Up To Some Bullying Politicians For His Two Gay Moms And Makes Us All Proud", "impressions": 3911, "clicks": 56, "ctr": 0.0143185885962669, "ctr_select": 0.009713701431492843, "ctr_holdout": 0.018925831202046037, "n_select": 1956, "n_holdout": 1955, "eb": 0.010772361935954475, "labelled_winner": false }, { "headline": "A Straight College Kid Stands Up To Some Bullying Politicians For His Two Gay Moms And Wins", "impressions": 3846, "clicks": 52, "ctr": 0.0135205408216328, "ctr_select": 0.010400416016640665, "ctr_holdout": 0.016640665626625067, "n_select": 1923, "n_holdout": 1923, "eb": 0.011091999891379885, "labelled_winner": false }, { "headline": "A Straight Kid Stands Up To Some Bullying Politicians For His Two Moms And Shuts Down Their Hate", "impressions": 3855, "clicks": 64, "ctr": 0.0166018158236057, "ctr_select": 0.017634854771784232, "ctr_holdout": 0.015568240788790867, "n_select": 1928, "n_holdout": 1927, "eb": 0.014376392933617135, "labelled_winner": false }, { "headline": "A Straight College Kid Stands Up To Bullying Politicians Who Don't Want His Moms To Stay Married", "impressions": 3960, "clicks": 36, "ctr": 0.009090909090909, "ctr_select": 0.007575757575757576, "ctr_holdout": 0.010606060606060607, "n_select": 1980, "n_holdout": 1980, "eb": 0.009781474028256599, "labelled_winner": false }, { "headline": "A Straight Kid Stands Up For His 2 Moms When Some Politicians Decide To Ban Their Marriage", "impressions": 3831, "clicks": 34, "ctr": 0.0088749673714434, "ctr_select": 0.007828810020876827, "ctr_holdout": 0.009921671018276762, "n_select": 1916, "n_holdout": 1915, "eb": 0.009929338703885942, "labelled_winner": false } ], "winner": 3, "pool_select": 0.011665808886601476, "pool_holdout": 0.013985413985413985, "lift_select": 0.5116701244813278, "lift_holdout": 0.11317697173838992, "eb_lift": 0.23235285897138636, "p_value": 0.007544455804351582, "held_up": false, "z_path": [ 0.02, 1.004, 2.003, 1.955, 0.687, 1.133, 1.253, 0.99, 0.655, 0.72 ], "first_cross": 2, "final_z": 0.72 }, { "id": "541b59e02a027740b30000ba", "variation": "image", "week": 201437, "impressions": 23099, "pooled_ctr": 0.009697389497380839, "arms": [ { "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living", "impressions": 4686, "clicks": 48, "ctr": 0.0102432778489116, "ctr_select": 0.011096884336320957, "ctr_holdout": 0.009389671361502348, "n_select": 2343, "n_holdout": 2343, "eb": 0.010315027018294181, "labelled_winner": false }, { "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living", "impressions": 4653, "clicks": 44, "ctr": 0.0094562647754137, "ctr_select": 0.009888220120378332, "ctr_holdout": 0.009024495058014611, "n_select": 2326, "n_holdout": 2327, "eb": 0.009706752164857843, "labelled_winner": false }, { "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living", "impressions": 4494, "clicks": 70, "ctr": 0.0155763239875389, "ctr_select": 0.013796172674677348, "ctr_holdout": 0.017356475300400534, "n_select": 2247, "n_holdout": 2247, "eb": 0.0116273247828701, "labelled_winner": false }, { "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living", "impressions": 4622, "clicks": 35, "ctr": 0.0075724794461272, "ctr_select": 0.006490696668109044, "ctr_holdout": 0.00865426222414539, "n_select": 2311, "n_holdout": 2311, "eb": 0.00800986046470135, "labelled_winner": false }, { "headline": "He Accepted That He Was Going To Be Murdered \u2026 That\u2019s When He Really Started Living", "impressions": 4644, "clicks": 27, "ctr": 0.005813953488372, "ctr_select": 0.006459948320413436, "ctr_holdout": 0.00516795865633075, "n_select": 2322, "n_holdout": 2322, "eb": 0.007990870334153898, "labelled_winner": false } ], "winner": 2, "pool_select": 0.009524634167460386, "pool_holdout": 0.009870129870129871, "lift_select": 0.44847271108953346, "lift_holdout": 0.7584849975405803, "eb_lift": 0.22076339924878896, "p_value": 0.020186427831472026, "held_up": true, "z_path": [ 0.588, 0.851, 1.406, 2.073, 2.435, 1.821, 1.408, 1.741, 2.082, 2.268 ], "first_cross": 3, "final_z": 2.268 }, { "id": "542dbdbbd3047b688f000060", "variation": "headline", "week": 201439, "impressions": 22583, "pooled_ctr": 0.016826816631979807, "arms": [ { "headline": "After You See These Images From Space, You'll Realize That The US Map Is *Incorrect*", "impressions": 5625, "clicks": 111, "ctr": 0.0197333333333333, "ctr_select": 0.02027027027027027, "ctr_holdout": 0.019196587273373623, "n_select": 2812, "n_holdout": 2813, "eb": 0.018474691857848717, "labelled_winner": false }, { "headline": "These Images From Space Prove That The US Map Is *WRONG*", "impressions": 5650, "clicks": 87, "ctr": 0.0153982300884955, "ctr_select": 0.01663716814159292, "ctr_holdout": 0.01415929203539823, "n_select": 2825, "n_holdout": 2825, "eb": 0.016483485766749007, "labelled_winner": false }, { "headline": "Is The US Map *Incorrect*? After Looking At These Images From Space, Yes, It Is.", "impressions": 5613, "clicks": 84, "ctr": 0.0149652592196686, "ctr_select": 0.012829650748396294, "ctr_holdout": 0.017100106875667972, "n_select": 2806, "n_holdout": 2807, "eb": 0.014397723806052927, "labelled_winner": false }, { "headline": "After Seeing These 8 Satellite Images I'm Like, \"Nooooooooo!\"", "impressions": 5695, "clicks": 98, "ctr": 0.017208077260755, "ctr_select": 0.01544943820224719, "ctr_holdout": 0.018967334035827187, "n_select": 2848, "n_holdout": 2847, "eb": 0.015829339828320632, "labelled_winner": false } ], "winner": 0, "pool_select": 0.01629616508723762, "pool_holdout": 0.017357421183138506, "lift_select": 0.24386750881316122, "lift_holdout": 0.10595848719864764, "eb_lift": 0.13368340090744502, "p_value": 0.05476592727539358, "held_up": true, "z_path": [ 1.014, 1.534, 1.216, 2.049, 1.505, 1.055, 1.041, 1.067, 1.39, 1.752 ], "first_cross": 3, "final_z": 1.752 }, { "id": "5330e65300d2ea8cf0000001", "variation": "image", "week": 201412, "impressions": 22392, "pooled_ctr": 0.00817256162915327, "arms": [ { "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of", "impressions": 3691, "clicks": 37, "ctr": 0.010024383635871, "ctr_select": 0.010834236186348862, "ctr_holdout": 0.00921409214092141, "n_select": 1846, "n_holdout": 1845, "eb": 0.009526644208933437, "labelled_winner": false }, { "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of", "impressions": 3798, "clicks": 33, "ctr": 0.0086887835703001, "ctr_select": 0.006845708267509215, "ctr_holdout": 0.010531858873091101, "n_select": 1899, "n_holdout": 1899, "eb": 0.00774683624811482, "labelled_winner": false }, { "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of", "impressions": 3717, "clicks": 31, "ctr": 0.0083400591875168, "ctr_select": 0.011302475780409042, "ctr_holdout": 0.0053792361484669175, "n_select": 1858, "n_holdout": 1859, "eb": 0.009738747749979973, "labelled_winner": false }, { "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of", "impressions": 3705, "clicks": 23, "ctr": 0.0062078272604588, "ctr_select": 0.004859611231101512, "ctr_holdout": 0.007555315704263357, "n_select": 1852, "n_holdout": 1853, "eb": 0.0068748548830634155, "labelled_winner": false }, { "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of", "impressions": 3682, "clicks": 29, "ctr": 0.0078761542639869, "ctr_select": 0.008690928843020097, "ctr_holdout": 0.007061379684953829, "n_select": 1841, "n_holdout": 1841, "eb": 0.008576258520992933, "labelled_winner": false }, { "headline": "One Of The Best Soul Singers You\u2019ve Ever Heard But Never Heard Of", "impressions": 3799, "clicks": 30, "ctr": 0.0078968149513029, "ctr_select": 0.008421052631578947, "ctr_holdout": 0.00737230121116377, "n_select": 1900, "n_holdout": 1899, "eb": 0.008456288032784967, "labelled_winner": false } ], "winner": 2, "pool_select": 0.008485173276170061, "pool_holdout": 0.007859949982136478, "lift_select": 0.33202651407852235, "lift_holdout": -0.31561445547459543, "eb_lift": 0.14773705061869236, "p_value": 0.147139641184714, "held_up": false, "z_path": [ -0.387, -0.014, -1.065, -0.897, -1.296, -1.265, -1.211, -1.545, -0.783, -0.76 ], "first_cross": null, "final_z": -0.76 }, { "id": "5245cbe3eb36284c7700014b", "variation": "other", "week": 201338, "impressions": 22254, "pooled_ctr": 0.014963602049069831, "arms": [ { "headline": "6 Things We Could Be Doing To Kick-Start This Country", "impressions": 6033, "clicks": 91, "ctr": 0.015083706282115, "ctr_select": 0.01293103448275862, "ctr_holdout": 0.01723566456745111, "n_select": 3016, "n_holdout": 3017, "eb": 0.013951544529054918, "labelled_winner": false }, { "headline": "6 Things We Could Be Doing To Kick-Start This Country", "impressions": 5373, "clicks": 63, "ctr": 0.0117252931323283, "ctr_select": 0.012658227848101266, "ctr_holdout": 0.010792705619650168, "n_select": 2686, "n_holdout": 2687, "eb": 0.013872404994896982, "labelled_winner": false }, { "headline": "6 Things We Could Be Doing To Kick-Start This Country", "impressions": 5296, "clicks": 101, "ctr": 0.0190709969788519, "ctr_select": 0.020770392749244714, "ctr_holdout": 0.017371601208459216, "n_select": 2648, "n_holdout": 2648, "eb": 0.018207576539817284, "labelled_winner": false }, { "headline": "6 Things We Could Be Doing To Kick-Start This Country", "impressions": 5552, "clicks": 78, "ctr": 0.0140489913544668, "ctr_select": 0.015129682997118156, "ctr_holdout": 0.012968299711815562, "n_select": 2776, "n_holdout": 2776, "eb": 0.015197863226216222, "labelled_winner": false } ], "winner": 2, "pool_select": 0.015279525435915873, "pool_holdout": 0.014647735442127965, "lift_select": 0.35936111604762755, "lift_holdout": 0.18595814875910532, "eb_lift": 0.19163233283533576, "p_value": 0.00831927894562641, "held_up": true, "z_path": [ 0.749, 1.655, 2.27, 2.695, 2.659, 1.941, 1.997, 1.529, 2.121, 2.052 ], "first_cross": 2, "final_z": 2.052 }, { "id": "5450f8327ccd9d83d8000055", "variation": "headline", "week": 201443, "impressions": 22029, "pooled_ctr": 0.01411775386989877, "arms": [ { "headline": "Only 7 Countries In The World Don't Have Paid Maternity Leave. Bet You Can Guess 1 Of Them.", "impressions": 3682, "clicks": 37, "ctr": 0.0100488864747419, "ctr_select": 0.00977729494839761, "ctr_holdout": 0.010320478001086366, "n_select": 1841, "n_holdout": 1841, "eb": 0.012625998218564444, "labelled_winner": false }, { "headline": "That Moment You Realize That Something Normal For The U.S. Would Seem Ridiculous Anywhere Else", "impressions": 3659, "clicks": 94, "ctr": 0.0256900792566274, "ctr_select": 0.029508196721311476, "ctr_holdout": 0.02186987424822307, "n_select": 1830, "n_holdout": 1829, "eb": 0.021338997225224633, "labelled_winner": false }, { "headline": "One Thing's For Sure, I've Never Wanted To Be Swedish More Than I Did At 1:40", "impressions": 3690, "clicks": 59, "ctr": 0.0159891598915989, "ctr_select": 0.016260162601626018, "ctr_holdout": 0.015718157181571817, "n_select": 1845, "n_holdout": 1845, "eb": 0.015496611665662187, "labelled_winner": false }, { "headline": "If You're A New Mom In America, Good Luck! You're Gonna Need It.", "impressions": 3626, "clicks": 21, "ctr": 0.0057915057915057, "ctr_select": 0.007722007722007722, "ctr_holdout": 0.003861003861003861, "n_select": 1813, "n_holdout": 1813, "eb": 0.011743224336130667, "labelled_winner": false }, { "headline": "The More She Explains, The Madder I Get. Why Can't America Have Nice Things?", "impressions": 3698, "clicks": 54, "ctr": 0.0146024878312601, "ctr_select": 0.014602487831260141, "ctr_holdout": 0.014602487831260141, "n_select": 1849, "n_holdout": 1849, "eb": 0.014761740392601717, "labelled_winner": false }, { "headline": "American Parents Are At 1 Huge Disadvantage Compared To Moms And Dads Around The World", "impressions": 3674, "clicks": 46, "ctr": 0.0125204137180185, "ctr_select": 0.011431682090364725, "ctr_holdout": 0.013609145345672292, "n_select": 1837, "n_holdout": 1837, "eb": 0.0133602282900981, "labelled_winner": false } ], "winner": 1, "pool_select": 0.014888788016341352, "pool_holdout": 0.01334664971854004, "lift_select": 0.981907237105158, "lift_holdout": 0.6386040474144823, "eb_lift": 0.4332259416820081, "p_value": 1.5580954056943683e-08, "held_up": true, "z_path": [ 0.495, 0.381, 0.888, 1.03, 1.073, 1.769, 2.182, 1.89, 2.878, 2.912 ], "first_cross": 6, "final_z": 2.912 }, { "id": "5296566858f57f80560003ad", "variation": "headline", "week": 201347, "impressions": 21949, "pooled_ctr": 0.01075219827782587, "arms": [ { "headline": "He Starts With 17 Lego Pieces And Ends Up With 6. The Reason Why Involves All Of Us.", "impressions": 5195, "clicks": 71, "ctr": 0.0136669874879692, "ctr_select": 0.013471901462663588, "ctr_holdout": 0.013862148633038121, "n_select": 2598, "n_holdout": 2597, "eb": 0.012275280755923995, "labelled_winner": true }, { "headline": "Why Does This Guy Start With 17 Lego Pieces And End Up With 6?", "impressions": 5277, "clicks": 50, "ctr": 0.0094750805381845, "ctr_select": 0.01023502653525398, "ctr_holdout": 0.008715422508525956, "n_select": 2638, "n_holdout": 2639, "eb": 0.010561897144511043, "labelled_winner": false }, { "headline": "Why Are Only 6 Lego Pieces Left In His Version Of The Future?", "impressions": 5750, "clicks": 45, "ctr": 0.0078260869565217, "ctr_select": 0.008347826086956521, "ctr_holdout": 0.007304347826086957, "n_select": 2875, "n_holdout": 2875, "eb": 0.00950210704382063, "labelled_winner": false }, { "headline": "He Starts With 17 Legos And Ends Up With 6. The Missing Pieces Are Not Stuck In Couch Cushions.", "impressions": 5727, "clicks": 70, "ctr": 0.0122228042605203, "ctr_select": 0.011871508379888268, "ctr_holdout": 0.012574222843171499, "n_select": 2864, "n_holdout": 2863, "eb": 0.011452148083158996, "labelled_winner": false } ], "winner": 0, "pool_select": 0.010933940774487472, "pool_holdout": 0.010570439219974486, "lift_select": 0.23211765460610723, "lift_holdout": 0.3114070611979338, "eb_lift": 0.12267671913554867, "p_value": 0.15449253407685468, "held_up": true, "z_path": [ 0.45, -0.541, -0.352, 0.345, -0.347, -0.328, 0.382, 0.131, 0.731, 0.668 ], "first_cross": null, "final_z": 0.668 }, { "id": "533845f710ea6c8ea300000e", "variation": "headline", "week": 201413, "impressions": 21821, "pooled_ctr": 0.008798863480133817, "arms": [ { "headline": "How 1 Little Studio Is Saving A Rich Culture, Rebuilding A City, And Changing 2 Young Women\u2019s Lives", "impressions": 3647, "clicks": 23, "ctr": 0.0063065533315053, "ctr_select": 0.005482456140350877, "ctr_holdout": 0.007131102578167855, "n_select": 1824, "n_holdout": 1823, "eb": 0.0071325114870102214, "labelled_winner": false }, { "headline": "Making Jewelry Makes Her Happy. It\u2019s Also Helping To Heal Her War-Torn Home.", "impressions": 3615, "clicks": 14, "ctr": 0.0038727524204702, "ctr_select": 0.00497787610619469, "ctr_holdout": 0.002767017155506364, "n_select": 1808, "n_holdout": 1807, "eb": 0.0069177890619491755, "labelled_winner": false }, { "headline": "I\u2019m Not Sure Which Is More Beautiful, This Woman\u2019s Jewelry Or The Hope And Freedom It Represents", "impressions": 3711, "clicks": 41, "ctr": 0.0110482349770951, "ctr_select": 0.011853448275862068, "ctr_holdout": 0.010242587601078167, "n_select": 1856, "n_holdout": 1855, "eb": 0.009952974411132682, "labelled_winner": false }, { "headline": "A Lovely Reminder That Freedom Is The Freedom To Pursue Your Very Individual Dream", "impressions": 3682, "clicks": 20, "ctr": 0.0054318305268875, "ctr_select": 0.005431830526887561, "ctr_holdout": 0.005431830526887561, "n_select": 1841, "n_holdout": 1841, "eb": 0.007103355077025418, "labelled_winner": false }, { "headline": "Life Under The Taliban Was Ugly For Storai. Her Life After It Is All About Making Beauty.", "impressions": 3562, "clicks": 48, "ctr": 0.0134755755193711, "ctr_select": 0.01347557551937114, "ctr_holdout": 0.01347557551937114, "n_select": 1781, "n_holdout": 1781, "eb": 0.010623058735168014, "labelled_winner": false }, { "headline": "The Hopes And Dreams Of One Young Woman After The Taliban Are As Beautiful As Her Art", "impressions": 3604, "clicks": 46, "ctr": 0.0127635960044395, "ctr_select": 0.009433962264150943, "ctr_holdout": 0.01609322974472808, "n_select": 1802, "n_holdout": 1802, "eb": 0.008869754305740236, "labelled_winner": false } ], "winner": 4, "pool_select": 0.00843108504398827, "pool_holdout": 0.009166743056192136, "lift_select": 0.5983204355149772, "lift_holdout": 0.47005053340819747, "eb_lift": 0.2599871404147107, "p_value": 0.010918368974714003, "held_up": false, "z_path": [ 0.915, 0.99, 1.287, 0.292, 0.754, 1.594, 1.062, 1.503, 0.968, 0.941 ], "first_cross": null, "final_z": 0.941 }, { "id": "5292bcd69c05ba48da002063", "variation": "image", "week": 201347, "impressions": 21666, "pooled_ctr": 0.04107818702113911, "arms": [ { "headline": "The Day After His Dad Dies, He Waits Until The End Of His Show To Tell Us About It. Just Beautiful.", "impressions": 5141, "clicks": 211, "ctr": 0.041042598716203, "ctr_select": 0.042801556420233464, "ctr_holdout": 0.03928432516530533, "n_select": 2570, "n_holdout": 2571, "eb": 0.04233636426928261, "labelled_winner": false }, { "headline": "The Day After His Dad Dies, He Waits Until The End Of His Show To Tell Us About It. Just Beautiful.", "impressions": 5511, "clicks": 224, "ctr": 0.0406459807657412, "ctr_select": 0.040638606676342524, "ctr_holdout": 0.040653357531760435, "n_select": 2756, "n_holdout": 2755, "eb": 0.0411785176461539, "labelled_winner": false }, { "headline": "The Day After His Dad Dies, He Waits Until The End Of His Show To Tell Us About It. Just Beautiful.", "impressions": 5437, "clicks": 235, "ctr": 0.0432223652749678, "ctr_select": 0.044518027961736574, "ctr_holdout": 0.0419271791099669, "n_select": 2718, "n_holdout": 2719, "eb": 0.043276495779517914, "labelled_winner": true }, { "headline": "The Day After His Dad Dies, He Waits Until The End Of His Show To Tell Us About It. Just Beautiful.", "impressions": 5577, "clicks": 220, "ctr": 0.039447731755424, "ctr_select": 0.03945480631276901, "ctr_holdout": 0.03944065973467192, "n_select": 2788, "n_holdout": 2789, "eb": 0.040528711731936604, "labelled_winner": false } ], "winner": 2, "pool_select": 0.0418205317577548, "pool_holdout": 0.04033597932434927, "lift_select": 0.06450171938527727, "lift_holdout": 0.03944864640133039, "eb_lift": 0.03481457457778814, "p_value": 0.4169545252779342, "held_up": true, "z_path": [ 0.782, 0.437, 1.137, 1.361, 1.063, 1.5, 1.065, 0.506, 0.767, 0.558 ], "first_cross": null, "final_z": 0.558 }, { "id": "540ff3111aec8007c5000045", "variation": "headline", "week": 201436, "impressions": 21553, "pooled_ctr": 0.00909386164339071, "arms": [ { "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Being Run On Americans Today", "impressions": 3606, "clicks": 40, "ctr": 0.0110926234054353, "ctr_select": 0.011092623405435386, "ctr_holdout": 0.011092623405435386, "n_select": 1803, "n_holdout": 1803, "eb": 0.009394085945374355, "labelled_winner": false }, { "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Being Run On People Today", "impressions": 3612, "clicks": 23, "ctr": 0.0063676633444075, "ctr_select": 0.005537098560354375, "ctr_holdout": 0.007198228128460687, "n_select": 1806, "n_holdout": 1806, "eb": 0.006962243706655797, "labelled_winner": false }, { "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Run On College Kids Today", "impressions": 3544, "clicks": 27, "ctr": 0.0076185101580135, "ctr_select": 0.006772009029345372, "ctr_holdout": 0.008465011286681716, "n_select": 1772, "n_holdout": 1772, "eb": 0.007509158527155074, "labelled_winner": false }, { "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Being Run On American Teens", "impressions": 3656, "clicks": 32, "ctr": 0.0087527352297593, "ctr_select": 0.007111597374179431, "ctr_holdout": 0.010393873085339168, "n_select": 1828, "n_holdout": 1828, "eb": 0.007648938888330842, "labelled_winner": false }, { "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams Being Run On Americans Today.", "impressions": 3595, "clicks": 47, "ctr": 0.0130737134909596, "ctr_select": 0.01167964404894327, "ctr_holdout": 0.014468558708959377, "n_select": 1798, "n_holdout": 1797, "eb": 0.009648470622010838, "labelled_winner": false }, { "headline": "John Oliver Tears Into The Awful Folks Behind 1 Of The Biggest Scams In The World, Student Loans", "impressions": 3540, "clicks": 27, "ctr": 0.0076271186440677, "ctr_select": 0.0062146892655367235, "ctr_holdout": 0.00903954802259887, "n_select": 1770, "n_holdout": 1770, "eb": 0.007268196070017152, "labelled_winner": false } ], "winner": 4, "pool_select": 0.008072747517862114, "pool_holdout": 0.010115070527097253, "lift_select": 0.44679912546507605, "lift_holdout": 0.4303962261261123, "eb_lift": 0.195190435556446, "p_value": 0.06114278912043014, "held_up": true, "z_path": [ 0.004, 0.612, 0.86, 1.039, 1.313, 1.138, 0.736, 0.827, 0.806, 0.769 ], "first_cross": null, "final_z": 0.769 }, { "id": "53ff424c6fd2bfb3da000004", "variation": "other", "week": 201434, "impressions": 21459, "pooled_ctr": 0.017428584742998274, "arms": [ { "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.", "impressions": 3524, "clicks": 72, "ctr": 0.0204313280363223, "ctr_select": 0.019863791146424517, "ctr_holdout": 0.020998864926220204, "n_select": 1762, "n_holdout": 1762, "eb": 0.01885049473520291, "labelled_winner": false }, { "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.", "impressions": 3614, "clicks": 69, "ctr": 0.0190924183729939, "ctr_select": 0.022689540675152185, "ctr_holdout": 0.01549529607083564, "n_select": 1807, "n_holdout": 1807, "eb": 0.020099489960023344, "labelled_winner": false }, { "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.", "impressions": 3576, "clicks": 60, "ctr": 0.0167785234899328, "ctr_select": 0.013982102908277404, "ctr_holdout": 0.019574944071588368, "n_select": 1788, "n_holdout": 1788, "eb": 0.016295481283707218, "labelled_winner": false }, { "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.", "impressions": 3670, "clicks": 59, "ctr": 0.0160762942779291, "ctr_select": 0.015258855585831062, "ctr_holdout": 0.016893732970027248, "n_select": 1835, "n_holdout": 1835, "eb": 0.01683347189573097, "labelled_winner": false }, { "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.", "impressions": 3567, "clicks": 68, "ctr": 0.0190636389122511, "ctr_select": 0.02242152466367713, "ctr_holdout": 0.015703869882220976, "n_select": 1784, "n_holdout": 1783, "eb": 0.019968395040397272, "labelled_winner": false }, { "headline": "A Teacher Gave A Student An 'F' And A Smiley Face. But She Wasn't Making Fun Of Him.", "impressions": 3508, "clicks": 46, "ctr": 0.0131128848346636, "ctr_select": 0.014253135689851768, "ctr_holdout": 0.011972633979475485, "n_select": 1754, "n_holdout": 1754, "eb": 0.01643155385191764, "labelled_winner": false } ], "winner": 1, "pool_select": 0.018080149114631874, "pool_holdout": 0.016776959642091527, "lift_select": 0.2549421208473348, "lift_holdout": -0.07639426920002446, "eb_lift": 0.11168828490232197, "p_value": 0.10683029735720204, "held_up": false, "z_path": [ 0.202, -1.31, -1.175, -1.604, -1.847, -1.463, -1.125, -0.762, -0.633, 0.009 ], "first_cross": null, "final_z": 0.009 }, { "id": "5476066d6163610018790000", "variation": "headline", "week": 201447, "impressions": 21384, "pooled_ctr": 0.01440329218106996, "arms": [ { "headline": "The Gap Between The Rich And The Poor Is So Large, You Can Literally See It From Space", "impressions": 3511, "clicks": 47, "ctr": 0.0133864995727712, "ctr_select": 0.014236902050113895, "ctr_holdout": 0.012535612535612535, "n_select": 1756, "n_holdout": 1755, "eb": 0.014543629857968816, "labelled_winner": false }, { "headline": "It's So Obvious, You Can Even See It From Outer Space", "impressions": 3626, "clicks": 51, "ctr": 0.0140650854936569, "ctr_select": 0.013237727523441808, "ctr_holdout": 0.014892443463872035, "n_select": 1813, "n_holdout": 1813, "eb": 0.014100850687793202, "labelled_winner": false }, { "headline": "6 Satellite Photos That Reveal The Striking Reality Of Rich And Poor Neighborhoods Around The World", "impressions": 3578, "clicks": 58, "ctr": 0.0162101732811626, "ctr_select": 0.01732811626607043, "ctr_holdout": 0.01509223029625489, "n_select": 1789, "n_holdout": 1789, "eb": 0.015887790205699786, "labelled_winner": false }, { "headline": "6 Pictures From Space That Identify A Striking Similarity In Cities Across The Globe", "impressions": 3518, "clicks": 29, "ctr": 0.0082433200682205, "ctr_select": 0.008527572484366117, "ctr_holdout": 0.007959067652075043, "n_select": 1759, "n_holdout": 1759, "eb": 0.012079961561406904, "labelled_winner": false }, { "headline": "These 6 Images Have Me Shaking My Head. Is This The World We Want To Live In?", "impressions": 3583, "clicks": 62, "ctr": 0.0173039352497906, "ctr_select": 0.01953125, "ctr_holdout": 0.01507537688442211, "n_select": 1792, "n_holdout": 1791, "eb": 0.016849500023170344, "labelled_winner": false }, { "headline": "These 6 Images Don't Shock Me. But I'm Still Shaking My Head.", "impressions": 3568, "clicks": 61, "ctr": 0.0170964125560538, "ctr_select": 0.01569506726457399, "ctr_holdout": 0.018497757847533634, "n_select": 1784, "n_holdout": 1784, "eb": 0.015175753078811617, "labelled_winner": false } ], "winner": 4, "pool_select": 0.01477602169643692, "pool_holdout": 0.014030492937985221, "lift_select": 0.32182060917721533, "lift_holdout": 0.07447236180904526, "eb_lift": 0.14032723890987664, "p_value": 0.06745589406513958, "held_up": false, "z_path": [ -0.715, -0.799, -1.169, -1.331, -0.801, -0.495, -0.35, 0.207, 0.093, 0.361 ], "first_cross": null, "final_z": 0.361 }, { "id": "545c301174bfe77ac7000003", "variation": "headline", "week": 201444, "impressions": 21339, "pooled_ctr": 0.012699751628473687, "arms": [ { "headline": "The U.S. Is The Only Country In The World That Does This To 13-Year-Olds. Hint: It's Not Good.", "impressions": 3544, "clicks": 61, "ctr": 0.0172121896162528, "ctr_select": 0.016930022573363433, "ctr_holdout": 0.017494356659142212, "n_select": 1772, "n_holdout": 1772, "eb": 0.013922139821731523, "labelled_winner": false }, { "headline": "The U.S. Is The Only Country That Does This To 13-Year-Olds. Um ... Doesn't That Tell Us Something?", "impressions": 3606, "clicks": 56, "ctr": 0.0155296727676095, "ctr_select": 0.013865779256794232, "ctr_holdout": 0.017193566278424846, "n_select": 1803, "n_holdout": 1803, "eb": 0.01260401684261435, "labelled_winner": false }, { "headline": "Only 1 Country Does This Terrifying Thing To 13-Year-Olds. It's The U.S.", "impressions": 3437, "clicks": 66, "ctr": 0.0192027931335466, "ctr_select": 0.017462165308498253, "ctr_holdout": 0.020942408376963352, "n_select": 1718, "n_holdout": 1719, "eb": 0.014108426210206611, "labelled_winner": false }, { "headline": "The U.S. Is The Only Country That Sentences 13-Year-Olds To Die In Prison. That's Terrifying.", "impressions": 3577, "clicks": 22, "ctr": 0.0061504053676265, "ctr_select": 0.0044742729306487695, "ctr_holdout": 0.007825600894354388, "n_select": 1788, "n_holdout": 1789, "eb": 0.008509477751636945, "labelled_winner": false }, { "headline": "Only 1 Country Sentences Kids To Die In Prison. I Wish This Surprised Me.", "impressions": 3551, "clicks": 28, "ctr": 0.007885102787947, "ctr_select": 0.007882882882882882, "ctr_holdout": 0.007887323943661971, "n_select": 1776, "n_holdout": 1775, "eb": 0.0100000894609754, "labelled_winner": false }, { "headline": "The U.S. Is The Only Country In The World That Does This To Kids. Hint: It's Not Good.", "impressions": 3624, "clicks": 38, "ctr": 0.010485651214128, "ctr_select": 0.009381898454746136, "ctr_holdout": 0.011589403973509934, "n_select": 1812, "n_holdout": 1812, "eb": 0.010639360654225792, "labelled_winner": false } ], "winner": 2, "pool_select": 0.011622457587402756, "pool_holdout": 0.013776944704779756, "lift_select": 0.5024503360997408, "lift_holdout": 0.5201054243687004, "eb_lift": 0.2138935422314059, "p_value": 0.013679324670803135, "held_up": true, "z_path": [ 1.22, 0.649, 1.183, 1.491, 1.002, 0.902, 0.874, 0.739, 0.448, 0.622 ], "first_cross": null, "final_z": 0.622 }, { "id": "548634da3831610015740000", "variation": "headline", "week": 201449, "impressions": 21282, "pooled_ctr": 0.00817592331547787, "arms": [ { "headline": "Country Music Is Actually Pretty Progressive, And These 11 Moments In 2014 Prove It", "impressions": 3011, "clicks": 23, "ctr": 0.007638658253072, "ctr_select": 0.004648074369189907, "ctr_holdout": 0.0106312292358804, "n_select": 1506, "n_holdout": 1505, "eb": 0.006388135364749224, "labelled_winner": false }, { "headline": "11 Reasons You Should Definitely Love (Or At Least Respect) Country Music", "impressions": 3041, "clicks": 25, "ctr": 0.0082209799408089, "ctr_select": 0.0059210526315789476, "ctr_holdout": 0.01051939513477975, "n_select": 1520, "n_holdout": 1521, "eb": 0.006885977105276738, "labelled_winner": false }, { "headline": "11 Times Country Music Proved It Ain't What You Thought It Was In 2014", "impressions": 2866, "clicks": 10, "ctr": 0.0034891835310537, "ctr_select": 0.0027913468248429866, "ctr_holdout": 0.00418702023726448, "n_select": 1433, "n_holdout": 1433, "eb": 0.005712616749841165, "labelled_winner": false }, { "headline": "11 Times The Most Popular Music Genre Totally Surprised Us In 2014", "impressions": 3121, "clicks": 21, "ctr": 0.0067286126241589, "ctr_select": 0.007692307692307693, "ctr_holdout": 0.005765534913516977, "n_select": 1560, "n_holdout": 1561, "eb": 0.007588600446062911, "labelled_winner": false }, { "headline": "11 Times Country Acted Like It Was 2014 In 2014", "impressions": 3172, "clicks": 26, "ctr": 0.0081967213114754, "ctr_select": 0.008827238335435058, "ctr_holdout": 0.007566204287515763, "n_select": 1586, "n_holdout": 1586, "eb": 0.00805039245066981, "labelled_winner": false }, { "headline": "That Time Garth Brooks Took Us All By Surprise \u2014 And 10 Other Great Moments In 2014 Country", "impressions": 2989, "clicks": 14, "ctr": 0.0046838407494145, "ctr_select": 0.004016064257028112, "ctr_holdout": 0.005351170568561873, "n_select": 1494, "n_holdout": 1495, "eb": 0.006145896224299961, "labelled_winner": false }, { "headline": "When Two Famous Singers Just Came Out As Gay \u2014 And 10 Other Great Moments In 2014", "impressions": 3082, "clicks": 55, "ctr": 0.017845554834523, "ctr_select": 0.018170019467878003, "ctr_holdout": 0.01752109020116807, "n_select": 1541, "n_holdout": 1541, "eb": 0.011772442934964435, "labelled_winner": false } ], "winner": 6, "pool_select": 0.007518796992481203, "pool_holdout": 0.008832926141702687, "lift_select": 1.4166125892277743, "lift_holdout": 0.9836110842641554, "eb_lift": 0.56573491035027, "p_value": 1.6581862847817923e-07, "held_up": true, "z_path": [ 0.035, 0.956, 1.603, 2.217, 2.638, 2.888, 3.119, 3.238, 3.108, 3.374 ], "first_cross": 3, "final_z": 3.374 }, { "id": "53ffc120bd37b6a7da00002a", "variation": "headline", "week": 201434, "impressions": 20624, "pooled_ctr": 0.01134600465477114, "arms": [ { "headline": "Come For The Major \u201890s Fashion Swag, Stay For The Story Of A Little Girl Called 'Mike'", "impressions": 5237, "clicks": 20, "ctr": 0.0038189803322512, "ctr_select": 0.004201680672268907, "ctr_holdout": 0.003436426116838488, "n_select": 2618, "n_holdout": 2619, "eb": 0.00769311316082316, "labelled_winner": false }, { "headline": "It Begins With A Girl Sweeping Away The Gender Binary And Ends With A Man Saying 'Thank You'", "impressions": 5170, "clicks": 32, "ctr": 0.0061895551257253, "ctr_select": 0.005415860735009671, "ctr_holdout": 0.006963249516441006, "n_select": 2585, "n_holdout": 2585, "eb": 0.008356803424906037, "labelled_winner": false }, { "headline": "This Is His Fifth Year Taking Testosterone. But It\u2019s Just One Part Of Why He Couldn\u2019t Be Happier.", "impressions": 5147, "clicks": 43, "ctr": 0.0083543811929279, "ctr_select": 0.009324009324009324, "ctr_holdout": 0.007384376214535562, "n_select": 2574, "n_holdout": 2573, "eb": 0.010419875480260838, "labelled_winner": false }, { "headline": "5 Years' Worth Of Photos Show What Testosterone Did To One Guy's Life", "impressions": 5070, "clicks": 139, "ctr": 0.0274161735700197, "ctr_select": 0.028007889546351084, "ctr_holdout": 0.026824457593688362, "n_select": 2535, "n_holdout": 2535, "eb": 0.020188951987635012, "labelled_winner": false } ], "winner": 3, "pool_select": 0.011636927851047323, "pool_holdout": 0.011055081458494957, "lift_select": 1.40681130834977, "lift_holdout": 1.426436900930828, "eb_lift": 0.734903940804102, "p_value": 8.73934596872334e-19, "held_up": true, "z_path": [ 2.119, 2.151, 3.368, 3.843, 4.406, 5.619, 6.182, 6.445, 6.577, 7.283 ], "first_cross": 0, "final_z": 7.283 }, { "id": "53a993bbf98fb2657b00007d", "variation": "headline", "week": 201425, "impressions": 20420, "pooled_ctr": 0.011998041136141039, "arms": [ { "headline": "The List She's Rattling Off Made Me Exhausted, But I Think That's The Point", "impressions": 4119, "clicks": 50, "ctr": 0.0121388686574411, "ctr_select": 0.01262135922330097, "ctr_holdout": 0.011656143759106362, "n_select": 2060, "n_holdout": 2059, "eb": 0.011746314498081327, "labelled_winner": false }, { "headline": "The List She's Rattling Off Is Exhausting, But I Think That's The Point", "impressions": 4027, "clicks": 57, "ctr": 0.0141544574124658, "ctr_select": 0.013902681231380337, "ctr_holdout": 0.014406358668653751, "n_select": 2014, "n_holdout": 2013, "eb": 0.012332765963564255, "labelled_winner": true }, { "headline": "The List She's Rattling Off Is Sorta Overwhelming, But Spoiler Alert: That's The Point", "impressions": 4050, "clicks": 56, "ctr": 0.0138271604938271, "ctr_select": 0.012839506172839505, "ctr_holdout": 0.014814814814814815, "n_select": 2025, "n_holdout": 2025, "eb": 0.011840983995107135, "labelled_winner": false }, { "headline": "The Stats She's Rattling Off Are A Bit Overwhelming, But I Think That's The Point", "impressions": 4085, "clicks": 35, "ctr": 0.0085679314565483, "ctr_select": 0.007345739471106758, "ctr_holdout": 0.00978952520802741, "n_select": 2042, "n_holdout": 2043, "eb": 0.009271701102852851, "labelled_winner": false }, { "headline": "Just When You Think There Can't Possibly Be More Stats On Body Image, She Keeps Going", "impressions": 4139, "clicks": 47, "ctr": 0.0113553998550374, "ctr_select": 0.00821256038647343, "ctr_holdout": 0.014499758337361043, "n_select": 2070, "n_holdout": 2069, "eb": 0.009668353854769185, "labelled_winner": false } ], "winner": 1, "pool_select": 0.010968563314073058, "pool_holdout": 0.013027720638652169, "lift_select": 0.26750248262164833, "lift_holdout": 0.10582342592696348, "eb_lift": 0.12437386833888042, "p_value": 0.15823787741797757, "held_up": false, "z_path": [ -0.83, -0.889, -0.657, -0.28, -0.789, -0.604, -0.56, 0.135, -0.074, 0.125 ], "first_cross": null, "final_z": 0.125 }, { "id": "5351565b6948478edc00002d", "variation": "headline", "week": 201415, "impressions": 20325, "pooled_ctr": 0.011168511685116851, "arms": [ { "headline": "The Numbers Don't Lie, These Students Are Keeping Their Classmates Out Of Juvie", "impressions": 4092, "clicks": 26, "ctr": 0.0063538611925708, "ctr_select": 0.003421309872922776, "ctr_holdout": 0.009286412512218964, "n_select": 2046, "n_holdout": 2046, "eb": 0.00745740970711921, "labelled_winner": false }, { "headline": "How Some Kids Put Their Faculty To Shame AND Made Them Proud With A New Kind Of School Discipline", "impressions": 4114, "clicks": 53, "ctr": 0.0128828390860476, "ctr_select": 0.013612056392805057, "ctr_holdout": 0.012153621779290228, "n_select": 2057, "n_holdout": 2057, "eb": 0.012238990036901327, "labelled_winner": false }, { "headline": "This School Struggled With Detentions, So They Asked For The Students Help. Guess What? It's Working.", "impressions": 4044, "clicks": 78, "ctr": 0.0192878338278931, "ctr_select": 0.02126607319485658, "ctr_holdout": 0.01730959446092977, "n_select": 2022, "n_holdout": 2022, "eb": 0.015794160396997214, "labelled_winner": false }, { "headline": "In One School, Students Serve As Judge, Jury, And Encouragers For Their Troubled Classmates", "impressions": 4000, "clicks": 31, "ctr": 0.00775, "ctr_select": 0.005, "ctr_holdout": 0.0105, "n_select": 2000, "n_holdout": 2000, "eb": 0.008231674390572943, "labelled_winner": false }, { "headline": "Instead Of Detention, These Students Are Going To Trial With Their Fellow Classmates As Jury", "impressions": 4075, "clicks": 39, "ctr": 0.0095705521472392, "ctr_select": 0.011776251226692836, "ctr_holdout": 0.007363770250368188, "n_select": 2038, "n_holdout": 2037, "eb": 0.011374040390535703, "labelled_winner": false } ], "winner": 2, "pool_select": 0.011020368001574339, "pool_holdout": 0.011316669946860853, "lift_select": 0.9297062667797089, "lift_holdout": 0.529566077495377, "eb_lift": 0.4331790367382382, "p_value": 8.191541128689619e-07, "held_up": true, "z_path": [ 1.429, 1.571, 1.473, 2.019, 1.808, 2.156, 1.917, 2.528, 2.443, 2.301 ], "first_cross": 3, "final_z": 2.301 }, { "id": "5283e278a4e712a50a001a2d", "variation": "image", "week": 201345, "impressions": 20297, "pooled_ctr": 0.018968320441444548, "arms": [ { "headline": "A Dude Dresses Up As A Superhero and New Yorkers Can\u2019t Even Handle It", "impressions": 5184, "clicks": 109, "ctr": 0.0210262345679012, "ctr_select": 0.019675925925925927, "ctr_holdout": 0.022376543209876542, "n_select": 2592, "n_holdout": 2592, "eb": 0.01917951794211326, "labelled_winner": false }, { "headline": "A Dude Dresses Up As A Superhero and New Yorkers Can\u2019t Even Handle It", "impressions": 5289, "clicks": 91, "ctr": 0.0172055208924182, "ctr_select": 0.01664145234493192, "ctr_holdout": 0.01776937618147448, "n_select": 2644, "n_holdout": 2645, "eb": 0.017567703718622683, "labelled_winner": false }, { "headline": "A Dude Dresses Up As A Superhero and New Yorkers Can\u2019t Even Handle It", "impressions": 4793, "clicks": 79, "ctr": 0.0164823701230961, "ctr_select": 0.017529215358931552, "ctr_holdout": 0.01543596161869003, "n_select": 2396, "n_holdout": 2397, "eb": 0.018067691344027977, "labelled_winner": false }, { "headline": "A Dude Dresses Up As A Superhero and New Yorkers Can\u2019t Even Handle It", "impressions": 5031, "clicks": 106, "ctr": 0.0210693699065792, "ctr_select": 0.02066772655007949, "ctr_holdout": 0.02147117296222664, "n_select": 2516, "n_holdout": 2515, "eb": 0.019687957549589005, "labelled_winner": false } ], "winner": 3, "pool_select": 0.018624359479700432, "pool_holdout": 0.019312247512070155, "lift_select": 0.10971475677358034, "lift_holdout": 0.11179048160019467, "eb_lift": 0.05710790059909643, "p_value": 0.3820053508156698, "held_up": false, "z_path": [ 0.486, -0.203, -0.838, -1.147, -0.701, -0.779, -0.299, -0.334, 0.07, 0.015 ], "first_cross": null, "final_z": 0.015 }, { "id": "54b86431356636001f4b0000", "variation": "other", "week": 201502, "impressions": 20135, "pooled_ctr": 0.007648373479016638, "arms": [ { "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.", "impressions": 3411, "clicks": 34, "ctr": 0.0099677513925535, "ctr_select": 0.009378663540445486, "ctr_holdout": 0.010557184750733138, "n_select": 1706, "n_holdout": 1705, "eb": 0.008153343393340997, "labelled_winner": false }, { "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.", "impressions": 3434, "clicks": 19, "ctr": 0.0055329062317996, "ctr_select": 0.0052417006406523005, "ctr_holdout": 0.005824111822947, "n_select": 1717, "n_holdout": 1717, "eb": 0.006396164219995819, "labelled_winner": false }, { "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.", "impressions": 3313, "clicks": 27, "ctr": 0.00814971325083, "ctr_select": 0.007246376811594203, "ctr_holdout": 0.009052504526252263, "n_select": 1656, "n_holdout": 1657, "eb": 0.0072493156848055604, "labelled_winner": false }, { "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.", "impressions": 3350, "clicks": 32, "ctr": 0.0095522388059701, "ctr_select": 0.00955223880597015, "ctr_holdout": 0.00955223880597015, "n_select": 1675, "n_holdout": 1675, "eb": 0.008216647370724563, "labelled_winner": false }, { "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.", "impressions": 3261, "clicks": 13, "ctr": 0.0039865072063784, "ctr_select": 0.005521472392638037, "ctr_holdout": 0.002452483139178418, "n_select": 1630, "n_holdout": 1631, "eb": 0.006537123920756696, "labelled_winner": false }, { "headline": "Some Might See These Ladies As Novelties, But They're As Real As It Gets. Just Ask Them.", "impressions": 3366, "clicks": 29, "ctr": 0.0086155674390968, "ctr_select": 0.006535947712418301, "ctr_holdout": 0.0106951871657754, "n_select": 1683, "n_holdout": 1683, "eb": 0.006950434936553902, "labelled_winner": false } ], "winner": 3, "pool_select": 0.007251415516042515, "pool_holdout": 0.008045292014302742, "lift_select": 0.3172929871191985, "lift_holdout": 0.18730790491984517, "eb_lift": 0.13310943946690657, "p_value": 0.2241520111019315, "held_up": false, "z_path": [ 0.033, -0.222, 0.255, 0.247, 0.784, 0.382, -0.084, -0.207, -0.331, -0.174 ], "first_cross": null, "final_z": -0.174 }, { "id": "5343745daf7ae25546000034", "variation": "image", "week": 201414, "impressions": 20124, "pooled_ctr": 0.030659908566885313, "arms": [ { "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return", "impressions": 2560, "clicks": 69, "ctr": 0.026953125, "ctr_select": 0.0265625, "ctr_holdout": 0.02734375, "n_select": 1280, "n_holdout": 1280, "eb": 0.030258528066759122, "labelled_winner": false }, { "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return", "impressions": 2494, "clicks": 92, "ctr": 0.036888532477947, "ctr_select": 0.03608660785886127, "ctr_holdout": 0.03769045709703288, "n_select": 1247, "n_holdout": 1247, "eb": 0.033624460445991385, "labelled_winner": false }, { "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return", "impressions": 2427, "clicks": 57, "ctr": 0.0234857849196538, "ctr_select": 0.0214168039538715, "ctr_holdout": 0.025556471558120363, "n_select": 1214, "n_holdout": 1213, "eb": 0.028558796761976986, "labelled_winner": false }, { "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return", "impressions": 2537, "clicks": 83, "ctr": 0.0327158060701616, "ctr_select": 0.031545741324921134, "ctr_holdout": 0.0338849487785658, "n_select": 1268, "n_holdout": 1269, "eb": 0.0320331066455884, "labelled_winner": false }, { "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return", "impressions": 2542, "clicks": 94, "ctr": 0.036978756884343, "ctr_select": 0.03540519276160504, "ctr_holdout": 0.03855232100708104, "n_select": 1271, "n_holdout": 1271, "eb": 0.033399591394005834, "labelled_winner": false }, { "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return", "impressions": 2547, "clicks": 78, "ctr": 0.0306242638398115, "ctr_select": 0.033751962323390894, "ctr_holdout": 0.02749410840534171, "n_select": 1274, "n_holdout": 1273, "eb": 0.032814854329003884, "labelled_winner": false }, { "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return", "impressions": 2401, "clicks": 71, "ctr": 0.0295710120783007, "ctr_select": 0.041666666666666664, "ctr_holdout": 0.017485428809325562, "n_select": 1200, "n_holdout": 1201, "eb": 0.03549510147131108, "labelled_winner": false }, { "headline": "This Guy Was Really Nice To A Homeless Little Girl And Got The Best Reward Ever In Return", "impressions": 2616, "clicks": 73, "ctr": 0.0279051987767584, "ctr_select": 0.03211009174311927, "ctr_holdout": 0.023700305810397553, "n_select": 1308, "n_holdout": 1308, "eb": 0.03223132386961618, "labelled_winner": false } ], "winner": 6, "pool_select": 0.03229974160206718, "pool_holdout": 0.029020075531703438, "lift_select": 0.29000000000000004, "lift_holdout": -0.3974712853444048, "eb_lift": 0.0989283415517912, "p_value": 0.05050487627726831, "held_up": false, "z_path": [ 0.859, 0.115, -0.448, -0.72, -1.099, -1.602, -2.035, -1.955, -1.695, -1.426 ], "first_cross": 6, "final_z": -1.426 }, { "id": "54cc4259343963000c710000", "variation": "both", "week": 201504, "impressions": 19858, "pooled_ctr": 0.007855776009668648, "arms": [ { "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was Money.", "impressions": 2838, "clicks": 24, "ctr": 0.0084566596194503, "ctr_select": 0.010570824524312896, "ctr_holdout": 0.006342494714587738, "n_select": 1419, "n_holdout": 1419, "eb": 0.009324089476860154, "labelled_winner": false }, { "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.", "impressions": 2859, "clicks": 24, "ctr": 0.0083945435466946, "ctr_select": 0.00909090909090909, "ctr_holdout": 0.007697690692792162, "n_select": 1430, "n_holdout": 1429, "eb": 0.008763061040216088, "labelled_winner": false }, { "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.", "impressions": 2843, "clicks": 19, "ctr": 0.0066830812521983, "ctr_select": 0.006329113924050633, "ctr_holdout": 0.007037297677691766, "n_select": 1422, "n_holdout": 1421, "eb": 0.00771220190077056, "labelled_winner": false }, { "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.", "impressions": 2908, "clicks": 23, "ctr": 0.0079092159559834, "ctr_select": 0.006189821182943604, "ctr_holdout": 0.009628610729023384, "n_select": 1454, "n_holdout": 1454, "eb": 0.007646769705469904, "labelled_winner": false }, { "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.", "impressions": 2745, "clicks": 19, "ctr": 0.0069216757741347, "ctr_select": 0.008746355685131196, "ctr_holdout": 0.005098324836125273, "n_select": 1372, "n_holdout": 1373, "eb": 0.008629786582785645, "labelled_winner": false }, { "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.", "impressions": 2860, "clicks": 24, "ctr": 0.0083916083916083, "ctr_select": 0.009790209790209791, "ctr_holdout": 0.006993006993006993, "n_select": 1430, "n_holdout": 1430, "eb": 0.009029891526240471, "labelled_winner": false }, { "headline": "Her Husband Left Her And Then The Government Studied Her. The Result Was To Offer Her Money.", "impressions": 2805, "clicks": 23, "ctr": 0.0081996434937611, "ctr_select": 0.009272467902995721, "ctr_holdout": 0.007127583749109052, "n_select": 1402, "n_holdout": 1403, "eb": 0.008829024924544742, "labelled_winner": false } ], "winner": 0, "pool_select": 0.008560781548997885, "pool_holdout": 0.00715077047033941, "lift_select": 0.23479666708120872, "lift_holdout": -0.11303337998392049, "eb_lift": 0.08916334606758203, "p_value": 0.37467075637699543, "held_up": false, "z_path": [ 1.434, 1.282, 1.405, 2.206, 1.907, 1.548, 0.984, 0.021, 0.188, 0.027 ], "first_cross": 3, "final_z": 0.027 }, { "id": "542aea408de8acc10600004b", "variation": "image", "week": 201439, "impressions": 19685, "pooled_ctr": 0.0063500127000254, "arms": [ { "headline": "There's An Incredibly Simple Reason Kids Are Dying Of Preventable Things. Hint: Rhymes With Bunny.", "impressions": 4929, "clicks": 23, "ctr": 0.0046662609048488, "ctr_select": 0.00487012987012987, "ctr_holdout": 0.004462474645030426, "n_select": 2464, "n_holdout": 2465, "eb": 0.0063509369453476645, "labelled_winner": false }, { "headline": "There's An Incredibly Simple Reason Kids Are Dying Of Preventable Things. Hint: Rhymes With Bunny.", "impressions": 4886, "clicks": 24, "ctr": 0.0049119934506753, "ctr_select": 0.005321326238231682, "ctr_holdout": 0.0045026606631191155, "n_select": 2443, "n_holdout": 2443, "eb": 0.006589004923230203, "labelled_winner": false }, { "headline": "There's An Incredibly Simple Reason Kids Are Dying Of Preventable Things. Hint: Rhymes With Bunny.", "impressions": 4922, "clicks": 49, "ctr": 0.0099553027224705, "ctr_select": 0.01381552214546932, "ctr_holdout": 0.006095083299471759, "n_select": 2461, "n_holdout": 2461, "eb": 0.01095868871618744, "labelled_winner": false }, { "headline": "There's An Incredibly Simple Reason Kids Are Dying Of Preventable Things. Hint: Rhymes With Bunny.", "impressions": 4948, "clicks": 29, "ctr": 0.005860953920776, "ctr_select": 0.007679870654810024, "ctr_holdout": 0.004042037186742118, "n_select": 2474, "n_holdout": 2474, "eb": 0.007798543004435674, "labelled_winner": false } ], "winner": 2, "pool_select": 0.007925218451534242, "pool_holdout": 0.004774966981611298, "lift_select": 0.7432354994321673, "lift_holdout": 0.27646606205745794, "eb_lift": 0.38276172236816386, "p_value": 0.0001415780554587344, "held_up": true, "z_path": [ 1.426, 1.106, 1.241, 1.847, 1.831, 1.834, 1.78, 1.667, 1.943, 2.297 ], "first_cross": 9, "final_z": 2.297 }, { "id": "544e5bc36fba94112e000026", "variation": "headline", "week": 201443, "impressions": 19533, "pooled_ctr": 0.01668970460246762, "arms": [ { "headline": "John Oliver Just Came Up With The Best Way To Hold A Massively Shady Industry Accountable", "impressions": 3941, "clicks": 71, "ctr": 0.0180157320477036, "ctr_select": 0.02182741116751269, "ctr_holdout": 0.014205986808726534, "n_select": 1970, "n_holdout": 1971, "eb": 0.019604204303222503, "labelled_winner": false }, { "headline": "John Oliver Just Found The Best Way To Call Out An Industry That Sells Cocaine-Like Products To Kids", "impressions": 3902, "clicks": 61, "ctr": 0.0156330087134802, "ctr_select": 0.01742696053305997, "ctr_holdout": 0.013839056893900564, "n_select": 1951, "n_holdout": 1951, "eb": 0.017583091374829236, "labelled_winner": false }, { "headline": "John Oliver Just Started #ShowUsYourPeanuts To Call Out An Massively Shady Industry. Sign Me Up!", "impressions": 3914, "clicks": 59, "ctr": 0.015074092999489, "ctr_select": 0.013796627491057742, "ctr_holdout": 0.016351558507920288, "n_select": 1957, "n_holdout": 1957, "eb": 0.015920868911578567, "labelled_winner": false }, { "headline": "John Oliver Wants YOU To Tweet A Massively Shady Industry #ShowUsYourPeanuts For The Best Reason", "impressions": 3877, "clicks": 60, "ctr": 0.0154758834150116, "ctr_select": 0.01805985552115583, "ctr_holdout": 0.012893243940175348, "n_select": 1938, "n_holdout": 1939, "eb": 0.017871782036887995, "labelled_winner": true }, { "headline": "Just In Time For Halloween, John Oliver Came Up With A Genius Way To Expose This Shady Industry", "impressions": 3899, "clicks": 75, "ctr": 0.0192357014619133, "ctr_select": 0.017435897435897435, "ctr_holdout": 0.02103642893791688, "n_select": 1950, "n_holdout": 1949, "eb": 0.01758721141626416, "labelled_winner": false } ], "winner": 0, "pool_select": 0.017714519762441122, "pool_holdout": 0.015664994368792873, "lift_select": 0.2321762859071037, "lift_holdout": -0.09313808391613032, "eb_lift": 0.10667433078191313, "p_value": 0.12140930497342717, "held_up": false, "z_path": [ 1.479, 2.579, 2.819, 2.179, 1.842, 1.488, 0.899, 0.832, 1.323, 0.875 ], "first_cross": 1, "final_z": 0.875 }, { "id": "5258cf57b018770da500029f", "variation": "image", "week": 201340, "impressions": 19259, "pooled_ctr": 0.012980943974245807, "arms": [ { "headline": "Heaven Caught On Tape. More Beautiful Than You Imagined, Closer Than You\u2019d Expect.", "impressions": 4797, "clicks": 71, "ctr": 0.0148009172399416, "ctr_select": 0.013761467889908258, "ctr_holdout": 0.015839933305543976, "n_select": 2398, "n_holdout": 2399, "eb": 0.013225056189516589, "labelled_winner": false }, { "headline": "Heaven Caught On Tape. More Beautiful Than You Imagined, Closer Than You\u2019d Expect.", "impressions": 4816, "clicks": 55, "ctr": 0.0114202657807308, "ctr_select": 0.009966777408637873, "ctr_holdout": 0.01287375415282392, "n_select": 2408, "n_holdout": 2408, "eb": 0.011292590100342115, "labelled_winner": false }, { "headline": "Heaven Caught On Tape. More Beautiful Than You Imagined, Closer Than You\u2019d Expect.", "impressions": 4904, "clicks": 51, "ctr": 0.0103996737357259, "ctr_select": 0.010603588907014683, "ctr_holdout": 0.010195758564437194, "n_select": 2452, "n_holdout": 2452, "eb": 0.011607730834339688, "labelled_winner": false }, { "headline": "Heaven Caught On Tape. More Beautiful Than You Imagined, Closer Than You\u2019d Expect.", "impressions": 4742, "clicks": 73, "ctr": 0.0153943483762125, "ctr_select": 0.016448755799240825, "ctr_holdout": 0.01433994095318431, "n_select": 2371, "n_holdout": 2371, "eb": 0.014580886084635187, "labelled_winner": false } ], "winner": 3, "pool_select": 0.012670059196178212, "pool_holdout": 0.013291796469366563, "lift_select": 0.2982382753351631, "lift_holdout": 0.07885649514972592, "eb_lift": 0.1508143615487887, "p_value": 0.05811539921166236, "held_up": false, "z_path": [ -1.492, -0.683, -0.252, 0.045, 0.168, 0.383, 0.882, 1.0, 0.155, 0.238 ], "first_cross": null, "final_z": 0.238 }, { "id": "51a92833b8852eea050083c4", "variation": "image", "week": 201321, "impressions": 19205, "pooled_ctr": 0.016974746159854205, "arms": [ { "headline": "Obama's Speech Writer/Stand-Up Comedian Gives A Graduation Speech. Not Surprisingly, It's Badass.", "impressions": 4831, "clicks": 88, "ctr": 0.0182156903332643, "ctr_select": 0.018625827814569538, "ctr_holdout": 0.01780538302277433, "n_select": 2416, "n_holdout": 2415, "eb": 0.017919845598968823, "labelled_winner": false }, { "headline": "Obama's Speech Writer/Stand-Up Comedian Gives A Graduation Speech. Not Surprisingly, It's Badass.", "impressions": 4852, "clicks": 99, "ctr": 0.0204039571310799, "ctr_select": 0.018961253091508656, "ctr_holdout": 0.021846661170651278, "n_select": 2426, "n_holdout": 2426, "eb": 0.018092875499927406, "labelled_winner": true }, { "headline": "Obama's Speech Writer/Stand-Up Comedian Gives A Graduation Speech. Not Surprisingly, It's Badass.", "impressions": 4697, "clicks": 69, "ctr": 0.0146902278049819, "ctr_select": 0.014054514480408859, "ctr_holdout": 0.01532567049808429, "n_select": 2348, "n_holdout": 2349, "eb": 0.015609054977330061, "labelled_winner": false }, { "headline": "Obama's Speech Writer/Stand-Up Comedian Gives A Graduation Speech. Not Surprisingly, It's Badass.", "impressions": 4825, "clicks": 70, "ctr": 0.0145077720207253, "ctr_select": 0.016998341625207296, "ctr_holdout": 0.012018234562784915, "n_select": 2412, "n_holdout": 2413, "eb": 0.017089280758904066, "labelled_winner": false } ], "winner": 1, "pool_select": 0.017183920016663193, "pool_holdout": 0.016765594085181714, "lift_select": 0.10343001324040069, "lift_holdout": 0.3030651380233802, "eb_lift": 0.052895700304866455, "p_value": 0.4358538102105517, "held_up": true, "z_path": [ -0.441, -0.913, -0.407, -0.375, -0.231, -0.409, 0.067, 0.99, 0.993, 0.782 ], "first_cross": null, "final_z": 0.782 }, { "id": "5342b02edf3e0e1d32000008", "variation": "headline", "week": 201414, "impressions": 19139, "pooled_ctr": 0.011547102774439625, "arms": [ { "headline": "Half Of All Human Deaths Since The Stone Age Can Be Linked To ... Mosquitoes?", "impressions": 3835, "clicks": 17, "ctr": 0.0044328552803129, "ctr_select": 0.0031282586027111575, "ctr_holdout": 0.005738132498695879, "n_select": 1918, "n_holdout": 1917, "eb": 0.007991864434868825, "labelled_winner": false }, { "headline": "We Now Have The Technology To Eliminate The Mosquito. Should We?", "impressions": 3792, "clicks": 33, "ctr": 0.0087025316455696, "ctr_select": 0.00949367088607595, "ctr_holdout": 0.007911392405063292, "n_select": 1896, "n_holdout": 1896, "eb": 0.01088144548008152, "labelled_winner": false }, { "headline": "If You Could Press A Button And Murder Ever Mosquito, Would You? Because That's Kinda Possible.", "impressions": 3802, "clicks": 66, "ctr": 0.0173592845870594, "ctr_select": 0.016833245660178853, "ctr_holdout": 0.01788532351394003, "n_select": 1901, "n_holdout": 1901, "eb": 0.014187108447418717, "labelled_winner": false }, { "headline": "It's Not Often That Humanity Has The Chance To Cause The Extinction Of Something That's Actually Bad", "impressions": 3880, "clicks": 50, "ctr": 0.0128865979381443, "ctr_select": 0.014948453608247423, "ctr_holdout": 0.010824742268041237, "n_select": 1940, "n_holdout": 1940, "eb": 0.01335255008373267, "labelled_winner": false }, { "headline": "They Suck. They Itch. They Murder. Now We Can Murder Them All. Should We?", "impressions": 3830, "clicks": 55, "ctr": 0.0143603133159268, "ctr_select": 0.015665796344647518, "ctr_holdout": 0.013054830287206266, "n_select": 1915, "n_holdout": 1915, "eb": 0.013667671513414026, "labelled_winner": false } ], "winner": 2, "pool_select": 0.012016718913270637, "pool_holdout": 0.011077437558783572, "lift_select": 0.4008187910253185, "lift_holdout": 0.6145722708008694, "eb_lift": 0.18061415514606205, "p_value": 0.03131873465938484, "held_up": true, "z_path": [ -0.373, 0.011, 1.283, 1.264, 0.717, 0.525, 0.493, 0.035, 0.625, 1.049 ], "first_cross": null, "final_z": 1.049 }, { "id": "54b7c3d465656200132d0000", "variation": "headline", "week": 201502, "impressions": 19032, "pooled_ctr": 0.007356031946195881, "arms": [ { "headline": "The Video That Made You Cry, Laugh, Spit Nails, Or Go 'Awwwwww' Is Being Missed By A Lot Of People", "impressions": 3782, "clicks": 29, "ctr": 0.0076679005817028, "ctr_select": 0.011634056054997356, "ctr_holdout": 0.0037017451084082496, "n_select": 1891, "n_holdout": 1891, "eb": 0.009914752502363839, "labelled_winner": false }, { "headline": "The Videos That Make You Cry, Laugh, Spit Nails, Or Go 'Awwwwww' Are Being Missed By A Lot Of People", "impressions": 3939, "clicks": 29, "ctr": 0.0073622746890073, "ctr_select": 0.007614213197969543, "ctr_holdout": 0.007110208227526663, "n_select": 1970, "n_holdout": 1969, "eb": 0.00809949732724397, "labelled_winner": false }, { "headline": "What If The Videos That Make You Cry, Laugh, Or Spit Nails Are Being Missed By A Lot Of People?", "impressions": 3790, "clicks": 16, "ctr": 0.004221635883905, "ctr_select": 0.005277044854881266, "ctr_holdout": 0.0031662269129287598, "n_select": 1895, "n_holdout": 1895, "eb": 0.007056807343667719, "labelled_winner": false }, { "headline": "You Know, It Hadn't Even Occurred To Me That Doing Something So Simple Would Include So Many People", "impressions": 3760, "clicks": 24, "ctr": 0.0063829787234042, "ctr_select": 0.005319148936170213, "ctr_holdout": 0.007446808510638298, "n_select": 1880, "n_holdout": 1880, "eb": 0.007082024049967974, "labelled_winner": false }, { "headline": "Pretty Much A No-Brainer. So Why Are So Many People WITH Brains Forgetting Do To This?", "impressions": 3761, "clicks": 42, "ctr": 0.0111672427545865, "ctr_select": 0.01276595744680851, "ctr_holdout": 0.009569377990430622, "n_select": 1880, "n_holdout": 1881, "eb": 0.01041718560401326, "labelled_winner": false } ], "winner": 4, "pool_select": 0.008511979823455234, "pool_holdout": 0.006200084068936528, "lift_select": 0.4997635933806144, "lift_holdout": 0.5434271348633524, "eb_lift": 0.22382639762703915, "p_value": 0.02500352120313054, "held_up": true, "z_path": [ 0.008, 0.591, 0.443, 0.775, 1.022, 1.117, 1.157, 1.438, 1.892, 1.574 ], "first_cross": null, "final_z": 1.574 }, { "id": "52791ac5ea4d32dc8b004160", "variation": "image", "week": 201344, "impressions": 18713, "pooled_ctr": 0.016352268476460215, "arms": [ { "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.", "impressions": 3979, "clicks": 51, "ctr": 0.012817290776577, "ctr_select": 0.012060301507537688, "ctr_holdout": 0.013574660633484163, "n_select": 1990, "n_holdout": 1989, "eb": 0.015290119061016731, "labelled_winner": false }, { "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.", "impressions": 3861, "clicks": 56, "ctr": 0.0145040145040145, "ctr_select": 0.018134715025906734, "ctr_holdout": 0.010875194199896427, "n_select": 1930, "n_holdout": 1931, "eb": 0.018095734707972804, "labelled_winner": false }, { "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.", "impressions": 3813, "clicks": 54, "ctr": 0.014162077104642, "ctr_select": 0.014690451206715634, "ctr_holdout": 0.013633980073413739, "n_select": 1906, "n_holdout": 1907, "eb": 0.016541243163796977, "labelled_winner": false }, { "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.", "impressions": 3415, "clicks": 80, "ctr": 0.0234260614934114, "ctr_select": 0.026346604215456676, "ctr_holdout": 0.020503807850029292, "n_select": 1708, "n_holdout": 1707, "eb": 0.021577678493526813, "labelled_winner": false }, { "headline": "Growing Up, He Was Told That Folks Making His Clothes Were 'Being Taken Care Of.' He Had No Idea.", "impressions": 3645, "clicks": 65, "ctr": 0.017832647462277, "ctr_select": 0.02030735455543359, "ctr_holdout": 0.015359297860669226, "n_select": 1822, "n_holdout": 1823, "eb": 0.01905095888872262, "labelled_winner": false } ], "winner": 3, "pool_select": 0.018063274903805045, "pool_holdout": 0.014641444907555841, "lift_select": 0.4585729528982998, "lift_holdout": 0.4003951098738985, "eb_lift": 0.19456070997299912, "p_value": 0.004468816016140567, "held_up": true, "z_path": [ 1.433, 0.74, 0.061, 0.786, 0.758, 1.29, 0.931, 0.73, 1.087, 1.656 ], "first_cross": null, "final_z": 1.656 }, { "id": "5272e33f6a50c7c8ed0058f7", "variation": "image", "week": 201343, "impressions": 18616, "pooled_ctr": 0.033358401375161155, "arms": [ { "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop", "impressions": 4087, "clicks": 208, "ctr": 0.0508930756055786, "ctr_select": 0.053816046966731895, "ctr_holdout": 0.047968673519334315, "n_select": 2044, "n_holdout": 2043, "eb": 0.0429149176927707, "labelled_winner": false }, { "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop", "impressions": 3574, "clicks": 80, "ctr": 0.0223838836038052, "ctr_select": 0.020705092333519866, "ctr_holdout": 0.024062674874090653, "n_select": 1787, "n_holdout": 1787, "eb": 0.027817370781339574, "labelled_winner": false }, { "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop", "impressions": 3783, "clicks": 95, "ctr": 0.0251123446999735, "ctr_select": 0.025898520084566595, "ctr_holdout": 0.024325753569539928, "n_select": 1892, "n_holdout": 1891, "eb": 0.02997409953123644, "labelled_winner": false }, { "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop", "impressions": 3158, "clicks": 101, "ctr": 0.031982267257758, "ctr_select": 0.03356554781507283, "ctr_holdout": 0.030398986700443317, "n_select": 1579, "n_holdout": 1579, "eb": 0.03340826203693005, "labelled_winner": false }, { "headline": "A Catalog That Believes Reality Can Sell Clothes Better Than Photoshop", "impressions": 4014, "clicks": 137, "ctr": 0.0341305430991529, "ctr_select": 0.030393622321873443, "ctr_holdout": 0.037867463876432486, "n_select": 2007, "n_holdout": 2007, "eb": 0.03195180505843267, "labelled_winner": false } ], "winner": 0, "pool_select": 0.03330110645611774, "pool_holdout": 0.03341570860642527, "lift_select": 0.616043810365507, "lift_holdout": 0.4355126831011076, "eb_lift": 0.2886934477483949, "p_value": 4.870700208930216e-09, "held_up": true, "z_path": [ 0.928, 0.911, 1.441, 2.529, 3.176, 3.563, 3.79, 3.649, 3.75, 3.95 ], "first_cross": 3, "final_z": 3.95 }, { "id": "5411b14a5856e96e050000d5", "variation": "image", "week": 201436, "impressions": 18513, "pooled_ctr": 0.01226165397288392, "arms": [ { "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive", "impressions": 3621, "clicks": 50, "ctr": 0.0138083402375034, "ctr_select": 0.012154696132596685, "ctr_holdout": 0.015461071231363888, "n_select": 1810, "n_holdout": 1811, "eb": 0.012609435780192224, "labelled_winner": false }, { "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive", "impressions": 3734, "clicks": 56, "ctr": 0.0149973219068023, "ctr_select": 0.017675415104445636, "ctr_holdout": 0.012319228709159078, "n_select": 1867, "n_holdout": 1867, "eb": 0.015066306714774547, "labelled_winner": false }, { "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive", "impressions": 3717, "clicks": 44, "ctr": 0.011837503362927, "ctr_select": 0.010764262648008612, "ctr_holdout": 0.012910166756320602, "n_select": 1858, "n_holdout": 1859, "eb": 0.011985527369734747, "labelled_winner": false }, { "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive", "impressions": 3710, "clicks": 32, "ctr": 0.0086253369272237, "ctr_select": 0.009164420485175202, "ctr_holdout": 0.008086253369272238, "n_select": 1855, "n_holdout": 1855, "eb": 0.011275185133679892, "labelled_winner": false }, { "headline": "A Student Puts Herself On Tape, Admits That Her Family Committed A Crime ... But They Had To Survive", "impressions": 3731, "clicks": 45, "ctr": 0.0120611096220852, "ctr_select": 0.015005359056806002, "ctr_holdout": 0.009115281501340482, "n_select": 1866, "n_holdout": 1865, "eb": 0.013874792880549967, "labelled_winner": false } ], "winner": 1, "pool_select": 0.01296456352636128, "pool_holdout": 0.01155882035216593, "lift_select": 0.36336368505624006, "lift_holdout": 0.0657859828101457, "eb_lift": 0.16211445793294343, "p_value": 0.04401652968976107, "held_up": false, "z_path": [ 1.139, 1.047, 1.063, 0.926, 1.576, 1.699, 1.84, 1.363, 0.64, 1.098 ], "first_cross": null, "final_z": 1.098 }, { "id": "5395561485b18f33ec000011", "variation": "image", "week": 201423, "impressions": 18473, "pooled_ctr": 0.04065392735343474, "arms": [ { "headline": "Have You Ever Wondered What Makes You Sexually Attracted To Certain People? Here Are Some Answers.", "impressions": 4540, "clicks": 144, "ctr": 0.0317180616740088, "ctr_select": 0.032158590308370046, "ctr_holdout": 0.03127753303964758, "n_select": 2270, "n_holdout": 2270, "eb": 0.036752376746970086, "labelled_winner": false }, { "headline": "Have You Ever Wondered What Makes You Sexually Attracted To Certain People? Here Are Some Answers.", "impressions": 4625, "clicks": 221, "ctr": 0.0477837837837837, "ctr_select": 0.04801038062283737, "ctr_holdout": 0.04755728491137051, "n_select": 2312, "n_holdout": 2313, "eb": 0.04462684339738134, "labelled_winner": false }, { "headline": "Have You Ever Wondered What Makes You Sexually Attracted To Certain People? Here Are Some Answers.", "impressions": 4671, "clicks": 220, "ctr": 0.0470991222436309, "ctr_select": 0.04708904109589041, "ctr_holdout": 0.047109207708779445, "n_select": 2336, "n_holdout": 2335, "eb": 0.04418181143076374, "labelled_winner": false }, { "headline": "Have You Ever Wondered What Makes You Sexually Attracted To Certain People? Here Are Some Answers.", "impressions": 4637, "clicks": 166, "ctr": 0.0357990079792969, "ctr_select": 0.037532355478861086, "ctr_holdout": 0.03406640793445451, "n_select": 2318, "n_holdout": 2319, "eb": 0.03939186851191274, "labelled_winner": false } ], "winner": 1, "pool_select": 0.04125162407968817, "pool_holdout": 0.040056295333982896, "lift_select": 0.16384219273628875, "lift_holdout": 0.1872611911522417, "eb_lift": 0.08182027721316043, "p_value": 0.05911393220439828, "held_up": true, "z_path": [ 0.525, 0.161, -0.131, 0.139, 1.115, 0.969, 0.492, -0.186, 0.1, 0.155 ], "first_cross": null, "final_z": 0.155 }, { "id": "543d6d2bdd6e021068000040", "variation": "headline", "week": 201441, "impressions": 18472, "pooled_ctr": 0.00698354265915981, "arms": [ { "headline": "Thanks, Joss Whedon! 16 Really Important Life Lessons 'Buffy The Vampire Slayer' Taught Me.", "impressions": 3090, "clicks": 25, "ctr": 0.0080906148867313, "ctr_select": 0.007766990291262136, "ctr_holdout": 0.008414239482200648, "n_select": 1545, "n_holdout": 1545, "eb": 0.007329388374407745, "labelled_winner": false }, { "headline": "16 Things That Got Drilled Into My Skull Because Of Buffy The Vampire Slayer", "impressions": 3150, "clicks": 23, "ctr": 0.0073015873015873, "ctr_select": 0.006349206349206349, "ctr_holdout": 0.008253968253968255, "n_select": 1575, "n_holdout": 1575, "eb": 0.006759120190317952, "labelled_winner": false }, { "headline": "Make A Little Room For The Cheese Slices And These 16 Important Lessons From Buffy", "impressions": 3074, "clicks": 12, "ctr": 0.0039037085230969, "ctr_select": 0.004554326610279766, "ctr_holdout": 0.0032530904359141183, "n_select": 1537, "n_holdout": 1537, "eb": 0.006047481135076326, "labelled_winner": false }, { "headline": "Buffy The Vampire Slayer Taught Me Countless Things That Shaped My Attitudes. Here Are 16.", "impressions": 3081, "clicks": 23, "ctr": 0.0074651087309315, "ctr_select": 0.007792207792207792, "ctr_holdout": 0.007138221933809215, "n_select": 1540, "n_holdout": 1541, "eb": 0.007338888066887023, "labelled_winner": false }, { "headline": "I Don't Live On A Hellmouth But These 16 Things I Learned From Buffy Came In Pretty Handy Anyway", "impressions": 3055, "clicks": 28, "ctr": 0.009165302782324, "ctr_select": 0.00981675392670157, "ctr_holdout": 0.008513425016371971, "n_select": 1528, "n_holdout": 1527, "eb": 0.008141880630000198, "labelled_winner": false }, { "headline": "They Got The Mustard Out + 15 Other GIF-Lessons That Will Make You Miss Buffy The Vampire Slayer", "impressions": 3022, "clicks": 18, "ctr": 0.0059563203176704, "ctr_select": 0.005956320317670417, "ctr_holdout": 0.005956320317670417, "n_select": 1511, "n_holdout": 1511, "eb": 0.006610919326318461, "labelled_winner": false } ], "winner": 4, "pool_select": 0.0070376786487656995, "pool_holdout": 0.006929406669553919, "lift_select": 0.3948852194925494, "lift_holdout": 0.2285936476751802, "eb_lift": 0.15689860767202823, "p_value": 0.15488141639581227, "held_up": true, "z_path": [ 0.587, -0.368, 0.592, 0.745, 1.111, 1.825, 1.159, 0.872, 0.487, 0.733 ], "first_cross": null, "final_z": 0.733 }, { "id": "53da7b0e4d33e277ec00001c", "variation": "image", "week": 201430, "impressions": 18426, "pooled_ctr": 0.005047215890589384, "arms": [ { "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.", "impressions": 3076, "clicks": 19, "ctr": 0.0061768530559167, "ctr_select": 0.007802340702210663, "ctr_holdout": 0.004551365409622887, "n_select": 1538, "n_holdout": 1538, "eb": 0.006439814380415746, "labelled_winner": false }, { "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.", "impressions": 3146, "clicks": 20, "ctr": 0.0063572790845518, "ctr_select": 0.008264462809917356, "ctr_holdout": 0.004450095359186269, "n_select": 1573, "n_holdout": 1573, "eb": 0.0066389063056924965, "labelled_winner": false }, { "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.", "impressions": 3078, "clicks": 16, "ctr": 0.0051981806367771, "ctr_select": 0.004548408057179987, "ctr_holdout": 0.005847953216374269, "n_select": 1539, "n_holdout": 1539, "eb": 0.005141698687198532, "labelled_winner": false }, { "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.", "impressions": 3001, "clicks": 10, "ctr": 0.0033322225924691, "ctr_select": 0.004, "ctr_holdout": 0.002664890073284477, "n_select": 1500, "n_holdout": 1501, "eb": 0.004932286152982215, "labelled_winner": false }, { "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.", "impressions": 3007, "clicks": 13, "ctr": 0.0043232457598935, "ctr_select": 0.005319148936170213, "ctr_holdout": 0.00332667997338656, "n_select": 1504, "n_holdout": 1503, "eb": 0.005450451379129435, "labelled_winner": false }, { "headline": "They've Been Left In The Dark For So Long. Here's What The Sun Has To Do With It.", "impressions": 3118, "clicks": 15, "ctr": 0.0048107761385503, "ctr_select": 0.003207184092366902, "ctr_holdout": 0.006414368184733804, "n_select": 1559, "n_holdout": 1559, "eb": 0.0045992695028878065, "labelled_winner": false } ], "winner": 1, "pool_select": 0.005535656138065777, "pool_holdout": 0.004558775643112993, "lift_select": 0.4929508993680116, "lift_holdout": -0.02383979656706925, "eb_lift": 0.19929889792833277, "p_value": 0.10919695828262725, "held_up": false, "z_path": [ 1.322, 1.234, 0.494, 1.297, 0.992, 0.907, 0.907, 0.871, 0.278, 0.09 ], "first_cross": null, "final_z": 0.09 }, { "id": "53f27699f1592f712e000032", "variation": "headline", "week": 201433, "impressions": 18404, "pooled_ctr": 0.012442947185394479, "arms": [ { "headline": "What's The One Thing White Women Are Never Supposed To Do (At Least Not On Camera)?", "impressions": 3095, "clicks": 42, "ctr": 0.0135702746365105, "ctr_select": 0.013565891472868217, "ctr_holdout": 0.013574660633484163, "n_select": 1548, "n_holdout": 1547, "eb": 0.012924388058133559, "labelled_winner": false }, { "headline": "Think Racism Doesn't Exist? Well Eff You. (Although Apparently I Shouldn't If We're Different Races)", "impressions": 3146, "clicks": 44, "ctr": 0.0139860139860139, "ctr_select": 0.012714558169103624, "ctr_holdout": 0.015257469802924348, "n_select": 1573, "n_holdout": 1573, "eb": 0.012584318023939589, "labelled_winner": false }, { "headline": "Here's A Thing White Women Can Do That Could Cost Them Money At Work. Hint: It's Sexist And Racist.", "impressions": 3007, "clicks": 45, "ctr": 0.0149650814765547, "ctr_select": 0.016622340425531915, "ctr_holdout": 0.01330671989354624, "n_select": 1504, "n_holdout": 1503, "eb": 0.014119844482909892, "labelled_winner": false }, { "headline": "Too Many People Have Told Her Who She Can't Have Sex With. So She's Speaking Out.", "impressions": 2984, "clicks": 33, "ctr": 0.0110589812332439, "ctr_select": 0.010723860589812333, "ctr_holdout": 0.011394101876675604, "n_select": 1492, "n_holdout": 1492, "eb": 0.011801927800055114, "labelled_winner": false }, { "headline": "Turns Out There Aren't Just Outdated Stereotypes About Porn. They're Also IN Porn. Go Figure.", "impressions": 3067, "clicks": 41, "ctr": 0.0133681121617215, "ctr_select": 0.012385919165580182, "ctr_holdout": 0.014350945857795172, "n_select": 1534, "n_holdout": 1533, "eb": 0.012452113568627922, "labelled_winner": false }, { "headline": "A Lot Of People Aren't Cool With The 'Relations' In Interracial Relationships. Especially On Film.", "impressions": 3105, "clicks": 24, "ctr": 0.0077294685990338, "ctr_select": 0.00902061855670103, "ctr_holdout": 0.00643915003219575, "n_select": 1552, "n_holdout": 1553, "eb": 0.01110210164444606, "labelled_winner": false } ], "winner": 2, "pool_select": 0.012495925241768988, "pool_holdout": 0.012389957613302902, "lift_select": 0.3302208603145236, "lift_holdout": 0.07399236614490312, "eb_lift": 0.1299559024019108, "p_value": 0.11524792580880683, "held_up": false, "z_path": [ 0.635, 0.563, -0.302, 0.413, 0.837, 0.809, 0.625, 0.241, 0.242, 0.459 ], "first_cross": null, "final_z": 0.459 }, { "id": "53f2409127e3e436d300000f", "variation": "headline", "week": 201433, "impressions": 18398, "pooled_ctr": 0.007392107837808457, "arms": [ { "headline": "The Number Of Black Men Getting Killed By White Police Officers In America Per Week Is How Many?", "impressions": 3070, "clicks": 21, "ctr": 0.0068403908794788, "ctr_select": 0.006514657980456026, "ctr_holdout": 0.0071661237785016286, "n_select": 1535, "n_holdout": 1535, "eb": 0.007239205222555899, "labelled_winner": false }, { "headline": "If This Many Police Officers Were Killed Each Week, People Would Freak Out", "impressions": 2997, "clicks": 11, "ctr": 0.0036703370036703, "ctr_select": 0.004005340453938585, "ctr_holdout": 0.00333555703802535, "n_select": 1498, "n_holdout": 1499, "eb": 0.006261101165142563, "labelled_winner": false }, { "headline": "How Many Black Men Are Killed Every Week By The Police?", "impressions": 3132, "clicks": 14, "ctr": 0.0044699872286079, "ctr_select": 0.005747126436781609, "ctr_holdout": 0.0031928480204342275, "n_select": 1566, "n_holdout": 1566, "eb": 0.00692393471104358, "labelled_winner": false }, { "headline": "Some Folks In America Have A Less American Dream-Like Experience Than Others. Please Hear Her Out.", "impressions": 3030, "clicks": 11, "ctr": 0.0036303630363036, "ctr_select": 0.0026402640264026403, "ctr_holdout": 0.0046204620462046205, "n_select": 1515, "n_holdout": 1515, "eb": 0.005711504280091988, "labelled_winner": false }, { "headline": "Ferguson Has Opened Some Raw Wounds That Most People Are Afraid To Look At, Let Alone Heal", "impressions": 3120, "clicks": 22, "ctr": 0.007051282051282, "ctr_select": 0.0057692307692307696, "ctr_holdout": 0.008333333333333333, "n_select": 1560, "n_holdout": 1560, "eb": 0.00693464818368396, "labelled_winner": false }, { "headline": "If You Aren't Black, You May Want To Watch This. It's Painful. It Also Happens To Be True.", "impressions": 3049, "clicks": 57, "ctr": 0.018694653984913, "ctr_select": 0.021653543307086614, "ctr_holdout": 0.015737704918032787, "n_select": 1524, "n_holdout": 1525, "eb": 0.013246869810430063, "labelled_winner": false } ], "winner": 5, "pool_select": 0.007719069362904979, "pool_holdout": 0.007065217391304348, "lift_select": 1.805201286458911, "lift_holdout": 1.2274905422446407, "eb_lift": 0.716122655159658, "p_value": 1.0114889966721197e-11, "held_up": true, "z_path": [ 1.689, 1.625, 2.782, 3.935, 4.061, 4.28, 4.256, 4.304, 4.299, 4.133 ], "first_cross": 2, "final_z": 4.133 }, { "id": "54836027663239001e260000", "variation": "other", "week": 201448, "impressions": 18301, "pooled_ctr": 0.01021802087317633, "arms": [ { "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.", "impressions": 3065, "clicks": 32, "ctr": 0.0104404567699836, "ctr_select": 0.013054830287206266, "ctr_holdout": 0.007827788649706457, "n_select": 1532, "n_holdout": 1533, "eb": 0.012497932049296142, "labelled_winner": false }, { "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.", "impressions": 3054, "clicks": 30, "ctr": 0.0098231827111984, "ctr_select": 0.010478061558611657, "ctr_holdout": 0.0091683038637852, "n_select": 1527, "n_holdout": 1527, "eb": 0.011473791624677716, "labelled_winner": false }, { "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.", "impressions": 2943, "clicks": 28, "ctr": 0.0095141012572205, "ctr_select": 0.010190217391304348, "ctr_holdout": 0.008837525492861998, "n_select": 1472, "n_holdout": 1471, "eb": 0.011376437827779464, "labelled_winner": false }, { "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.", "impressions": 3018, "clicks": 27, "ctr": 0.0089463220675944, "ctr_select": 0.008614976805831677, "ctr_holdout": 0.00927766732935719, "n_select": 1509, "n_holdout": 1509, "eb": 0.01074379621705174, "labelled_winner": false }, { "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.", "impressions": 3183, "clicks": 36, "ctr": 0.0113100848256361, "ctr_select": 0.013190954773869347, "ctr_holdout": 0.009428032683846637, "n_select": 1592, "n_holdout": 1591, "eb": 0.012561907347628225, "labelled_winner": true }, { "headline": "They Let A Rape Survivor Tell Her Story. But Then They Took 2 Steps Backward.", "impressions": 3038, "clicks": 34, "ctr": 0.0111915734035549, "ctr_select": 0.017116524028966424, "ctr_holdout": 0.005266622778143515, "n_select": 1519, "n_holdout": 1519, "eb": 0.014104124038098777, "labelled_winner": false } ], "winner": 5, "pool_select": 0.01212982187739045, "pool_holdout": 0.008306010928961749, "lift_select": 0.4111109134150608, "lift_holdout": -0.36592633657877416, "eb_lift": 0.16276431596974694, "p_value": 0.051877098634353976, "held_up": false, "z_path": [ 0.444, 0.091, 0.108, 0.131, 0.64, 0.456, 0.742, 0.174, -0.058, -0.044 ], "first_cross": null, "final_z": -0.044 }, { "id": "553ffd1c316638000c290200", "variation": "image", "week": 201517, "impressions": 18290, "pooled_ctr": 0.004264625478403499, "arms": [ { "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.", "impressions": 3104, "clicks": 15, "ctr": 0.0048324742268041, "ctr_select": 0.003865979381443299, "ctr_holdout": 0.005798969072164949, "n_select": 1552, "n_holdout": 1552, "eb": 0.003973756164608848, "labelled_winner": false }, { "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.", "impressions": 2961, "clicks": 8, "ctr": 0.0027017899358324, "ctr_select": 0.0013513513513513514, "ctr_holdout": 0.004051316677920324, "n_select": 1480, "n_holdout": 1481, "eb": 0.0029958243108922484, "labelled_winner": false }, { "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.", "impressions": 3107, "clicks": 17, "ctr": 0.0054715159317669, "ctr_select": 0.007722007722007722, "ctr_holdout": 0.003219575016097875, "n_select": 1554, "n_holdout": 1553, "eb": 0.005521411210223256, "labelled_winner": false }, { "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.", "impressions": 3066, "clicks": 17, "ctr": 0.0055446836268754, "ctr_select": 0.006523157208088715, "ctr_holdout": 0.0045662100456621, "n_select": 1533, "n_holdout": 1533, "eb": 0.005032136146559357, "labelled_winner": false }, { "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.", "impressions": 3063, "clicks": 12, "ctr": 0.0039177277179236, "ctr_select": 0.0026109660574412533, "ctr_holdout": 0.005225342913128674, "n_select": 1532, "n_holdout": 1531, "eb": 0.003474879363348337, "labelled_winner": false }, { "headline": "More than 100 clergy members joined arm-in-arm in peaceful protest through Baltimore.", "impressions": 2989, "clicks": 9, "ctr": 0.0030110404817664, "ctr_select": 0.002008032128514056, "ctr_holdout": 0.004013377926421404, "n_select": 1494, "n_holdout": 1495, "eb": 0.003247171226069369, "labelled_winner": false } ], "winner": 2, "pool_select": 0.004045926735921268, "pool_holdout": 0.00448332422088573, "lift_select": 0.908588124804341, "lift_holdout": -0.2818777189703643, "eb_lift": 0.3646839329051803, "p_value": 0.012221883065665079, "held_up": false, "z_path": [ -0.016, -0.93, -0.561, -0.802, -0.973, -0.886, -0.968, -0.769, -0.392, -0.039 ], "first_cross": null, "final_z": -0.039 }, { "id": "51cdd24c558059c42800309f", "variation": "headline", "week": 201325, "impressions": 18288, "pooled_ctr": 0.003937007874015748, "arms": [ { "headline": "I Never Thought I'd See Something So Hardcore on Sesame Street's Lesson Plan", "impressions": 4225, "clicks": 24, "ctr": 0.005680473372781, "ctr_select": 0.007102272727272727, "ctr_holdout": 0.004259346900141978, "n_select": 2112, "n_holdout": 2113, "eb": 0.005503368907174159, "labelled_winner": false }, { "headline": "Sesame Street Takes The Phrase Educational Programming To A Whole New Level", "impressions": 4975, "clicks": 12, "ctr": 0.0024120603015075, "ctr_select": 0.0028135048231511255, "ctr_holdout": 0.002010454362685967, "n_select": 2488, "n_holdout": 2487, "eb": 0.0034080921486323913, "labelled_winner": false }, { "headline": "Can You Tell Me How To Get, How To Get To... Jail?", "impressions": 4113, "clicks": 9, "ctr": 0.0021881838074398, "ctr_select": 0.0029182879377431907, "ctr_holdout": 0.0014584346135148275, "n_select": 2056, "n_holdout": 2057, "eb": 0.0035160776199896757, "labelled_winner": false }, { "headline": "Whoa. Apparently No Topic Is Off Limits For Sesame Street.", "impressions": 4975, "clicks": 27, "ctr": 0.0054271356783919, "ctr_select": 0.003617363344051447, "ctr_holdout": 0.007237635705669481, "n_select": 2488, "n_holdout": 2487, "eb": 0.0038242648092788128, "labelled_winner": false } ], "winner": 0, "pool_select": 0.004046369203849519, "pool_holdout": 0.0038276465441819773, "lift_select": 0.75522113022113, "lift_holdout": 0.11278480156852133, "eb_lift": 0.36007581857298687, "p_value": 0.011646279577591007, "held_up": false, "z_path": [ 1.023, 1.133, 1.512, 1.266, 2.044, 1.394, 0.819, 0.333, 0.544, 0.163 ], "first_cross": 4, "final_z": 0.163 }, { "id": "5362dbb2f0b630c49100002f", "variation": "headline", "week": 201417, "impressions": 18200, "pooled_ctr": 0.0035164835164835165, "arms": [ { "headline": "The NFL Makes Over 9 Billion Dollars Annually. Guess How Much Money It Pays In Taxes?", "impressions": 3050, "clicks": 16, "ctr": 0.0052459016393442, "ctr_select": 0.003934426229508197, "ctr_holdout": 0.006557377049180328, "n_select": 1525, "n_holdout": 1525, "eb": 0.0033509493816641766, "labelled_winner": false }, { "headline": "UNFAIR: The NFL Gets Taxpayers To Pay For Stadiums, But Doesn't Pay Any Taxes Itself", "impressions": 3088, "clicks": 7, "ctr": 0.0022668393782383, "ctr_select": 0.0019430051813471502, "ctr_holdout": 0.0025906735751295338, "n_select": 1544, "n_holdout": 1544, "eb": 0.00255760192865792, "labelled_winner": false }, { "headline": "The Most Financially Successful Sports League In The World Pays No Taxes. None.", "impressions": 3011, "clicks": 11, "ctr": 0.0036532713384257, "ctr_select": 0.0033200531208499337, "ctr_holdout": 0.003986710963455149, "n_select": 1506, "n_holdout": 1505, "eb": 0.003106073339132471, "labelled_winner": false }, { "headline": "The NFL Has Avoided Paying Taxes For Over The Past 40 Years", "impressions": 2986, "clicks": 12, "ctr": 0.0040187541862022, "ctr_select": 0.003348961821835231, "ctr_holdout": 0.004688546550569324, "n_select": 1493, "n_holdout": 1493, "eb": 0.003116669549206972, "labelled_winner": false }, { "headline": "NFL Ticket Prices Are High, But The NFL Itself? Get's A Free Ride From The Government.", "impressions": 3068, "clicks": 7, "ctr": 0.0022816166883963, "ctr_select": 0.001303780964797914, "ctr_holdout": 0.003259452411994785, "n_select": 1534, "n_holdout": 1534, "eb": 0.0023046163478993255, "labelled_winner": false }, { "headline": "What Do The NFL And Organized Crime Have In Common?", "impressions": 2997, "clicks": 11, "ctr": 0.0036703370036703, "ctr_select": 0.004005340453938585, "ctr_holdout": 0.00333555703802535, "n_select": 1498, "n_holdout": 1499, "eb": 0.003374660807544442, "labelled_winner": false } ], "winner": 5, "pool_select": 0.0029670329670329672, "pool_holdout": 0.004065934065934066, "lift_select": 0.34994807892004154, "lift_holdout": -0.1796332690261977, "eb_lift": 0.13738567957979342, "p_value": 0.4188642021490535, "held_up": false, "z_path": [ 0.402, 0.401, 0.359, 0.934, 0.567, -0.194, -0.411, -0.586, -0.959, -0.919 ], "first_cross": null, "final_z": -0.919 }, { "id": "5358433c1a09bfcf21000036", "variation": "image", "week": 201416, "impressions": 18140, "pooled_ctr": 0.0185226019845645, "arms": [ { "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him", "impressions": 3030, "clicks": 53, "ctr": 0.0174917491749174, "ctr_select": 0.01782178217821782, "ctr_holdout": 0.017161716171617162, "n_select": 1515, "n_holdout": 1515, "eb": 0.018245579874176965, "labelled_winner": false }, { "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him", "impressions": 2994, "clicks": 45, "ctr": 0.0150300601202404, "ctr_select": 0.013360053440213761, "ctr_holdout": 0.016700066800267203, "n_select": 1497, "n_holdout": 1497, "eb": 0.01649666552267801, "labelled_winner": false }, { "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him", "impressions": 3130, "clicks": 70, "ctr": 0.0223642172523961, "ctr_select": 0.021086261980830672, "ctr_holdout": 0.023642172523961662, "n_select": 1565, "n_holdout": 1565, "eb": 0.01955593710343505, "labelled_winner": false }, { "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him", "impressions": 3017, "clicks": 47, "ctr": 0.0155783891282731, "ctr_select": 0.015915119363395226, "ctr_holdout": 0.015241882041086813, "n_select": 1508, "n_holdout": 1509, "eb": 0.01749479370061142, "labelled_winner": false }, { "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him", "impressions": 2978, "clicks": 73, "ctr": 0.0245130960376091, "ctr_select": 0.0241773002014775, "ctr_holdout": 0.024848891873740765, "n_select": 1489, "n_holdout": 1489, "eb": 0.020734452333157554, "labelled_winner": true }, { "headline": "This Little Boy Hated That His Dad Was Poor, Which Exactly Why Everyone Else Loved Him", "impressions": 2991, "clicks": 48, "ctr": 0.0160481444332999, "ctr_select": 0.01871657754010695, "ctr_holdout": 0.013377926421404682, "n_select": 1496, "n_holdout": 1495, "eb": 0.01859869281649879, "labelled_winner": false } ], "winner": 4, "pool_select": 0.0185226019845645, "pool_holdout": 0.0185226019845645, "lift_select": 0.30528638587738643, "lift_holdout": 0.3415443410406471, "eb_lift": 0.11941358727225593, "p_value": 0.07670489980645687, "held_up": true, "z_path": [ 0.41, 0.92, 1.265, 1.538, 1.58, 1.426, 1.139, 1.116, 0.642, 0.555 ], "first_cross": null, "final_z": 0.555 }, { "id": "53396e3a09ccd48b4900002c", "variation": "image", "week": 201413, "impressions": 18131, "pooled_ctr": 0.007059731950802493, "arms": [ { "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening", "impressions": 3573, "clicks": 23, "ctr": 0.0064371676462356, "ctr_select": 0.007278835386338186, "ctr_holdout": 0.005595970900951315, "n_select": 1786, "n_holdout": 1787, "eb": 0.007342681014997483, "labelled_winner": false }, { "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening", "impressions": 3581, "clicks": 40, "ctr": 0.0111700642278693, "ctr_select": 0.012849162011173185, "ctr_holdout": 0.009491903964265773, "n_select": 1790, "n_holdout": 1791, "eb": 0.009769984578424628, "labelled_winner": true }, { "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening", "impressions": 3625, "clicks": 23, "ctr": 0.0063448275862068, "ctr_select": 0.00772626931567329, "ctr_holdout": 0.004964147821290678, "n_select": 1812, "n_holdout": 1813, "eb": 0.007538600995460494, "labelled_winner": false }, { "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening", "impressions": 3648, "clicks": 20, "ctr": 0.0054824561403508, "ctr_select": 0.00712719298245614, "ctr_holdout": 0.003837719298245614, "n_select": 1824, "n_holdout": 1824, "eb": 0.007275312052202901, "labelled_winner": false }, { "headline": "A Doctor Gets The Green Light To Keep Being A Super Human, And That's Exactly What's Happening", "impressions": 3704, "clicks": 22, "ctr": 0.0059395248380129, "ctr_select": 0.0021598272138228943, "ctr_holdout": 0.009719222462203024, "n_select": 1852, "n_holdout": 1852, "eb": 0.005068027642296455, "labelled_winner": false } ], "winner": 1, "pool_select": 0.0073918799646954985, "pool_holdout": 0.006727693834785485, "lift_select": 0.7382806637205037, "lift_holdout": 0.41087038104914364, "eb_lift": 0.32171851072896773, "p_value": 0.0026218286752617594, "held_up": false, "z_path": [ 1.023, 1.181, 0.72, 1.497, 1.87, 1.713, 2.079, 1.523, 1.994, 2.2 ], "first_cross": 6, "final_z": 2.2 }, { "id": "52474008bf43c26f6f0000ee", "variation": "image", "week": 201338, "impressions": 17532, "pooled_ctr": 0.03165639972621492, "arms": [ { "headline": "A Man Who Clearly Hasn't Seen Senator Warren In Action Tries To One Up Her. Good Luck With That.", "impressions": 4246, "clicks": 139, "ctr": 0.032736693358455, "ctr_select": 0.037211493170042395, "ctr_holdout": 0.02826189354686764, "n_select": 2123, "n_holdout": 2123, "eb": 0.03499687040791555, "labelled_winner": false }, { "headline": "A Man Who Clearly Hasn't Seen Senator Warren In Action Tries To One Up Her. Good Luck With That.", "impressions": 4229, "clicks": 148, "ctr": 0.0349964530621896, "ctr_select": 0.03500473036896878, "ctr_holdout": 0.034988179669030735, "n_select": 2114, "n_holdout": 2115, "eb": 0.03393970710875151, "labelled_winner": false }, { "headline": "A Man Who Clearly Hasn't Seen Senator Warren In Action Tries To One Up Her. Good Luck With That.", "impressions": 4250, "clicks": 121, "ctr": 0.0284705882352941, "ctr_select": 0.027294117647058823, "ctr_holdout": 0.029647058823529412, "n_select": 2125, "n_holdout": 2125, "eb": 0.030254257891462362, "labelled_winner": false }, { "headline": "A Man Who Clearly Hasn't Seen Senator Warren In Action Tries To One Up Her. Good Luck With That.", "impressions": 4807, "clicks": 147, "ctr": 0.0305804035781152, "ctr_select": 0.0324459234608985, "ctr_holdout": 0.02871410736579276, "n_select": 2404, "n_holdout": 2403, "eb": 0.03270233118445129, "labelled_winner": false } ], "winner": 0, "pool_select": 0.03296828656171572, "pool_holdout": 0.03034451289071412, "lift_select": 0.1287057063273067, "lift_holdout": -0.0686324855945799, "eb_lift": 0.06153137022763899, "p_value": 0.2084560854860471, "held_up": false, "z_path": [ 0.574, 0.264, 0.789, 0.078, 0.765, 0.523, -0.032, -0.233, -0.286, -0.575 ], "first_cross": null, "final_z": -0.575 }, { "id": "54755e743762650015310000", "variation": "headline", "week": 201447, "impressions": 17127, "pooled_ctr": 0.002218718981724762, "arms": [ { "headline": "If You Take Playing In The Dirt Seriously You'll Love This Message", "impressions": 4339, "clicks": 13, "ctr": 0.0029960820465545, "ctr_select": 0.002304147465437788, "ctr_holdout": 0.0036883356385431073, "n_select": 2170, "n_holdout": 2169, "eb": 0.0021997818449777467, "labelled_winner": false }, { "headline": "This Is A Reminder Of Who We All Need To Thank For That Groaning T-Day Table", "impressions": 4214, "clicks": 8, "ctr": 0.0018984337921214, "ctr_select": 0.0023730422401518746, "ctr_holdout": 0.0014238253440911248, "n_select": 2107, "n_holdout": 2107, "eb": 0.002231102912649432, "labelled_winner": false }, { "headline": "The Earth's Skin Is A Lot Like Ours. It's Not Very Thick, It's Alive, And It Needs Proper Treatment.", "impressions": 4305, "clicks": 3, "ctr": 0.0006968641114982, "ctr_select": 0.00046468401486988845, "ctr_holdout": 0.0009289363678588017, "n_select": 2152, "n_holdout": 2153, "eb": 0.0013137255189125028, "labelled_winner": false }, { "headline": "Think About The Best Food You've Ever Tasted. Now, Here's The Backstory.", "impressions": 4269, "clicks": 14, "ctr": 0.0032794565472007, "ctr_select": 0.0032802249297094657, "ctr_holdout": 0.003278688524590164, "n_select": 2134, "n_holdout": 2135, "eb": 0.0026668378502080177, "labelled_winner": false } ], "winner": 3, "pool_select": 0.002102067032582039, "pool_holdout": 0.002335357309668379, "lift_select": 0.5604758929501197, "lift_holdout": 0.40393442622950815, "eb_lift": 0.26867402840729193, "p_value": 0.1702390050023943, "held_up": false, "z_path": [ 1.408, 0.568, 1.33, 0.984, 0.885, 1.134, 1.134, 1.187, 1.12, 1.25 ], "first_cross": null, "final_z": 1.25 }, { "id": "54b06a2f3663330012c00000", "variation": "image", "week": 201501, "impressions": 17083, "pooled_ctr": 0.016273488263185622, "arms": [ { "headline": "What Happens If You Try To Kill Yourself And Survive?", "impressions": 2869, "clicks": 45, "ctr": 0.0156849076333217, "ctr_select": 0.01394700139470014, "ctr_holdout": 0.017421602787456445, "n_select": 1434, "n_holdout": 1435, "eb": 0.015530685598719212, "labelled_winner": false }, { "headline": "What Happens If You Try To Kill Yourself And Survive?", "impressions": 2734, "clicks": 64, "ctr": 0.0234089246525237, "ctr_select": 0.02121433796634967, "ctr_holdout": 0.025603511338697878, "n_select": 1367, "n_holdout": 1367, "eb": 0.018255619023376114, "labelled_winner": false }, { "headline": "What Happens If You Try To Kill Yourself And Survive?", "impressions": 2860, "clicks": 45, "ctr": 0.0157342657342657, "ctr_select": 0.016083916083916083, "ctr_holdout": 0.015384615384615385, "n_select": 1430, "n_holdout": 1430, "eb": 0.016347753298338756, "labelled_winner": false }, { "headline": "What Happens If You Try To Kill Yourself And Survive?", "impressions": 2890, "clicks": 32, "ctr": 0.0110726643598615, "ctr_select": 0.011072664359861591, "ctr_holdout": 0.011072664359861591, "n_select": 1445, "n_holdout": 1445, "eb": 0.014422215594219759, "labelled_winner": false }, { "headline": "What Happens If You Try To Kill Yourself And Survive?", "impressions": 2821, "clicks": 30, "ctr": 0.010634526763559, "ctr_select": 0.01347517730496454, "ctr_holdout": 0.007795889440113395, "n_select": 1410, "n_holdout": 1411, "eb": 0.015362414565139707, "labelled_winner": false }, { "headline": "What Happens If You Try To Kill Yourself And Survive?", "impressions": 2909, "clicks": 62, "ctr": 0.0213131660364386, "ctr_select": 0.023383768913342505, "ctr_holdout": 0.019243986254295534, "n_select": 1454, "n_holdout": 1455, "eb": 0.01916018810927458, "labelled_winner": false } ], "winner": 5, "pool_select": 0.01651053864168618, "pool_holdout": 0.016036521128409222, "lift_select": 0.41629352141804965, "lift_holdout": 0.20001003336092515, "eb_lift": 0.1604823152709569, "p_value": 0.02395147998986142, "held_up": false, "z_path": [ 0.1, 0.012, -0.054, -0.108, 0.084, 0.336, -0.517, -0.522, -0.622, -0.533 ], "first_cross": null, "final_z": -0.533 }, { "id": "54babe753032350013290000", "variation": "other", "week": 201502, "impressions": 17082, "pooled_ctr": 0.006029738906451235, "arms": [ { "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These", "impressions": 2839, "clicks": 16, "ctr": 0.0056357872490313, "ctr_select": 0.004225352112676056, "ctr_holdout": 0.007047216349541931, "n_select": 1420, "n_holdout": 1419, "eb": 0.00530792827337932, "labelled_winner": false }, { "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These", "impressions": 2794, "clicks": 11, "ctr": 0.0039370078740157, "ctr_select": 0.005010737294201861, "ctr_holdout": 0.002863278453829635, "n_select": 1397, "n_holdout": 1397, "eb": 0.0056099938624279, "labelled_winner": false }, { "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These", "impressions": 2801, "clicks": 21, "ctr": 0.0074973223848625, "ctr_select": 0.007142857142857143, "ctr_holdout": 0.007851534618129907, "n_select": 1400, "n_holdout": 1401, "eb": 0.006412417904246618, "labelled_winner": false }, { "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These", "impressions": 2804, "clicks": 17, "ctr": 0.0060627674750356, "ctr_select": 0.007132667617689016, "ctr_holdout": 0.004992867332382311, "n_select": 1402, "n_holdout": 1402, "eb": 0.006408970087561806, "labelled_winner": false }, { "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These", "impressions": 2930, "clicks": 17, "ctr": 0.0058020477815699, "ctr_select": 0.005460750853242321, "ctr_holdout": 0.0061433447098976105, "n_select": 1465, "n_holdout": 1465, "eb": 0.005773506906728837, "labelled_winner": false }, { "headline": "Every Child Is Full Of 'Why' Questions. But No One Wants Their Child To Ask Questions Like These", "impressions": 2914, "clicks": 21, "ctr": 0.0072065888812628, "ctr_select": 0.0068634179821551134, "ctr_holdout": 0.0075497597803706245, "n_select": 1457, "n_holdout": 1457, "eb": 0.006315586886469134, "labelled_winner": false } ], "winner": 2, "pool_select": 0.00597119775201967, "pool_holdout": 0.0060882800608828, "lift_select": 0.19621848739495795, "lift_holdout": 0.28961456102783734, "eb_lift": 0.07389139843471293, "p_value": 0.5337349863618881, "held_up": true, "z_path": [ 0.0, 0.382, 1.301, 0.952, 0.902, 1.288, 0.791, 0.544, 0.338, 0.654 ], "first_cross": null, "final_z": 0.654 }, { "id": "54b965fd3536310024ae0000", "variation": "other", "week": 201502, "impressions": 17077, "pooled_ctr": 0.020846752942554312, "arms": [ { "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore", "impressions": 2817, "clicks": 48, "ctr": 0.0170394036208732, "ctr_select": 0.020596590909090908, "ctr_holdout": 0.0134847409510291, "n_select": 1408, "n_holdout": 1409, "eb": 0.021262996745371805, "labelled_winner": false }, { "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore", "impressions": 2835, "clicks": 55, "ctr": 0.019400352733686, "ctr_select": 0.021156558533145273, "ctr_holdout": 0.017642907551164433, "n_select": 1418, "n_holdout": 1417, "eb": 0.02147376594962769, "labelled_winner": false }, { "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore", "impressions": 2865, "clicks": 54, "ctr": 0.018848167539267, "ctr_select": 0.017458100558659217, "ctr_holdout": 0.020237264480111653, "n_select": 1432, "n_holdout": 1433, "eb": 0.020060149935530378, "labelled_winner": false }, { "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore", "impressions": 2841, "clicks": 84, "ctr": 0.0295670538542766, "ctr_select": 0.03169014084507042, "ctr_holdout": 0.02744546094299789, "n_select": 1420, "n_holdout": 1421, "eb": 0.02547544121875294, "labelled_winner": true }, { "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore", "impressions": 2908, "clicks": 52, "ctr": 0.0178817056396148, "ctr_select": 0.02063273727647868, "ctr_holdout": 0.015130674002751032, "n_select": 1454, "n_holdout": 1454, "eb": 0.021268803719161348, "labelled_winner": false }, { "headline": "When Billie Holiday Performed This Song Live, She Refused To Come Back For An Encore", "impressions": 2811, "clicks": 63, "ctr": 0.0224119530416221, "ctr_select": 0.01849217638691323, "ctr_holdout": 0.026334519572953737, "n_select": 1406, "n_holdout": 1405, "eb": 0.020468766321449727, "labelled_winner": false } ], "winner": 3, "pool_select": 0.02166783790114781, "pool_holdout": 0.020025764141000116, "lift_select": 0.46254282451465545, "lift_holdout": 0.37050754966233335, "eb_lift": 0.17572603851736535, "p_value": 0.0044982836010920854, "held_up": true, "z_path": [ 1.301, 0.183, -0.155, 0.386, 0.474, 1.439, 1.852, 2.201, 2.595, 2.478 ], "first_cross": 7, "final_z": 2.478 }, { "id": "549ae7786335660018720000", "variation": "headline", "week": 201451, "impressions": 17068, "pooled_ctr": 0.021092102179517225, "arms": [ { "headline": "She Tries Playing The Video Game Using A Female Avatar. What Happens Next Is Way Too Realistic.", "impressions": 2850, "clicks": 89, "ctr": 0.0312280701754385, "ctr_select": 0.03368421052631579, "ctr_holdout": 0.028771929824561403, "n_select": 1425, "n_holdout": 1425, "eb": 0.025378493382975018, "labelled_winner": true }, { "headline": "She Tries To Play The Game Using A Female Avatar. How Real It Is Will Make You Laugh With Anger.", "impressions": 2867, "clicks": 60, "ctr": 0.0209277990931287, "ctr_select": 0.02092050209205021, "ctr_holdout": 0.0209351011863224, "n_select": 1434, "n_holdout": 1433, "eb": 0.02051978463948251, "labelled_winner": false }, { "headline": "She Tries Playing A Video Game Using A Female Avatar. Brace Yourself. You'll Be Smacked By Reality.", "impressions": 2849, "clicks": 68, "ctr": 0.0238680238680238, "ctr_select": 0.025280898876404494, "ctr_holdout": 0.02245614035087719, "n_select": 1424, "n_holdout": 1425, "eb": 0.02217817565239346, "labelled_winner": false }, { "headline": "Amy Schumer Tries Playing A Video Game Using A Female Avatar. Get Ready To Be Smacked By Reality.", "impressions": 2850, "clicks": 44, "ctr": 0.015438596491228, "ctr_select": 0.01263157894736842, "ctr_holdout": 0.018245614035087718, "n_select": 1425, "n_holdout": 1425, "eb": 0.01736288475851829, "labelled_winner": false }, { "headline": "She Tries To Play A Video Game Using A Female Character. It's Funny, But Not Ha-Ha Funny.", "impressions": 2794, "clicks": 39, "ctr": 0.0139584824624194, "ctr_select": 0.008589835361488905, "ctr_holdout": 0.019327129563350035, "n_select": 1397, "n_holdout": 1397, "eb": 0.015878554224735456, "labelled_winner": false }, { "headline": "Amy Schumer Tries Playing A Game Using A Female Avatar. Her Boyfriend Doesn't Make It Any Easier.", "impressions": 2858, "clicks": 60, "ctr": 0.0209937018894331, "ctr_select": 0.020293911826452064, "ctr_holdout": 0.021693491952414275, "n_select": 1429, "n_holdout": 1429, "eb": 0.020280266763329165, "labelled_winner": false } ], "winner": 0, "pool_select": 0.020271853761424888, "pool_holdout": 0.021912350597609563, "lift_select": 0.6616245816854274, "lift_holdout": 0.31304625199362035, "eb_lift": 0.2519078758977389, "p_value": 8.275727817878576e-05, "held_up": true, "z_path": [ 1.522, 1.85, 2.391, 2.32, 1.885, 1.944, 2.117, 1.821, 1.442, 1.697 ], "first_cross": 2, "final_z": 1.697 }, { "id": "5342f20eaf7ae2a1fd000005", "variation": "headline", "week": 201414, "impressions": 16972, "pooled_ctr": 0.022507659674758426, "arms": [ { "headline": "The Scientific Truth About Why Some Dogs Are Mean", "impressions": 2859, "clicks": 85, "ctr": 0.0297306750612102, "ctr_select": 0.02727272727272727, "ctr_holdout": 0.03219034289713086, "n_select": 1430, "n_holdout": 1429, "eb": 0.023961471128794282, "labelled_winner": false }, { "headline": "Here's A New Study On Dog Behavior That May Give You Paws", "impressions": 2866, "clicks": 75, "ctr": 0.026168876482903, "ctr_select": 0.02372644801116539, "ctr_holdout": 0.028611304954640614, "n_select": 1433, "n_holdout": 1433, "eb": 0.02260922021101975, "labelled_winner": false }, { "headline": "An Internet TV Host Just Dropped Some Science On The Notion Of \u2018Aggressive Dog Breeds\u2019", "impressions": 2820, "clicks": 42, "ctr": 0.0148936170212765, "ctr_select": 0.01347517730496454, "ctr_holdout": 0.016312056737588652, "n_select": 1410, "n_holdout": 1410, "eb": 0.01872478829553866, "labelled_winner": false }, { "headline": "Here's The Truth About Those 'Aggressive' Dog Breeds", "impressions": 2817, "clicks": 52, "ctr": 0.0184593539226127, "ctr_select": 0.022017045454545456, "ctr_holdout": 0.014904187366926898, "n_select": 1408, "n_holdout": 1409, "eb": 0.02195571322045096, "labelled_winner": false }, { "headline": "If You're Afraid Of Pit Bulls And Not Poodles, It Might Be Time To Examine Your Prejudices", "impressions": 2778, "clicks": 73, "ctr": 0.0262778977681785, "ctr_select": 0.020158387329013677, "ctr_holdout": 0.032397408207343416, "n_select": 1389, "n_holdout": 1389, "eb": 0.021258909311790614, "labelled_winner": false }, { "headline": "The Truth About Why Some Dogs Are Mean", "impressions": 2832, "clicks": 55, "ctr": 0.0194209039548022, "ctr_select": 0.024717514124293787, "ctr_holdout": 0.014124293785310734, "n_select": 1416, "n_holdout": 1416, "eb": 0.022979993865745226, "labelled_winner": false } ], "winner": 0, "pool_select": 0.0219184539241103, "pool_holdout": 0.02309686542540655, "lift_select": 0.2442815249266861, "lift_holdout": 0.393710458291084, "eb_lift": 0.09320991397284017, "p_value": 0.12938778246600205, "held_up": false, "z_path": [ 2.541, 2.571, 3.145, 2.892, 3.588, 3.5, 3.345, 3.12, 2.708, 2.51 ], "first_cross": 0, "final_z": 2.51 }, { "id": "52881022a060cda2b7007f35", "variation": "headline", "week": 201346, "impressions": 16919, "pooled_ctr": 0.006915302322832319, "arms": [ { "headline": "Do You Think Video Games Are Racist?", "impressions": 4608, "clicks": 31, "ctr": 0.0067274305555555, "ctr_select": 0.007378472222222222, "ctr_holdout": 0.006076388888888889, "n_select": 2304, "n_holdout": 2304, "eb": 0.006880007164228276, "labelled_winner": false }, { "headline": "Yet Another Reason To Question The Thing We Spend 3 Billion Hours A Week Doing", "impressions": 4373, "clicks": 44, "ctr": 0.0100617425108621, "ctr_select": 0.00777676120768527, "ctr_holdout": 0.012345679012345678, "n_select": 2186, "n_holdout": 2187, "eb": 0.007060268118364603, "labelled_winner": false }, { "headline": "Standing In Line To Get A Video Game? Chew On This While You Wait.", "impressions": 3781, "clicks": 21, "ctr": 0.0055540862205765, "ctr_select": 0.006349206349206349, "ctr_holdout": 0.004759386567953464, "n_select": 1890, "n_holdout": 1891, "eb": 0.0063686403348955185, "labelled_winner": false }, { "headline": "Look Around You: Are The Video Games You\u2019re Playing Totally Racist?", "impressions": 4157, "clicks": 21, "ctr": 0.0050517199903776, "ctr_select": 0.0038498556304138597, "ctr_holdout": 0.006253006253006253, "n_select": 2078, "n_holdout": 2079, "eb": 0.005186278686713683, "labelled_winner": false } ], "winner": 1, "pool_select": 0.006384488058642705, "pool_holdout": 0.0074459283772603715, "lift_select": 0.21807122767781517, "lift_holdout": 0.6580442876739172, "eb_lift": 0.10584718046532982, "p_value": 0.3425740359456335, "held_up": true, "z_path": [ 1.702, 1.396, 1.134, 1.243, 1.496, 1.386, 1.561, 1.641, 2.011, 1.736 ], "first_cross": 8, "final_z": 1.736 }, { "id": "53756c883a55adcf1c00000e", "variation": "image", "week": 201419, "impressions": 16881, "pooled_ctr": 0.009478111486286358, "arms": [ { "headline": "A Sweet Little Girl Reads A Sweet Story About How Her Two Dads Met And Created A Family", "impressions": 5589, "clicks": 51, "ctr": 0.0091250670960815, "ctr_select": 0.00894774516821761, "ctr_holdout": 0.009302325581395349, "n_select": 2794, "n_holdout": 2795, "eb": 0.009188473445826791, "labelled_winner": false }, { "headline": "A Sweet Little Girl Reads A Sweet Story About How Her Two Dads Met And Created A Family", "impressions": 5721, "clicks": 57, "ctr": 0.0099632931305715, "ctr_select": 0.01048951048951049, "ctr_holdout": 0.009437259699405802, "n_select": 2860, "n_holdout": 2861, "eb": 0.010037028344781802, "labelled_winner": false }, { "headline": "A Sweet Little Girl Reads A Sweet Story About How Her Two Dads Met And Created A Family", "impressions": 5571, "clicks": 52, "ctr": 0.0093340513372823, "ctr_select": 0.008973438621679828, "ctr_holdout": 0.00969479353680431, "n_select": 2786, "n_holdout": 2785, "eb": 0.009202876294863063, "labelled_winner": false } ], "winner": 1, "pool_select": 0.009478672985781991, "pool_holdout": 0.00947755005331122, "lift_select": 0.10664335664335667, "lift_holdout": -0.00425113596644533, "eb_lift": 0.058906490374480036, "p_value": 0.4926247661873232, "held_up": false, "z_path": [ -0.747, 0.675, 0.488, -0.082, 0.198, 0.916, 0.33, 1.032, 0.263, 0.342 ], "first_cross": null, "final_z": 0.342 }, { "id": "537b91dde51c404662000015", "variation": "headline", "week": 201420, "impressions": 16428, "pooled_ctr": 0.008400292184075968, "arms": [ { "headline": "Guess What, Ronald? The Folks Who Work For You Have Had Enough.", "impressions": 4126, "clicks": 42, "ctr": 0.0101793504604944, "ctr_select": 0.0116335433834222, "ctr_holdout": 0.00872515753756665, "n_select": 2063, "n_holdout": 2063, "eb": 0.010051747351084175, "labelled_winner": false }, { "headline": "Ronald McDonald Now Has A Bigger Problem On His Rather Large, Clownish Hands", "impressions": 4098, "clicks": 37, "ctr": 0.0090287945339189, "ctr_select": 0.010248901903367497, "ctr_holdout": 0.007808687164470474, "n_select": 2049, "n_holdout": 2049, "eb": 0.009396956080772421, "labelled_winner": false }, { "headline": "Would You Like A Large Order Of Fries With Your Injustice?", "impressions": 4118, "clicks": 31, "ctr": 0.0075279261777561, "ctr_select": 0.006313744536182613, "ctr_holdout": 0.008742107819329771, "n_select": 2059, "n_holdout": 2059, "eb": 0.007547623522977165, "labelled_winner": false }, { "headline": "That One Time When Some People Decided To Take On McDonald\u2019s Around The World", "impressions": 4086, "clicks": 28, "ctr": 0.0068526676456191, "ctr_select": 0.006363191385217817, "ctr_holdout": 0.007342143906020558, "n_select": 2043, "n_holdout": 2043, "eb": 0.0075753167923912085, "labelled_winner": false } ], "winner": 0, "pool_select": 0.008643778914049185, "pool_holdout": 0.008156805454102752, "lift_select": 0.3458862725553513, "lift_holdout": 0.06967826885929052, "eb_lift": 0.16288806678599177, "p_value": 0.09003494119720921, "held_up": false, "z_path": [ 1.338, 0.567, -0.017, 0.335, 0.46, 0.708, 0.381, 0.353, 0.093, 0.535 ], "first_cross": null, "final_z": 0.535 }, { "id": "53fb519f019157fe6900001a", "variation": "image", "week": 201434, "impressions": 16386, "pooled_ctr": 0.02038325399731478, "arms": [ { "headline": "They Were At A Beauty Pageant, So She Decided To Tell A Room Full Of Strangers Her Deepest Secret", "impressions": 4181, "clicks": 100, "ctr": 0.0239177230327672, "ctr_select": 0.022488038277511963, "ctr_holdout": 0.025346724055475847, "n_select": 2090, "n_holdout": 2091, "eb": 0.021251627439111943, "labelled_winner": true }, { "headline": "They Were At A Beauty Pageant, So She Decided To Tell A Room Full Of Strangers Her Deepest Secret", "impressions": 4107, "clicks": 98, "ctr": 0.0238616995373752, "ctr_select": 0.025803310613437196, "ctr_holdout": 0.021919142717973697, "n_select": 2054, "n_holdout": 2053, "eb": 0.022799094589147684, "labelled_winner": false }, { "headline": "They Were At A Beauty Pageant, So She Decided To Tell A Room Full Of Strangers Her Deepest Secret", "impressions": 4123, "clicks": 85, "ctr": 0.0206160562697065, "ctr_select": 0.02133850630455868, "ctr_holdout": 0.019893255701115962, "n_select": 2062, "n_holdout": 2061, "eb": 0.02070251318650949, "labelled_winner": false }, { "headline": "They Were At A Beauty Pageant, So She Decided To Tell A Room Full Of Strangers Her Deepest Secret", "impressions": 3975, "clicks": 51, "ctr": 0.0128301886792452, "ctr_select": 0.01056338028169014, "ctr_holdout": 0.01509813789632612, "n_select": 1988, "n_holdout": 1987, "eb": 0.015716558412886487, "labelled_winner": false } ], "winner": 1, "pool_select": 0.02013668537954601, "pool_holdout": 0.0206298828125, "lift_select": 0.2814080434333599, "lift_holdout": 0.06249477600970721, "eb_lift": 0.13221685493015833, "p_value": 0.03467900442739771, "held_up": false, "z_path": [ -0.386, -1.816, -1.024, -1.229, -0.625, -0.475, -0.587, -0.376, -0.186, -0.017 ], "first_cross": null, "final_z": -0.017 }, { "id": "53c87edad52a7babcf000099", "variation": "image", "week": 201428, "impressions": 16332, "pooled_ctr": 0.012674504041146216, "arms": [ { "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them", "impressions": 3291, "clicks": 46, "ctr": 0.0139775144333029, "ctr_select": 0.015188335358444714, "ctr_holdout": 0.01276595744680851, "n_select": 1646, "n_holdout": 1645, "eb": 0.012965745738948178, "labelled_winner": false }, { "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them", "impressions": 3219, "clicks": 34, "ctr": 0.0105622864243553, "ctr_select": 0.008695652173913044, "ctr_holdout": 0.012430080795525171, "n_select": 1610, "n_holdout": 1609, "eb": 0.010283962682943118, "labelled_winner": false }, { "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them", "impressions": 3342, "clicks": 52, "ctr": 0.0155595451825254, "ctr_select": 0.013165769000598444, "ctr_holdout": 0.017953321364452424, "n_select": 1671, "n_holdout": 1671, "eb": 0.012132355021067593, "labelled_winner": false }, { "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them", "impressions": 3192, "clicks": 38, "ctr": 0.0119047619047619, "ctr_select": 0.009398496240601503, "ctr_holdout": 0.014411027568922305, "n_select": 1596, "n_holdout": 1596, "eb": 0.01057626310160004, "labelled_winner": false }, { "headline": "Half Of All Teens Are Sipping Something Everyday And Have No Idea How It Hurts Them", "impressions": 3288, "clicks": 37, "ctr": 0.0112530413625304, "ctr_select": 0.010340632603406326, "ctr_holdout": 0.012165450121654502, "n_select": 1644, "n_holdout": 1644, "eb": 0.010952954985328965, "labelled_winner": false } ], "winner": 0, "pool_select": 0.011387290314681034, "pool_holdout": 0.013962033067973056, "lift_select": 0.3337971491657845, "lift_holdout": -0.0856662933930572, "eb_lift": 0.13861554247300822, "p_value": 0.10383159141852126, "held_up": false, "z_path": [ 0.358, 0.453, 0.588, 0.876, 1.218, 0.737, 0.573, -0.287, -1.014, -0.534 ], "first_cross": null, "final_z": -0.534 }, { "id": "52b0d9903e602968240015ad", "variation": "other", "week": 201350, "impressions": 16114, "pooled_ctr": 0.008812212982499689, "arms": [ { "headline": "The Shocking Reality Of What Is 'Hidden In The Harvest'", "impressions": 4190, "clicks": 35, "ctr": 0.0083532219570405, "ctr_select": 0.009069212410501194, "ctr_holdout": 0.0076372315035799524, "n_select": 2095, "n_holdout": 2095, "eb": 0.00847789945920465, "labelled_winner": false }, { "headline": "The Shocking Reality Of What Is 'Hidden In The Harvest'", "impressions": 4162, "clicks": 21, "ctr": 0.0050456511292647, "ctr_select": 0.0028832292167227293, "ctr_holdout": 0.007208073041806823, "n_select": 2081, "n_holdout": 2081, "eb": 0.005549462391569773, "labelled_winner": false }, { "headline": "The Shocking Reality Of What Is 'Hidden In The Harvest'", "impressions": 3754, "clicks": 37, "ctr": 0.0098561534363345, "ctr_select": 0.008524240809802876, "ctr_holdout": 0.011188066062866276, "n_select": 1877, "n_holdout": 1877, "eb": 0.008203310442546574, "labelled_winner": false }, { "headline": "The Shocking Reality Of What Is 'Hidden In The Harvest'", "impressions": 4008, "clicks": 49, "ctr": 0.0122255489021956, "ctr_select": 0.011477045908183632, "ctr_holdout": 0.012974051896207584, "n_select": 2004, "n_holdout": 2004, "eb": 0.009581976878981407, "labelled_winner": false } ], "winner": 3, "pool_select": 0.007943403251830706, "pool_holdout": 0.009681022713168673, "lift_select": 0.44485248253493026, "lift_holdout": 0.34015302727877583, "eb_lift": 0.20628105803051877, "p_value": 0.03979246831994764, "held_up": true, "z_path": [ 1.395, 1.973, 2.074, 1.702, 1.218, 1.293, 1.488, 1.387, 1.335, 1.74 ], "first_cross": 1, "final_z": 1.74 }, { "id": "52b1ed8b196a3278ac0002cb", "variation": "other", "week": 201350, "impressions": 16106, "pooled_ctr": 0.0201788153483174, "arms": [ { "headline": "See Why We Have An Absolutely Ridiculous Standard Of Beauty In Just 37 Seconds", "impressions": 3842, "clicks": 75, "ctr": 0.0195210827693909, "ctr_select": 0.019260801665799063, "ctr_holdout": 0.01978136387298282, "n_select": 1921, "n_holdout": 1921, "eb": 0.019593017961662424, "labelled_winner": false }, { "headline": "See Why We Have An Absolutely Ridiculous Standard Of Beauty In Just 37 Seconds", "impressions": 4132, "clicks": 76, "ctr": 0.0183930300096805, "ctr_select": 0.017909002904162634, "ctr_holdout": 0.01887705711519845, "n_select": 2066, "n_holdout": 2066, "eb": 0.0189449377800756, "labelled_winner": false }, { "headline": "See Why We Have An Absolutely Ridiculous Standard Of Beauty In Just 37 Seconds", "impressions": 3591, "clicks": 73, "ctr": 0.0203285992759676, "ctr_select": 0.021158129175946547, "ctr_holdout": 0.019498607242339833, "n_select": 1796, "n_holdout": 1795, "eb": 0.0204314673307841, "labelled_winner": false }, { "headline": "See Why We Have An Absolutely Ridiculous Standard Of Beauty In Just 37 Seconds", "impressions": 4541, "clicks": 101, "ctr": 0.0222417969610218, "ctr_select": 0.021145374449339206, "ctr_holdout": 0.023337736679876705, "n_select": 2270, "n_holdout": 2271, "eb": 0.020500234827145, "labelled_winner": false } ], "winner": 2, "pool_select": 0.01986837203526636, "pool_holdout": 0.020489258661368435, "lift_select": 0.0649150890868595, "lift_holdout": -0.04834979319658994, "eb_lift": 0.02834129009252706, "p_value": 0.6567833803052845, "held_up": false, "z_path": [ -1.013, -1.911, -1.873, -1.964, -1.493, -1.277, -0.814, -0.731, -0.216, -0.592 ], "first_cross": 3, "final_z": -0.592 }, { "id": "53d014f6e54a0f95e8000088", "variation": "image", "week": 201429, "impressions": 16064, "pooled_ctr": 0.014379980079681274, "arms": [ { "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out", "impressions": 2698, "clicks": 34, "ctr": 0.0126019273535952, "ctr_select": 0.01111934766493699, "ctr_holdout": 0.014084507042253521, "n_select": 1349, "n_holdout": 1349, "eb": 0.013455813304447523, "labelled_winner": false }, { "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out", "impressions": 2640, "clicks": 39, "ctr": 0.0147727272727272, "ctr_select": 0.015909090909090907, "ctr_holdout": 0.013636363636363636, "n_select": 1320, "n_holdout": 1320, "eb": 0.015212478949778167, "labelled_winner": false }, { "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out", "impressions": 2740, "clicks": 41, "ctr": 0.014963503649635, "ctr_select": 0.016058394160583942, "ctr_holdout": 0.013868613138686132, "n_select": 1370, "n_holdout": 1370, "eb": 0.015277391009194069, "labelled_winner": false }, { "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out", "impressions": 2681, "clicks": 33, "ctr": 0.0123088399850801, "ctr_select": 0.014925373134328358, "ctr_holdout": 0.009694258016405667, "n_select": 1340, "n_holdout": 1341, "eb": 0.014855902303943513, "labelled_winner": false }, { "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out", "impressions": 2658, "clicks": 33, "ctr": 0.0124153498871331, "ctr_select": 0.013544018058690745, "ctr_holdout": 0.011286681715575621, "n_select": 1329, "n_holdout": 1329, "eb": 0.014352272657778239, "labelled_winner": false }, { "headline": "If You Think A Woman Owes You Sex, Here's The Perfect Way To Find Out", "impressions": 2647, "clicks": 51, "ctr": 0.0192670948243294, "ctr_select": 0.017371601208459216, "ctr_holdout": 0.021164021164021163, "n_select": 1324, "n_holdout": 1323, "eb": 0.01574496413703939, "labelled_winner": false } ], "winner": 5, "pool_select": 0.01481573705179283, "pool_holdout": 0.013944223107569721, "lift_select": 0.17251009164995312, "lift_holdout": 0.5177626606198034, "eb_lift": 0.06271892393865874, "p_value": 0.3996092861143552, "held_up": true, "z_path": [ 0.903, 0.571, 0.481, -0.071, 0.557, 1.24, 2.068, 2.284, 1.597, 1.219 ], "first_cross": 6, "final_z": 1.219 }, { "id": "52f27c51c67e22334600001e", "variation": "image", "week": 201405, "impressions": 15959, "pooled_ctr": 0.022181840967479164, "arms": [ { "headline": "Here Is The Harrowing Footage Of Gay People Being Beaten On Camera In Russia", "impressions": 3989, "clicks": 77, "ctr": 0.0193030834795688, "ctr_select": 0.023069207622868605, "ctr_holdout": 0.015538847117794486, "n_select": 1994, "n_holdout": 1995, "eb": 0.022660331249792377, "labelled_winner": false }, { "headline": "Here Is The Harrowing Footage Of Gay People Being Beaten On Camera In Russia", "impressions": 4060, "clicks": 109, "ctr": 0.026847290640394, "ctr_select": 0.026600985221674877, "ctr_holdout": 0.027093596059113302, "n_select": 2030, "n_holdout": 2030, "eb": 0.024312752499629935, "labelled_winner": false }, { "headline": "Here Is The Harrowing Footage Of Gay People Being Beaten On Camera In Russia", "impressions": 3982, "clicks": 90, "ctr": 0.0226017076845806, "ctr_select": 0.020090406830738324, "ctr_holdout": 0.025113008538422903, "n_select": 1991, "n_holdout": 1991, "eb": 0.02128357673766438, "labelled_winner": false }, { "headline": "Here Is The Harrowing Footage Of Gay People Being Beaten On Camera In Russia", "impressions": 3928, "clicks": 78, "ctr": 0.0198574338085539, "ctr_select": 0.019348268839103868, "ctr_holdout": 0.020366598778004074, "n_select": 1964, "n_holdout": 1964, "eb": 0.020950684617200017, "labelled_winner": false } ], "winner": 1, "pool_select": 0.02230855996992104, "pool_holdout": 0.02205513784461153, "lift_select": 0.19241157912215656, "lift_holdout": 0.22844827586206895, "eb_lift": 0.0898396190704902, "p_value": 0.12937186818373697, "held_up": true, "z_path": [ 0.47, 0.132, -1.02, -0.427, -0.082, 1.168, 1.402, 1.979, 1.928, 2.252 ], "first_cross": 7, "final_z": 2.252 }, { "id": "52858f3907156c96cc001f74", "variation": "headline", "week": 201345, "impressions": 15647, "pooled_ctr": 0.01910909439509171, "arms": [ { "headline": "A Suicide Bomber Destroyed His Leg But An Artist Brought It Back To Life", "impressions": 4117, "clicks": 79, "ctr": 0.0191887296575176, "ctr_select": 0.019436345966958212, "ctr_holdout": 0.01894123360854784, "n_select": 2058, "n_holdout": 2059, "eb": 0.019501956180535008, "labelled_winner": false }, { "headline": "Think Your Scars Are Ugly? This Artist Thinks You're A Walking Work Of Art.", "impressions": 3905, "clicks": 94, "ctr": 0.0240717029449423, "ctr_select": 0.02612704918032787, "ctr_holdout": 0.022017409114183307, "n_select": 1952, "n_holdout": 1953, "eb": 0.022562408760340383, "labelled_winner": false }, { "headline": "How One Artist Is Telling The Story Of People's Lives Through Their Scars", "impressions": 3824, "clicks": 63, "ctr": 0.0164748953974895, "ctr_select": 0.013598326359832637, "ctr_holdout": 0.019351464435146442, "n_select": 1912, "n_holdout": 1912, "eb": 0.016865192237470544, "labelled_winner": false }, { "headline": "After A Suicide Bomber Destroyed This Man's Leg, An Artist Transformed It Into A Work Of Art", "impressions": 3801, "clicks": 63, "ctr": 0.0165745856353591, "ctr_select": 0.018947368421052633, "ctr_holdout": 0.014203051025775907, "n_select": 1900, "n_holdout": 1901, "eb": 0.019284138075063375, "labelled_winner": false } ], "winner": 1, "pool_select": 0.019560214778828945, "pool_holdout": 0.01865814696485623, "lift_select": 0.3357240437158471, "lift_holdout": 0.18004264601701636, "eb_lift": 0.1534847145319116, "p_value": 0.015586443794496661, "held_up": true, "z_path": [ -1.241, -0.165, 0.47, 0.58, 1.5, 1.522, 1.022, 1.425, 1.546, 1.505 ], "first_cross": null, "final_z": 1.505 }, { "id": "544ada1437d2889992000090", "variation": "headline", "week": 201442, "impressions": 15611, "pooled_ctr": 0.006597911728909103, "arms": [ { "headline": "Which Religions Are Protected In The United States?", "impressions": 3876, "clicks": 31, "ctr": 0.0079979360165118, "ctr_select": 0.008255933952528379, "ctr_holdout": 0.007739938080495356, "n_select": 1938, "n_holdout": 1938, "eb": 0.008085293481375159, "labelled_winner": false }, { "headline": "What Religion Is Tops When It Comes To We The People?", "impressions": 4001, "clicks": 31, "ctr": 0.0077480629842539, "ctr_select": 0.011, "ctr_holdout": 0.004497751124437781, "n_select": 2000, "n_holdout": 2001, "eb": 0.009358822148806647, "labelled_winner": false }, { "headline": "A Video About Church And State May Make You A Believer In Why It Exists", "impressions": 3815, "clicks": 28, "ctr": 0.0073394495412844, "ctr_select": 0.00890985324947589, "ctr_holdout": 0.0057682223387519665, "n_select": 1908, "n_holdout": 1907, "eb": 0.00837934168152885, "labelled_winner": false }, { "headline": "We Protect Religion But We Don't Recognize It. How Do We Separate Church And State?", "impressions": 3919, "clicks": 13, "ctr": 0.00331717274815, "ctr_select": 0.0035714285714285713, "ctr_holdout": 0.0030627871362940277, "n_select": 1960, "n_holdout": 1959, "eb": 0.0059397757074923585, "labelled_winner": false } ], "winner": 1, "pool_select": 0.007942608250064053, "pool_holdout": 0.005253042921204356, "lift_select": 0.38493548387096777, "lift_holdout": -0.14378176765275907, "eb_lift": 0.1783058982836241, "p_value": 0.07409231981375168, "held_up": false, "z_path": [ -0.503, -0.381, -0.094, 0.112, 0.434, -0.31, -0.309, -0.161, -0.041, 0.209 ], "first_cross": null, "final_z": 0.209 }, { "id": "5447163bcdf1931e1400002b", "variation": "headline", "week": 201442, "impressions": 15580, "pooled_ctr": 0.00853658536585366, "arms": [ { "headline": "Like Coke Or Pepsi? You Might Not Like Them As Much After You Hear What They're Doing.", "impressions": 3923, "clicks": 40, "ctr": 0.0101962783583991, "ctr_select": 0.008664627930682976, "ctr_holdout": 0.011728709841917389, "n_select": 1962, "n_holdout": 1961, "eb": 0.008560525941624401, "labelled_winner": false }, { "headline": "Coke And Pepsi Might Not Want Us To Know About This Little Secret. Ahh, Well \u2026 We Do Now.", "impressions": 3921, "clicks": 38, "ctr": 0.0096914052537617, "ctr_select": 0.010714285714285714, "ctr_holdout": 0.008669046404895462, "n_select": 1960, "n_holdout": 1961, "eb": 0.009499610778669192, "labelled_winner": false }, { "headline": "Pepsi And Coke Don\u2019t Want Us To Know About How They\u2019re Playing Voters. Secret\u2019s Out. Bah Dah BOOM!", "impressions": 3956, "clicks": 23, "ctr": 0.005813953488372, "ctr_select": 0.005055611729019211, "ctr_holdout": 0.006572295247724975, "n_select": 1978, "n_holdout": 1978, "eb": 0.006899103535814354, "labelled_winner": false }, { "headline": "Pepsi And Coke Don\u2019t Want Us To Know A Secret About How They\u2019re Playing Voters. Secret\u2019s Out.", "impressions": 3780, "clicks": 32, "ctr": 0.0084656084656084, "ctr_select": 0.009523809523809525, "ctr_holdout": 0.007407407407407408, "n_select": 1890, "n_holdout": 1890, "eb": 0.008944669000669916, "labelled_winner": false } ], "winner": 1, "pool_select": 0.008472400513478818, "pool_holdout": 0.008600770218228498, "lift_select": 0.26461038961038974, "lift_holdout": 0.007938380509487253, "eb_lift": 0.12124193887625778, "p_value": 0.2106582277897937, "held_up": false, "z_path": [ 0.277, 0.365, -0.291, 0.068, -0.114, -0.123, 0.138, 0.387, 0.452, 0.567 ], "first_cross": null, "final_z": 0.567 }, { "id": "52a5592e0fe0295fc000816d", "variation": "image", "week": 201349, "impressions": 15396, "pooled_ctr": 0.014744089373863341, "arms": [ { "headline": "Say Hello To The Ingenious Invention That's Maybe Gonna Be Around Longer Than All Of Us", "impressions": 3688, "clicks": 28, "ctr": 0.0075921908893709, "ctr_select": 0.009219088937093275, "ctr_holdout": 0.00596529284164859, "n_select": 1844, "n_holdout": 1844, "eb": 0.012043852240492764, "labelled_winner": false }, { "headline": "Say Hello To The Ingenious Invention That's Maybe Gonna Be Around Longer Than All Of Us", "impressions": 4001, "clicks": 65, "ctr": 0.0162459385153711, "ctr_select": 0.016, "ctr_holdout": 0.01649175412293853, "n_select": 2000, "n_holdout": 2001, "eb": 0.01508277713706444, "labelled_winner": false }, { "headline": "Say Hello To The Ingenious Invention That's Maybe Gonna Be Around Longer Than All Of Us", "impressions": 4001, "clicks": 72, "ctr": 0.0179955011247188, "ctr_select": 0.0155, "ctr_holdout": 0.02048975512243878, "n_select": 2000, "n_holdout": 2001, "eb": 0.014851172224352116, "labelled_winner": true }, { "headline": "Say Hello To The Ingenious Invention That's Maybe Gonna Be Around Longer Than All Of Us", "impressions": 3706, "clicks": 62, "ctr": 0.0167296276308688, "ctr_select": 0.016189962223421478, "ctr_holdout": 0.017269293038316244, "n_select": 1853, "n_holdout": 1853, "eb": 0.015134847139382557, "labelled_winner": false } ], "winner": 3, "pool_select": 0.01429128231778615, "pool_holdout": 0.015196778802441876, "lift_select": 0.13285581121522827, "lift_holdout": 0.1363785222392886, "eb_lift": 0.05902653119843215, "p_value": 0.4293597867880051, "held_up": false, "z_path": [ -1.674, -1.856, -1.538, -1.524, -0.057, -0.561, -0.501, -0.338, -0.284, 0.167 ], "first_cross": null, "final_z": 0.167 }, { "id": "53a4685862e6f52d4f000069", "variation": "both", "week": 201424, "impressions": 15394, "pooled_ctr": 0.03488372093023256, "arms": [ { "headline": "A Customer Wants To Make A Waitress's Life Miserable. She Responds In The Best Way Possible.", "impressions": 2564, "clicks": 81, "ctr": 0.031591263650546, "ctr_select": 0.031981279251170044, "ctr_holdout": 0.031201248049921998, "n_select": 1282, "n_holdout": 1282, "eb": 0.03154703093723764, "labelled_winner": false }, { "headline": "A Customer Wants To Make A Waitress's Life Miserable. So The Waitress Blurts Out The Truth.", "impressions": 2574, "clicks": 85, "ctr": 0.033022533022533, "ctr_select": 0.03418803418803419, "ctr_holdout": 0.031857031857031856, "n_select": 1287, "n_holdout": 1287, "eb": 0.032335520036243036, "labelled_winner": false }, { "headline": "A Customer Wants To Make A Waitress's Life Miserable. She Responds In The Best Way Possible.", "impressions": 2592, "clicks": 106, "ctr": 0.040895061728395, "ctr_select": 0.030864197530864196, "ctr_holdout": 0.05092592592592592, "n_select": 1296, "n_holdout": 1296, "eb": 0.0311480882057323, "labelled_winner": false }, { "headline": "A Customer Wants To Make A Waitress's Life Miserable. So The Waitress Blurts Out The Truth.", "impressions": 2538, "clicks": 72, "ctr": 0.0283687943262411, "ctr_select": 0.030732860520094562, "ctr_holdout": 0.026004728132387706, "n_select": 1269, "n_holdout": 1269, "eb": 0.031103757283101215, "labelled_winner": false }, { "headline": "A Customer Wants To Make A Waitress's Life Miserable. She Responds In The Best Way Possible.", "impressions": 2575, "clicks": 81, "ctr": 0.0314563106796116, "ctr_select": 0.024844720496894408, "ctr_holdout": 0.03807303807303807, "n_select": 1288, "n_holdout": 1287, "eb": 0.028998486123341275, "labelled_winner": false }, { "headline": "A Customer Wants To Make A Waitress's Life Miserable. So The Waitress Blurts Out The Truth.", "impressions": 2551, "clicks": 112, "ctr": 0.0439043512348098, "ctr_select": 0.03526645768025078, "ctr_holdout": 0.05254901960784314, "n_select": 1276, "n_holdout": 1275, "eb": 0.03271276120448333, "labelled_winner": false } ], "winner": 5, "pool_select": 0.03130683294362172, "pool_holdout": 0.038461538461538464, "lift_select": 0.1264779718778859, "lift_holdout": 0.3662745098039215, "eb_lift": 0.04490803216644257, "p_value": 0.37387246759247594, "held_up": true, "z_path": [ 0.93, 1.174, 1.255, 1.558, 1.324, 1.696, 2.169, 2.419, 2.578, 2.026 ], "first_cross": 6, "final_z": 2.026 }, { "id": "54392e97f50d2746c0000001", "variation": "image", "week": 201440, "impressions": 15393, "pooled_ctr": 0.006756317806795297, "arms": [ { "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?", "impressions": 3067, "clicks": 20, "ctr": 0.006521030322791, "ctr_select": 0.007170795306388526, "ctr_holdout": 0.005870841487279843, "n_select": 1534, "n_holdout": 1533, "eb": 0.007078030003862936, "labelled_winner": false }, { "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?", "impressions": 3080, "clicks": 21, "ctr": 0.0068181818181818, "ctr_select": 0.005844155844155844, "ctr_holdout": 0.007792207792207792, "n_select": 1540, "n_holdout": 1540, "eb": 0.006548577377865412, "labelled_winner": false }, { "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?", "impressions": 3093, "clicks": 25, "ctr": 0.0080827675396055, "ctr_select": 0.01034928848641656, "ctr_holdout": 0.005817711700064641, "n_select": 1546, "n_holdout": 1547, "eb": 0.0083501439257307, "labelled_winner": false }, { "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?", "impressions": 3093, "clicks": 23, "ctr": 0.0074361461364371, "ctr_select": 0.007761966364812419, "ctr_holdout": 0.007110536522301228, "n_select": 1546, "n_holdout": 1547, "eb": 0.007314866244663066, "labelled_winner": false }, { "headline": "Guess What Profession Gets You Death Threats And Hacking Attempts Just For Being A Woman?", "impressions": 3060, "clicks": 15, "ctr": 0.0049019607843137, "ctr_select": 0.00392156862745098, "ctr_holdout": 0.0058823529411764705, "n_select": 1530, "n_holdout": 1530, "eb": 0.0057859098162513535, "labelled_winner": false } ], "winner": 2, "pool_select": 0.007016632016632017, "pool_holdout": 0.006496037417175523, "lift_select": 0.47496526280484885, "lift_holdout": -0.10442146089204907, "eb_lift": 0.19005014171154566, "p_value": 0.0790674108497883, "held_up": false, "z_path": [ 0.0, -0.58, 0.502, 0.41, 0.41, 0.733, 0.874, 0.688, 0.765, 0.29 ], "first_cross": null, "final_z": 0.29 }, { "id": "53e3e69f6dcda749e100001e", "variation": "image", "week": 201431, "impressions": 15388, "pooled_ctr": 0.012152326488172603, "arms": [ { "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.", "impressions": 2518, "clicks": 38, "ctr": 0.0150913423351866, "ctr_select": 0.013502779984114376, "ctr_holdout": 0.016679904686258934, "n_select": 1259, "n_holdout": 1259, "eb": 0.011828503496807205, "labelled_winner": false }, { "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.", "impressions": 2538, "clicks": 35, "ctr": 0.0137903861308116, "ctr_select": 0.014972419227738377, "ctr_holdout": 0.01260835303388495, "n_select": 1269, "n_holdout": 1269, "eb": 0.012353140675625112, "labelled_winner": false }, { "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.", "impressions": 2545, "clicks": 31, "ctr": 0.012180746561886, "ctr_select": 0.008647798742138365, "ctr_holdout": 0.015710919088766692, "n_select": 1272, "n_holdout": 1273, "eb": 0.010114216932366358, "labelled_winner": false }, { "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.", "impressions": 2582, "clicks": 23, "ctr": 0.0089078233927188, "ctr_select": 0.006971340046475601, "ctr_holdout": 0.010844306738962044, "n_select": 1291, "n_holdout": 1291, "eb": 0.00950674844895755, "labelled_winner": false }, { "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.", "impressions": 2560, "clicks": 33, "ctr": 0.012890625, "ctr_select": 0.01015625, "ctr_holdout": 0.015625, "n_select": 1280, "n_holdout": 1280, "eb": 0.010647637586933614, "labelled_winner": false }, { "headline": "An Abortion Joke Shows How Ridiculous Some Laws Are Getting. So Absurd It's Funny.", "impressions": 2645, "clicks": 27, "ctr": 0.0102079395085066, "ctr_select": 0.0113464447806354, "ctr_holdout": 0.009070294784580499, "n_select": 1322, "n_holdout": 1323, "eb": 0.011074266210341466, "labelled_winner": false } ], "winner": 1, "pool_select": 0.01091901728844404, "pool_holdout": 0.013385315139701105, "lift_select": 0.37122406094037297, "lift_holdout": -0.05804585829374087, "eb_lift": 0.13134180020933317, "p_value": 0.12838407947755892, "held_up": false, "z_path": [ 1.411, 0.567, 0.883, 0.521, 0.294, 0.123, -0.313, -0.543, -0.402, -0.388 ], "first_cross": null, "final_z": -0.388 }, { "id": "538fd67daa2af376fb000061", "variation": "headline", "week": 201422, "impressions": 15387, "pooled_ctr": 0.007733801260804575, "arms": [ { "headline": "Neil DeGrasse Tyson: 'If People Begin To Lose Their Wealth, They Change Their Mind Real Fast'", "impressions": 3765, "clicks": 31, "ctr": 0.0082337317397078, "ctr_select": 0.005844845908607864, "ctr_holdout": 0.010621348911311737, "n_select": 1882, "n_holdout": 1883, "eb": 0.0064930325025082955, "labelled_winner": false }, { "headline": "Neil DeGrasse Tyson: 'If People Begin To Lose Their Wealth, They Change Their Mind Real Fast'", "impressions": 3838, "clicks": 23, "ctr": 0.0059927045336112, "ctr_select": 0.005211047420531527, "ctr_holdout": 0.006774361646690985, "n_select": 1919, "n_holdout": 1919, "eb": 0.006200294552090872, "labelled_winner": false }, { "headline": "'If People Begin To Lose Their Wealth, They Change Their Mind Real Fast.' Oh, Yes.", "impressions": 3839, "clicks": 29, "ctr": 0.0075540505339932, "ctr_select": 0.008333333333333333, "ctr_holdout": 0.006774361646690985, "n_select": 1920, "n_holdout": 1919, "eb": 0.00761469456908438, "labelled_winner": false }, { "headline": "'If People Begin To Lose Their Wealth, They Change Their Mind Real Fast.' Oh, Yes ... What He Said.", "impressions": 3945, "clicks": 36, "ctr": 0.0091254752851711, "ctr_select": 0.008620689655172414, "ctr_holdout": 0.00963000506842372, "n_select": 1972, "n_holdout": 1973, "eb": 0.007755505396326373, "labelled_winner": false } ], "winner": 3, "pool_select": 0.007019368256856883, "pool_holdout": 0.008448141408890044, "lift_select": 0.2281289910600255, "lift_holdout": 0.1398962922531093, "eb_lift": 0.10487227803590349, "p_value": 0.3233002646613852, "held_up": false, "z_path": [ 0.671, 0.723, 0.593, 1.16, 0.951, 1.091, 0.908, 1.039, 1.118, 0.762 ], "first_cross": null, "final_z": 0.762 }, { "id": "5387a93cd006a99c9f00003a", "variation": "headline", "week": 201421, "impressions": 15347, "pooled_ctr": 0.01407441193718642, "arms": [ { "headline": "If A Restaurant Gets An \u2018A\u2019 That Means It\u2019s Clean. If It Gets One Of These That Means It\u2019s Awesome.", "impressions": 3166, "clicks": 50, "ctr": 0.0157927984838913, "ctr_select": 0.017056222362602652, "ctr_holdout": 0.014529374605180037, "n_select": 1583, "n_holdout": 1583, "eb": 0.016445365507732353, "labelled_winner": false }, { "headline": "Leftover Food, Meet Hungry People. There, That Was Easy, Wasn\u2019t It?", "impressions": 3043, "clicks": 21, "ctr": 0.0069010844561288, "ctr_select": 0.00788436268068331, "ctr_holdout": 0.005917159763313609, "n_select": 1522, "n_holdout": 1521, "eb": 0.012800070023367676, "labelled_winner": false }, { "headline": "Did Your Mom Ever Tell You To Finish Your Meal Because Of Starving Kids? This Makes Way More Sense.", "impressions": 3034, "clicks": 70, "ctr": 0.023071852340145, "ctr_select": 0.024390243902439025, "ctr_holdout": 0.021753460777851022, "n_select": 1517, "n_holdout": 1517, "eb": 0.019336178546296993, "labelled_winner": false }, { "headline": "We Care A Lot About Where Our Food Comes From, But What About Where The Uneaten Stuff Goes?", "impressions": 3051, "clicks": 13, "ctr": 0.0042608980662078, "ctr_select": 0.007208387942332897, "ctr_holdout": 0.0013114754098360656, "n_select": 1526, "n_holdout": 1525, "eb": 0.012526583302800112, "labelled_winner": false }, { "headline": "The Best Restaurant Window Sticker Since The One With The Big \u2018A\u2019 On It", "impressions": 3053, "clicks": 62, "ctr": 0.0203078938748771, "ctr_select": 0.023591087811271297, "ctr_holdout": 0.017026850032743943, "n_select": 1526, "n_holdout": 1527, "eb": 0.019030736834400347, "labelled_winner": false } ], "winner": 2, "pool_select": 0.01602814698983581, "pool_holdout": 0.012120422259872279, "lift_select": 0.5217132659131469, "lift_holdout": 0.794777468262913, "eb_lift": 0.20638889564457807, "p_value": 0.0037873404799747736, "held_up": true, "z_path": [ 1.532, 0.917, 0.386, 1.21, 1.066, 0.469, 0.235, 0.798, 1.054, 0.74 ], "first_cross": null, "final_z": 0.74 }, { "id": "546d11cd545240e554000041", "variation": "image", "week": 201446, "impressions": 15294, "pooled_ctr": 0.004184647574212109, "arms": [ { "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead", "impressions": 3049, "clicks": 13, "ctr": 0.0042636930141029, "ctr_select": 0.003937007874015748, "ctr_holdout": 0.0045901639344262295, "n_select": 1524, "n_holdout": 1525, "eb": 0.004559771405450846, "labelled_winner": false }, { "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead", "impressions": 3000, "clicks": 22, "ctr": 0.0073333333333333, "ctr_select": 0.008666666666666666, "ctr_holdout": 0.006, "n_select": 1500, "n_holdout": 1500, "eb": 0.006422002274359335, "labelled_winner": false }, { "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead", "impressions": 3092, "clicks": 10, "ctr": 0.0032341526520051, "ctr_select": 0.00646830530401035, "ctr_holdout": 0.0, "n_select": 1546, "n_holdout": 1546, "eb": 0.005569085623900896, "labelled_winner": false }, { "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead", "impressions": 3098, "clicks": 11, "ctr": 0.0035506778566817, "ctr_select": 0.0032278889606197547, "ctr_holdout": 0.003873466752743706, "n_select": 1549, "n_holdout": 1549, "eb": 0.004271671747750383, "labelled_winner": false }, { "headline": "Say No To The Kind Of News That WANTS You To Be Scared And Watch This Instead", "impressions": 3055, "clicks": 8, "ctr": 0.0026186579378068, "ctr_select": 0.002617801047120419, "ctr_holdout": 0.0026195153896529143, "n_select": 1528, "n_holdout": 1527, "eb": 0.004034967003068992, "labelled_winner": false } ], "winner": 1, "pool_select": 0.00496926899437688, "pool_holdout": 0.0034000261540473387, "lift_select": 0.7440526315789473, "lift_holdout": 0.7646923076923078, "eb_lift": 0.29234345768489045, "p_value": 0.023123530464628953, "held_up": true, "z_path": [ 1.184, 1.322, 0.566, 0.795, 0.758, 0.727, 1.119, 2.131, 2.066, 2.213 ], "first_cross": 7, "final_z": 2.213 }, { "id": "5474a87ea341b993aa00004f", "variation": "headline", "week": 201447, "impressions": 15278, "pooled_ctr": 0.014072522581489723, "arms": [ { "headline": "1 Ugly Article That Should Never Exist And 1 Beautiful Response To It", "impressions": 3042, "clicks": 42, "ctr": 0.0138067061143984, "ctr_select": 0.015779092702169626, "ctr_holdout": 0.011834319526627219, "n_select": 1521, "n_holdout": 1521, "eb": 0.015025346934253475, "labelled_winner": false }, { "headline": "Ugly Things Are Said About Women With Eating Disorders. I've Waited For This Kind Of Response.", "impressions": 3070, "clicks": 35, "ctr": 0.011400651465798, "ctr_select": 0.011074918566775244, "ctr_holdout": 0.011726384364820847, "n_select": 1535, "n_holdout": 1535, "eb": 0.013153838861246171, "labelled_winner": false }, { "headline": "1 Ugly Article About Body Issues That Should Never Exist And 1 Beautiful Response To It", "impressions": 2950, "clicks": 36, "ctr": 0.0122033898305084, "ctr_select": 0.009491525423728813, "ctr_holdout": 0.014915254237288136, "n_select": 1475, "n_holdout": 1475, "eb": 0.01257093723277629, "labelled_winner": false }, { "headline": "The Misunderstood Truth About Dating A Woman With An Ugly Eating Disorder", "impressions": 3109, "clicks": 27, "ctr": 0.008684464458025, "ctr_select": 0.009009009009009009, "ctr_holdout": 0.008360128617363344, "n_select": 1554, "n_holdout": 1555, "eb": 0.012314433725232133, "labelled_winner": false }, { "headline": "A Beautiful And Real Response To A Disgusting Article That Should Never Exist", "impressions": 3107, "clicks": 75, "ctr": 0.0241390408754425, "ctr_select": 0.02702702702702703, "ctr_holdout": 0.021249195106245976, "n_select": 1554, "n_holdout": 1553, "eb": 0.01954640329932514, "labelled_winner": false } ], "winner": 4, "pool_select": 0.014530697735305667, "pool_holdout": 0.013614347427673779, "lift_select": 0.8599951302654008, "lift_holdout": 0.5607942443905098, "eb_lift": 0.3451799531850879, "p_value": 3.979609515560447e-06, "held_up": true, "z_path": [ 1.792, 1.946, 1.938, 2.123, 2.545, 2.553, 2.822, 2.78, 2.72, 2.965 ], "first_cross": 3, "final_z": 2.965 }, { "id": "5494308437323200121e0000", "variation": "headline", "week": 201450, "impressions": 15266, "pooled_ctr": 0.01978252325429058, "arms": [ { "headline": "A Tampon Commercial That Shows Just How Confusing Actual Tampon Commercials Are", "impressions": 3077, "clicks": 61, "ctr": 0.0198245043873903, "ctr_select": 0.01950585175552666, "ctr_holdout": 0.02014294996751137, "n_select": 1538, "n_holdout": 1539, "eb": 0.019202645797161494, "labelled_winner": false }, { "headline": "A Tampon Commercial That Totally Explains Why Actual Tampon Commercials Are So Confusing", "impressions": 3085, "clicks": 66, "ctr": 0.0213938411669367, "ctr_select": 0.017509727626459144, "ctr_holdout": 0.025275437459494492, "n_select": 1542, "n_holdout": 1543, "eb": 0.018405482180048564, "labelled_winner": false }, { "headline": "A Tampon Commercial That Shows Just How Mystifying Actual Tampon Commercials Are", "impressions": 3121, "clicks": 68, "ctr": 0.0217878884972765, "ctr_select": 0.01730769230769231, "ctr_holdout": 0.02626521460602178, "n_select": 1560, "n_holdout": 1561, "eb": 0.018320045226351802, "labelled_winner": false }, { "headline": "Why Tampon Commercials Are So Confusing", "impressions": 2981, "clicks": 38, "ctr": 0.0127474002012747, "ctr_select": 0.012080536912751677, "ctr_holdout": 0.01341381623071764, "n_select": 1490, "n_holdout": 1491, "eb": 0.01629320448614213, "labelled_winner": false }, { "headline": "A Tampon Commercial That Shows Just How Baffling Actual Tampon Commercials Are", "impressions": 3002, "clicks": 69, "ctr": 0.0229846768820786, "ctr_select": 0.028647568287808126, "ctr_holdout": 0.017321785476349102, "n_select": 1501, "n_holdout": 1501, "eb": 0.022793005379135388, "labelled_winner": false } ], "winner": 4, "pool_select": 0.019001441488664658, "pool_holdout": 0.020563195808775376, "lift_select": 0.5076523696845778, "lift_holdout": -0.15763164259920126, "eb_lift": 0.19954085550470446, "p_value": 0.00225773388675334, "held_up": false, "z_path": [ -0.497, 0.975, 0.823, 0.892, 1.187, 1.245, 1.354, 0.91, 0.79, 0.852 ], "first_cross": null, "final_z": 0.852 }, { "id": "53626ef4d3d68209a800009a", "variation": "image", "week": 201417, "impressions": 15226, "pooled_ctr": 0.017470116905293577, "arms": [ { "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness", "impressions": 2947, "clicks": 44, "ctr": 0.014930437733288, "ctr_select": 0.016960651289009497, "ctr_holdout": 0.012898845892735914, "n_select": 1474, "n_holdout": 1473, "eb": 0.01783262634375151, "labelled_winner": false }, { "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness", "impressions": 3083, "clicks": 54, "ctr": 0.0175154070710347, "ctr_select": 0.017509727626459144, "ctr_holdout": 0.01752109020116807, "n_select": 1542, "n_holdout": 1541, "eb": 0.01803662714093299, "labelled_winner": false }, { "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness", "impressions": 3012, "clicks": 50, "ctr": 0.0166002656042496, "ctr_select": 0.016600265604249667, "ctr_holdout": 0.016600265604249667, "n_select": 1506, "n_holdout": 1506, "eb": 0.01768338754741349, "labelled_winner": false }, { "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness", "impressions": 3107, "clicks": 73, "ctr": 0.023495333118764, "ctr_select": 0.027670527670527672, "ctr_holdout": 0.019317450096587252, "n_select": 1554, "n_holdout": 1553, "eb": 0.022113278180970787, "labelled_winner": true }, { "headline": "Wrap Your Brain Around This: An Infographic On The United States Of Mental Illness", "impressions": 3077, "clicks": 45, "ctr": 0.0146246343841403, "ctr_select": 0.013003901170351105, "ctr_holdout": 0.016244314489928524, "n_select": 1538, "n_holdout": 1539, "eb": 0.016239843777156916, "labelled_winner": false } ], "winner": 3, "pool_select": 0.018387181507748884, "pool_holdout": 0.01655281135049921, "lift_select": 0.5048814120242693, "lift_holdout": 0.16701928678747757, "eb_lift": 0.202646429070797, "p_value": 0.0022632359549176876, "held_up": true, "z_path": [ -0.553, -0.218, -0.021, -0.025, 0.379, 0.472, 0.758, 0.98, 1.763, 1.659 ], "first_cross": null, "final_z": 1.659 }, { "id": "54709a23bc296ffe23000024", "variation": "other", "week": 201446, "impressions": 15219, "pooled_ctr": 0.04914908995334779, "arms": [ { "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.", "impressions": 3045, "clicks": 132, "ctr": 0.0433497536945812, "ctr_select": 0.04204993429697766, "ctr_holdout": 0.04464871963230466, "n_select": 1522, "n_holdout": 1523, "eb": 0.044040093291538455, "labelled_winner": false }, { "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.", "impressions": 3089, "clicks": 147, "ctr": 0.0475882162512139, "ctr_select": 0.04145077720207254, "ctr_holdout": 0.05372168284789644, "n_select": 1544, "n_holdout": 1545, "eb": 0.043789197927240224, "labelled_winner": false }, { "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.", "impressions": 3070, "clicks": 170, "ctr": 0.0553745928338762, "ctr_select": 0.05472312703583062, "ctr_holdout": 0.056026058631921824, "n_select": 1535, "n_holdout": 1535, "eb": 0.04908265833987898, "labelled_winner": true }, { "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.", "impressions": 2993, "clicks": 163, "ctr": 0.0544604076177748, "ctr_select": 0.0427807486631016, "ctr_holdout": 0.06613226452905811, "n_select": 1496, "n_holdout": 1497, "eb": 0.04434033796593638, "labelled_winner": false }, { "headline": "They Gave Each Kid A Barbie And A Doll With Real Proportions. What They Say Next Really Says It All.", "impressions": 3022, "clicks": 136, "ctr": 0.0450033090668431, "ctr_select": 0.045665122435473195, "ctr_holdout": 0.04434149569821311, "n_select": 1511, "n_holdout": 1511, "eb": 0.04547254929859503, "labelled_winner": false } ], "winner": 2, "pool_select": 0.045347003154574135, "pool_holdout": 0.052949678097490474, "lift_select": 0.2067639144597082, "lift_holdout": 0.05810008001875189, "eb_lift": 0.082379317825505, "p_value": 0.0481407967967455, "held_up": false, "z_path": [ 1.875, 1.941, 1.103, 1.156, 1.526, 1.198, 1.986, 1.874, 1.843, 1.853 ], "first_cross": 6, "final_z": 1.853 }, { "id": "547fa599373665001e0a0000", "variation": "image", "week": 201448, "impressions": 15200, "pooled_ctr": 0.013421052631578948, "arms": [ { "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans", "impressions": 3141, "clicks": 45, "ctr": 0.0143266475644699, "ctr_select": 0.01592356687898089, "ctr_holdout": 0.012730744748567792, "n_select": 1570, "n_holdout": 1571, "eb": 0.014197357775596121, "labelled_winner": false }, { "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans", "impressions": 2984, "clicks": 46, "ctr": 0.0154155495978552, "ctr_select": 0.014745308310991957, "ctr_holdout": 0.0160857908847185, "n_select": 1492, "n_holdout": 1492, "eb": 0.013700571353093015, "labelled_winner": true }, { "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans", "impressions": 3004, "clicks": 31, "ctr": 0.0103195739014647, "ctr_select": 0.010652463382157125, "ctr_holdout": 0.009986684420772303, "n_select": 1502, "n_holdout": 1502, "eb": 0.012093898210847923, "labelled_winner": false }, { "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans", "impressions": 3037, "clicks": 39, "ctr": 0.0128416200197563, "ctr_select": 0.011198945981554678, "ctr_holdout": 0.014483212639894667, "n_select": 1518, "n_holdout": 1519, "eb": 0.012304159209166965, "labelled_winner": false }, { "headline": "A Homeless Man Escapes The Streets By Taking Out 37 Thousand Dollars In Student Loans", "impressions": 3034, "clicks": 43, "ctr": 0.0141727092946605, "ctr_select": 0.012524719841793012, "ctr_holdout": 0.015820698747528016, "n_select": 1517, "n_holdout": 1517, "eb": 0.012828921345378467, "labelled_winner": false } ], "winner": 0, "pool_select": 0.013028030003947888, "pool_holdout": 0.013813971845809762, "lift_select": 0.22225439104419986, "lift_holdout": -0.07841532539177354, "eb_lift": 0.08975476501772639, "p_value": 0.25599554720899975, "held_up": false, "z_path": [ 0.257, -1.543, -1.213, -0.641, -0.469, -1.01, -0.826, -0.572, -0.456, -0.352 ], "first_cross": null, "final_z": -0.352 }, { "id": "547f42b734333600212d0000", "variation": "headline", "week": 201448, "impressions": 15198, "pooled_ctr": 0.016449532833267534, "arms": [ { "headline": "The Ancient Greeks, Pacific Islanders, and Maya All Ignored This One Thing. Big Mistake!", "impressions": 3064, "clicks": 48, "ctr": 0.0156657963446475, "ctr_select": 0.015013054830287207, "ctr_holdout": 0.016318537859007835, "n_select": 1532, "n_holdout": 1532, "eb": 0.016275372655567517, "labelled_winner": false }, { "headline": "The Ancient Greeks, Pacific Islanders and Maya All Ignored This One Thing. Big Mistake!", "impressions": 3049, "clicks": 56, "ctr": 0.0183666775992128, "ctr_select": 0.01706036745406824, "ctr_holdout": 0.019672131147540985, "n_select": 1524, "n_holdout": 1525, "eb": 0.017090169451647708, "labelled_winner": false }, { "headline": "The Ancient Greeks, Pacific Islanders and Maya All Ignored This One Thing. Big Mistake!", "impressions": 3085, "clicks": 32, "ctr": 0.0103727714748784, "ctr_select": 0.009079118028534372, "ctr_holdout": 0.011665586519766688, "n_select": 1542, "n_holdout": 1543, "eb": 0.013901416344026751, "labelled_winner": false }, { "headline": "The Ancient Greeks, Pacific Islanders and Maya All Ignored This One Thing. Big Mistake!", "impressions": 2972, "clicks": 53, "ctr": 0.0178331090174966, "ctr_select": 0.01816958277254374, "ctr_holdout": 0.017496635262449527, "n_select": 1486, "n_holdout": 1486, "eb": 0.017523807076728445, "labelled_winner": false }, { "headline": "The Ancient Greeks, Pacific Islanders and Maya All Ignored This One Thing. Big Mistake!", "impressions": 3028, "clicks": 61, "ctr": 0.0201453104359313, "ctr_select": 0.026420079260237782, "ctr_holdout": 0.013870541611624834, "n_select": 1514, "n_holdout": 1514, "eb": 0.020788504442264998, "labelled_winner": false } ], "winner": 4, "pool_select": 0.01710976572782311, "pool_holdout": 0.015789473684210527, "lift_select": 0.544152017071436, "lift_holdout": -0.12153236459709382, "eb_lift": 0.21500812886407283, "p_value": 0.0017973729636608256, "held_up": false, "z_path": [ -0.02, -0.031, 1.311, 1.395, 1.486, 1.604, 0.923, 0.667, 0.517, 0.656 ], "first_cross": null, "final_z": 0.656 }, { "id": "54753ffe3762650018130000", "variation": "headline", "week": 201447, "impressions": 14748, "pooled_ctr": 0.03356387306753458, "arms": [ { "headline": "The Truth About The Ferguson Case That Most People Won't Acknowledge", "impressions": 3747, "clicks": 122, "ctr": 0.0325593808380037, "ctr_select": 0.0368196371398079, "ctr_holdout": 0.02829684997330486, "n_select": 1874, "n_holdout": 1873, "eb": 0.03430710130431008, "labelled_winner": false }, { "headline": "The Truth About The Ferguson Case That Some People Won't Acknowledge", "impressions": 3680, "clicks": 125, "ctr": 0.0339673913043478, "ctr_select": 0.02717391304347826, "ctr_holdout": 0.04076086956521739, "n_select": 1840, "n_holdout": 1840, "eb": 0.030017814342495553, "labelled_winner": false }, { "headline": "The Truth About The Ferguson Case That Some People Really Can't Accept", "impressions": 3681, "clicks": 129, "ctr": 0.0350448247758761, "ctr_select": 0.03206521739130435, "ctr_holdout": 0.03802281368821293, "n_select": 1840, "n_holdout": 1841, "eb": 0.03218247388230977, "labelled_winner": true }, { "headline": "The Truth About The Ferguson Case That Many People Are Unwilling To Hear", "impressions": 3640, "clicks": 119, "ctr": 0.0326923076923076, "ctr_select": 0.03296703296703297, "ctr_holdout": 0.03241758241758242, "n_select": 1820, "n_holdout": 1820, "eb": 0.03257971157219836, "labelled_winner": false } ], "winner": 0, "pool_select": 0.032275562788174665, "pool_holdout": 0.034852183346894494, "lift_select": 0.14078993390312378, "lift_holdout": -0.18808960426789867, "eb_lift": 0.06294355049572498, "p_value": 0.19746540669340096, "held_up": false, "z_path": [ 0.139, 0.341, 0.241, 0.396, -0.26, -0.096, 0.198, 0.155, -0.011, -0.032 ], "first_cross": null, "final_z": -0.032 }, { "id": "52a01fae8f48676d170045ff", "variation": "image", "week": 201348, "impressions": 14385, "pooled_ctr": 0.019256169621133124, "arms": [ { "headline": "You Know You Shouldn't Do It, But You Do Anyway. These People Will Convince You To Stop. Right Now.", "impressions": 3576, "clicks": 71, "ctr": 0.0198545861297539, "ctr_select": 0.01901565995525727, "ctr_holdout": 0.02069351230425056, "n_select": 1788, "n_holdout": 1788, "eb": 0.0184065202360505, "labelled_winner": false }, { "headline": "You Know You Shouldn't Do It, But You Do Anyway. These People Will Convince You To Stop. Right Now.", "impressions": 3454, "clicks": 56, "ctr": 0.0162130862767805, "ctr_select": 0.015055008685581933, "ctr_holdout": 0.017371163867979156, "n_select": 1727, "n_holdout": 1727, "eb": 0.016706219554068397, "labelled_winner": false }, { "headline": "You Know You Shouldn't Do It, But You Do Anyway. These People Will Convince You To Stop. Right Now.", "impressions": 3645, "clicks": 80, "ctr": 0.0219478737997256, "ctr_select": 0.021405049396267837, "ctr_holdout": 0.022490400438837082, "n_select": 1822, "n_holdout": 1823, "eb": 0.019463162199804915, "labelled_winner": false }, { "headline": "You Know You Shouldn't Do It, But You Do Anyway. These People Will Convince You To Stop. Right Now.", "impressions": 3710, "clicks": 70, "ctr": 0.0188679245283018, "ctr_select": 0.016172506738544475, "ctr_holdout": 0.0215633423180593, "n_select": 1855, "n_holdout": 1855, "eb": 0.017152358725619637, "labelled_winner": false } ], "winner": 2, "pool_select": 0.01793659621802002, "pool_holdout": 0.020575559571805922, "lift_select": 0.1933729864958007, "lift_holdout": 0.09306385376050774, "eb_lift": 0.08510901194571274, "p_value": 0.1967225439640914, "held_up": true, "z_path": [ -0.031, 0.346, 0.232, 0.456, 0.621, 0.46, 0.201, 0.454, 0.761, 0.622 ], "first_cross": null, "final_z": 0.622 }, { "id": "537c01c2db41d8a90c00006e", "variation": "headline", "week": 201420, "impressions": 14342, "pooled_ctr": 0.014642309301352671, "arms": [ { "headline": "The Whole Problem With Education In America, Explained By One Teacher In 6 Minutes", "impressions": 3539, "clicks": 67, "ctr": 0.0189319016671376, "ctr_select": 0.01864406779661017, "ctr_holdout": 0.01921989824759751, "n_select": 1770, "n_holdout": 1769, "eb": 0.016532052544685278, "labelled_winner": false }, { "headline": "Her Job Owes Her $11,000 ... So Why Does She Still Go In Everyday?", "impressions": 3523, "clicks": 45, "ctr": 0.0127732046551234, "ctr_select": 0.01362088535754824, "ctr_holdout": 0.01192504258943782, "n_select": 1762, "n_holdout": 1761, "eb": 0.014358424932627262, "labelled_winner": false }, { "headline": "A Teacher Warns About A Crisis And I Hope Someone Is Listening", "impressions": 3626, "clicks": 48, "ctr": 0.0132377275234418, "ctr_select": 0.012134583563154992, "ctr_holdout": 0.014340871483728626, "n_select": 1813, "n_holdout": 1813, "eb": 0.013696967748651128, "labelled_winner": false }, { "headline": "Teachers Have Infinite Patience, So When They\u2019re Pissed, You Know Something Is Really Wrong", "impressions": 3654, "clicks": 50, "ctr": 0.0136836343732895, "ctr_select": 0.01532567049808429, "ctr_holdout": 0.0120415982484948, "n_select": 1827, "n_holdout": 1827, "eb": 0.015098334715661707, "labelled_winner": false } ], "winner": 0, "pool_select": 0.014919129949804797, "pool_holdout": 0.014365411436541143, "lift_select": 0.2496752732456835, "lift_holdout": 0.3379288391774191, "eb_lift": 0.10811103598582061, "p_value": 0.13635546839561022, "held_up": true, "z_path": [ -0.196, 0.639, 1.212, 0.816, 1.377, 1.327, 1.723, 1.82, 1.928, 1.759 ], "first_cross": null, "final_z": 1.759 }, { "id": "51b7848d8c43697706000ef2", "variation": "headline", "week": 201323, "impressions": 14315, "pooled_ctr": 0.01683548725113517, "arms": [ { "headline": "Let The Daily Show Sum Up The NSA Scandal In Less Tham 5 Minutes", "impressions": 3544, "clicks": 71, "ctr": 0.0200338600451467, "ctr_select": 0.01636568848758465, "ctr_holdout": 0.023702031602708805, "n_select": 1772, "n_holdout": 1772, "eb": 0.015246886736382654, "labelled_winner": false }, { "headline": "The NSA Scandal Summed Up In Less Tham 5 Minutes", "impressions": 3688, "clicks": 64, "ctr": 0.0173535791757049, "ctr_select": 0.016811279826464208, "ctr_holdout": 0.01789587852494577, "n_select": 1844, "n_holdout": 1844, "eb": 0.015463679069467249, "labelled_winner": false }, { "headline": "NAILED IT: Why You Should Feel Paranoid About The NSA", "impressions": 3510, "clicks": 60, "ctr": 0.017094017094017, "ctr_select": 0.013105413105413105, "ctr_holdout": 0.021082621082621083, "n_select": 1755, "n_holdout": 1755, "eb": 0.013837304355900729, "labelled_winner": false }, { "headline": "That Guy Sitting In Jon Stewart's Seat Nails Why You SHOULD Be Paranoid About The NSA", "impressions": 3573, "clicks": 46, "ctr": 0.0128743352924713, "ctr_select": 0.011198208286674132, "ctr_holdout": 0.014549524342473419, "n_select": 1786, "n_holdout": 1787, "eb": 0.013001727107810373, "labelled_winner": false } ], "winner": 1, "pool_select": 0.014391504820455498, "pool_holdout": 0.019279128248113998, "lift_select": 0.1681391234757703, "lift_holdout": -0.07174856172781274, "eb_lift": 0.07450049611822429, "p_value": 0.31124256988573895, "held_up": false, "z_path": [ 1.229, 0.104, -0.651, -0.737, -0.284, 0.052, -0.404, -0.7, -0.497, -0.842 ], "first_cross": null, "final_z": -0.842 }, { "id": "545d64fa81f41f555c000007", "variation": "image", "week": 201444, "impressions": 14259, "pooled_ctr": 0.01718213058419244, "arms": [ { "headline": "He Was Just 'Being Friendly', Until His Neighbor Started Being More Friendly To Him Than He Wanted", "impressions": 3556, "clicks": 59, "ctr": 0.0165916760404949, "ctr_select": 0.0140607424071991, "ctr_holdout": 0.019122609673790775, "n_select": 1778, "n_holdout": 1778, "eb": 0.0151530608001254, "labelled_winner": false }, { "headline": "He Was Just 'Being Friendly', Until His Neighbor Started Being More Friendly To Him Than He Wanted", "impressions": 3605, "clicks": 62, "ctr": 0.0171983356449375, "ctr_select": 0.016648168701442843, "ctr_holdout": 0.01774819744869662, "n_select": 1802, "n_holdout": 1803, "eb": 0.01627846534997421, "labelled_winner": false }, { "headline": "He Was Just 'Being Friendly', Until His Neighbor Started Being More Friendly To Him Than He Wanted", "impressions": 3504, "clicks": 42, "ctr": 0.011986301369863, "ctr_select": 0.014269406392694063, "ctr_holdout": 0.009703196347031963, "n_select": 1752, "n_holdout": 1752, "eb": 0.015249868870256662, "labelled_winner": false }, { "headline": "He Was Just 'Being Friendly', Until His Neighbor Started Being More Friendly To Him Than He Wanted", "impressions": 3594, "clicks": 82, "ctr": 0.0228158041179744, "ctr_select": 0.018920422927100722, "ctr_holdout": 0.02671118530884808, "n_select": 1797, "n_holdout": 1797, "eb": 0.01727037110337943, "labelled_winner": false } ], "winner": 3, "pool_select": 0.015991022583812596, "pool_holdout": 0.018373071528751752, "lift_select": 0.18319030743246545, "lift_holdout": 0.4538225286418842, "eb_lift": 0.08000417189466624, "p_value": 0.25234145506332073, "held_up": true, "z_path": [ 0.285, 0.637, 1.645, 0.85, 1.275, 1.82, 1.417, 1.353, 1.383, 1.702 ], "first_cross": null, "final_z": 1.702 }, { "id": "5535537b646339001c260000", "variation": "image", "week": 201516, "impressions": 14246, "pooled_ctr": 0.035518742103046466, "arms": [ { "headline": "Canceling out the harmful effects of sitting is easier than you think.", "impressions": 2079, "clicks": 17, "ctr": 0.0081770081770081, "ctr_select": 0.006730769230769231, "ctr_holdout": 0.009624639076034648, "n_select": 1040, "n_holdout": 1039, "eb": 0.023594952519395473, "labelled_winner": false }, { "headline": "Canceling out the harmful effects of sitting is easier than you think.", "impressions": 2044, "clicks": 19, "ctr": 0.0092954990215264, "ctr_select": 0.005870841487279843, "ctr_holdout": 0.012720156555772993, "n_select": 1022, "n_holdout": 1022, "eb": 0.023422694157851972, "labelled_winner": false }, { "headline": "Canceling out the harmful effects of sitting is easier than you think.", "impressions": 2069, "clicks": 22, "ctr": 0.0106331561140647, "ctr_select": 0.010638297872340425, "ctr_holdout": 0.010628019323671498, "n_select": 1034, "n_holdout": 1035, "eb": 0.024830615780252835, "labelled_winner": false }, { "headline": "Canceling out the harmful effects of sitting is easier than you think.", "impressions": 2087, "clicks": 44, "ctr": 0.0210828941063727, "ctr_select": 0.017241379310344827, "ctr_holdout": 0.024928092042186004, "n_select": 1044, "n_holdout": 1043, "eb": 0.02683903367960474, "labelled_winner": false }, { "headline": "Canceling out the harmful effects of sitting is easier than you think.", "impressions": 2008, "clicks": 25, "ctr": 0.0124501992031872, "ctr_select": 0.010956175298804782, "ctr_holdout": 0.013944223107569721, "n_select": 1004, "n_holdout": 1004, "eb": 0.025054874138371434, "labelled_winner": false }, { "headline": "Canceling out the harmful effects of sitting is easier than you think.", "impressions": 2011, "clicks": 29, "ctr": 0.0144206862257583, "ctr_select": 0.013916500994035786, "ctr_holdout": 0.014925373134328358, "n_select": 1006, "n_holdout": 1005, "eb": 0.025942406794428056, "labelled_winner": false }, { "headline": "Canceling out the harmful effects of sitting is easier than you think.", "impressions": 1948, "clicks": 350, "ctr": 0.1796714579055441, "ctr_select": 0.1591375770020534, "ctr_holdout": 0.2002053388090349, "n_select": 974, "n_holdout": 974, "eb": 0.0690296431520524, "labelled_winner": false } ], "winner": 6, "pool_select": 0.03116226838854576, "pool_holdout": 0.03987643920247121, "lift_select": 4.106739182714542, "lift_holdout": 4.020642334499812, "eb_lift": 1.2151674676361321, "p_value": 4.027004748096685e-135, "held_up": true, "z_path": [ 5.567, 8.105, 9.51, 10.963, 11.702, 12.623, 13.745, 15.22, 16.132, 16.959 ], "first_cross": 0, "final_z": 16.959 }, { "id": "54ac7e253137331dd60f0000", "variation": "image", "week": 201501, "impressions": 14240, "pooled_ctr": 0.0125, "arms": [ { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2715, "clicks": 31, "ctr": 0.0114180478821362, "ctr_select": 0.013991163475699559, "ctr_holdout": 0.008843036109064112, "n_select": 1358, "n_holdout": 1357, "eb": 0.013846735947562396, "labelled_winner": false }, { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2775, "clicks": 35, "ctr": 0.0126126126126126, "ctr_select": 0.016570605187319884, "ctr_holdout": 0.008651766402307137, "n_select": 1388, "n_holdout": 1387, "eb": 0.014814056600051239, "labelled_winner": false }, { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2901, "clicks": 39, "ctr": 0.0134436401240951, "ctr_select": 0.011724137931034483, "ctr_holdout": 0.015161957270847692, "n_select": 1450, "n_holdout": 1451, "eb": 0.012977796886143287, "labelled_winner": false }, { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2903, "clicks": 39, "ctr": 0.0134343782294178, "ctr_select": 0.015151515151515152, "ctr_holdout": 0.01171605789110958, "n_select": 1452, "n_holdout": 1451, "eb": 0.014297277862061602, "labelled_winner": false }, { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2946, "clicks": 34, "ctr": 0.0115410726408689, "ctr_select": 0.011541072640868975, "ctr_holdout": 0.011541072640868975, "n_select": 1473, "n_holdout": 1473, "eb": 0.012899054303210231, "labelled_winner": false } ], "winner": 1, "pool_select": 0.013762112062912512, "pool_holdout": 0.011237533361427167, "lift_select": 0.20407428100923375, "lift_holdout": -0.2301009372746936, "eb_lift": 0.07643772498943746, "p_value": 0.3168489205325362, "held_up": false, "z_path": [ 0.877, 1.821, 0.503, -0.046, -0.329, 0.013, -0.108, -0.078, -0.184, -0.273 ], "first_cross": null, "final_z": -0.273 }, { "id": "54ac36206233650024a00000", "variation": "image", "week": 201501, "impressions": 14220, "pooled_ctr": 0.026160337552742614, "arms": [ { "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.", "impressions": 2795, "clicks": 84, "ctr": 0.0300536672629695, "ctr_select": 0.02575107296137339, "ctr_holdout": 0.03435934144595562, "n_select": 1398, "n_holdout": 1397, "eb": 0.024249712527628392, "labelled_winner": false }, { "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.", "impressions": 2942, "clicks": 69, "ctr": 0.0234534330387491, "ctr_select": 0.016315431679129844, "ctr_holdout": 0.030591434398368457, "n_select": 1471, "n_holdout": 1471, "eb": 0.020615157836797883, "labelled_winner": false }, { "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.", "impressions": 2903, "clicks": 65, "ctr": 0.022390630382363, "ctr_select": 0.02203856749311295, "ctr_holdout": 0.022742935906271536, "n_select": 1452, "n_holdout": 1451, "eb": 0.0228412482706762, "labelled_winner": false }, { "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.", "impressions": 2743, "clicks": 86, "ctr": 0.0313525337222019, "ctr_select": 0.02696793002915452, "ctr_holdout": 0.03574033552151714, "n_select": 1372, "n_holdout": 1371, "eb": 0.024691616622825808, "labelled_winner": true }, { "headline": "People Say She's Too Pretty To Be Deaf. That's When She Hits Them With This Face.", "impressions": 2837, "clicks": 68, "ctr": 0.0239689813182939, "ctr_select": 0.02609308885754584, "ctr_holdout": 0.021846370683579985, "n_select": 1418, "n_holdout": 1419, "eb": 0.024387573187180305, "labelled_winner": false } ], "winner": 3, "pool_select": 0.023344114751792995, "pool_holdout": 0.028977352651568436, "lift_select": 0.1552346411886616, "lift_holdout": 0.23338856904109395, "eb_lift": 0.05772340846333934, "p_value": 0.32240180654825834, "held_up": true, "z_path": [ 2.585, 2.766, 1.844, 1.479, 1.201, 0.795, 1.335, 1.115, 1.44, 1.683 ], "first_cross": 0, "final_z": 1.683 }, { "id": "51b7cf31ac05f91e9a0049e7", "variation": "headline", "week": 201323, "impressions": 14217, "pooled_ctr": 0.010199057466413449, "arms": [ { "headline": "You Know The Name Of One Of These Athletes But The Other Two Are The Ones You Should Remember", "impressions": 3484, "clicks": 50, "ctr": 0.0143513203214695, "ctr_select": 0.011481056257175661, "ctr_holdout": 0.01722158438576349, "n_select": 1742, "n_holdout": 1742, "eb": 0.010066137124591212, "labelled_winner": false }, { "headline": "RESPECT: These Two Little Boys Came In Last But Won Gold At LIFE!", "impressions": 3596, "clicks": 42, "ctr": 0.0116796440489432, "ctr_select": 0.012235817575083427, "ctr_holdout": 0.011123470522803115, "n_select": 1798, "n_holdout": 1798, "eb": 0.010415117157649922, "labelled_winner": false }, { "headline": "When Faced With His Brother\u2019s Disability, This Big Brother Decided To Run With It.", "impressions": 3494, "clicks": 33, "ctr": 0.0094447624499141, "ctr_select": 0.005724098454493417, "ctr_holdout": 0.01316542644533486, "n_select": 1747, "n_holdout": 1747, "eb": 0.007593547176717703, "labelled_winner": false }, { "headline": "I Agree With This Superstar, Love Watching These Two Brothers Bond Over Disability", "impressions": 3643, "clicks": 20, "ctr": 0.0054899807850672, "ctr_select": 0.006586169045005488, "ctr_holdout": 0.004393190554640308, "n_select": 1822, "n_holdout": 1821, "eb": 0.007939099935351461, "labelled_winner": false } ], "winner": 1, "pool_select": 0.009002672668448446, "pool_holdout": 0.011395610579628587, "lift_select": 0.35913167408231383, "lift_holdout": -0.023881129924882116, "eb_lift": 0.15689168552708277, "p_value": 0.09310424148884665, "held_up": false, "z_path": [ -0.044, -0.971, -1.393, -1.471, -0.998, -1.537, -1.543, -1.433, -1.121, -0.992 ], "first_cross": null, "final_z": -0.992 }, { "id": "552c5646363535000c070000", "variation": "image", "week": 201515, "impressions": 14217, "pooled_ctr": 0.0046423296054019835, "arms": [ { "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.", "impressions": 2039, "clicks": 18, "ctr": 0.0088278567925453, "ctr_select": 0.011764705882352941, "ctr_holdout": 0.005888125613346418, "n_select": 1020, "n_holdout": 1019, "eb": 0.007306573200067666, "labelled_winner": false }, { "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.", "impressions": 1983, "clicks": 6, "ctr": 0.0030257186081694, "ctr_select": 0.004032258064516129, "ctr_holdout": 0.0020181634712411706, "n_select": 992, "n_holdout": 991, "eb": 0.004951247534076134, "labelled_winner": false }, { "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.", "impressions": 2114, "clicks": 3, "ctr": 0.0014191106906338, "ctr_select": 0.000946073793755913, "ctr_holdout": 0.001892147587511826, "n_select": 1057, "n_holdout": 1057, "eb": 0.003966913119425191, "labelled_winner": false }, { "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.", "impressions": 2051, "clicks": 4, "ctr": 0.0019502681618722, "ctr_select": 0.0009746588693957114, "ctr_holdout": 0.002926829268292683, "n_select": 1026, "n_holdout": 1025, "eb": 0.00400369105737447, "labelled_winner": false }, { "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.", "impressions": 1950, "clicks": 7, "ctr": 0.0035897435897435, "ctr_select": 0.003076923076923077, "ctr_holdout": 0.0041025641025641026, "n_select": 975, "n_holdout": 975, "eb": 0.004673108186321323, "labelled_winner": false }, { "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.", "impressions": 2029, "clicks": 9, "ctr": 0.0044356826022671, "ctr_select": 0.004930966469428008, "ctr_holdout": 0.003940886699507389, "n_select": 1014, "n_holdout": 1015, "eb": 0.005218700536493161, "labelled_winner": false }, { "headline": "When a PSA from the 1970s can illustrate a problem still happening today. that's a PROBLEM.", "impressions": 2051, "clicks": 19, "ctr": 0.0092637737688932, "ctr_select": 0.011695906432748537, "ctr_holdout": 0.006829268292682927, "n_select": 1026, "n_holdout": 1025, "eb": 0.007293462134692454, "labelled_winner": false } ], "winner": 0, "pool_select": 0.005344585091420534, "pool_holdout": 0.003939777683973547, "lift_select": 1.2012383900928794, "lift_holdout": 0.4945324547876069, "eb_lift": 0.3670983013810818, "p_value": 0.0023766595616396833, "held_up": false, "z_path": [ -0.439, 0.648, 0.291, -0.719, -0.679, -0.196, 0.014, -0.172, -0.329, -0.147 ], "first_cross": null, "final_z": -0.147 }, { "id": "545d2b0674bfe7db21000063", "variation": "other", "week": 201444, "impressions": 14212, "pooled_ctr": 0.013720799324514494, "arms": [ { "headline": "If You've Ever Wondered What It's Like To Have ADD ... Here's The General Idea", "impressions": 3455, "clicks": 69, "ctr": 0.0199710564399421, "ctr_select": 0.022569444444444444, "ctr_holdout": 0.017371163867979156, "n_select": 1728, "n_holdout": 1727, "eb": 0.018427356224183487, "labelled_winner": false }, { "headline": "If You've Ever Wondered What It's Like To Have ADD ... Here's The General Idea", "impressions": 3598, "clicks": 39, "ctr": 0.0108393551973318, "ctr_select": 0.012229016120066704, "ctr_holdout": 0.009449694274596998, "n_select": 1799, "n_holdout": 1799, "eb": 0.0139800191580425, "labelled_winner": false }, { "headline": "If You've Ever Wondered What It's Like To Have ADD ... Here's The General Idea", "impressions": 3586, "clicks": 43, "ctr": 0.0119910764082543, "ctr_select": 0.012827663134411601, "ctr_holdout": 0.011154489682097044, "n_select": 1793, "n_holdout": 1793, "eb": 0.01424369218700942, "labelled_winner": false }, { "headline": "If You've Ever Wondered What It's Like To Have ADD ... Here's The General Idea", "impressions": 3573, "clicks": 44, "ctr": 0.0123145815841029, "ctr_select": 0.013997760358342666, "ctr_holdout": 0.010632344711807499, "n_select": 1786, "n_holdout": 1787, "eb": 0.014755354106681998, "labelled_winner": false } ], "winner": 0, "pool_select": 0.015339150014072614, "pool_holdout": 0.012102448634956376, "lift_select": 0.4713621304791029, "lift_holdout": 0.4353429121611614, "eb_lift": 0.20132837916557667, "p_value": 0.004936272800119368, "held_up": true, "z_path": [ 0.604, 0.713, 0.776, 1.44, 1.293, 1.569, 1.255, 1.717, 2.348, 2.551 ], "first_cross": 8, "final_z": 2.551 }, { "id": "54ad55df3236640022250000", "variation": "image", "week": 201501, "impressions": 14199, "pooled_ctr": 0.011761391647299106, "arms": [ { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2808, "clicks": 42, "ctr": 0.0149572649572649, "ctr_select": 0.01282051282051282, "ctr_holdout": 0.017094017094017096, "n_select": 1404, "n_holdout": 1404, "eb": 0.012467417149813951, "labelled_winner": false }, { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2868, "clicks": 43, "ctr": 0.0149930264993026, "ctr_select": 0.01394700139470014, "ctr_holdout": 0.01603905160390516, "n_select": 1434, "n_holdout": 1434, "eb": 0.012900814937566719, "labelled_winner": false }, { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2761, "clicks": 16, "ctr": 0.005795001810938, "ctr_select": 0.007246376811594203, "ctr_holdout": 0.004344677769732078, "n_select": 1380, "n_holdout": 1381, "eb": 0.010384828846623723, "labelled_winner": false }, { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2891, "clicks": 43, "ctr": 0.0148737461086129, "ctr_select": 0.017289073305670817, "ctr_holdout": 0.01245674740484429, "n_select": 1446, "n_holdout": 1445, "eb": 0.014188163338258787, "labelled_winner": false }, { "headline": "An Actress Reveals Why It\u2019s Hard To Make Films About Women Who Aren\u2019t Obsessed With Men", "impressions": 2871, "clicks": 23, "ctr": 0.0080111459421804, "ctr_select": 0.009749303621169917, "ctr_holdout": 0.0062717770034843206, "n_select": 1436, "n_holdout": 1435, "eb": 0.011295517413891351, "labelled_winner": false } ], "winner": 3, "pool_select": 0.012253521126760564, "pool_holdout": 0.011269192844062545, "lift_select": 0.4109473617271586, "lift_holdout": 0.10538062283737015, "eb_lift": 0.1578845942716942, "p_value": 0.051125448545898705, "held_up": false, "z_path": [ 0.697, 0.432, 0.584, 1.169, 1.138, 0.652, 0.738, 0.327, 0.201, -0.037 ], "first_cross": null, "final_z": -0.037 }, { "id": "5243023a2dec5eddc9006a29", "variation": "headline", "week": 201338, "impressions": 14190, "pooled_ctr": 0.0817477096546864, "arms": [ { "headline": "A Girl Is Hit By A Car And Is Dying. The Response She's Met With Is A Shrug And \"Not My Problem.\"", "impressions": 3627, "clicks": 342, "ctr": 0.0942928039702233, "ctr_select": 0.0926130099228225, "ctr_holdout": 0.09597352454495311, "n_select": 1814, "n_holdout": 1813, "eb": 0.08493679287552869, "labelled_winner": false }, { "headline": "A Young Girl Is Hit By A Car And Is Dying. The Response She's Met With Is: \"It's Not My Problem.\"", "impressions": 3540, "clicks": 326, "ctr": 0.0920903954802259, "ctr_select": 0.08870056497175141, "ctr_holdout": 0.09548022598870057, "n_select": 1770, "n_holdout": 1770, "eb": 0.0831600516294128, "labelled_winner": false }, { "headline": "A Young Girl Is Hit By A Car And Is Dying. Instead Of Saving Her, They Say \"It's Not My Problem.\"", "impressions": 3290, "clicks": 267, "ctr": 0.0811550151975683, "ctr_select": 0.0729483282674772, "ctr_holdout": 0.08936170212765958, "n_select": 1645, "n_holdout": 1645, "eb": 0.07644619265520684, "labelled_winner": false }, { "headline": "A Girl Is Hit By A Car And Is Dying. So Why Would Someone Shrug And Say \"It's Not My Problem\"?", "impressions": 3733, "clicks": 225, "ctr": 0.0602732386820251, "ctr_select": 0.06162915326902465, "ctr_holdout": 0.05891805034815212, "n_select": 1866, "n_holdout": 1867, "eb": 0.07121287750613461, "labelled_winner": false } ], "winner": 0, "pool_select": 0.07892882311486962, "pool_holdout": 0.08456659619450317, "lift_select": 0.1733737596471887, "lift_holdout": 0.13488692774407052, "eb_lift": 0.07611883116406437, "p_value": 0.012228451244630794, "held_up": true, "z_path": [ 1.264, 0.713, -0.049, 0.485, 0.159, 0.724, 0.775, 0.375, 0.408, 0.321 ], "first_cross": null, "final_z": 0.321 }, { "id": "52f82fbcab954f6fba000013", "variation": "other", "week": 201406, "impressions": 14145, "pooled_ctr": 0.0272180982679392, "arms": [ { "headline": "Please Don't Ever Ask This Question. These Ladies Tell Us Why.", "impressions": 3597, "clicks": 101, "ctr": 0.0280789546844592, "ctr_select": 0.029477196885428252, "ctr_holdout": 0.02668148971650917, "n_select": 1798, "n_holdout": 1799, "eb": 0.028721448609586846, "labelled_winner": false }, { "headline": "Please Don't Ever Ask This Question. These Ladies Tell Us Why.", "impressions": 3471, "clicks": 112, "ctr": 0.0322673581100547, "ctr_select": 0.03341013824884793, "ctr_holdout": 0.03112391930835735, "n_select": 1736, "n_holdout": 1735, "eb": 0.03039417566223379, "labelled_winner": true }, { "headline": "Please Don't Ever Ask This Question. These Ladies Tell Us Why.", "impressions": 3527, "clicks": 101, "ctr": 0.0286362347604196, "ctr_select": 0.031746031746031744, "ctr_holdout": 0.025524673851389675, "n_select": 1764, "n_holdout": 1763, "eb": 0.029695682749516217, "labelled_winner": false }, { "headline": "Please Don't Ever Ask This Question. These Ladies Tell Us Why.", "impressions": 3550, "clicks": 71, "ctr": 0.02, "ctr_select": 0.018028169014084508, "ctr_holdout": 0.021971830985915493, "n_select": 1775, "n_holdout": 1775, "eb": 0.0237517664708476, "labelled_winner": false } ], "winner": 1, "pool_select": 0.02813516188321787, "pool_holdout": 0.026300904977375566, "lift_select": 0.18748697404071057, "lift_holdout": 0.1833782653156084, "eb_lift": 0.08029147969336492, "p_value": 0.12599097812917776, "held_up": true, "z_path": [ 0.486, 1.365, 1.301, 0.826, 1.133, 1.344, 1.105, 0.574, 0.479, 0.884 ], "first_cross": null, "final_z": 0.884 }, { "id": "52b9d34721757196f500485b", "variation": "headline", "week": 201351, "impressions": 14075, "pooled_ctr": 0.013783303730017762, "arms": [ { "headline": "Parents Worrying About Their Kids Is Natural. So Is Trying To Make Money Off That Fear.", "impressions": 3406, "clicks": 42, "ctr": 0.0123311802701115, "ctr_select": 0.014092777451556078, "ctr_holdout": 0.010569583088667059, "n_select": 1703, "n_holdout": 1703, "eb": 0.014405244571863492, "labelled_winner": false }, { "headline": "Parents Worrying About Their Kids Is Natural. So Is A Corporation Getting Rich Off That Fear.", "impressions": 3623, "clicks": 64, "ctr": 0.0176649185757659, "ctr_select": 0.018211920529801324, "ctr_holdout": 0.017117614577581448, "n_select": 1812, "n_holdout": 1811, "eb": 0.016204366171812434, "labelled_winner": false }, { "headline": "It's Natural To Worry About Your Kid. But Don't Fall For Every Ad Claiming To Want To Help.", "impressions": 3552, "clicks": 33, "ctr": 0.0092905405405405, "ctr_select": 0.009572072072072071, "ctr_holdout": 0.009009009009009009, "n_select": 1776, "n_holdout": 1776, "eb": 0.012438420581750237, "labelled_winner": false }, { "headline": "What Kind Of Company Would Exploit Your Children For Profit? One That Could Make A Ton Of Money.", "impressions": 3494, "clicks": 55, "ctr": 0.0157412707498568, "ctr_select": 0.01659988551803091, "ctr_holdout": 0.014882655981682884, "n_select": 1747, "n_holdout": 1747, "eb": 0.015479412826457218, "labelled_winner": false } ], "winner": 1, "pool_select": 0.014634839443023587, "pool_holdout": 0.012931647008668466, "lift_select": 0.24442229794894876, "lift_holdout": 0.3236994921147325, "eb_lift": 0.10724591375937775, "p_value": 0.14116056121565634, "held_up": true, "z_path": [ -0.646, 1.006, 0.054, -0.271, -0.27, -0.272, 0.068, 0.246, 0.502, 0.633 ], "first_cross": null, "final_z": 0.633 }, { "id": "52bf66c349f74f8c140017b8", "variation": "headline", "week": 201352, "impressions": 13879, "pooled_ctr": 0.032351033936162545, "arms": [ { "headline": "Feel Like Having Your Teeny-Little Mind Blown? Just Start Watching. It Won't Take Long.", "impressions": 3245, "clicks": 90, "ctr": 0.0277349768875192, "ctr_select": 0.02651048088779285, "ctr_holdout": 0.02895871842267406, "n_select": 1622, "n_holdout": 1623, "eb": 0.030159762151385584, "labelled_winner": false }, { "headline": "Feel Like Having Your Teeny-Little Mind Blown? Just Start Watching. It Won't Take Long.", "impressions": 3577, "clicks": 129, "ctr": 0.036063740564719, "ctr_select": 0.035794183445190156, "ctr_holdout": 0.03633314700950251, "n_select": 1788, "n_holdout": 1789, "eb": 0.0340551973815093, "labelled_winner": true }, { "headline": "Feel Like Having Your Teeny-Little Mind Blown? Just Start Watching. It Won't Take Long.", "impressions": 3475, "clicks": 102, "ctr": 0.0293525179856115, "ctr_select": 0.02991944764096663, "ctr_holdout": 0.028785261945883708, "n_select": 1738, "n_holdout": 1737, "eb": 0.0315162408613627, "labelled_winner": false }, { "headline": "Halfway Through You'll Feel Smarter. By The End, You'll Be Picking Up Your Brain Pieces.", "impressions": 3582, "clicks": 128, "ctr": 0.0357342266890005, "ctr_select": 0.03796761585706309, "ctr_holdout": 0.03350083752093802, "n_select": 1791, "n_holdout": 1791, "eb": 0.03500387618147532, "labelled_winner": false } ], "winner": 3, "pool_select": 0.03271364749963972, "pool_holdout": 0.031988472622478385, "lift_select": 0.16060478604476125, "lift_holdout": 0.04727843421310762, "eb_lift": 0.07000835604959121, "p_value": 0.14672870169283841, "held_up": false, "z_path": [ -0.211, -0.43, -0.955, -1.056, 0.086, 0.574, -0.158, -0.078, -0.409, -0.075 ], "first_cross": null, "final_z": -0.075 }, { "id": "51c069a938e7046c89005a28", "variation": "headline", "week": 201324, "impressions": 13828, "pooled_ctr": 0.008244142319930576, "arms": [ { "headline": "Waaaaaiiiit, Shouldn't My Lunch Be Worth More Than Where It Came From?", "impressions": 3505, "clicks": 22, "ctr": 0.0062767475035663, "ctr_select": 0.00684931506849315, "ctr_holdout": 0.005704506560182544, "n_select": 1752, "n_holdout": 1753, "eb": 0.007726731210874642, "labelled_winner": false }, { "headline": "Waaaaaiiiit, Shouldn't My Lunch Be Worth More Than Where It Came From?", "impressions": 3448, "clicks": 34, "ctr": 0.009860788863109, "ctr_select": 0.009280742459396751, "ctr_holdout": 0.010440835266821345, "n_select": 1724, "n_holdout": 1724, "eb": 0.008769943440495396, "labelled_winner": false }, { "headline": "Holy Crap! Am I The Only One Who Thinks My Lunch Should Be Worth More Than Where It Came From?", "impressions": 3350, "clicks": 28, "ctr": 0.0083582089552238, "ctr_select": 0.008955223880597015, "ctr_holdout": 0.007761194029850746, "n_select": 1675, "n_holdout": 1675, "eb": 0.00862711450286913, "labelled_winner": false }, { "headline": "Holy Crap! Am I The Only One Who Thinks My Lunch Should Be Worth More Than Where It Came From?", "impressions": 3525, "clicks": 30, "ctr": 0.0085106382978723, "ctr_select": 0.00851305334846765, "ctr_holdout": 0.008508224617129892, "n_select": 1762, "n_holdout": 1763, "eb": 0.008443140342014748, "labelled_winner": false } ], "winner": 1, "pool_select": 0.008389989874150152, "pool_holdout": 0.008098336948662329, "lift_select": 0.10616849347947821, "lift_holdout": 0.2892567119655285, "eb_lift": 0.04528653455421838, "p_value": 0.6397690669597684, "held_up": true, "z_path": [ -0.03, 0.668, 0.782, -0.066, 0.098, 0.581, 0.712, 1.113, 0.428, 0.652 ], "first_cross": null, "final_z": 0.652 }, { "id": "52b8930dd45683a980004370", "variation": "image", "week": 201351, "impressions": 13713, "pooled_ctr": 0.03405527601545978, "arms": [ { "headline": "What's It Like Being An Unmarried Woman Over 30? Looks Pretty Cool To Me.", "impressions": 3467, "clicks": 129, "ctr": 0.0372079607730026, "ctr_select": 0.03863898500576701, "ctr_holdout": 0.03577611079053664, "n_select": 1734, "n_holdout": 1733, "eb": 0.037141749711242, "labelled_winner": false }, { "headline": "What's It Like Being An Unmarried Woman Over 30? Looks Pretty Cool To Me.", "impressions": 3522, "clicks": 147, "ctr": 0.0417376490630323, "ctr_select": 0.04258943781942078, "ctr_holdout": 0.04088586030664395, "n_select": 1761, "n_holdout": 1761, "eb": 0.03885729061780303, "labelled_winner": true }, { "headline": "What's It Like Being An Unmarried Woman Over 30? Looks Pretty Cool To Me.", "impressions": 3313, "clicks": 89, "ctr": 0.0268638696045879, "ctr_select": 0.029589371980676328, "ctr_holdout": 0.024140012070006035, "n_select": 1656, "n_holdout": 1657, "eb": 0.03334102186400039, "labelled_winner": false }, { "headline": "What's It Like Being An Unmarried Woman Over 30? Looks Pretty Cool To Me.", "impressions": 3411, "clicks": 102, "ctr": 0.0299032541776605, "ctr_select": 0.032825322391559206, "ctr_holdout": 0.026979472140762465, "n_select": 1706, "n_holdout": 1705, "eb": 0.0346664069446584, "labelled_winner": false } ], "winner": 1, "pool_select": 0.03602158378299548, "pool_holdout": 0.03208868144690782, "lift_select": 0.18233107339177468, "lift_holdout": 0.27415208301068605, "eb_lift": 0.07872243630070996, "p_value": 0.08621759607191591, "held_up": true, "z_path": [ 2.114, 1.324, 0.973, 0.71, 0.528, 0.774, 0.995, 0.842, 1.036, 0.972 ], "first_cross": 0, "final_z": 0.972 }, { "id": "53a30adbb5839f8d24000056", "variation": "image", "week": 201424, "impressions": 13398, "pooled_ctr": 0.005896402448126586, "arms": [ { "headline": "An Infographic About How Your Tax Dollars Are Spent That Will Tick You Off", "impressions": 3331, "clicks": 20, "ctr": 0.0060042029420594, "ctr_select": 0.007202881152460984, "ctr_holdout": 0.004804804804804805, "n_select": 1666, "n_holdout": 1665, "eb": 0.006833594175026042, "labelled_winner": false }, { "headline": "An Infographic About How Your Tax Dollars Are Spent That Will Tick You Off", "impressions": 3388, "clicks": 18, "ctr": 0.0053128689492325, "ctr_select": 0.007674144037780401, "ctr_holdout": 0.0029515938606847697, "n_select": 1694, "n_holdout": 1694, "eb": 0.007035169523578636, "labelled_winner": false }, { "headline": "An Infographic About How Your Tax Dollars Are Spent That Will Tick You Off", "impressions": 3314, "clicks": 22, "ctr": 0.0066385033192516, "ctr_select": 0.004224502112251056, "ctr_holdout": 0.009052504526252263, "n_select": 1657, "n_holdout": 1657, "eb": 0.005591110338666137, "labelled_winner": false }, { "headline": "An Infographic About How Your Tax Dollars Are Spent That Will Tick You Off", "impressions": 3365, "clicks": 19, "ctr": 0.0056463595839524, "ctr_select": 0.007134363852556481, "ctr_holdout": 0.0041592394533571005, "n_select": 1682, "n_holdout": 1683, "eb": 0.0068062577324059575, "labelled_winner": false } ], "winner": 1, "pool_select": 0.006568144499178982, "pool_holdout": 0.00522466039707419, "lift_select": 0.16838842975206614, "lift_holdout": -0.43506493506493504, "eb_lift": 0.07110455996484744, "p_value": 0.5144238242612851, "held_up": false, "z_path": [ -0.015, -0.028, -0.278, -0.509, -1.134, -0.669, -0.645, -0.978, -0.573, -0.378 ], "first_cross": null, "final_z": -0.378 }, { "id": "5359593d67617a1d64000029", "variation": "image", "week": 201416, "impressions": 13346, "pooled_ctr": 0.014536190618912034, "arms": [ { "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.", "impressions": 2248, "clicks": 51, "ctr": 0.0226868327402135, "ctr_select": 0.017793594306049824, "ctr_holdout": 0.027580071174377226, "n_select": 1124, "n_holdout": 1124, "eb": 0.014893586581392228, "labelled_winner": false }, { "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.", "impressions": 2090, "clicks": 12, "ctr": 0.0057416267942583, "ctr_select": 0.004784688995215311, "ctr_holdout": 0.0066985645933014355, "n_select": 1045, "n_holdout": 1045, "eb": 0.010782777976437981, "labelled_winner": false }, { "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.", "impressions": 2236, "clicks": 27, "ctr": 0.0120751341681574, "ctr_select": 0.01073345259391771, "ctr_holdout": 0.013416815742397137, "n_select": 1118, "n_holdout": 1118, "eb": 0.012591102730778259, "labelled_winner": false }, { "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.", "impressions": 2246, "clicks": 41, "ctr": 0.0182546749777382, "ctr_select": 0.018699910952804988, "ctr_holdout": 0.017809439002671415, "n_select": 1123, "n_holdout": 1123, "eb": 0.015188553961181062, "labelled_winner": false }, { "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.", "impressions": 2244, "clicks": 35, "ctr": 0.0155971479500891, "ctr_select": 0.016042780748663103, "ctr_holdout": 0.015151515151515152, "n_select": 1122, "n_holdout": 1122, "eb": 0.014320799960003138, "labelled_winner": false }, { "headline": "Hate Animal Cruelty And Don't Want To Watch A Video Of It? Watch This Instead.", "impressions": 2282, "clicks": 28, "ctr": 0.0122699386503067, "ctr_select": 0.012269938650306749, "ctr_holdout": 0.012269938650306749, "n_select": 1141, "n_holdout": 1141, "eb": 0.013085625462527957, "labelled_winner": false } ], "winner": 3, "pool_select": 0.013487187172186423, "pool_holdout": 0.015585194065637645, "lift_select": 0.38649450875630764, "lift_holdout": 0.14271525446948408, "eb_lift": 0.1261468953662359, "p_value": 0.09680028677081567, "held_up": false, "z_path": [ 0.381, -0.245, -0.412, -1.536, -1.355, -1.205, -1.318, -1.14, -1.319, -1.049 ], "first_cross": null, "final_z": -1.049 }, { "id": "5236b8bf9d26831210005b30", "variation": "headline", "week": 201337, "impressions": 13321, "pooled_ctr": 0.021394790180917347, "arms": [ { "headline": "A Young Thief Gets Caught And Receives Nothing But Generosity, What He Learns is Invaluable", "impressions": 3342, "clicks": 85, "ctr": 0.0254338719329742, "ctr_select": 0.022142429682824656, "ctr_holdout": 0.02872531418312388, "n_select": 1671, "n_holdout": 1671, "eb": 0.021925153631637197, "labelled_winner": false }, { "headline": "A Young Thief Gets Heart Punched With Generosity, Watch What Lesson He Learns", "impressions": 3240, "clicks": 79, "ctr": 0.0243827160493827, "ctr_select": 0.025308641975308643, "ctr_holdout": 0.02345679012345679, "n_select": 1620, "n_holdout": 1620, "eb": 0.023224944326385055, "labelled_winner": false }, { "headline": "Think Your Time Is Expensive? Learn The Value Of A Simple Act Of Kindness.", "impressions": 3236, "clicks": 54, "ctr": 0.0166872682323856, "ctr_select": 0.0203955500618047, "ctr_holdout": 0.012978986402966625, "n_select": 1618, "n_holdout": 1618, "eb": 0.02120407008546402, "labelled_winner": false }, { "headline": "Probably The Best Reason I've Seen To Release A Young Thief", "impressions": 3503, "clicks": 67, "ctr": 0.0191264630316871, "ctr_select": 0.019406392694063926, "ctr_holdout": 0.018846373500856654, "n_select": 1752, "n_holdout": 1751, "eb": 0.020751617464878604, "labelled_winner": false } ], "winner": 1, "pool_select": 0.021768503227743582, "pool_holdout": 0.021021021021021023, "lift_select": 0.16262664963814388, "lift_holdout": 0.1158730158730159, "eb_lift": 0.06690589074517828, "p_value": 0.2616853798033174, "held_up": false, "z_path": [ -1.287, -1.398, -1.444, -0.476, 0.148, 0.156, 1.044, 0.803, 0.525, -0.274 ], "first_cross": null, "final_z": -0.274 }, { "id": "52e6b42bacd398e056000001", "variation": "other", "week": 201404, "impressions": 13224, "pooled_ctr": 0.05565638233514821, "arms": [ { "headline": "The Amusing Response SNL Give To Victoria's Secret After They Ban A Mom From Breastfeeding In Store", "impressions": 3319, "clicks": 170, "ctr": 0.0512202470623681, "ctr_select": 0.05, "ctr_holdout": 0.05244122965641953, "n_select": 1660, "n_holdout": 1659, "eb": 0.05241458630933749, "labelled_winner": false }, { "headline": "The Amusing Response SNL Give To Victoria's Secret After They Ban A Mom From Breastfeeding In Store", "impressions": 3301, "clicks": 192, "ctr": 0.0581641926688882, "ctr_select": 0.052121212121212124, "ctr_holdout": 0.06420351302241066, "n_select": 1650, "n_holdout": 1651, "eb": 0.05330279557461928, "labelled_winner": false }, { "headline": "The Amusing Response SNL Give To Victoria's Secret After They Ban A Mom From Breastfeeding In Store", "impressions": 3332, "clicks": 177, "ctr": 0.0531212484993997, "ctr_select": 0.05342136854741897, "ctr_holdout": 0.05282112845138055, "n_select": 1666, "n_holdout": 1666, "eb": 0.05384178106749651, "labelled_winner": false }, { "headline": "The Amusing Response SNL Give To Victoria's Secret After They Ban A Mom From Breastfeeding In Store", "impressions": 3272, "clicks": 197, "ctr": 0.0602078239608801, "ctr_select": 0.061124694376528114, "ctr_holdout": 0.059290953545232276, "n_select": 1636, "n_holdout": 1636, "eb": 0.057032529165723246, "labelled_winner": false } ], "winner": 3, "pool_select": 0.05414398064125832, "pool_holdout": 0.05716878402903811, "lift_select": 0.12892871289833496, "lift_holdout": 0.03712112391818989, "eb_lift": 0.05334939341832978, "p_value": 0.15036714809725585, "held_up": false, "z_path": [ 1.618, 1.717, 0.295, -0.25, -0.748, -0.863, -0.58, 0.098, 0.397, 1.246 ], "first_cross": null, "final_z": 1.246 }, { "id": "543e5ef84b9f3dd0fa000026", "variation": "other", "week": 201441, "impressions": 13126, "pooled_ctr": 0.01973182995581289, "arms": [ { "headline": "Her Bracelet Came From The Bottom Of The Sea, And It\u2019s A Big Friggin\u2019 Red Flag For All Of Humanity", "impressions": 3389, "clicks": 63, "ctr": 0.018589554440838, "ctr_select": 0.017119244391971666, "ctr_holdout": 0.020058997050147492, "n_select": 1694, "n_holdout": 1695, "eb": 0.018056437156611232, "labelled_winner": false }, { "headline": "Her Bracelet Came From The Bottom Of The Sea, And It\u2019s A Big Friggin\u2019 Red Flag For All Of Humanity", "impressions": 3236, "clicks": 61, "ctr": 0.0188504326328801, "ctr_select": 0.0173053152039555, "ctr_holdout": 0.0203955500618047, "n_select": 1618, "n_holdout": 1618, "eb": 0.018151030098407576, "labelled_winner": false }, { "headline": "Her Bracelet Came From The Bottom Of The Sea, And It\u2019s A Big Friggin\u2019 Red Flag For All Of Humanity", "impressions": 3206, "clicks": 73, "ctr": 0.0227698066126013, "ctr_select": 0.020586400499064253, "ctr_holdout": 0.024953212726138492, "n_select": 1603, "n_holdout": 1603, "eb": 0.019495756432509906, "labelled_winner": false }, { "headline": "Her Bracelet Came From The Bottom Of The Sea, And It\u2019s A Big Friggin\u2019 Red Flag For All Of Humanity", "impressions": 3295, "clicks": 62, "ctr": 0.0188163884673748, "ctr_select": 0.020024271844660196, "ctr_holdout": 0.017607771706132362, "n_select": 1648, "n_holdout": 1647, "eb": 0.019274532046587155, "labelled_winner": false } ], "winner": 2, "pool_select": 0.018741429224440043, "pool_holdout": 0.020722230687185737, "lift_select": 0.09844346727933884, "lift_holdout": 0.2041759935415215, "eb_lift": 0.04024918265497979, "p_value": 0.5309373058246877, "held_up": true, "z_path": [ 0.959, 1.717, 1.008, 0.881, 1.198, 1.124, 0.854, 0.826, 1.606, 1.118 ], "first_cross": null, "final_z": 1.118 }, { "id": "52efd159f41748cff200001a", "variation": "other", "week": 201405, "impressions": 13113, "pooled_ctr": 0.014336917562724014, "arms": [ { "headline": "It's Amazing What People Can Do When They Expect Their Children To Live Past Kindergarten", "impressions": 3295, "clicks": 69, "ctr": 0.0209408194233687, "ctr_select": 0.02366504854368932, "ctr_holdout": 0.018214936247723135, "n_select": 1648, "n_holdout": 1647, "eb": 0.018390974546732667, "labelled_winner": false }, { "headline": "It's Amazing What People Can Do When They Expect Their Children To Live Past Kindergarten", "impressions": 3227, "clicks": 44, "ctr": 0.0136349550666253, "ctr_select": 0.014250309789343246, "ctr_holdout": 0.013019218846869188, "n_select": 1614, "n_holdout": 1613, "eb": 0.014480524654491601, "labelled_winner": false }, { "headline": "It's Amazing What People Can Do When They Expect Their Children To Live Past Kindergarten", "impressions": 3329, "clicks": 40, "ctr": 0.0120156203063983, "ctr_select": 0.010817307692307692, "ctr_holdout": 0.013213213213213212, "n_select": 1664, "n_holdout": 1665, "eb": 0.013042940323048615, "labelled_winner": false }, { "headline": "It's Amazing What People Can Do When They Expect Their Children To Live Past Kindergarten", "impressions": 3262, "clicks": 35, "ctr": 0.0107296137339055, "ctr_select": 0.009809932556713672, "ctr_holdout": 0.011649294911097487, "n_select": 1631, "n_holdout": 1631, "eb": 0.012645446494261128, "labelled_winner": false } ], "winner": 0, "pool_select": 0.014640841848406283, "pool_holdout": 0.014032946918852958, "lift_select": 0.6163721177184467, "lift_holdout": 0.29801219608774865, "eb_lift": 0.256141876072147, "p_value": 0.0004234271447894082, "held_up": true, "z_path": [ 2.121, 2.383, 2.035, 1.761, 1.362, 2.009, 1.992, 1.933, 2.07, 2.261 ], "first_cross": 0, "final_z": 2.261 }, { "id": "547cd788366131001b130000", "variation": "headline", "week": 201448, "impressions": 13053, "pooled_ctr": 0.004290201486248372, "arms": [ { "headline": "His Movie Is About To Be Released In Theaters. Here's Why It Means So Much To Him.", "impressions": 3296, "clicks": 17, "ctr": 0.0051577669902912, "ctr_select": 0.0054611650485436895, "ctr_holdout": 0.0048543689320388345, "n_select": 1648, "n_holdout": 1648, "eb": 0.004776619358260943, "labelled_winner": false }, { "headline": "At His Movie Screening, Someone Asked Him About His Anger. His Answer Was Brilliant.", "impressions": 3243, "clicks": 14, "ctr": 0.0043169904409497, "ctr_select": 0.004315659679408138, "ctr_holdout": 0.00431832202344232, "n_select": 1622, "n_holdout": 1621, "eb": 0.004300489452537783, "labelled_winner": false }, { "headline": "How A Camera Helps Make One Black Man A Better Husband, Father And American", "impressions": 3304, "clicks": 13, "ctr": 0.0039346246973365, "ctr_select": 0.0036319612590799033, "ctr_holdout": 0.00423728813559322, "n_select": 1652, "n_holdout": 1652, "eb": 0.004016081226123429, "labelled_winner": false }, { "headline": "Everyone Has A Coping Mechanism. It Just So Happens That His Makes Beautiful Art For The Rest Of Us.", "impressions": 3210, "clicks": 12, "ctr": 0.0037383177570093, "ctr_select": 0.003738317757009346, "ctr_holdout": 0.003738317757009346, "n_select": 1605, "n_holdout": 1605, "eb": 0.004064200105486682, "labelled_winner": false } ], "winner": 0, "pool_select": 0.004289872835912364, "pool_holdout": 0.004290530186944529, "lift_select": 0.27303658113730944, "lift_holdout": 0.13141470180305137, "eb_lift": 0.11346409112032774, "p_value": 0.40007606065102785, "held_up": true, "z_path": [ 1.723, 1.875, 1.711, 1.042, 0.744, 0.193, 0.388, 0.351, 0.15, 0.495 ], "first_cross": null, "final_z": 0.495 }, { "id": "53c7fbfa4273de563b000054", "variation": "headline", "week": 201428, "impressions": 13018, "pooled_ctr": 0.0202796128437548, "arms": [ { "headline": "These Sexy Young Pop Stars Wrote A Catchy Song. May Its Unsexy Message Never Ever Catch On.", "impressions": 3268, "clicks": 75, "ctr": 0.0229498164014687, "ctr_select": 0.020195838433292534, "ctr_holdout": 0.025703794369645042, "n_select": 1634, "n_holdout": 1634, "eb": 0.01889174740981986, "labelled_winner": false }, { "headline": "These Sexy Young Pop Stars Wrote A Catchy Song, But I Hope Its Unsexy Message Never Ever Catches On", "impressions": 3243, "clicks": 70, "ctr": 0.0215849522047486, "ctr_select": 0.02281134401972873, "ctr_holdout": 0.020357803824799507, "n_select": 1622, "n_holdout": 1621, "eb": 0.019964596467235123, "labelled_winner": false }, { "headline": "May The Unsexy Message Behind These Sexy Young Pop Stars' Catchy Song Never Ever Catch On", "impressions": 3259, "clicks": 60, "ctr": 0.0184105553850874, "ctr_select": 0.016564417177914112, "ctr_holdout": 0.020257826887661142, "n_select": 1630, "n_holdout": 1629, "eb": 0.017391016235949557, "labelled_winner": false }, { "headline": "Some Sexy Young Pop Stars Wrote A Song With Some Profoundly Unsexy Advice", "impressions": 3248, "clicks": 59, "ctr": 0.0181650246305418, "ctr_select": 0.012315270935960592, "ctr_holdout": 0.02401477832512315, "n_select": 1624, "n_holdout": 1624, "eb": 0.015641604074160174, "labelled_winner": false } ], "winner": 1, "pool_select": 0.017972350230414748, "pool_holdout": 0.02258758451137062, "lift_select": 0.2692465775079831, "lift_holdout": -0.09871709325309386, "eb_lift": 0.1108506239461593, "p_value": 0.09046724145172004, "held_up": false, "z_path": [ -0.772, -1.13, -0.454, -0.815, -0.34, -0.506, -1.078, -1.003, -0.849, -0.373 ], "first_cross": null, "final_z": -0.373 }, { "id": "547bd484346330001b090000", "variation": "other", "week": 201448, "impressions": 13014, "pooled_ctr": 0.013523897341324727, "arms": [ { "headline": "A Father Wrote His Kid 14 Things To Always Remember. And I Forget 9 Of Them Every Day.", "impressions": 3227, "clicks": 39, "ctr": 0.0120855283545088, "ctr_select": 0.01486988847583643, "ctr_holdout": 0.009299442033477991, "n_select": 1614, "n_holdout": 1613, "eb": 0.015162166775797372, "labelled_winner": false }, { "headline": "A Father Wrote His Kid 14 Things To Always Remember. And I Forget 9 Of Them Every Day.", "impressions": 3239, "clicks": 48, "ctr": 0.0148193887002161, "ctr_select": 0.012962962962962963, "ctr_holdout": 0.016676961087090797, "n_select": 1620, "n_holdout": 1619, "eb": 0.014377197173631846, "labelled_winner": false }, { "headline": "A Father Wrote His Kid 14 Things To Always Remember. And I Forget 9 Of Them Every Day.", "impressions": 3304, "clicks": 40, "ctr": 0.0121065375302663, "ctr_select": 0.01513317191283293, "ctr_holdout": 0.009079903147699757, "n_select": 1652, "n_holdout": 1652, "eb": 0.015268935022161551, "labelled_winner": false }, { "headline": "A Father Wrote His Kid 14 Things To Always Remember. And I Forget 9 Of Them Every Day.", "impressions": 3244, "clicks": 49, "ctr": 0.0151048088779284, "ctr_select": 0.018495684340320593, "ctr_holdout": 0.011713933415536375, "n_select": 1622, "n_holdout": 1622, "eb": 0.01665433777634867, "labelled_winner": true } ], "winner": 3, "pool_select": 0.015365703749231715, "pool_holdout": 0.01168152474638795, "lift_select": 0.2036991368680643, "lift_holdout": 0.0027743526510481953, "eb_lift": 0.08386430248477161, "p_value": 0.23690020250145705, "held_up": false, "z_path": [ 0.615, 0.041, 1.359, 0.884, 1.275, 1.052, 0.851, 0.925, 1.343, 1.048 ], "first_cross": null, "final_z": 1.048 }, { "id": "5398abbb326359ba6c00007f", "variation": "image", "week": 201423, "impressions": 12997, "pooled_ctr": 0.033007617142417484, "arms": [ { "headline": "What It Looks Like When We Judge People At First Glance", "impressions": 4376, "clicks": 132, "ctr": 0.0301645338208409, "ctr_select": 0.031992687385740404, "ctr_holdout": 0.0283363802559415, "n_select": 2188, "n_holdout": 2188, "eb": 0.032713970458522396, "labelled_winner": false }, { "headline": "What It Looks Like When We Judge People At First Glance", "impressions": 4144, "clicks": 135, "ctr": 0.03257722007722, "ctr_select": 0.032818532818532815, "ctr_holdout": 0.032335907335907334, "n_select": 2072, "n_holdout": 2072, "eb": 0.0331228415685521, "labelled_winner": false }, { "headline": "What It Looks Like When We Judge People At First Glance", "impressions": 4477, "clicks": 162, "ctr": 0.0361849452758543, "ctr_select": 0.035299374441465595, "ctr_holdout": 0.03707012058954891, "n_select": 2238, "n_holdout": 2239, "eb": 0.034330474000797105, "labelled_winner": true } ], "winner": 2, "pool_select": 0.03339489073561096, "pool_holdout": 0.03262040313894445, "lift_select": 0.0570291941043477, "lift_holdout": 0.1364090269409357, "eb_lift": 0.028015760632164044, "p_value": 0.535693647889956, "held_up": true, "z_path": [ 0.501, 1.392, 0.919, 1.022, 0.9, 0.644, 0.204, 0.745, 0.965, 0.918 ], "first_cross": null, "final_z": 0.918 }, { "id": "51ccb50a38cf80cec6000270", "variation": "both", "week": 201325, "impressions": 12903, "pooled_ctr": 0.009455165465395644, "arms": [ { "headline": "Sometimes We Inspire Our Kids In Ways We Don't Even Realize", "impressions": 3116, "clicks": 43, "ctr": 0.0137997432605905, "ctr_select": 0.01540436456996149, "ctr_holdout": 0.012195121951219513, "n_select": 1558, "n_holdout": 1558, "eb": 0.01258774380554559, "labelled_winner": true }, { "headline": "Sometimes We Inspire Our Kids In Ways We Don't Even Realize", "impressions": 3395, "clicks": 20, "ctr": 0.0058910162002945, "ctr_select": 0.005889281507656066, "ctr_holdout": 0.0058927519151443725, "n_select": 1698, "n_holdout": 1697, "eb": 0.008662576800423039, "labelled_winner": false }, { "headline": "Just When We Think We're Not Enough For Our Kids, It Turns Out We're More Than We Imagined", "impressions": 3315, "clicks": 32, "ctr": 0.0096530920060331, "ctr_select": 0.013268998793727383, "ctr_holdout": 0.006035003017501509, "n_select": 1658, "n_holdout": 1657, "eb": 0.011768070223030487, "labelled_winner": false }, { "headline": "Just When We Think We're Not Enough For Our Kids, It Turns Out We're More Than We Imagined", "impressions": 3077, "clicks": 27, "ctr": 0.0087747806304842, "ctr_select": 0.008452535760728219, "ctr_holdout": 0.009096816114359974, "n_select": 1538, "n_holdout": 1539, "eb": 0.009800117298682572, "labelled_winner": false } ], "winner": 0, "pool_select": 0.01069435833849969, "pool_holdout": 0.008215780499147418, "lift_select": 0.44041971312161654, "lift_holdout": 0.4843534284399449, "eb_lift": 0.17704526135333554, "p_value": 0.0379601069814802, "held_up": true, "z_path": [ 0.519, 1.192, 1.005, 0.004, 0.672, 0.796, 1.324, 1.145, 1.244, 1.548 ], "first_cross": null, "final_z": 1.548 }, { "id": "537e69c542275460e3000026", "variation": "headline", "week": 201420, "impressions": 12793, "pooled_ctr": 0.01438286562964121, "arms": [ { "headline": "The Reason These Girls Are Missing Up To 50 Days Of School A Year Shouldn't Be A Thing. Period.", "impressions": 2505, "clicks": 46, "ctr": 0.0183632734530938, "ctr_select": 0.01437699680511182, "ctr_holdout": 0.0223463687150838, "n_select": 1252, "n_holdout": 1253, "eb": 0.013266192293871151, "labelled_winner": false }, { "headline": "You Can't Stick A Band-Aid On Global Poverty, But You Can Give It A Maxi Pad", "impressions": 2533, "clicks": 22, "ctr": 0.0086853533359652, "ctr_select": 0.0071090047393364926, "ctr_holdout": 0.010260457774269928, "n_select": 1266, "n_holdout": 1267, "eb": 0.010702994320940589, "labelled_winner": false }, { "headline": "Half The World Goes Through It Every Month. Why's It Taking 5 Years Off These Girls' Lives?", "impressions": 2580, "clicks": 45, "ctr": 0.0174418604651162, "ctr_select": 0.017829457364341085, "ctr_holdout": 0.017054263565891473, "n_select": 1290, "n_holdout": 1290, "eb": 0.01451238448094321, "labelled_winner": false }, { "headline": "They're Spinning Banana Fibers Into Gold For Women And Girls. No, It's Not A Fairytale.", "impressions": 2637, "clicks": 47, "ctr": 0.0178232840348881, "ctr_select": 0.012898330804248861, "ctr_holdout": 0.022744503411675512, "n_select": 1318, "n_holdout": 1319, "eb": 0.012750316430758317, "labelled_winner": false }, { "headline": "Can These Banana Fibers Change The World?", "impressions": 2538, "clicks": 24, "ctr": 0.0094562647754137, "ctr_select": 0.01103230890464933, "ctr_holdout": 0.007880220646178092, "n_select": 1269, "n_holdout": 1269, "eb": 0.0120880820607349, "labelled_winner": false } ], "winner": 2, "pool_select": 0.012666145426114151, "pool_holdout": 0.01609878086902157, "lift_select": 0.407646664752608, "lift_holdout": 0.05935124557838489, "eb_lift": 0.14576171303249175, "p_value": 0.06344539675420757, "held_up": false, "z_path": [ 0.419, 0.953, 0.325, 0.991, 0.533, -0.242, 0.029, -0.0, -0.245, -0.248 ], "first_cross": null, "final_z": -0.248 }, { "id": "54b7e3856565620016530000", "variation": "headline", "week": 201502, "impressions": 12619, "pooled_ctr": 0.011886837308820033, "arms": [ { "headline": "Jay Walking Can Be Extremely Dangerous. So They Made Waiting For The Light More Interesting.", "impressions": 3119, "clicks": 29, "ctr": 0.0092978518756011, "ctr_select": 0.010897435897435897, "ctr_holdout": 0.007697241821680564, "n_select": 1560, "n_holdout": 1559, "eb": 0.011298792436400393, "labelled_winner": false }, { "headline": "If You Saw This At Your Crosswalk, You Might Not Make It Across. Point Made.", "impressions": 3094, "clicks": 44, "ctr": 0.0142210730446024, "ctr_select": 0.011635423400129283, "ctr_holdout": 0.01680672268907563, "n_select": 1547, "n_holdout": 1547, "eb": 0.011595551559448485, "labelled_winner": false }, { "headline": "The Traffic Light Started Dancing, And Things Got A Whole Lots Safer", "impressions": 3262, "clicks": 37, "ctr": 0.0113427345187001, "ctr_select": 0.009809932556713672, "ctr_holdout": 0.012875536480686695, "n_select": 1631, "n_holdout": 1631, "eb": 0.010842385185759414, "labelled_winner": false }, { "headline": "A Crosswalk Experiment Stops People In Their Tracks For All The Right Reasons", "impressions": 3144, "clicks": 40, "ctr": 0.0127226463104325, "ctr_select": 0.013994910941475827, "ctr_holdout": 0.011450381679389313, "n_select": 1572, "n_holdout": 1572, "eb": 0.012549381777820875, "labelled_winner": false } ], "winner": 3, "pool_select": 0.011568938193343899, "pool_holdout": 0.012204786812490093, "lift_select": 0.20969709644811596, "lift_holdout": -0.06181223356795873, "eb_lift": 0.08474793175410578, "p_value": 0.299256653523877, "held_up": false, "z_path": [ 0.886, 0.823, -0.047, -0.205, -0.496, -0.868, -0.708, -0.568, 0.044, -0.513 ], "first_cross": null, "final_z": -0.513 }, { "id": "53988492326359ba6c00003f", "variation": "headline", "week": 201423, "impressions": 12426, "pooled_ctr": 0.005391920167390954, "arms": [ { "headline": "Why Raising The Minimum Wage Is Also Critical To Our Children", "impressions": 3071, "clicks": 7, "ctr": 0.0022793878215564, "ctr_select": 0.0026041666666666665, "ctr_holdout": 0.001954397394136808, "n_select": 1536, "n_holdout": 1535, "eb": 0.004425983132926598, "labelled_winner": false }, { "headline": "It Used To Be That When You Got To A Certain Age, Your Income Began To Go Up. Ummm\u2026", "impressions": 3077, "clicks": 15, "ctr": 0.0048748781280467, "ctr_select": 0.003250975292587776, "ctr_holdout": 0.00649772579597141, "n_select": 1538, "n_holdout": 1539, "eb": 0.004683043750656792, "labelled_winner": false }, { "headline": "People Working At Minimum Wage Or Close To It Are Just High-School Kids, Right? In A Word, No.", "impressions": 3176, "clicks": 20, "ctr": 0.0062972292191435, "ctr_select": 0.00818639798488665, "ctr_holdout": 0.004408060453400504, "n_select": 1588, "n_holdout": 1588, "eb": 0.006671381917036867, "labelled_winner": false }, { "headline": "Hey, America! Don\u2019t Look Now, But We\u2019re Dooming An Entire Generation To Poverty.", "impressions": 3102, "clicks": 25, "ctr": 0.0080593165699548, "ctr_select": 0.008381689232753063, "ctr_holdout": 0.007736943907156673, "n_select": 1551, "n_holdout": 1551, "eb": 0.006735186625953432, "labelled_winner": false } ], "winner": 3, "pool_select": 0.005633349428617415, "pool_holdout": 0.005150490906164494, "lift_select": 0.4878695772312793, "lift_holdout": 0.5021760154738879, "eb_lift": 0.1955918430585335, "p_value": 0.0950199860421498, "held_up": true, "z_path": [ 0.848, 0.368, 0.823, 0.879, 0.456, 0.78, 0.569, 0.568, 0.556, 0.827 ], "first_cross": null, "final_z": 0.827 }, { "id": "52d4d061d4d15ffc2f002a66", "variation": "headline", "week": 201402, "impressions": 12369, "pooled_ctr": 0.0063060878001455255, "arms": [ { "headline": "I'm Glad We Can Always Rely On Local News ... For Laughs", "impressions": 3125, "clicks": 20, "ctr": 0.0064, "ctr_select": 0.005761843790012804, "ctr_holdout": 0.007037747920665387, "n_select": 1562, "n_holdout": 1563, "eb": 0.0058946476417575736, "labelled_winner": false }, { "headline": "Why Local News Is A Joke. A Really Unfunny One. =(", "impressions": 3133, "clicks": 28, "ctr": 0.0089371209703159, "ctr_select": 0.006385696040868455, "ctr_holdout": 0.01148691767708998, "n_select": 1566, "n_holdout": 1567, "eb": 0.006146063041125004, "labelled_winner": false }, { "headline": "Yuck. Do People Actually Call This News?", "impressions": 3090, "clicks": 18, "ctr": 0.0058252427184466, "ctr_select": 0.006472491909385114, "ctr_holdout": 0.005177993527508091, "n_select": 1545, "n_holdout": 1545, "eb": 0.006179476819197458, "labelled_winner": false }, { "headline": "The Teleprompter Your Newscaster Reads Has A Secret", "impressions": 3021, "clicks": 12, "ctr": 0.0039721946375372, "ctr_select": 0.005298013245033113, "ctr_holdout": 0.0026472534745201853, "n_select": 1510, "n_holdout": 1511, "eb": 0.0057134738981285986, "labelled_winner": false } ], "winner": 2, "pool_select": 0.005984150088953582, "pool_holdout": 0.00662786938247656, "lift_select": 0.0816058777223827, "lift_holdout": -0.21875443997158417, "eb_lift": 0.03264068035399692, "p_value": 0.7738375961909268, "held_up": false, "z_path": [ 1.75, 1.158, 0.03, -1.499, -2.161, -1.882, -2.3, -2.307, -1.944, -1.433 ], "first_cross": 4, "final_z": -1.433 }, { "id": "5441525e5c55ba2e07000045", "variation": "headline", "week": 201441, "impressions": 12360, "pooled_ctr": 0.017233009708737864, "arms": [ { "headline": "An Albino Model Had To Explain Why He's Black. And Not. At The Same Time.", "impressions": 3030, "clicks": 67, "ctr": 0.0221122112211221, "ctr_select": 0.02046204620462046, "ctr_holdout": 0.023762376237623763, "n_select": 1515, "n_holdout": 1515, "eb": 0.01963467529720976, "labelled_winner": false }, { "headline": "A Model Explains Why He's Black \u2014 And Not Black \u2014 At The Same Time", "impressions": 3136, "clicks": 43, "ctr": 0.0137117346938775, "ctr_select": 0.01594387755102041, "ctr_holdout": 0.011479591836734694, "n_select": 1568, "n_holdout": 1568, "eb": 0.017822738722124337, "labelled_winner": false }, { "headline": "An Albino Model Has Had To Explain Why He's Black. And Not. At The Same Time.", "impressions": 3070, "clicks": 59, "ctr": 0.0192182410423452, "ctr_select": 0.02280130293159609, "ctr_holdout": 0.015635179153094463, "n_select": 1535, "n_holdout": 1535, "eb": 0.020570981950523568, "labelled_winner": false }, { "headline": "A Model Has Had To Explain Why He's Black. And Not. At The Same Time.", "impressions": 3124, "clicks": 44, "ctr": 0.0140845070422535, "ctr_select": 0.017285531370038413, "ctr_holdout": 0.01088348271446863, "n_select": 1562, "n_holdout": 1562, "eb": 0.018365805886820206, "labelled_winner": false } ], "winner": 2, "pool_select": 0.019093851132686083, "pool_holdout": 0.015372168284789644, "lift_select": 0.19416993319715137, "lift_holdout": 0.017109549117092504, "eb_lift": 0.07736159706979362, "p_value": 0.2208574281336798, "held_up": false, "z_path": [ -1.543, -1.62, -1.641, -1.169, -1.474, -1.341, -1.441, -1.35, -1.125, -0.795 ], "first_cross": null, "final_z": -0.795 }, { "id": "53b1ee6cd98e8a020400000f", "variation": "headline", "week": 201426, "impressions": 12351, "pooled_ctr": 0.01659784632823253, "arms": [ { "headline": "If They Stopped Walking, They'd Be Shot. So Her 4 Closest Friends Formed A Chair And Carried Her.", "impressions": 3006, "clicks": 57, "ctr": 0.0189620758483033, "ctr_select": 0.015968063872255488, "ctr_holdout": 0.021956087824351298, "n_select": 1503, "n_holdout": 1503, "eb": 0.016989459002276885, "labelled_winner": false }, { "headline": "She Was Forced Into A Death March. She Survived By Leaning On Her Best Friends ... Literally.", "impressions": 3138, "clicks": 56, "ctr": 0.0178457616316124, "ctr_select": 0.0165710643722116, "ctr_holdout": 0.019120458891013385, "n_select": 1569, "n_holdout": 1569, "eb": 0.01721553670953002, "labelled_winner": false }, { "headline": "If They Stopped Walking, They'd Be Shot. But The Girls Used Their Unbreakable Bond To Survive.", "impressions": 3089, "clicks": 43, "ctr": 0.0139203625768857, "ctr_select": 0.019430051813471502, "ctr_holdout": 0.008414239482200648, "n_select": 1544, "n_holdout": 1545, "eb": 0.018362801139995836, "labelled_winner": false }, { "headline": "If The Girl Collapsed, Her Captors Would Shoot Her. But Her Best Friends Came To The Rescue.", "impressions": 3118, "clicks": 49, "ctr": 0.0157152020525978, "ctr_select": 0.01860166773572803, "ctr_holdout": 0.012828736369467608, "n_select": 1559, "n_holdout": 1559, "eb": 0.01803379895455969, "labelled_winner": false } ], "winner": 2, "pool_select": 0.017651821862348177, "pool_holdout": 0.015544041450777202, "lift_select": 0.10073917383657371, "lift_holdout": -0.4586839266450916, "eb_lift": 0.040277954490589796, "p_value": 0.5400585915233771, "held_up": false, "z_path": [ -0.765, -0.638, 0.026, -1.099, -1.043, 0.036, -0.457, -0.196, -0.837, -0.585 ], "first_cross": null, "final_z": -0.585 }, { "id": "544104243637f0339500002e", "variation": "headline", "week": 201441, "impressions": 12314, "pooled_ctr": 0.007227545882735098, "arms": [ { "headline": "Listen To Some Of The Gross Things We Used To Do To Cure Diseases Before This Guy Invented Vaccines", "impressions": 3038, "clicks": 25, "ctr": 0.0082290980908492, "ctr_select": 0.008558262014483212, "ctr_holdout": 0.007899934167215274, "n_select": 1519, "n_holdout": 1519, "eb": 0.008195906277052547, "labelled_winner": false }, { "headline": "Listen To Some Of The Icky Things We Used To Do To Cure Diseases Before This Guy Invented Vaccines", "impressions": 3123, "clicks": 21, "ctr": 0.0067243035542747, "ctr_select": 0.0076824583866837385, "ctr_holdout": 0.005765534913516977, "n_select": 1562, "n_holdout": 1561, "eb": 0.007847316239148864, "labelled_winner": false }, { "headline": "Hear Some Of The Astonishing Things We Used To Do To Cure Diseases Before This Guy Invented Vaccines", "impressions": 3117, "clicks": 22, "ctr": 0.0070580686557587, "ctr_select": 0.00834403080872914, "ctr_holdout": 0.005772931366260423, "n_select": 1558, "n_holdout": 1559, "eb": 0.008113433295627073, "labelled_winner": false }, { "headline": "Hear Some Of The Ridiculous Things We Used To Do To Cure Diseases Before This Guy Invented Vaccines", "impressions": 3036, "clicks": 21, "ctr": 0.0069169960474308, "ctr_select": 0.007246376811594203, "ctr_holdout": 0.006587615283267457, "n_select": 1518, "n_holdout": 1518, "eb": 0.007676625497621855, "labelled_winner": false } ], "winner": 0, "pool_select": 0.007958421309079098, "pool_holdout": 0.0064966704563911, "lift_select": 0.0753718208810843, "lift_holdout": 0.21599736668861103, "eb_lift": 0.029840713220663728, "p_value": 0.7617765873966197, "held_up": true, "z_path": [ -0.42, 0.045, -0.952, -0.598, 0.064, -0.125, 0.247, 0.399, 0.388, 0.528 ], "first_cross": null, "final_z": 0.528 }, { "id": "543c0908a9b5dab1e4000021", "variation": "image", "week": 201441, "impressions": 12303, "pooled_ctr": 0.01715028854750874, "arms": [ { "headline": "They Were Shooting A Beautiful Video From Space. Then They Sped It Up. Just WOW.", "impressions": 3055, "clicks": 49, "ctr": 0.0160392798690671, "ctr_select": 0.01767015706806283, "ctr_holdout": 0.014407334643091027, "n_select": 1528, "n_holdout": 1527, "eb": 0.018288493354124375, "labelled_winner": false }, { "headline": "They Were Shooting A Beautiful Video From Space. Then They Sped It Up. Just WOW.", "impressions": 3037, "clicks": 76, "ctr": 0.0250246954231149, "ctr_select": 0.025691699604743084, "ctr_holdout": 0.024358130348913758, "n_select": 1518, "n_holdout": 1519, "eb": 0.021464678202795396, "labelled_winner": false }, { "headline": "They Were Shooting A Beautiful Video From Space. Then They Sped It Up. Just WOW.", "impressions": 3134, "clicks": 48, "ctr": 0.0153158902361199, "ctr_select": 0.016592214422463305, "ctr_holdout": 0.014039566049776643, "n_select": 1567, "n_holdout": 1567, "eb": 0.017847467727145197, "labelled_winner": false }, { "headline": "They Were Shooting A Beautiful Video From Space. Then They Sped It Up. Just WOW.", "impressions": 3077, "clicks": 38, "ctr": 0.0123496912577185, "ctr_select": 0.01495448634590377, "ctr_holdout": 0.009746588693957114, "n_select": 1538, "n_holdout": 1539, "eb": 0.017203635241800415, "labelled_winner": false } ], "winner": 1, "pool_select": 0.01869614696797269, "pool_holdout": 0.015604681404421327, "lift_select": 0.37417081972847566, "lift_holdout": 0.5609501865262232, "eb_lift": 0.1480803097860388, "p_value": 0.020417897701711415, "held_up": true, "z_path": [ -1.669, 0.623, 1.332, 2.31, 1.99, 1.985, 2.346, 2.631, 2.492, 2.474 ], "first_cross": 3, "final_z": 2.474 }, { "id": "540e19a64c8041124a00000d", "variation": "headline", "week": 201436, "impressions": 12301, "pooled_ctr": 0.0042272985936102755, "arms": [ { "headline": "These People Are Using Their Bodies To Share Some Really Great Words About Sex", "impressions": 3108, "clicks": 10, "ctr": 0.0032175032175032, "ctr_select": 0.002574002574002574, "ctr_holdout": 0.003861003861003861, "n_select": 1554, "n_holdout": 1554, "eb": 0.003758588800317962, "labelled_winner": false }, { "headline": "75 People Use Their Bodies To Share Some Terrific Messages About Sex", "impressions": 3082, "clicks": 18, "ctr": 0.0058403634003893, "ctr_select": 0.007138221933809215, "ctr_holdout": 0.004542504866969501, "n_select": 1541, "n_holdout": 1541, "eb": 0.005585335074310531, "labelled_winner": false }, { "headline": "They Want Us To Talk About Sex, And They're Using Their Bodies To Do It", "impressions": 3094, "clicks": 19, "ctr": 0.0061409179056237, "ctr_select": 0.007110536522301228, "ctr_holdout": 0.005171299288946348, "n_select": 1547, "n_holdout": 1547, "eb": 0.005576663759499626, "labelled_winner": false }, { "headline": "These People Want Women To Get Equal Treatment For Sexual Health. You'll Love Where They Put It.", "impressions": 3017, "clicks": 5, "ctr": 0.0016572754391779, "ctr_select": 0.001326259946949602, "ctr_holdout": 0.0019880715705765406, "n_select": 1508, "n_holdout": 1509, "eb": 0.003281001427700372, "labelled_winner": false } ], "winner": 1, "pool_select": 0.0045528455284552845, "pool_holdout": 0.003901804584620387, "lift_select": 0.5678594604616669, "lift_holdout": 0.16420614319705829, "eb_lift": 0.22677895382177726, "p_value": 0.08160632319437633, "held_up": false, "z_path": [ 0.582, 0.004, -0.703, -1.063, -0.22, 0.011, 0.011, -0.158, -0.153, -0.153 ], "first_cross": null, "final_z": -0.153 }, { "id": "53e9508caa1cffe874000027", "variation": "headline", "week": 201432, "impressions": 12300, "pooled_ctr": 0.007723577235772358, "arms": [ { "headline": "Big Companies Are Not Even Cuddling After They Screw Us. Geez.", "impressions": 3064, "clicks": 18, "ctr": 0.0058746736292428, "ctr_select": 0.0045691906005221935, "ctr_holdout": 0.007180156657963447, "n_select": 1532, "n_holdout": 1532, "eb": 0.006516461546893176, "labelled_winner": false }, { "headline": "This Whole Tomfoolery Thing Happening Between You And Big Information Companies Is, Well, Insulting", "impressions": 3110, "clicks": 25, "ctr": 0.0080385852090032, "ctr_select": 0.00964630225080386, "ctr_holdout": 0.006430868167202572, "n_select": 1555, "n_holdout": 1555, "eb": 0.008543503671825317, "labelled_winner": false }, { "headline": "Everything I Hoped Wasn't True Is True. The Deal Is Sealed And Now We Have To Change It.", "impressions": 3075, "clicks": 31, "ctr": 0.0100813008130081, "ctr_select": 0.011053315994798439, "ctr_holdout": 0.009108653220559532, "n_select": 1538, "n_holdout": 1537, "eb": 0.009099885350773504, "labelled_winner": false }, { "headline": "Companies Are Trying To Keep You In The Slow Lane. You Can Stop It. So, Um, Let's Stop It.", "impressions": 3051, "clicks": 21, "ctr": 0.0068829891838741, "ctr_select": 0.005897771952817824, "ctr_holdout": 0.007868852459016393, "n_select": 1526, "n_holdout": 1525, "eb": 0.007046966005353298, "labelled_winner": false } ], "winner": 2, "pool_select": 0.007803609169240774, "pool_holdout": 0.007643519271426248, "lift_select": 0.4164363892501084, "lift_holdout": 0.19168316283447995, "eb_lift": 0.166112391512663, "p_value": 0.0944327611699148, "held_up": true, "z_path": [ -1.629, -0.929, -0.671, -0.797, -0.875, -0.315, 0.034, 0.63, 0.181, 0.848 ], "first_cross": null, "final_z": 0.848 }, { "id": "53ed1533441a0bc7d2000046", "variation": "other", "week": 201432, "impressions": 12271, "pooled_ctr": 0.01026811180832858, "arms": [ { "headline": "This Kind Of Conversation Often Happens In Private \u2014 But Turns Out, It's Not So Rare", "impressions": 3127, "clicks": 41, "ctr": 0.0131116085705148, "ctr_select": 0.015345268542199489, "ctr_holdout": 0.010876519513755598, "n_select": 1564, "n_holdout": 1563, "eb": 0.012020406219054282, "labelled_winner": true }, { "headline": "This Kind Of Conversation Often Happens In Private \u2014 But Turns Out, It's Not So Rare", "impressions": 3090, "clicks": 30, "ctr": 0.0097087378640776, "ctr_select": 0.009061488673139158, "ctr_holdout": 0.010355987055016181, "n_select": 1545, "n_holdout": 1545, "eb": 0.009490668447932626, "labelled_winner": false }, { "headline": "This Kind Of Conversation Often Happens In Private \u2014 But Turns Out, It's Not So Rare", "impressions": 3035, "clicks": 29, "ctr": 0.0095551894563426, "ctr_select": 0.00922266139657444, "ctr_holdout": 0.00988793671720501, "n_select": 1518, "n_holdout": 1517, "eb": 0.009557474559317417, "labelled_winner": false }, { "headline": "This Kind Of Conversation Often Happens In Private \u2014 But Turns Out, It's Not So Rare", "impressions": 3019, "clicks": 26, "ctr": 0.0086121232196091, "ctr_select": 0.005298013245033113, "ctr_holdout": 0.011928429423459244, "n_select": 1510, "n_holdout": 1509, "eb": 0.00800992805470346, "labelled_winner": false } ], "winner": 0, "pool_select": 0.009776763891152028, "pool_holdout": 0.010759700032605152, "lift_select": 0.5695652173913044, "lift_holdout": 0.010857131778436946, "eb_lift": 0.2294872161056023, "p_value": 0.009519599171415954, "held_up": false, "z_path": [ 0.67, 1.242, 1.505, 2.124, 1.817, 2.273, 2.096, 1.564, 1.063, 1.317 ], "first_cross": 3, "final_z": 1.317 }, { "id": "53ed3cc90d60d0875600006e", "variation": "headline", "week": 201432, "impressions": 12271, "pooled_ctr": 0.004726591149865536, "arms": [ { "headline": "Another Unarmed Black Kid Was Shot By A Cop. These Stats Explain Why, Yes It's 'A Racial Thing'.", "impressions": 3038, "clicks": 8, "ctr": 0.0026333113890717, "ctr_select": 0.0026333113890717576, "ctr_holdout": 0.0026333113890717576, "n_select": 1519, "n_holdout": 1519, "eb": 0.004193463238629587, "labelled_winner": false }, { "headline": "How Does One City Turn Into A Racially Charged Powder Keg? Well, Uh, Just Look At The Numbers.", "impressions": 3141, "clicks": 10, "ctr": 0.003183699458771, "ctr_select": 0.0025477707006369425, "ctr_holdout": 0.0038192234245703373, "n_select": 1570, "n_holdout": 1571, "eb": 0.0041384521134294, "labelled_winner": false }, { "headline": "The Numbers Don't Lie. Here's What It's Like To Live In One Of Most Segregated Cities In The US.", "impressions": 3019, "clicks": 23, "ctr": 0.0076184166942696, "ctr_select": 0.007947019867549669, "ctr_holdout": 0.00728959575878065, "n_select": 1510, "n_holdout": 1509, "eb": 0.006293352464316464, "labelled_winner": false }, { "headline": "Here Are Some Statics That Give Context To Why Ferguson Residents Are So Upset About Mike Brown", "impressions": 3073, "clicks": 17, "ctr": 0.0055320533680442, "ctr_select": 0.0078125, "ctr_holdout": 0.0032530904359141183, "n_select": 1536, "n_holdout": 1537, "eb": 0.006250892682409995, "labelled_winner": false } ], "winner": 2, "pool_select": 0.0052159739201303994, "pool_holdout": 0.00423728813559322, "lift_select": 0.5235927152317881, "lift_holdout": 0.7203445990722332, "eb_lift": 0.20655366776817208, "p_value": 0.08972994717979814, "held_up": true, "z_path": [ -0.984, -0.305, 0.335, 0.038, 0.699, 0.246, 0.607, 0.575, 0.548, 1.008 ], "first_cross": null, "final_z": 1.008 }, { "id": "543b622147df54d1ac00002f", "variation": "image", "week": 201441, "impressions": 12257, "pooled_ctr": 0.018193685241086725, "arms": [ { "headline": "The Dirty Little Secret About What It Means When Men Are Silent", "impressions": 3012, "clicks": 43, "ctr": 0.0142762284196547, "ctr_select": 0.01593625498007968, "ctr_holdout": 0.012616201859229747, "n_select": 1506, "n_holdout": 1506, "eb": 0.016957535139036268, "labelled_winner": false }, { "headline": "The Dirty Little Secret About What It Means When Men Are Silent", "impressions": 3120, "clicks": 62, "ctr": 0.0198717948717948, "ctr_select": 0.019230769230769232, "ctr_holdout": 0.020512820512820513, "n_select": 1560, "n_holdout": 1560, "eb": 0.018268697970635, "labelled_winner": false }, { "headline": "The Dirty Little Secret About What It Means When Men Are Silent", "impressions": 3090, "clicks": 68, "ctr": 0.0220064724919093, "ctr_select": 0.018770226537216828, "ctr_holdout": 0.02524271844660194, "n_select": 1545, "n_holdout": 1545, "eb": 0.01808075431416675, "labelled_winner": false }, { "headline": "The Dirty Little Secret About What It Means When Men Are Silent", "impressions": 3035, "clicks": 50, "ctr": 0.0164744645799011, "ctr_select": 0.016469038208168644, "ctr_holdout": 0.016479894528675015, "n_select": 1518, "n_holdout": 1517, "eb": 0.01716519216512809, "labelled_winner": false } ], "winner": 1, "pool_select": 0.01762114537444934, "pool_holdout": 0.018766318537859008, "lift_select": 0.09134615384615374, "lift_holdout": 0.0930657748049053, "eb_lift": 0.03674860983353612, "p_value": 0.575720080785344, "held_up": false, "z_path": [ -1.25, -1.563, -0.741, -1.416, -0.171, -0.043, -0.258, -0.051, -0.513, -0.588 ], "first_cross": null, "final_z": -0.588 }, { "id": "52e91de0b06959e12f00000b", "variation": "headline", "week": 201404, "impressions": 12250, "pooled_ctr": 0.012489795918367347, "arms": [ { "headline": "PRETTY URGENT STUFF: How One Big Oil Company Is Trying To Make It A Crime For People To Speak Up", "impressions": 3047, "clicks": 49, "ctr": 0.016081391532655, "ctr_select": 0.015091863517060367, "ctr_holdout": 0.017071569271175313, "n_select": 1524, "n_holdout": 1523, "eb": 0.013373082198286484, "labelled_winner": false }, { "headline": "THIS IS NOT GOOD: How One Big Oil Company Is Trying To Make It A Crime For People To Speak Up", "impressions": 3174, "clicks": 31, "ctr": 0.0097668557025834, "ctr_select": 0.010081915563957152, "ctr_holdout": 0.00945179584120983, "n_select": 1587, "n_holdout": 1587, "eb": 0.011364602983830413, "labelled_winner": false }, { "headline": "UH OH: How One Big Oil Company Is Trying To Make It A Crime For People To Speak Up", "impressions": 3086, "clicks": 40, "ctr": 0.0129617627997407, "ctr_select": 0.012313674659753726, "ctr_holdout": 0.013609850939727802, "n_select": 1543, "n_holdout": 1543, "eb": 0.012271185886471893, "labelled_winner": false }, { "headline": "AWFUL STUFF ALERT: How One Big Oil Company Is Trying To Make It A Crime For People To Speak Up", "impressions": 2943, "clicks": 33, "ctr": 0.0112130479102956, "ctr_select": 0.01154891304347826, "ctr_holdout": 0.010876954452753228, "n_select": 1472, "n_holdout": 1471, "eb": 0.011973340018533606, "labelled_winner": false } ], "winner": 0, "pool_select": 0.012242899118511263, "pool_holdout": 0.012736773350751143, "lift_select": 0.23270341207349077, "lift_holdout": 0.3403370540599695, "eb_lift": 0.09231335395604012, "p_value": 0.2432574415910621, "held_up": true, "z_path": [ 0.326, 0.242, 1.509, 1.494, 0.902, 0.969, 1.292, 1.352, 0.938, 1.021 ], "first_cross": null, "final_z": 1.021 }, { "id": "53d9554b45ba39bd63000056", "variation": "headline", "week": 201430, "impressions": 12249, "pooled_ctr": 0.01665442076904237, "arms": [ { "headline": "When It\u2019s Appropriate To Use The Word \u2018Rape\u2019 And Other Facts About Rape Culture", "impressions": 3061, "clicks": 44, "ctr": 0.01437438745508, "ctr_select": 0.01241830065359477, "ctr_holdout": 0.016329196603527107, "n_select": 1530, "n_holdout": 1531, "eb": 0.015265868857133958, "labelled_winner": false }, { "headline": "There's Only One Cause For Sexual Assault", "impressions": 3115, "clicks": 38, "ctr": 0.012199036918138, "ctr_select": 0.012195121951219513, "ctr_holdout": 0.012202954399486191, "n_select": 1558, "n_holdout": 1557, "eb": 0.01515558050118487, "labelled_winner": false }, { "headline": "\u2018What Are Women\u2019s Bodies For?' Plus 3 Other Thoughts On Rape Culture", "impressions": 3000, "clicks": 42, "ctr": 0.014, "ctr_select": 0.017333333333333333, "ctr_holdout": 0.010666666666666666, "n_select": 1500, "n_holdout": 1500, "eb": 0.017219395993832355, "labelled_winner": false }, { "headline": "3 Facts About Rape Culture That Are Totally Worth Sharing", "impressions": 3073, "clicks": 80, "ctr": 0.0260331923202082, "ctr_select": 0.026692708333333332, "ctr_holdout": 0.025374105400130124, "n_select": 1536, "n_holdout": 1537, "eb": 0.020950903456633505, "labelled_winner": false } ], "winner": 3, "pool_select": 0.01714565643370346, "pool_holdout": 0.01616326530612245, "lift_select": 0.5568204365079366, "lift_holdout": 0.5698625815737071, "eb_lift": 0.22193650255641528, "p_value": 0.0008683425416905227, "held_up": true, "z_path": [ 0.96, 1.92, 1.988, 2.839, 3.036, 3.079, 3.457, 3.674, 3.682, 3.342 ], "first_cross": 2, "final_z": 3.342 }, { "id": "54f36ac739383600191f0000", "variation": "headline", "week": 201509, "impressions": 12249, "pooled_ctr": 0.0032655726998122294, "arms": [ { "headline": "Think you're not capable of changing the world? Maybe you should watch this.", "impressions": 3089, "clicks": 10, "ctr": 0.0032372936225315, "ctr_select": 0.0045336787564766836, "ctr_holdout": 0.001941747572815534, "n_select": 1544, "n_holdout": 1545, "eb": 0.004360775512356986, "labelled_winner": false }, { "headline": "A solid reality check about your place in the world.", "impressions": 3042, "clicks": 8, "ctr": 0.0026298487836949, "ctr_select": 0.0019723865877712033, "ctr_holdout": 0.003287310979618672, "n_select": 1521, "n_holdout": 1521, "eb": 0.0033448835450087928, "labelled_winner": false }, { "headline": "Listen to this 2-minute reality check about what you have to offer.", "impressions": 3098, "clicks": 14, "ctr": 0.0045190445448676, "ctr_select": 0.006455777921239509, "ctr_holdout": 0.0025823111684958036, "n_select": 1549, "n_holdout": 1549, "eb": 0.005130992464892195, "labelled_winner": false }, { "headline": "\"People think that if they can't tackle all of the world problems ... there's no purpose.\" NOT TRUE!", "impressions": 3020, "clicks": 8, "ctr": 0.0026490066225165, "ctr_select": 0.003973509933774834, "ctr_holdout": 0.0013245033112582781, "n_select": 1510, "n_holdout": 1510, "eb": 0.004138256991223857, "labelled_winner": false } ], "winner": 2, "pool_select": 0.004245591116917048, "pool_holdout": 0.002285714285714286, "lift_select": 0.5205839996027213, "lift_holdout": 0.12976113621691399, "eb_lift": 0.20854607134614622, "p_value": 0.12165613302743672, "held_up": false, "z_path": [ 0.999, 0.575, -0.003, 0.444, 0.813, 1.262, 0.827, 0.465, 0.442, 0.811 ], "first_cross": null, "final_z": 0.811 }, { "id": "54cab7bb383837000c320000", "variation": "other", "week": 201504, "impressions": 12211, "pooled_ctr": 0.0258783064450086, "arms": [ { "headline": "After Seeing His Mom Harassed, This Little Kid Decided To Send A Big Message Out To The World", "impressions": 3070, "clicks": 91, "ctr": 0.0296416938110749, "ctr_select": 0.028013029315960912, "ctr_holdout": 0.031270358306188926, "n_select": 1535, "n_holdout": 1535, "eb": 0.02820538871856726, "labelled_winner": false }, { "headline": "After Seeing His Mom Harassed, This Little Kid Decided To Send A Big Message Out To The World", "impressions": 3040, "clicks": 70, "ctr": 0.0230263157894736, "ctr_select": 0.024342105263157894, "ctr_holdout": 0.021710526315789475, "n_select": 1520, "n_holdout": 1520, "eb": 0.026752194746961014, "labelled_winner": false }, { "headline": "After Seeing His Mom Harassed, This Little Kid Decided To Send A Big Message Out To The World", "impressions": 3114, "clicks": 74, "ctr": 0.0237636480411046, "ctr_select": 0.028259473346178548, "ctr_holdout": 0.019267822736030827, "n_select": 1557, "n_holdout": 1557, "eb": 0.028303327055211602, "labelled_winner": false }, { "headline": "After Seeing His Mom Harassed, This Little Kid Decided To Send A Big Message Out To The World", "impressions": 2987, "clicks": 81, "ctr": 0.0271175092065617, "ctr_select": 0.03279785809906292, "ctr_holdout": 0.021433355659745478, "n_select": 1494, "n_holdout": 1493, "eb": 0.03008287783024594, "labelled_winner": true } ], "winner": 3, "pool_select": 0.028332787422207666, "pool_holdout": 0.023423423423423424, "lift_select": 0.1575937662016078, "lift_holdout": -0.08496058529548156, "eb_lift": 0.061769086887177416, "p_value": 0.23137121693022744, "held_up": false, "z_path": [ 1.006, -0.083, 0.452, 0.296, 0.063, 0.631, 0.411, 0.977, 0.943, 0.832 ], "first_cross": null, "final_z": 0.832 }, { "id": "54d508b4376437000c1a0100", "variation": "image", "week": 201505, "impressions": 12210, "pooled_ctr": 0.01891891891891892, "arms": [ { "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer", "impressions": 2049, "clicks": 54, "ctr": 0.0263543191800878, "ctr_select": 0.02734375, "ctr_holdout": 0.025365853658536587, "n_select": 1024, "n_holdout": 1025, "eb": 0.021218635290760048, "labelled_winner": false }, { "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer", "impressions": 2091, "clicks": 50, "ctr": 0.0239120038259206, "ctr_select": 0.02676864244741874, "ctr_holdout": 0.021052631578947368, "n_select": 1046, "n_holdout": 1045, "eb": 0.021079856477267504, "labelled_winner": false }, { "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer", "impressions": 2029, "clicks": 23, "ctr": 0.0113356333169048, "ctr_select": 0.007889546351084813, "ctr_holdout": 0.014778325123152709, "n_select": 1014, "n_holdout": 1015, "eb": 0.015279376870174978, "labelled_winner": false }, { "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer", "impressions": 2046, "clicks": 30, "ctr": 0.0146627565982404, "ctr_select": 0.011730205278592375, "ctr_holdout": 0.017595307917888565, "n_select": 1023, "n_holdout": 1023, "eb": 0.016435568166026274, "labelled_winner": false }, { "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer", "impressions": 1981, "clicks": 33, "ctr": 0.01665825340737, "ctr_select": 0.01818181818181818, "ctr_holdout": 0.015136226034308779, "n_select": 990, "n_holdout": 991, "eb": 0.018413491886296374, "labelled_winner": false }, { "headline": "A Football Team Is Taking Their Towels Off To Show Men How To Check For Cancer", "impressions": 2014, "clicks": 41, "ctr": 0.0203574975173783, "ctr_select": 0.018867924528301886, "ctr_holdout": 0.021847070506454815, "n_select": 1007, "n_holdout": 1007, "eb": 0.018620118382586672, "labelled_winner": false } ], "winner": 0, "pool_select": 0.018512450851900394, "pool_holdout": 0.019325253848673438, "lift_select": 0.47704646017699104, "lift_holdout": 0.3125754443985118, "eb_lift": 0.14618185676813567, "p_value": 0.021554080511398552, "held_up": true, "z_path": [ 0.679, 0.916, 0.428, 0.225, 0.636, 0.962, 0.553, 0.535, 0.724, 0.502 ], "first_cross": null, "final_z": 0.502 }, { "id": "5536963931363319a04f0100", "variation": "image", "week": 201516, "impressions": 12161, "pooled_ctr": 0.0037003535893429815, "arms": [ { "headline": "Most workers have the same rights, right? Umm \u2026 no.", "impressions": 2048, "clicks": 7, "ctr": 0.00341796875, "ctr_select": 0.0029296875, "ctr_holdout": 0.00390625, "n_select": 1024, "n_holdout": 1024, "eb": 0.0034069589945267416, "labelled_winner": false }, { "headline": "Most workers have the same rights, right? Umm \u2026 no.", "impressions": 2019, "clicks": 1, "ctr": 0.0004952947003467, "ctr_select": 0.0009900990099009901, "ctr_holdout": 0.0, "n_select": 1010, "n_holdout": 1009, "eb": 0.0028202763245961638, "labelled_winner": false }, { "headline": "Most workers have the same rights, right? Umm \u2026 no.", "impressions": 2008, "clicks": 16, "ctr": 0.0079681274900398, "ctr_select": 0.00796812749003984, "ctr_holdout": 0.00796812749003984, "n_select": 1004, "n_holdout": 1004, "eb": 0.004932726728544162, "labelled_winner": false }, { "headline": "Most workers have the same rights, right? Umm \u2026 no.", "impressions": 1936, "clicks": 6, "ctr": 0.003099173553719, "ctr_select": 0.004132231404958678, "ctr_holdout": 0.002066115702479339, "n_select": 968, "n_holdout": 968, "eb": 0.003769375081074814, "labelled_winner": false }, { "headline": "Most workers have the same rights, right? Umm \u2026 no.", "impressions": 1998, "clicks": 11, "ctr": 0.0055055055055055, "ctr_select": 0.004004004004004004, "ctr_holdout": 0.007007007007007007, "n_select": 999, "n_holdout": 999, "eb": 0.003734144065794147, "labelled_winner": false }, { "headline": "Most workers have the same rights, right? Umm \u2026 no.", "impressions": 2152, "clicks": 4, "ctr": 0.0018587360594795, "ctr_select": 0.0018587360594795538, "ctr_holdout": 0.0018587360594795538, "n_select": 1076, "n_holdout": 1076, "eb": 0.003060091957398493, "labelled_winner": false } ], "winner": 2, "pool_select": 0.003617826015457984, "pool_holdout": 0.0037828947368421053, "lift_select": 1.2024628757696485, "lift_holdout": 1.106357179975749, "eb_lift": 0.3634505107398658, "p_value": 0.011982555804691151, "held_up": true, "z_path": [ -1.041, -1.038, -0.426, 0.244, 0.518, 1.226, 1.224, 1.813, 1.991, 2.052 ], "first_cross": 8, "final_z": 2.052 }, { "id": "5318d184bc91b20a60000013", "variation": "headline", "week": 201409, "impressions": 12111, "pooled_ctr": 0.04062422591032945, "arms": [ { "headline": "Never Before Have I Laughed So Hard At One Guy's Non-Attempt To Save The Planet", "impressions": 4058, "clicks": 184, "ctr": 0.0453425332676195, "ctr_select": 0.040413997042878264, "ctr_holdout": 0.05027106949236077, "n_select": 2029, "n_holdout": 2029, "eb": 0.03938308971403499, "labelled_winner": false }, { "headline": "Never Before Have I Laughed So Hard At One Dude's Non-Attempt To Save The Planet", "impressions": 4092, "clicks": 147, "ctr": 0.0359237536656891, "ctr_select": 0.03470185728250244, "ctr_holdout": 0.03714565004887586, "n_select": 2046, "n_holdout": 2046, "eb": 0.03670886401944899, "labelled_winner": false }, { "headline": "Never Before Have I Laughed So Hard At One Man's Non-Attempt To Save The Planet", "impressions": 3961, "clicks": 161, "ctr": 0.0406463014390305, "ctr_select": 0.04040404040404041, "ctr_holdout": 0.0408884401817264, "n_select": 1980, "n_holdout": 1981, "eb": 0.03936674873129519, "labelled_winner": false } ], "winner": 0, "pool_select": 0.03848059454995871, "pool_holdout": 0.042767503302509904, "lift_select": 0.050243571221578964, "lift_holdout": 0.17545018087157094, "eb_lift": 0.0234532541565744, "p_value": 0.5787280405430407, "held_up": true, "z_path": [ 0.272, 0.153, 0.801, -0.407, 0.304, 0.592, -0.134, -0.031, 0.41, 1.036 ], "first_cross": null, "final_z": 1.036 }, { "id": "53149f4bd167d1a490000017", "variation": "headline", "week": 201409, "impressions": 12098, "pooled_ctr": 0.06968093899818152, "arms": [ { "headline": "Oscar Winner Lupita Nyong\u2019o's Speech On Beauty That Left An Entire Audience Speechless", "impressions": 2056, "clicks": 167, "ctr": 0.0812256809338521, "ctr_select": 0.08268482490272373, "ctr_holdout": 0.07976653696498054, "n_select": 1028, "n_holdout": 1028, "eb": 0.07384829861406061, "labelled_winner": true }, { "headline": "An Oscar Winner's Speech On Beauty That Left An Entire Audience Speechless", "impressions": 1973, "clicks": 142, "ctr": 0.0719716168271667, "ctr_select": 0.08417849898580122, "ctr_holdout": 0.05977710233029382, "n_select": 986, "n_holdout": 987, "eb": 0.07418175180508511, "labelled_winner": false }, { "headline": "Lupita Nyong\u2019o's Speech On Beauty That Left An Entire Audience Speechless", "impressions": 2067, "clicks": 144, "ctr": 0.06966618287373, "ctr_select": 0.07059961315280464, "ctr_holdout": 0.06873184898354308, "n_select": 1034, "n_holdout": 1033, "eb": 0.07013582485054663, "labelled_winner": false }, { "headline": "Listen To (Or Read) Oscar Winner Lupita Nyong\u2019o's Speech On Beauty That Left An Audience Speechless", "impressions": 1994, "clicks": 129, "ctr": 0.0646940822467402, "ctr_select": 0.058174523570712136, "ctr_holdout": 0.0712136409227683, "n_select": 997, "n_holdout": 997, "eb": 0.06639340982935578, "labelled_winner": false }, { "headline": "A Speech On Beauty That Left An Entire Audience Speechless, By Oscar Winner Lupita Nyong\u2019o", "impressions": 1968, "clicks": 123, "ctr": 0.0625, "ctr_select": 0.05589430894308943, "ctr_holdout": 0.06910569105691057, "n_select": 984, "n_holdout": 984, "eb": 0.06574620167664721, "labelled_winner": false }, { "headline": "Oscar Winner Lupita Nyong\u2019o Delivers A Speech On Beauty And It Left An Entire Audience Speechless", "impressions": 2040, "clicks": 138, "ctr": 0.0676470588235294, "ctr_select": 0.06764705882352941, "ctr_holdout": 0.06764705882352941, "n_select": 1020, "n_holdout": 1020, "eb": 0.0692315792539781, "labelled_winner": false } ], "winner": 1, "pool_select": 0.06992891387006117, "pool_holdout": 0.06943296412630187, "lift_select": 0.20377243585132754, "lift_holdout": -0.13906740000964934, "eb_lift": 0.06081658786988142, "p_value": 0.055142689705152174, "held_up": false, "z_path": [ 0.341, 0.291, 0.066, -0.626, -0.65, -0.617, -0.462, -0.462, -0.394, -1.103 ], "first_cross": null, "final_z": -1.103 }, { "id": "52eabc0217d1b029dc000009", "variation": "image", "week": 201404, "impressions": 12086, "pooled_ctr": 0.031689558166473605, "arms": [ { "headline": "Here Are Some Secrets About An Industry That Nobody Wants To Talk About", "impressions": 3038, "clicks": 103, "ctr": 0.0339038841342988, "ctr_select": 0.032916392363396975, "ctr_holdout": 0.03489137590520079, "n_select": 1519, "n_holdout": 1519, "eb": 0.03132557840743455, "labelled_winner": false }, { "headline": "Here Are Some Secrets About An Industry That Nobody Wants To Talk About", "impressions": 2952, "clicks": 104, "ctr": 0.035230352303523, "ctr_select": 0.034552845528455285, "ctr_holdout": 0.03590785907859079, "n_select": 1476, "n_holdout": 1476, "eb": 0.03194423599542653, "labelled_winner": false }, { "headline": "Here Are Some Secrets About An Industry That Nobody Wants To Talk About", "impressions": 3071, "clicks": 89, "ctr": 0.0289807880169325, "ctr_select": 0.03125, "ctr_holdout": 0.026710097719869708, "n_select": 1536, "n_holdout": 1535, "eb": 0.030668408326376826, "labelled_winner": false }, { "headline": "Here Are Some Secrets About An Industry That Nobody Wants To Talk About", "impressions": 3025, "clicks": 87, "ctr": 0.0287603305785124, "ctr_select": 0.022486772486772486, "ctr_holdout": 0.03502974223397224, "n_select": 1512, "n_holdout": 1513, "eb": 0.027204960497552954, "labelled_winner": false } ], "winner": 1, "pool_select": 0.030282972033758068, "pool_holdout": 0.03309614429918915, "lift_select": 0.14099915589319822, "lift_holdout": 0.08495596205962053, "eb_lift": 0.05485802251564209, "p_value": 0.27083056617654155, "held_up": true, "z_path": [ -0.421, 0.249, 0.8, -0.453, -0.659, -0.856, -0.874, -0.209, 0.35, 0.281 ], "first_cross": null, "final_z": 0.281 }, { "id": "52278c3138274a5b8100063c", "variation": "headline", "week": 201335, "impressions": 11725, "pooled_ctr": 0.04247334754797441, "arms": [ { "headline": "There's No Good Way To Say This, But I Think This Is A Sex Tape You Should Share. Trust Me.", "impressions": 2209, "clicks": 124, "ctr": 0.0561339972838388, "ctr_select": 0.057065217391304345, "ctr_holdout": 0.05520361990950226, "n_select": 1104, "n_holdout": 1105, "eb": 0.047761571528432065, "labelled_winner": false }, { "headline": "Alyssa Milano Made A Sex Tape And It's Really Important That You Watch It. I'm Not Joking.", "impressions": 2338, "clicks": 161, "ctr": 0.0688622754491018, "ctr_select": 0.07613344739093242, "ctr_holdout": 0.06159110350727117, "n_select": 1169, "n_holdout": 1169, "eb": 0.05432809945547935, "labelled_winner": false }, { "headline": "A Celebrity Sex Tape About Syria. Trust Me.", "impressions": 2177, "clicks": 55, "ctr": 0.0252641249425815, "ctr_select": 0.027573529411764705, "ctr_holdout": 0.02295684113865932, "n_select": 1088, "n_holdout": 1089, "eb": 0.03829631135232144, "labelled_winner": false }, { "headline": "Sharing Celebrity Sex Tapes Is Not A Good Idea, But I Have A Really Important Reason To Share This", "impressions": 2307, "clicks": 78, "ctr": 0.0338101430429128, "ctr_select": 0.030329289428076257, "ctr_holdout": 0.0372940156114484, "n_select": 1154, "n_holdout": 1153, "eb": 0.039008482647491086, "labelled_winner": false }, { "headline": "This Is A Totally Safe For Work Celebrity Sex Tape About A Really Important Issue", "impressions": 2694, "clicks": 80, "ctr": 0.0296956198960653, "ctr_select": 0.02746844840386043, "ctr_holdout": 0.03192279138827023, "n_select": 1347, "n_holdout": 1347, "eb": 0.03749986240195908, "labelled_winner": false } ], "winner": 1, "pool_select": 0.04332992152848857, "pool_holdout": 0.04161691966570016, "lift_select": 0.7570640496285268, "lift_holdout": 0.47995344206201174, "eb_lift": 0.2538240905827558, "p_value": 7.426059028263499e-10, "held_up": true, "z_path": [ 2.529, 1.987, 2.524, 2.216, 1.885, 1.733, 1.775, 1.796, 1.52, 1.77 ], "first_cross": 0, "final_z": 1.77 }, { "id": "5143606c220cb800020005dd", "variation": "headline", "week": 201310, "impressions": 11468, "pooled_ctr": 0.0177886292291594, "arms": [ { "headline": "Finally, An Answer To The Question: Do Feminist D**k Jokes Exist?", "impressions": 2806, "clicks": 32, "ctr": 0.0114041339985744, "ctr_select": 0.011404133998574484, "ctr_holdout": 0.011404133998574484, "n_select": 1403, "n_holdout": 1403, "eb": 0.015487850094401573, "labelled_winner": false }, { "headline": "Nothing Makes Me Happier Than Smart Comedy And Feminist D**k Jokes", "impressions": 2868, "clicks": 40, "ctr": 0.0139470013947001, "ctr_select": 0.014644351464435146, "ctr_holdout": 0.013249651324965132, "n_select": 1434, "n_holdout": 1434, "eb": 0.016692605115439255, "labelled_winner": false }, { "headline": "I Didn't Think This Kind Of Joke Was Possible, But Here We Are", "impressions": 2895, "clicks": 55, "ctr": 0.0189982728842832, "ctr_select": 0.020027624309392266, "ctr_holdout": 0.01796821008984105, "n_select": 1448, "n_holdout": 1447, "eb": 0.018754986150121472, "labelled_winner": false }, { "headline": "A Joke About Male Genitalia That Even Hillary Clinton Would Laugh At", "impressions": 2899, "clicks": 77, "ctr": 0.0265608830631252, "ctr_select": 0.025517241379310347, "ctr_holdout": 0.027605244996549344, "n_select": 1450, "n_holdout": 1449, "eb": 0.02086834307307899, "labelled_winner": true } ], "winner": 3, "pool_select": 0.017959895379250218, "pool_holdout": 0.017617303331589045, "lift_select": 0.42079009039169746, "lift_holdout": 0.5669393026259149, "eb_lift": 0.161941238098136, "p_value": 0.012181290252280666, "held_up": true, "z_path": [ 0.0, 0.202, -0.162, 0.281, 0.906, 1.299, 0.971, 1.887, 2.107, 1.929 ], "first_cross": 8, "final_z": 1.929 }, { "id": "54b004a736633300245c0000", "variation": "headline", "week": 201501, "impressions": 11448, "pooled_ctr": 0.0159853249475891, "arms": [ { "headline": "That Song That's Been Stuck In Your Head For The Last 6 Months? She Just Made It 10 Times Better.", "impressions": 2883, "clicks": 51, "ctr": 0.0176899063475546, "ctr_select": 0.018030513176144243, "ctr_holdout": 0.01734906315058987, "n_select": 1442, "n_holdout": 1441, "eb": 0.016823588034115163, "labelled_winner": false }, { "headline": "She Just Rewrote \"Rude\" In The Most Hilariously Perfect Way Possible", "impressions": 2838, "clicks": 27, "ctr": 0.0095137420718816, "ctr_select": 0.009161381254404511, "ctr_holdout": 0.009866102889358703, "n_select": 1419, "n_holdout": 1419, "eb": 0.013448132358486669, "labelled_winner": false }, { "headline": "It's One Of The Most Popular Songs In The World Right Now. Thank God Someone Finally Fixed It.", "impressions": 2873, "clicks": 62, "ctr": 0.0215802297250261, "ctr_select": 0.022284122562674095, "ctr_holdout": 0.020876826722338204, "n_select": 1436, "n_holdout": 1437, "eb": 0.01844890312559755, "labelled_winner": false }, { "headline": "Her Boyfriend And Father Are Fighting Over Her, So She Wrote Them This Hilarious Song", "impressions": 2854, "clicks": 43, "ctr": 0.0150665732305536, "ctr_select": 0.014716187806587245, "ctr_holdout": 0.015416958654519973, "n_select": 1427, "n_holdout": 1427, "eb": 0.015555756408183666, "labelled_winner": false } ], "winner": 2, "pool_select": 0.01607267645003494, "pool_holdout": 0.015897973445143255, "lift_select": 0.3864599733559404, "lift_holdout": 0.3131753424029, "eb_lift": 0.14784262490130828, "p_value": 0.03057557866492494, "held_up": true, "z_path": [ 0.287, 0.421, 0.339, 0.304, 0.661, 1.316, 1.355, 1.07, 0.818, 1.064 ], "first_cross": null, "final_z": 1.064 }, { "id": "51e59879cde6d2e201002399", "variation": "image", "week": 201328, "impressions": 11426, "pooled_ctr": 0.021529844214948364, "arms": [ { "headline": "How Can Our Kids Accept Diversity If These Books Are What We Use To Raise Them?", "impressions": 2891, "clicks": 71, "ctr": 0.024558976132826, "ctr_select": 0.022130013831258646, "ctr_holdout": 0.02698961937716263, "n_select": 1446, "n_holdout": 1445, "eb": 0.02165033609357659, "labelled_winner": false }, { "headline": "How Can Our Kids Accept Diversity If These Books Are What We Use To Raise Them?", "impressions": 2682, "clicks": 32, "ctr": 0.01193139448173, "ctr_select": 0.013422818791946308, "ctr_holdout": 0.010439970171513796, "n_select": 1341, "n_holdout": 1341, "eb": 0.018445175225729755, "labelled_winner": false }, { "headline": "How Can Our Kids Accept Diversity If These Books Are What We Use To Raise Them?", "impressions": 2739, "clicks": 69, "ctr": 0.0251916757940854, "ctr_select": 0.02408759124087591, "ctr_holdout": 0.026296566837107377, "n_select": 1370, "n_holdout": 1369, "eb": 0.02236770116929509, "labelled_winner": false }, { "headline": "How Can Our Kids Accept Diversity If These Books Are What We Use To Raise Them?", "impressions": 3114, "clicks": 74, "ctr": 0.0237636480411046, "ctr_select": 0.025048169556840076, "ctr_holdout": 0.0224791265253693, "n_select": 1557, "n_holdout": 1557, "eb": 0.02283670288758872, "labelled_winner": false } ], "winner": 3, "pool_select": 0.021351067553377668, "pool_holdout": 0.021708683473389355, "lift_select": 0.1731577118670835, "lift_holdout": 0.035490086394430964, "eb_lift": 0.06958131393181932, "p_value": 0.23672511803976526, "held_up": false, "z_path": [ -1.33, -0.335, -0.739, -0.896, -0.282, 0.113, -0.439, -0.294, 0.439, -0.353 ], "first_cross": null, "final_z": -0.353 }, { "id": "54a2cb0334303400154b0000", "variation": "image", "week": 201452, "impressions": 11411, "pooled_ctr": 0.007887126456927527, "arms": [ { "headline": "When He Mentioned Beyonc\u00e9, I Almost Rolled My Eyes, But Then I Saw Where He Was Going", "impressions": 2843, "clicks": 15, "ctr": 0.0052761167780513, "ctr_select": 0.004922644163150493, "ctr_holdout": 0.005629838142153413, "n_select": 1422, "n_holdout": 1421, "eb": 0.006975804494400704, "labelled_winner": false }, { "headline": "When He Mentioned Beyonc\u00e9, I Almost Rolled My Eyes, But Then I Saw Where He Was Going", "impressions": 2819, "clicks": 17, "ctr": 0.0060305072720822, "ctr_select": 0.0035460992907801418, "ctr_holdout": 0.008516678495386799, "n_select": 1410, "n_holdout": 1409, "eb": 0.006461736429362888, "labelled_winner": false }, { "headline": "When He Mentioned Beyonc\u00e9, I Almost Rolled My Eyes, But Then I Saw Where He Was Going", "impressions": 2898, "clicks": 26, "ctr": 0.0089717046238785, "ctr_select": 0.00966183574879227, "ctr_holdout": 0.008281573498964804, "n_select": 1449, "n_holdout": 1449, "eb": 0.008784193118279602, "labelled_winner": false }, { "headline": "When He Mentioned Beyonc\u00e9, I Almost Rolled My Eyes, But Then I Saw Where He Was Going", "impressions": 2851, "clicks": 32, "ctr": 0.0112241318835496, "ctr_select": 0.014726507713884993, "ctr_holdout": 0.0077192982456140355, "n_select": 1426, "n_holdout": 1425, "eb": 0.01070796929091377, "labelled_winner": false } ], "winner": 3, "pool_select": 0.008235500262835115, "pool_holdout": 0.007538569424964937, "lift_select": 0.7881740324072692, "lift_holdout": 0.023973888208894456, "eb_lift": 0.3002208668775508, "p_value": 0.0017390935993718255, "held_up": false, "z_path": [ 0.357, 0.029, 0.035, 0.451, 0.56, 0.663, 0.354, 0.483, 0.618, 0.854 ], "first_cross": null, "final_z": 0.854 }, { "id": "5230d6a46b6514d4750081c0", "variation": "other", "week": 201336, "impressions": 11404, "pooled_ctr": 0.03560154331813399, "arms": [ { "headline": "Why We're Going To Invade Syria Is Explained Perfectly By \u2014 A Dead Comedian?", "impressions": 2955, "clicks": 112, "ctr": 0.037901861252115, "ctr_select": 0.04736129905277402, "ctr_holdout": 0.02843601895734597, "n_select": 1478, "n_holdout": 1477, "eb": 0.04210822511825934, "labelled_winner": false }, { "headline": "Why We're Going To Invade Syria Is Explained Perfectly By \u2014 A Dead Comedian?", "impressions": 3016, "clicks": 90, "ctr": 0.029840848806366, "ctr_select": 0.03050397877984085, "ctr_holdout": 0.029177718832891247, "n_select": 1508, "n_holdout": 1508, "eb": 0.035504659258116814, "labelled_winner": false }, { "headline": "Why We're Going To Invade Syria Is Explained Perfectly By \u2014 A Dead Comedian?", "impressions": 2756, "clicks": 100, "ctr": 0.0362844702467343, "ctr_select": 0.034107402031930335, "ctr_holdout": 0.038461538461538464, "n_select": 1378, "n_holdout": 1378, "eb": 0.03702415737521486, "labelled_winner": false }, { "headline": "Why We're Going To Invade Syria Is Explained Perfectly By \u2014 A Dead Comedian?", "impressions": 2677, "clicks": 104, "ctr": 0.038849458348898, "ctr_select": 0.043348281016442454, "ctr_holdout": 0.03435399551904406, "n_select": 1338, "n_holdout": 1339, "eb": 0.04043827071374037, "labelled_winner": false } ], "winner": 0, "pool_select": 0.03875833041038232, "pool_holdout": 0.032444756225885656, "lift_select": 0.22196437646568978, "lift_holdout": -0.12355578327142314, "eb_lift": 0.08643031504214838, "p_value": 0.046497459022196576, "held_up": false, "z_path": [ -1.544, -2.149, -0.786, 0.217, 0.073, -0.139, -0.051, -0.179, 0.316, -0.185 ], "first_cross": 1, "final_z": -0.185 }, { "id": "54c6f5a83564300016020000", "variation": "headline", "week": 201504, "impressions": 11398, "pooled_ctr": 0.008159326197578523, "arms": [ { "headline": "5 Minutes Will Change How You Think About Love", "impressions": 2965, "clicks": 21, "ctr": 0.0070826306913996, "ctr_select": 0.004048582995951417, "ctr_holdout": 0.010114632501685773, "n_select": 1482, "n_holdout": 1483, "eb": 0.006288431608325599, "labelled_winner": false }, { "headline": "True Fact: If You're In Love, You Don't Exactly Know What Love Is", "impressions": 2786, "clicks": 28, "ctr": 0.0100502512562814, "ctr_select": 0.01148600143575018, "ctr_holdout": 0.008614501076812634, "n_select": 1393, "n_holdout": 1393, "eb": 0.009134168785987194, "labelled_winner": false }, { "headline": "True Fact: If You're In Love, You Don't Exactly Know What Love Is", "impressions": 2815, "clicks": 30, "ctr": 0.0106571936056838, "ctr_select": 0.01065340909090909, "ctr_holdout": 0.010660980810234541, "n_select": 1408, "n_holdout": 1407, "eb": 0.008828987683834019, "labelled_winner": false }, { "headline": "The Problem With Examining Love Is That You Only Care If You're In It (Or Falling Out Of It)", "impressions": 2832, "clicks": 14, "ctr": 0.0049435028248587, "ctr_select": 0.0049435028248587575, "ctr_holdout": 0.0049435028248587575, "n_select": 1416, "n_holdout": 1416, "eb": 0.006667422249263987, "labelled_winner": false } ], "winner": 1, "pool_select": 0.007720652746095806, "pool_holdout": 0.008597999649061239, "lift_select": 0.48769823141682433, "lift_holdout": 0.0019192170766368921, "eb_lift": 0.18308245253047772, "p_value": 0.06472729924384903, "held_up": false, "z_path": [ 0.352, 0.281, -0.195, -0.367, -0.338, -0.48, -0.277, 0.182, 0.178, -0.224 ], "first_cross": null, "final_z": -0.224 }, { "id": "54a310426565330015080000", "variation": "headline", "week": 201452, "impressions": 11372, "pooled_ctr": 0.009145269081955681, "arms": [ { "headline": "They Took Utah's Biggest Secret And Made Sure Everyone Had A Peek. (And Nope: Not THAT Utah Secret.)", "impressions": 2840, "clicks": 30, "ctr": 0.0105633802816901, "ctr_select": 0.01267605633802817, "ctr_holdout": 0.008450704225352112, "n_select": 1420, "n_holdout": 1420, "eb": 0.010159495974712603, "labelled_winner": true }, { "headline": "It's 1 Of Utah's Big Secrets \u2014 So, They Made Sure Everyone Had A Peek. (No, Not THAT Utah Secret.)", "impressions": 2869, "clicks": 31, "ctr": 0.0108051585918438, "ctr_select": 0.008368200836820083, "ctr_holdout": 0.0132404181184669, "n_select": 1434, "n_holdout": 1435, "eb": 0.008522308379331723, "labelled_winner": false }, { "headline": "They Took 1 Of Utah's Dirtiest Secrets And Made Sure Everyone Had A Peek. (Not THAT Utah Secret.)", "impressions": 2871, "clicks": 22, "ctr": 0.0076628352490421, "ctr_select": 0.006267409470752089, "ctr_holdout": 0.009059233449477353, "n_select": 1436, "n_holdout": 1435, "eb": 0.007718555692205654, "labelled_winner": false }, { "headline": "They Took 1 Of Our Government's Dirtiest Secrets And Made Sure Everyone Had A Peek \u2014 With A Blimp", "impressions": 2792, "clicks": 21, "ctr": 0.0075214899713467, "ctr_select": 0.0071633237822349575, "ctr_holdout": 0.007879656160458453, "n_select": 1396, "n_holdout": 1396, "eb": 0.008070965158936105, "labelled_winner": false } ], "winner": 0, "pool_select": 0.008617657404150545, "pool_holdout": 0.009672880759760817, "lift_select": 0.470939925265881, "lift_holdout": -0.12635083226632537, "eb_lift": 0.1789162063717522, "p_value": 0.056108674822637136, "held_up": false, "z_path": [ -1.728, 0.015, 0.297, 0.021, -0.384, 0.762, 0.701, 0.188, 0.178, -0.089 ], "first_cross": null, "final_z": -0.089 }, { "id": "5148f04dcb47730002005288", "variation": "headline", "week": 201311, "impressions": 11361, "pooled_ctr": 0.005633306927207112, "arms": [ { "headline": "Fate Says These Kids Won't Amount To Anything. Fortunately, They Aren't Listening.", "impressions": 3677, "clicks": 20, "ctr": 0.0054392167527875, "ctr_select": 0.00544069640914037, "ctr_holdout": 0.005437737901033171, "n_select": 1838, "n_holdout": 1839, "eb": 0.005646584255024341, "labelled_winner": false }, { "headline": "Meet The Cutest People Ever To Spit In The Face Of Fate", "impressions": 3942, "clicks": 20, "ctr": 0.0050735667174023, "ctr_select": 0.0050735667174023336, "ctr_holdout": 0.0050735667174023336, "n_select": 1971, "n_holdout": 1971, "eb": 0.005471473823457493, "labelled_winner": false }, { "headline": "Tiny And Terrifying: Why Fate Doesn't Stand A Chance Against These Kids", "impressions": 3742, "clicks": 24, "ctr": 0.0064136825227151, "ctr_select": 0.00694815606627472, "ctr_holdout": 0.005879208979155532, "n_select": 1871, "n_holdout": 1871, "eb": 0.006318311583657129, "labelled_winner": false } ], "winner": 2, "pool_select": 0.0058098591549295775, "pool_holdout": 0.00545678577715191, "lift_select": 0.1959250441345577, "lift_holdout": 0.0774124584058895, "eb_lift": 0.08751544833856029, "p_value": 0.4288726505073468, "held_up": true, "z_path": [ -0.47, -0.027, 0.307, 0.466, -0.258, 0.168, -0.243, 0.66, 0.427, 0.547 ], "first_cross": null, "final_z": 0.547 }, { "id": "5220d3fd762f11fc44008b62", "variation": "other", "week": 201334, "impressions": 11361, "pooled_ctr": 0.029398820526362116, "arms": [ { "headline": "The Warning Label That We Never Knew We Needed Until We Saw It", "impressions": 2548, "clicks": 79, "ctr": 0.0310047095761381, "ctr_select": 0.027472527472527472, "ctr_holdout": 0.03453689167974882, "n_select": 1274, "n_holdout": 1274, "eb": 0.029285260726749616, "labelled_winner": false }, { "headline": "The Warning Label That We Never Knew We Needed Until We Saw It", "impressions": 2955, "clicks": 103, "ctr": 0.0348561759729272, "ctr_select": 0.036535859269282815, "ctr_holdout": 0.03317535545023697, "n_select": 1478, "n_holdout": 1477, "eb": 0.03271698997005576, "labelled_winner": false }, { "headline": "The Warning Label That We Never Knew We Needed Until We Saw It", "impressions": 3081, "clicks": 79, "ctr": 0.0256410256410256, "ctr_select": 0.025324675324675326, "ctr_holdout": 0.025957170668397145, "n_select": 1540, "n_holdout": 1541, "eb": 0.028302840733404177, "labelled_winner": false }, { "headline": "The Warning Label That We Never Knew We Needed Until We Saw It", "impressions": 2777, "clicks": 73, "ctr": 0.026287360460929, "ctr_select": 0.03170028818443804, "ctr_holdout": 0.02087832973362131, "n_select": 1388, "n_holdout": 1389, "eb": 0.030813037876534544, "labelled_winner": false } ], "winner": 1, "pool_select": 0.03028169014084507, "pool_holdout": 0.02851610631930998, "lift_select": 0.20653302703213017, "lift_holdout": 0.16339008835059388, "eb_lift": 0.08042152924370183, "p_value": 0.10282204850119653, "held_up": false, "z_path": [ 1.302, 1.255, 0.052, -0.511, -0.099, 0.25, 0.389, 0.739, 1.335, 1.879 ], "first_cross": null, "final_z": 1.879 }, { "id": "549a190b63356600181f0000", "variation": "headline", "week": 201451, "impressions": 11358, "pooled_ctr": 0.004138052474027118, "arms": [ { "headline": "This 47 Seconds Is Just Between The President Of The United States And Your Daughters", "impressions": 2881, "clicks": 7, "ctr": 0.0024297119055883, "ctr_select": 0.001388888888888889, "ctr_holdout": 0.0034698126301179735, "n_select": 1440, "n_holdout": 1441, "eb": 0.002487534793471008, "labelled_winner": false }, { "headline": "What? Girls Don't Like Tools? The Leader Of The Free World Would Like To Know The Answer.", "impressions": 2816, "clicks": 7, "ctr": 0.0024857954545454, "ctr_select": 0.0007102272727272727, "ctr_holdout": 0.004261363636363636, "n_select": 1408, "n_holdout": 1408, "eb": 0.0022404941155694967, "labelled_winner": false }, { "headline": "Obama Separated Boy Toys From Girl Toys. He's Got A Wonderful Way Of Doing It.", "impressions": 2869, "clicks": 21, "ctr": 0.0073196235622168, "ctr_select": 0.0041841004184100415, "ctr_holdout": 0.010452961672473868, "n_select": 1434, "n_holdout": 1435, "eb": 0.003557697030636737, "labelled_winner": false }, { "headline": "Obama Walked Into This Miltary Event And Shook Things Up. He's Got Daughters.", "impressions": 2792, "clicks": 12, "ctr": 0.0042979942693409, "ctr_select": 0.0064469914040114614, "ctr_holdout": 0.002148997134670487, "n_select": 1396, "n_holdout": 1396, "eb": 0.00440192096497903, "labelled_winner": false } ], "winner": 3, "pool_select": 0.0031701303275801338, "pool_holdout": 0.0051056338028169015, "lift_select": 1.0336676217765044, "lift_holdout": -0.57909297500247, "eb_lift": 0.38856151328616284, "p_value": 0.012141966477187772, "held_up": false, "z_path": [ -0.422, 0.035, 0.041, -0.726, -0.632, -0.563, -0.686, -0.858, -1.184, -1.493 ], "first_cross": null, "final_z": -1.493 }, { "id": "54a5d6446135350018020000", "variation": "headline", "week": 201500, "impressions": 11348, "pooled_ctr": 0.006432851603806839, "arms": [ { "headline": "Christopher Columbus Called Native People \u2018Indians.\u2019 I Believe I\u2019ll Call These Guys Heroes.", "impressions": 2832, "clicks": 18, "ctr": 0.0063559322033898, "ctr_select": 0.007768361581920904, "ctr_holdout": 0.0049435028248587575, "n_select": 1416, "n_holdout": 1416, "eb": 0.006666490792782979, "labelled_winner": false }, { "headline": "Some People Call Native Americans \u2018Indians.\u2019 I Believe I\u2019ll Call These Guys Heroes.", "impressions": 2837, "clicks": 18, "ctr": 0.0063447303489601, "ctr_select": 0.005641748942172073, "ctr_holdout": 0.007047216349541931, "n_select": 1418, "n_holdout": 1419, "eb": 0.005859858880932215, "labelled_winner": false }, { "headline": "Some Folks Call Native Americans \u2018Indians.\u2019 I Believe I\u2019ll Call These Guys Straight-Up Heroes.", "impressions": 2890, "clicks": 16, "ctr": 0.0055363321799307, "ctr_select": 0.004152249134948097, "ctr_holdout": 0.006920415224913495, "n_select": 1445, "n_holdout": 1445, "eb": 0.005286276744071447, "labelled_winner": false }, { "headline": "Some Call Native Americans \u2018Indians.\u2019 I Believe I\u2019ll Call These Guys Simply 'Heroes.'", "impressions": 2789, "clicks": 21, "ctr": 0.007529580494801, "ctr_select": 0.006456241032998565, "ctr_holdout": 0.008602150537634409, "n_select": 1394, "n_holdout": 1395, "eb": 0.006167167477674416, "labelled_winner": false } ], "winner": 0, "pool_select": 0.005993301604089547, "pool_holdout": 0.006872246696035243, "lift_select": 0.2961739780658026, "lift_holdout": -0.28065696074170654, "eb_lift": 0.11232359610170106, "p_value": 0.3177879886874009, "held_up": false, "z_path": [ -0.594, -1.031, -1.188, -0.808, -0.933, -0.857, -0.826, -0.774, -0.397, -0.53 ], "first_cross": null, "final_z": -0.53 }, { "id": "54ae476463316300244d0000", "variation": "other", "week": 201501, "impressions": 11314, "pooled_ctr": 0.025720346473395793, "arms": [ { "headline": "Girls Said She'd Steal Their Boyfriends, So She Told Them The Truth. It Only Made Things Worse.", "impressions": 2825, "clicks": 58, "ctr": 0.0205309734513274, "ctr_select": 0.018413597733711047, "ctr_holdout": 0.02264685067232838, "n_select": 1412, "n_holdout": 1413, "eb": 0.023231633366076098, "labelled_winner": false }, { "headline": "Girls Said She'd Steal Their Boyfriends, So She Told Them The Truth. It Only Made Things Worse.", "impressions": 2845, "clicks": 75, "ctr": 0.0263620386643233, "ctr_select": 0.03164556962025317, "ctr_holdout": 0.02108222066057625, "n_select": 1422, "n_holdout": 1423, "eb": 0.028250136236798156, "labelled_winner": false }, { "headline": "Girls Said She'd Steal Their Boyfriends, So She Told Them The Truth. It Only Made Things Worse.", "impressions": 2785, "clicks": 88, "ctr": 0.0315978456014362, "ctr_select": 0.03089080459770115, "ctr_holdout": 0.03230437903804738, "n_select": 1392, "n_holdout": 1393, "eb": 0.0279394651243093, "labelled_winner": true }, { "headline": "Girls Said She'd Steal Their Boyfriends, So She Told Them The Truth. It Only Made Things Worse.", "impressions": 2859, "clicks": 70, "ctr": 0.024484085344526, "ctr_select": 0.023776223776223775, "ctr_holdout": 0.025192442267319804, "n_select": 1430, "n_holdout": 1429, "eb": 0.02525469690987138, "labelled_winner": false } ], "winner": 1, "pool_select": 0.02616690240452617, "pool_holdout": 0.02527394839165783, "lift_select": 0.20937393089291834, "lift_holdout": -0.16585171680041666, "eb_lift": 0.07961331456304288, "p_value": 0.13469497063318195, "held_up": false, "z_path": [ -1.214, -1.137, -2.079, -1.855, -1.524, -0.755, -1.558, -1.491, -0.803, -1.171 ], "first_cross": 2, "final_z": -1.171 }, { "id": "522e6e2a36430f3b45001e97", "variation": "headline", "week": 201336, "impressions": 11313, "pooled_ctr": 0.016352868381508, "arms": [ { "headline": "20 Years Ago We Thought Of It As Sci-fi, And Now 162 Countries Can Stare Me In The Face", "impressions": 3100, "clicks": 65, "ctr": 0.0209677419354838, "ctr_select": 0.02064516129032258, "ctr_holdout": 0.02129032258064516, "n_select": 1550, "n_holdout": 1550, "eb": 0.017911562602866898, "labelled_winner": true }, { "headline": "20 Years Ago It Was Only In Sci-Fi Films, But Now 162 Countries Can Stare Each Other In The Face", "impressions": 2955, "clicks": 35, "ctr": 0.0118443316412859, "ctr_select": 0.011502029769959404, "ctr_holdout": 0.012186865267433988, "n_select": 1478, "n_holdout": 1477, "eb": 0.014299481620797484, "labelled_winner": false }, { "headline": "20 Years Ago Only Sci-fi Movies Could Do What Now 162 Countries Can Do Just By Logging On", "impressions": 2811, "clicks": 38, "ctr": 0.0135183208822483, "ctr_select": 0.013513513513513514, "ctr_holdout": 0.013523131672597865, "n_select": 1406, "n_holdout": 1405, "eb": 0.015113071476098335, "labelled_winner": false }, { "headline": "Videochatting? No Biggie. But Watch when A Comedian Asks Everyone About Their Message To The World.", "impressions": 2447, "clicks": 47, "ctr": 0.0192071924805884, "ctr_select": 0.018790849673202614, "ctr_holdout": 0.019623875715453803, "n_select": 1224, "n_holdout": 1223, "eb": 0.01701910061004081, "labelled_winner": false } ], "winner": 0, "pool_select": 0.016083421703782255, "pool_holdout": 0.016622458001768346, "lift_select": 0.2836299184686282, "lift_holdout": 0.28081674673987655, "eb_lift": 0.11366616711011979, "p_value": 0.09383632488081853, "held_up": true, "z_path": [ 0.543, 1.209, 0.17, 0.092, 0.503, 0.054, 0.2, 0.402, 0.605, 0.463 ], "first_cross": null, "final_z": 0.463 }, { "id": "52deecd74ded95426b000009", "variation": "image", "week": 201403, "impressions": 11304, "pooled_ctr": 0.021142958244869073, "arms": [ { "headline": "There Are People In Our Lives Who Are So Good At What They Do, You Never See Them. Here\u2019s One Story.", "impressions": 2873, "clicks": 58, "ctr": 0.0201879568395405, "ctr_select": 0.02158774373259053, "ctr_holdout": 0.018789144050104383, "n_select": 1436, "n_holdout": 1437, "eb": 0.021588580703635483, "labelled_winner": false }, { "headline": "There Are People In Our Lives Who Are So Good At What They Do, You Never See Them. Here\u2019s One Story.", "impressions": 2806, "clicks": 56, "ctr": 0.0199572344975053, "ctr_select": 0.023521026372059873, "ctr_holdout": 0.01639344262295082, "n_select": 1403, "n_holdout": 1403, "eb": 0.022317589985911122, "labelled_winner": false }, { "headline": "There Are People In Our Lives Who Are So Good At What They Do, You Never See Them. Here\u2019s One Story.", "impressions": 2824, "clicks": 52, "ctr": 0.018413597733711, "ctr_select": 0.0169971671388102, "ctr_holdout": 0.019830028328611898, "n_select": 1412, "n_holdout": 1412, "eb": 0.01985067201064436, "labelled_winner": false }, { "headline": "There Are People In Our Lives Who Are So Good At What They Do, You Never See Them. Here\u2019s One Story.", "impressions": 2801, "clicks": 73, "ctr": 0.0260621206711888, "ctr_select": 0.024285714285714285, "ctr_holdout": 0.027837259100642397, "n_select": 1400, "n_holdout": 1401, "eb": 0.022604582865570964, "labelled_winner": false } ], "winner": 3, "pool_select": 0.021589099274464698, "pool_holdout": 0.020696975057491598, "lift_select": 0.1249063231850116, "lift_holdout": 0.3449916726147988, "eb_lift": 0.04703686699460241, "p_value": 0.42346297961794266, "held_up": true, "z_path": [ 0.305, 1.809, 1.435, 0.157, 0.517, 1.092, 1.205, 1.237, 1.258, 1.525 ], "first_cross": null, "final_z": 1.525 }, { "id": "53b8d9e555a05d3c30000018", "variation": "image", "week": 201427, "impressions": 11283, "pooled_ctr": 0.01701675086413188, "arms": [ { "headline": "What Every Dog Owner Should Know About Dog Poop", "impressions": 2821, "clicks": 47, "ctr": 0.0166607585962424, "ctr_select": 0.02056737588652482, "ctr_holdout": 0.012756909992912827, "n_select": 1410, "n_holdout": 1411, "eb": 0.019573129870720005, "labelled_winner": false }, { "headline": "What Every Dog Owner Should Know About Dog Poop", "impressions": 2827, "clicks": 52, "ctr": 0.0183940573045631, "ctr_select": 0.02050919377652051, "ctr_holdout": 0.01627742392073602, "n_select": 1414, "n_holdout": 1413, "eb": 0.0195521494683188, "labelled_winner": false }, { "headline": "What Every Dog Owner Should Know About Dog Poop", "impressions": 2773, "clicks": 44, "ctr": 0.0158672917417958, "ctr_select": 0.01948051948051948, "ctr_holdout": 0.012256669069935111, "n_select": 1386, "n_holdout": 1387, "eb": 0.019159963083055873, "labelled_winner": false }, { "headline": "What Every Dog Owner Should Know About Dog Poop", "impressions": 2862, "clicks": 49, "ctr": 0.017120894479385, "ctr_select": 0.015373864430468204, "ctr_holdout": 0.018867924528301886, "n_select": 1431, "n_holdout": 1431, "eb": 0.01759616711959618, "labelled_winner": false } ], "winner": 0, "pool_select": 0.01896826803758199, "pool_holdout": 0.015065579581708614, "lift_select": 0.08430436799893948, "lift_holdout": -0.1532413390586569, "eb_lift": 0.031888089726463065, "p_value": 0.6112689254942019, "held_up": false, "z_path": [ -1.902, -0.535, 0.006, -0.328, -0.131, 0.012, -0.24, 0.365, -0.212, -0.496 ], "first_cross": null, "final_z": -0.496 }, { "id": "519109d5f11e26001000aa4a", "variation": "headline", "week": 201319, "impressions": 11250, "pooled_ctr": 0.010666666666666666, "arms": [ { "headline": "It Doesn't Take A Rocket Scientist To Understand How Much This Epic Song Rocks. Especially IN SPACE.", "impressions": 2874, "clicks": 29, "ctr": 0.0100904662491301, "ctr_select": 0.010438413361169102, "ctr_holdout": 0.009742519137091163, "n_select": 1437, "n_holdout": 1437, "eb": 0.010250093731769381, "labelled_winner": false }, { "headline": "OMG. There's A Riff In The Space Time Continuum.", "impressions": 2810, "clicks": 22, "ctr": 0.0078291814946619, "ctr_select": 0.006405693950177936, "ctr_holdout": 0.009252669039145907, "n_select": 1405, "n_holdout": 1405, "eb": 0.008726468272328328, "labelled_winner": false }, { "headline": "Well, That's Never Happened Before. IN SPACE.", "impressions": 2786, "clicks": 32, "ctr": 0.0114860014357501, "ctr_select": 0.010050251256281407, "ctr_holdout": 0.012921751615218953, "n_select": 1393, "n_holdout": 1393, "eb": 0.010102144230219707, "labelled_winner": false }, { "headline": "The Final Act Of The Departing Commander Of The International Space Station Truly Rocked", "impressions": 2780, "clicks": 37, "ctr": 0.0133093525179856, "ctr_select": 0.013669064748201438, "ctr_holdout": 0.012949640287769784, "n_select": 1390, "n_holdout": 1390, "eb": 0.01145886390733993, "labelled_winner": false } ], "winner": 3, "pool_select": 0.010133333333333333, "pool_holdout": 0.0112, "lift_select": 0.34892086330935257, "lift_holdout": 0.15621788283658788, "eb_lift": 0.13080893822433537, "p_value": 0.12929214730272892, "held_up": true, "z_path": [ -1.308, -0.519, 0.529, 1.404, 2.027, 1.21, 1.413, 1.075, 1.275, 1.127 ], "first_cross": 4, "final_z": 1.127 }, { "id": "51eeb293145e76e55b00389c", "variation": "image", "week": 201329, "impressions": 11213, "pooled_ctr": 0.015517702666547757, "arms": [ { "headline": "If You Know Nothing About Keystone XL, Here's An Adorable Animation You Should Watch", "impressions": 2775, "clicks": 40, "ctr": 0.0144144144144144, "ctr_select": 0.01440922190201729, "ctr_holdout": 0.014419610670511895, "n_select": 1388, "n_holdout": 1387, "eb": 0.014657111676282985, "labelled_winner": false }, { "headline": "If You Know Nothing About Keystone XL, Here's An Adorable Animation You Should Watch", "impressions": 2824, "clicks": 47, "ctr": 0.0166430594900849, "ctr_select": 0.014164305949008499, "ctr_holdout": 0.019121813031161474, "n_select": 1412, "n_holdout": 1412, "eb": 0.014562795539135002, "labelled_winner": false }, { "headline": "If You Know Nothing About Keystone XL, Here's An Adorable Animation You Should Watch", "impressions": 2921, "clicks": 40, "ctr": 0.0136939404313591, "ctr_select": 0.013013698630136987, "ctr_holdout": 0.014373716632443531, "n_select": 1460, "n_holdout": 1461, "eb": 0.01411304695713643, "labelled_winner": false }, { "headline": "If You Know Nothing About Keystone XL, Here's An Adorable Animation You Should Watch", "impressions": 2693, "clicks": 47, "ctr": 0.0174526550315633, "ctr_select": 0.017830609212481426, "ctr_holdout": 0.017074981440237565, "n_select": 1346, "n_holdout": 1347, "eb": 0.01591693141647762, "labelled_winner": false } ], "winner": 3, "pool_select": 0.014805565465572601, "pool_holdout": 0.016229712858926344, "lift_select": 0.2043180150020587, "lift_holdout": 0.052081548740791295, "eb_lift": 0.0750640665153437, "p_value": 0.29181419278839826, "held_up": false, "z_path": [ 0.963, 1.013, 0.895, 0.442, 1.372, 0.964, 0.639, 0.368, 0.248, 0.898 ], "first_cross": null, "final_z": 0.898 }, { "id": "51436090220cb80002000a00", "variation": "headline", "week": 201307, "impressions": 11143, "pooled_ctr": 0.017051063447904514, "arms": [ { "headline": "A Congressperson Thinks Legalized Bribery Is A Good Idea. A Human Person Corrects Them.", "impressions": 2729, "clicks": 52, "ctr": 0.0190545987541223, "ctr_select": 0.018328445747800588, "ctr_holdout": 0.01978021978021978, "n_select": 1364, "n_holdout": 1365, "eb": 0.017188141736879197, "labelled_winner": false }, { "headline": "A Congressperson Endorses Legalized Bribery Of Congress. Honest Shellacking Ensues.", "impressions": 2761, "clicks": 38, "ctr": 0.0137631293009779, "ctr_select": 0.010144927536231883, "ctr_holdout": 0.017378711078928313, "n_select": 1380, "n_holdout": 1381, "eb": 0.014138944325836213, "labelled_winner": false }, { "headline": "2 Members Of Congress Debate Whether Accepting Legalized Bribes Is Cool. Righteous Honesty Prevails.", "impressions": 2820, "clicks": 26, "ctr": 0.0092198581560283, "ctr_select": 0.011347517730496455, "ctr_holdout": 0.0070921985815602835, "n_select": 1410, "n_holdout": 1410, "eb": 0.014561680232845488, "labelled_winner": false }, { "headline": "Watch A Real Person Correct An Elected Official On How Democracy Should Actually Function", "impressions": 2833, "clicks": 74, "ctr": 0.0261207200847158, "ctr_select": 0.026129943502824857, "ctr_holdout": 0.02611150317572336, "n_select": 1416, "n_holdout": 1417, "eb": 0.02016273087297812, "labelled_winner": true } ], "winner": 3, "pool_select": 0.01651705565529623, "pool_holdout": 0.017584783778934146, "lift_select": 0.5819976664210267, "lift_holdout": 0.484891910186799, "eb_lift": 0.22072185828791446, "p_value": 0.0010145035597310008, "held_up": true, "z_path": [ 0.164, 0.604, 1.226, 1.419, 1.589, 1.716, 2.384, 2.313, 1.846, 1.771 ], "first_cross": 6, "final_z": 1.771 }, { "id": "5143606c220cb800020005de", "variation": "both", "week": 201310, "impressions": 11134, "pooled_ctr": 0.02811208909646129, "arms": [ { "headline": "An 'Arab' Filmmaker Drives His Car Into Cardboard Twin Towers. Trust Me, You'll Be On His Side.", "impressions": 2216, "clicks": 42, "ctr": 0.0189530685920577, "ctr_select": 0.018050541516245487, "ctr_holdout": 0.019855595667870037, "n_select": 1108, "n_holdout": 1108, "eb": 0.023220260531002783, "labelled_winner": false }, { "headline": "Racists Spray-Paint A Comedian's Car. They Shouldn't Have Done That.", "impressions": 2224, "clicks": 73, "ctr": 0.0328237410071942, "ctr_select": 0.0341726618705036, "ctr_holdout": 0.03147482014388489, "n_select": 1112, "n_holdout": 1112, "eb": 0.02844145507933867, "labelled_winner": true }, { "headline": "Racists Spray Paint 'Go Home Arab' On A Car \u2014 Filmmaker Responds Perfectly", "impressions": 2225, "clicks": 70, "ctr": 0.0314606741573033, "ctr_select": 0.026079136690647483, "ctr_holdout": 0.036837376460017966, "n_select": 1112, "n_holdout": 1113, "eb": 0.025817317101277416, "labelled_winner": false }, { "headline": "An 'Arab' Filmmaker Drives His Car Into Cardboard Twin Towers. Trust Me, You'll Be On His Side.", "impressions": 2240, "clicks": 58, "ctr": 0.0258928571428571, "ctr_select": 0.020535714285714286, "ctr_holdout": 0.03125, "n_select": 1120, "n_holdout": 1120, "eb": 0.024011882455746036, "labelled_winner": false }, { "headline": "Idiot Racists Accidentally Make A Filmmaker Famous When They Spray Paint 'Go Home Arab' On His Car", "impressions": 2229, "clicks": 70, "ctr": 0.0314042171377299, "ctr_select": 0.0296229802513465, "ctr_holdout": 0.033183856502242155, "n_select": 1114, "n_holdout": 1115, "eb": 0.026967874552904225, "labelled_winner": false } ], "winner": 1, "pool_select": 0.025691699604743084, "pool_holdout": 0.0305316091954023, "lift_select": 0.3301051466519094, "lift_holdout": 0.030892932712653254, "eb_lift": 0.10702894385733575, "p_value": 0.045689773370043846, "held_up": false, "z_path": [ 0.481, 0.371, 0.146, 0.368, 0.336, 0.112, 0.01, 0.201, 0.369, 0.269 ], "first_cross": null, "final_z": 0.269 }, { "id": "51ef1b498faddd30ea00f6e3", "variation": "image", "week": 201329, "impressions": 11117, "pooled_ctr": 0.028694791760367006, "arms": [ { "headline": "Meet 4 Kenyan Women Who Want You To Keep Your 'Poverty Porn' To Yourself", "impressions": 3066, "clicks": 87, "ctr": 0.0283757338551859, "ctr_select": 0.028049575994781473, "ctr_holdout": 0.028701891715590344, "n_select": 1533, "n_holdout": 1533, "eb": 0.027191230694499902, "labelled_winner": false }, { "headline": "Meet 4 Kenyan Women Who Want You To Keep Your 'Poverty Porn' To Yourself", "impressions": 2687, "clicks": 84, "ctr": 0.0312616300707108, "ctr_select": 0.027529761904761904, "ctr_holdout": 0.034996276991809384, "n_select": 1344, "n_holdout": 1343, "eb": 0.026956132805984957, "labelled_winner": false }, { "headline": "Meet 4 Kenyan Women Who Want You To Keep Your 'Poverty Porn' To Yourself", "impressions": 2530, "clicks": 59, "ctr": 0.0233201581027668, "ctr_select": 0.02608695652173913, "ctr_holdout": 0.020553359683794466, "n_select": 1265, "n_holdout": 1265, "eb": 0.026434049822006773, "labelled_winner": false }, { "headline": "Meet 4 Kenyan Women Who Want You To Keep Your 'Poverty Porn' To Yourself", "impressions": 2834, "clicks": 89, "ctr": 0.0314043754410726, "ctr_select": 0.024700070571630206, "ctr_holdout": 0.038108680310515175, "n_select": 1417, "n_holdout": 1417, "eb": 0.025893718118468613, "labelled_winner": false } ], "winner": 0, "pool_select": 0.026623493434070875, "pool_holdout": 0.03076646275638719, "lift_select": 0.053564817263447306, "lift_holdout": -0.06710459558332671, "eb_lift": 0.02132467182922282, "p_value": 0.6835879884360658, "held_up": false, "z_path": [ -0.522, -1.27, -1.509, -1.048, -1.39, -0.589, -0.65, -0.456, -0.746, -0.643 ], "first_cross": null, "final_z": -0.643 }, { "id": "52253bf7661ea3d61f002f61", "variation": "other", "week": 201335, "impressions": 11100, "pooled_ctr": 0.011531531531531532, "arms": [ { "headline": "I Don't Know About You, But Most Folks I Know Are Working Harder While Falling More Behind", "impressions": 2884, "clicks": 32, "ctr": 0.0110957004160887, "ctr_select": 0.011095700416088766, "ctr_holdout": 0.011095700416088766, "n_select": 1442, "n_holdout": 1442, "eb": 0.01191974026433826, "labelled_winner": false }, { "headline": "I Don't Know About You, But Most Folks I Know Are Working Harder While Falling More Behind", "impressions": 2862, "clicks": 40, "ctr": 0.0139762403913347, "ctr_select": 0.014675052410901468, "ctr_holdout": 0.013277428371767994, "n_select": 1431, "n_holdout": 1431, "eb": 0.013288513514996643, "labelled_winner": false }, { "headline": "I Don't Know About You, But Most Folks I Know Are Working Harder While Falling More Behind", "impressions": 2766, "clicks": 22, "ctr": 0.0079537237888647, "ctr_select": 0.008676789587852495, "ctr_holdout": 0.0072306579898770785, "n_select": 1383, "n_holdout": 1383, "eb": 0.011028898702144262, "labelled_winner": false }, { "headline": "I Don't Know About You, But Most Folks I Know Are Working Harder While Falling More Behind", "impressions": 2588, "clicks": 34, "ctr": 0.0131375579598145, "ctr_select": 0.015455950540958269, "ctr_holdout": 0.010819165378670788, "n_select": 1294, "n_holdout": 1294, "eb": 0.013515699292616328, "labelled_winner": false } ], "winner": 3, "pool_select": 0.012432432432432432, "pool_holdout": 0.010630630630630631, "lift_select": 0.24319602177273048, "lift_holdout": 0.017735048332591097, "eb_lift": 0.08713233440609591, "p_value": 0.2623336593653335, "held_up": false, "z_path": [ 1.087, 0.732, 0.455, -0.704, -0.232, -0.727, -0.794, -0.584, -0.782, -0.267 ], "first_cross": null, "final_z": -0.267 }, { "id": "51708b0defe57b0002008c03", "variation": "other", "week": 201315, "impressions": 11086, "pooled_ctr": 0.00847916290817247, "arms": [ { "headline": "These Talented Artists Said Everything By Saying Almost Nothing At All", "impressions": 5581, "clicks": 52, "ctr": 0.0093173266439706, "ctr_select": 0.010752688172043012, "ctr_holdout": 0.007882479398065209, "n_select": 2790, "n_holdout": 2791, "eb": 0.009967367573197555, "labelled_winner": false }, { "headline": "These Talented Artists Said Everything By Saying Almost Nothing At All", "impressions": 5505, "clicks": 42, "ctr": 0.0076294277929155, "ctr_select": 0.007267441860465116, "ctr_holdout": 0.007991282237559027, "n_select": 2752, "n_holdout": 2753, "eb": 0.008069573938791343, "labelled_winner": false } ], "winner": 0, "pool_select": 0.009022013713460845, "pool_holdout": 0.007936507936507936, "lift_select": 0.19182795698924737, "lift_holdout": -0.006807595843783654, "eb_lift": 0.10478302181321686, "p_value": 0.17007457502193757, "held_up": false, "z_path": [ -0.327, 0.177, -0.217, 0.429, -0.186, 0.327, 0.188, 0.85, 0.905, 0.969 ], "first_cross": null, "final_z": 0.969 }, { "id": "5175a64c162a4e0002000e8f", "variation": "both", "week": 201316, "impressions": 11075, "pooled_ctr": 0.020225733634311512, "arms": [ { "headline": "Voices In Your Head? It May Not Be As Bad As Actually Going To A Doctor.", "impressions": 2772, "clicks": 33, "ctr": 0.0119047619047619, "ctr_select": 0.010822510822510822, "ctr_holdout": 0.012987012987012988, "n_select": 1386, "n_holdout": 1386, "eb": 0.015123748090861687, "labelled_winner": false }, { "headline": "A Woman With Voices In Her Head Does The Only Logical Thing And Becomes A Psychologist", "impressions": 2787, "clicks": 56, "ctr": 0.0200932902762827, "ctr_select": 0.01721664275466284, "ctr_holdout": 0.02297200287150036, "n_select": 1394, "n_holdout": 1393, "eb": 0.01751591758359473, "labelled_winner": false }, { "headline": "Having Voices In Your Head Is Tough, But It's Nothing Compared To A Doctor Mistreating It", "impressions": 2786, "clicks": 42, "ctr": 0.0150753768844221, "ctr_select": 0.014357501794687724, "ctr_holdout": 0.015793251974156496, "n_select": 1393, "n_holdout": 1393, "eb": 0.016442673574134512, "labelled_winner": false }, { "headline": "A Woman With Schizophrenia Is Given The Correct Answers To Test Questions By One Of The Voices", "impressions": 2730, "clicks": 93, "ctr": 0.034065934065934, "ctr_select": 0.02857142857142857, "ctr_holdout": 0.03956043956043956, "n_select": 1365, "n_holdout": 1365, "eb": 0.021726950596271725, "labelled_winner": true } ], "winner": 3, "pool_select": 0.01769591910436981, "pool_holdout": 0.022756005056890013, "lift_select": 0.6145772594752186, "lift_holdout": 0.7384615384615383, "eb_lift": 0.22779441226686536, "p_value": 0.00044672514843723947, "held_up": true, "z_path": [ 1.875, 2.882, 3.626, 3.438, 2.923, 3.069, 3.091, 3.234, 3.276, 3.201 ], "first_cross": 1, "final_z": 3.201 }, { "id": "51f94ff94dcfeddbb6001229", "variation": "image", "week": 201330, "impressions": 11035, "pooled_ctr": 0.033801540552786585, "arms": [ { "headline": "What World Do We Live In When Even An Act Of Kindness Faces This Kind of Discrimination?", "impressions": 2954, "clicks": 95, "ctr": 0.0321597833446174, "ctr_select": 0.03182125930941097, "ctr_holdout": 0.03249830737982397, "n_select": 1477, "n_holdout": 1477, "eb": 0.033309400522441475, "labelled_winner": false }, { "headline": "What World Do We Live In When Even An Act Of Kindness Faces This Kind of Discrimination?", "impressions": 2839, "clicks": 106, "ctr": 0.0373370905248326, "ctr_select": 0.036619718309859155, "ctr_holdout": 0.03805496828752643, "n_select": 1420, "n_holdout": 1419, "eb": 0.035155091740654745, "labelled_winner": false }, { "headline": "What World Do We Live In When Even An Act Of Kindness Faces This Kind of Discrimination?", "impressions": 2645, "clicks": 87, "ctr": 0.0328922495274102, "ctr_select": 0.03252647503782148, "ctr_holdout": 0.03325774754346183, "n_select": 1322, "n_holdout": 1323, "eb": 0.03362892076158049, "labelled_winner": false }, { "headline": "What World Do We Live In When Even An Act Of Kindness Faces This Kind of Discrimination?", "impressions": 2597, "clicks": 85, "ctr": 0.03273007316134, "ctr_select": 0.0362095531587057, "ctr_holdout": 0.029253271747498075, "n_select": 1298, "n_holdout": 1299, "eb": 0.03495842739929436, "labelled_winner": false } ], "winner": 1, "pool_select": 0.03425774877650897, "pool_holdout": 0.03334541500543675, "lift_select": 0.06894701542588866, "lift_holdout": 0.14123540766614573, "eb_lift": 0.026193868429588596, "p_value": 0.5701427590385715, "held_up": true, "z_path": [ -0.167, 0.465, 1.408, 0.863, 0.116, 0.888, 0.392, 0.533, 1.069, 0.921 ], "first_cross": null, "final_z": 0.921 }, { "id": "51f2bb3ffc2f41984e000314", "variation": "headline", "week": 201329, "impressions": 11008, "pooled_ctr": 0.024164244186046513, "arms": [ { "headline": "He\u2019s Caught Lots Of Criminals In The Act; Now He Has To Hide His Face So They Won't Catch Him", "impressions": 2665, "clicks": 83, "ctr": 0.0311444652908067, "ctr_select": 0.03153153153153153, "ctr_holdout": 0.03075768942235559, "n_select": 1332, "n_holdout": 1333, "eb": 0.027202006338853023, "labelled_winner": false }, { "headline": "The Bravest Man You\u2019ll Ever See Wearing A Mop On His Head", "impressions": 3152, "clicks": 58, "ctr": 0.0184010152284263, "ctr_select": 0.01649746192893401, "ctr_holdout": 0.02030456852791878, "n_select": 1576, "n_holdout": 1576, "eb": 0.021388178131353013, "labelled_winner": false }, { "headline": "Just Imagine How Much Justice We Could Get Done If We Had More Reporters Like Him", "impressions": 2361, "clicks": 61, "ctr": 0.0258365099534095, "ctr_select": 0.02711864406779661, "ctr_holdout": 0.024555461473327687, "n_select": 1180, "n_holdout": 1181, "eb": 0.025525104693315356, "labelled_winner": false }, { "headline": "Naming, Shaming, And Jailing: This Is What Justice Looks Like, People", "impressions": 2830, "clicks": 64, "ctr": 0.0226148409893992, "ctr_select": 0.025441696113074206, "ctr_holdout": 0.019787985865724382, "n_select": 1415, "n_holdout": 1415, "eb": 0.02498972793337895, "labelled_winner": false } ], "winner": 0, "pool_select": 0.02471379247683082, "pool_holdout": 0.023614895549500452, "lift_select": 0.2758677795442499, "lift_holdout": 0.30246984823128864, "eb_lift": 0.10068118296108963, "p_value": 0.06563235505527663, "held_up": true, "z_path": [ -0.209, -1.297, -0.871, -0.323, -0.049, 0.561, 0.836, 0.931, 0.903, 1.126 ], "first_cross": null, "final_z": 1.126 }, { "id": "54f8ec713831360024600100", "variation": "image", "week": 201509, "impressions": 10993, "pooled_ctr": 0.030292004002547075, "arms": [ { "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.", "impressions": 1786, "clicks": 72, "ctr": 0.0403135498320268, "ctr_select": 0.043673012318029114, "ctr_holdout": 0.036954087346024636, "n_select": 893, "n_holdout": 893, "eb": 0.034606737781897844, "labelled_winner": false }, { "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.", "impressions": 1869, "clicks": 23, "ctr": 0.0123060460139111, "ctr_select": 0.011777301927194861, "ctr_holdout": 0.012834224598930482, "n_select": 934, "n_holdout": 935, "eb": 0.0255595018696753, "labelled_winner": false }, { "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.", "impressions": 1804, "clicks": 52, "ctr": 0.0288248337028824, "ctr_select": 0.028824833702882482, "ctr_holdout": 0.028824833702882482, "n_select": 902, "n_holdout": 902, "eb": 0.03047235565624745, "labelled_winner": false }, { "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.", "impressions": 1829, "clicks": 57, "ctr": 0.0311645708037178, "ctr_select": 0.03282275711159737, "ctr_holdout": 0.029508196721311476, "n_select": 914, "n_holdout": 915, "eb": 0.03159694466929576, "labelled_winner": false }, { "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.", "impressions": 1867, "clicks": 63, "ctr": 0.0337439742903053, "ctr_select": 0.034261241970021415, "ctr_holdout": 0.03322615219721329, "n_select": 934, "n_holdout": 933, "eb": 0.032017666775371115, "labelled_winner": false }, { "headline": "This test will tell you whether you're prejudiced without knowing it. I dare you to take it.", "impressions": 1838, "clicks": 66, "ctr": 0.0359085963003264, "ctr_select": 0.035908596300326445, "ctr_holdout": 0.035908596300326445, "n_select": 919, "n_holdout": 919, "eb": 0.032475004820722464, "labelled_winner": false } ], "winner": 0, "pool_select": 0.03111353711790393, "pool_holdout": 0.029470620338366383, "lift_select": 0.403665939765427, "lift_holdout": 0.25392974161171233, "eb_lift": 0.11227269502520798, "p_value": 0.01817422469029952, "held_up": true, "z_path": [ 1.591, 1.231, 1.378, 0.935, 1.25, 0.585, 0.664, 0.448, 0.255, 0.693 ], "first_cross": null, "final_z": 0.693 }, { "id": "517568f4c270aa000200b91a", "variation": "image", "week": 201316, "impressions": 10951, "pooled_ctr": 0.03972239978084193, "arms": [ { "headline": "The One Where The Weather Lady Starts Telling The Truth. A Little Too Much Truth, To Be Honest.", "impressions": 2761, "clicks": 87, "ctr": 0.0315103223469757, "ctr_select": 0.035507246376811595, "ctr_holdout": 0.027516292541636494, "n_select": 1380, "n_holdout": 1381, "eb": 0.03855686912472771, "labelled_winner": false }, { "headline": "The One Where The Weather Lady Starts Telling The Truth. A Little Too Much Truth, To Be Honest.", "impressions": 2808, "clicks": 133, "ctr": 0.0473646723646723, "ctr_select": 0.046296296296296294, "ctr_holdout": 0.04843304843304843, "n_select": 1404, "n_holdout": 1404, "eb": 0.04260734187597669, "labelled_winner": false }, { "headline": "The One Where The Weather Lady Starts Telling The Truth. A Little Too Much Truth, To Be Honest.", "impressions": 2645, "clicks": 103, "ctr": 0.0389413988657845, "ctr_select": 0.036308623298033284, "ctr_holdout": 0.04157218442932729, "n_select": 1322, "n_holdout": 1323, "eb": 0.03889653975387367, "labelled_winner": false }, { "headline": "The One Where The Weather Lady Starts Telling The Truth. A Little Too Much Truth, To Be Honest.", "impressions": 2737, "clicks": 112, "ctr": 0.0409207161125319, "ctr_select": 0.04312865497076023, "ctr_holdout": 0.03871439006574142, "n_select": 1368, "n_holdout": 1369, "eb": 0.04139559418249401, "labelled_winner": false } ], "winner": 1, "pool_select": 0.040372670807453416, "pool_holdout": 0.039072484937009315, "lift_select": 0.14672364672364657, "lift_holdout": 0.23956918816731876, "eb_lift": 0.055351083389576505, "p_value": 0.190951409805865, "held_up": true, "z_path": [ 0.854, 0.693, 0.9, 1.447, 1.452, 2.09, 1.835, 1.406, 1.556, 1.167 ], "first_cross": 5, "final_z": 1.167 }, { "id": "51f1823e9a72d2ca6300081c", "variation": "headline", "week": 201329, "impressions": 10912, "pooled_ctr": 0.08165322580645161, "arms": [ { "headline": "The Most Anti-Gay Airline In The Entire World Has A New Safety Video", "impressions": 2861, "clicks": 264, "ctr": 0.0922754281719678, "ctr_select": 0.09300699300699301, "ctr_holdout": 0.09154437456324249, "n_select": 1430, "n_holdout": 1431, "eb": 0.08457758962906371, "labelled_winner": false }, { "headline": "The Most Anti-Gay Airline In The History Of The World Has A New Safety Video", "impressions": 2625, "clicks": 211, "ctr": 0.0803809523809523, "ctr_select": 0.07850609756097561, "ctr_holdout": 0.08225437928408226, "n_select": 1312, "n_holdout": 1313, "eb": 0.0790620216526423, "labelled_winner": false }, { "headline": "The Most Anti-Gay Airline In The Entire World Has A New Jerktacular Safety Video", "impressions": 2812, "clicks": 201, "ctr": 0.071479374110953, "ctr_select": 0.07610241820768136, "ctr_holdout": 0.06685633001422475, "n_select": 1406, "n_holdout": 1406, "eb": 0.07814040279215385, "labelled_winner": false }, { "headline": "The Most Anti-Gay Airline In The Entire World Has A New Scumtacular Safety Video", "impressions": 2614, "clicks": 215, "ctr": 0.0822494261667941, "ctr_select": 0.06885998469778118, "ctr_holdout": 0.09563886763580719, "n_select": 1307, "n_holdout": 1307, "eb": 0.07558457582988534, "labelled_winner": false } ], "winner": 0, "pool_select": 0.07937671860678276, "pool_holdout": 0.08392889866226864, "lift_select": 0.17171627448763727, "lift_holdout": 0.0907372314227386, "eb_lift": 0.06552136588116064, "p_value": 0.026437325468948777, "held_up": false, "z_path": [ 1.232, 0.936, 1.694, 1.488, 2.102, 1.632, 2.241, 2.252, 2.098, 1.565 ], "first_cross": 4, "final_z": 1.565 }, { "id": "51d47483c6d06f695c000505", "variation": "image", "week": 201326, "impressions": 10900, "pooled_ctr": 0.034128440366972476, "arms": [ { "headline": "The Next Time Someone Talks About Starving Kids In Africa, Show Them This", "impressions": 2563, "clicks": 60, "ctr": 0.0234100663285212, "ctr_select": 0.0249609984399376, "ctr_holdout": 0.02185792349726776, "n_select": 1282, "n_holdout": 1281, "eb": 0.030154702701359407, "labelled_winner": false }, { "headline": "The Next Time Someone Talks About Starving Kids In Africa, Show Them This", "impressions": 2645, "clicks": 90, "ctr": 0.0340264650283553, "ctr_select": 0.03328290468986384, "ctr_holdout": 0.03476946334089191, "n_select": 1322, "n_holdout": 1323, "eb": 0.03312028192465662, "labelled_winner": false }, { "headline": "The Next Time Someone Talks About Starving Kids In Africa, Show Them This", "impressions": 2923, "clicks": 118, "ctr": 0.040369483407458, "ctr_select": 0.03967168262653899, "ctr_holdout": 0.04106776180698152, "n_select": 1462, "n_holdout": 1461, "eb": 0.035597506629444046, "labelled_winner": false }, { "headline": "The Next Time Someone Talks About Starving Kids In Africa, Show Them This", "impressions": 2769, "clicks": 104, "ctr": 0.0375586854460093, "ctr_select": 0.033236994219653176, "ctr_holdout": 0.04187725631768953, "n_select": 1384, "n_holdout": 1385, "eb": 0.03310584058352231, "labelled_winner": false } ], "winner": 2, "pool_select": 0.03302752293577982, "pool_holdout": 0.03522935779816514, "lift_select": 0.20117039063687492, "lift_holdout": 0.16572553045858984, "eb_lift": 0.0778133951692781, "p_value": 0.09654473783038803, "held_up": false, "z_path": [ -0.217, 1.007, 1.098, 0.95, 0.94, 1.25, 1.045, 0.734, 0.995, 1.246 ], "first_cross": null, "final_z": 1.246 }, { "id": "516e6376eb1fc000020183d9", "variation": "both", "week": 201315, "impressions": 10871, "pooled_ctr": 0.00901481004507405, "arms": [ { "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Unless It's Friday.", "impressions": 2204, "clicks": 17, "ctr": 0.0077132486388384, "ctr_select": 0.007259528130671506, "ctr_holdout": 0.008166969147005444, "n_select": 1102, "n_holdout": 1102, "eb": 0.008448607234753339, "labelled_winner": false }, { "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Except On Friday.", "impressions": 2126, "clicks": 21, "ctr": 0.0098777046095954, "ctr_select": 0.011288805268109126, "ctr_holdout": 0.008466603951081843, "n_select": 1063, "n_holdout": 1063, "eb": 0.00972925876305196, "labelled_winner": false }, { "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Unless It's Friday.", "impressions": 2210, "clicks": 18, "ctr": 0.0081447963800904, "ctr_select": 0.009954751131221719, "ctr_holdout": 0.006334841628959276, "n_select": 1105, "n_holdout": 1105, "eb": 0.009317703612814482, "labelled_winner": false }, { "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Unless It's Friday.", "impressions": 2192, "clicks": 20, "ctr": 0.0091240875912408, "ctr_select": 0.010036496350364963, "ctr_holdout": 0.008211678832116789, "n_select": 1096, "n_holdout": 1096, "eb": 0.009342269154280408, "labelled_winner": false }, { "headline": "If You Saw Someone Bullying A Kid For Being Gay Would You Speak Out? Me Too. Unless It's Friday.", "impressions": 2139, "clicks": 22, "ctr": 0.0102851799906498, "ctr_select": 0.0065420560747663555, "ctr_holdout": 0.01403180542563143, "n_select": 1070, "n_holdout": 1069, "eb": 0.008233226574463081, "labelled_winner": false } ], "winner": 1, "pool_select": 0.009013980868285504, "pool_holdout": 0.009015639374425023, "lift_select": 0.25236623341716746, "lift_holdout": -0.060898112772860835, "eb_lift": 0.07935205379490728, "p_value": 0.3816129727632338, "held_up": false, "z_path": [ -1.381, -0.585, -0.832, -0.832, -0.319, -0.099, -0.439, -0.08, -0.072, 0.255 ], "first_cross": null, "final_z": 0.255 }, { "id": "515b3b374c9a93000200b70d", "variation": "image", "week": 201313, "impressions": 10868, "pooled_ctr": 0.03422892896577107, "arms": [ { "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It", "impressions": 2161, "clicks": 77, "ctr": 0.0356316520129569, "ctr_select": 0.03611111111111111, "ctr_holdout": 0.03515263644773358, "n_select": 1080, "n_holdout": 1081, "eb": 0.03645911208577955, "labelled_winner": false }, { "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It", "impressions": 2172, "clicks": 65, "ctr": 0.0299263351749539, "ctr_select": 0.03314917127071823, "ctr_holdout": 0.026703499079189688, "n_select": 1086, "n_holdout": 1086, "eb": 0.035513448001647664, "labelled_winner": false }, { "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It", "impressions": 2200, "clicks": 84, "ctr": 0.0381818181818181, "ctr_select": 0.04454545454545455, "ctr_holdout": 0.031818181818181815, "n_select": 1100, "n_holdout": 1100, "eb": 0.03917170359020483, "labelled_winner": false }, { "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It", "impressions": 2255, "clicks": 72, "ctr": 0.0319290465631929, "ctr_select": 0.03280141843971631, "ctr_holdout": 0.031055900621118012, "n_select": 1128, "n_holdout": 1127, "eb": 0.03537078754266193, "labelled_winner": false }, { "headline": "North Korea Doesn't Want You To See This And They'll Be Really Furious If You Share It", "impressions": 2080, "clicks": 74, "ctr": 0.035576923076923, "ctr_select": 0.03653846153846154, "ctr_holdout": 0.03461538461538462, "n_select": 1040, "n_holdout": 1040, "eb": 0.03659562362545704, "labelled_winner": false } ], "winner": 2, "pool_select": 0.03662127346337873, "pool_holdout": 0.03183658446816342, "lift_select": 0.21638190954773884, "lift_holdout": -0.0005780346820811744, "eb_lift": 0.0696434035636837, "p_value": 0.11717217104502756, "held_up": false, "z_path": [ -0.114, 0.959, 1.694, 1.387, 1.419, 0.552, 0.854, -0.051, 0.179, 0.452 ], "first_cross": null, "final_z": 0.452 }, { "id": "5194806d83c7ab001000a221", "variation": "image", "week": 201319, "impressions": 10849, "pooled_ctr": 0.017052262881371554, "arms": [ { "headline": "They Were Happily Married For 16 Years. Paid Taxes. Never Had A Parking Ticket. So We Deported Him.", "impressions": 2766, "clicks": 45, "ctr": 0.0162689804772234, "ctr_select": 0.018076644974692697, "ctr_holdout": 0.014461315979754157, "n_select": 1383, "n_holdout": 1383, "eb": 0.017722715188389155, "labelled_winner": false }, { "headline": "They Were Happily Married For 16 Years. Paid Taxes. Never Had A Parking Ticket. So We Deported Him.", "impressions": 2728, "clicks": 52, "ctr": 0.0190615835777126, "ctr_select": 0.017595307917888565, "ctr_holdout": 0.020527859237536656, "n_select": 1364, "n_holdout": 1364, "eb": 0.01754256231744982, "labelled_winner": false }, { "headline": "They Were Happily Married For 16 Years. Paid Taxes. Never Had A Parking Ticket. So We Deported Him.", "impressions": 2704, "clicks": 45, "ctr": 0.0166420118343195, "ctr_select": 0.017011834319526627, "ctr_holdout": 0.016272189349112426, "n_select": 1352, "n_holdout": 1352, "eb": 0.017327424909484686, "labelled_winner": false }, { "headline": "They Were Happily Married For 16 Years. Paid Taxes. Never Had A Parking Ticket. So We Deported Him.", "impressions": 2651, "clicks": 43, "ctr": 0.0162202942285929, "ctr_select": 0.017345399698340876, "ctr_holdout": 0.01509433962264151, "n_select": 1326, "n_holdout": 1325, "eb": 0.017451066628465278, "labelled_winner": false } ], "winner": 0, "pool_select": 0.017511520737327188, "pool_holdout": 0.016592920353982302, "lift_select": 0.03227156829166189, "lift_holdout": -0.12846469028681617, "eb_lift": 0.012060314705380826, "p_value": 0.8527406540647168, "held_up": false, "z_path": [ -0.023, 0.187, -0.602, -0.7, -0.197, -0.694, -0.298, -0.402, -0.807, -0.786 ], "first_cross": null, "final_z": -0.786 }, { "id": "520af3a05ac98b06e20036e6", "variation": "headline", "week": 201332, "impressions": 10804, "pooled_ctr": 0.023417252869307664, "arms": [ { "headline": "If You Don\u2019t Say, 'Holy S***' Once Over The Next Three Minutes, I\u2019ve Failed", "impressions": 2536, "clicks": 73, "ctr": 0.0287854889589905, "ctr_select": 0.033123028391167195, "ctr_holdout": 0.02444794952681388, "n_select": 1268, "n_holdout": 1268, "eb": 0.025949435360090842, "labelled_winner": false }, { "headline": "If You Don\u2019t Say, 'Holy Crap' Once Over The Next Three Minutes, I\u2019ve Failed", "impressions": 2686, "clicks": 72, "ctr": 0.0268056589724497, "ctr_select": 0.021593447505584513, "ctr_holdout": 0.03201787043931496, "n_select": 1343, "n_holdout": 1343, "eb": 0.02186655038625483, "labelled_winner": false }, { "headline": "If You Don\u2019t Say, 'Holy S***' At Least Once , I\u2019ve Failed", "impressions": 2855, "clicks": 61, "ctr": 0.0213660245183887, "ctr_select": 0.021708683473389355, "ctr_holdout": 0.02102312543798178, "n_select": 1428, "n_holdout": 1427, "eb": 0.02190428520318719, "labelled_winner": false }, { "headline": "If You Don\u2019t Say, 'Holy Crap' At Least Once, I\u2019ve Failed", "impressions": 2727, "clicks": 47, "ctr": 0.0172350568390172, "ctr_select": 0.012463343108504398, "ctr_holdout": 0.022010271460014674, "n_select": 1364, "n_holdout": 1363, "eb": 0.018482460067696372, "labelled_winner": false } ], "winner": 0, "pool_select": 0.022024801036461226, "pool_holdout": 0.0248102203295686, "lift_select": 0.5038968268695492, "lift_holdout": -0.014601676161777877, "eb_lift": 0.17819159034093124, "p_value": 0.0020837292088948705, "held_up": false, "z_path": [ 0.75, 0.185, 0.858, 1.753, 1.565, 1.846, 2.013, 1.874, 1.756, 1.746 ], "first_cross": 6, "final_z": 1.746 }, { "id": "521d6395188487ad41003a3b", "variation": "image", "week": 201334, "impressions": 10791, "pooled_ctr": 0.02075803910666296, "arms": [ { "headline": "If 287,000 Women Had Been Given A Few More Choices About Their Sex Lives They\u2019d Still Be Alive", "impressions": 2709, "clicks": 66, "ctr": 0.0243632336655592, "ctr_select": 0.024372230428360415, "ctr_holdout": 0.024354243542435424, "n_select": 1354, "n_holdout": 1355, "eb": 0.021273023355833864, "labelled_winner": false }, { "headline": "If 287,000 Women Had Been Given A Few More Choices About Their Sex Lives They\u2019d Still Be Alive", "impressions": 2614, "clicks": 52, "ctr": 0.0198928844682478, "ctr_select": 0.019127773527161437, "ctr_holdout": 0.020657995409334353, "n_select": 1307, "n_holdout": 1307, "eb": 0.01934178150542352, "labelled_winner": false }, { "headline": "If 287,000 Women Had Been Given A Few More Choices About Their Sex Lives They\u2019d Still Be Alive", "impressions": 2807, "clicks": 71, "ctr": 0.0252939080869255, "ctr_select": 0.022792022792022793, "ctr_holdout": 0.027797576621525304, "n_select": 1404, "n_holdout": 1403, "eb": 0.020718531472943405, "labelled_winner": false }, { "headline": "If 287,000 Women Had Been Given A Few More Choices About Their Sex Lives They\u2019d Still Be Alive", "impressions": 2661, "clicks": 35, "ctr": 0.0131529500187899, "ctr_select": 0.011278195488721804, "ctr_holdout": 0.015026296018031555, "n_select": 1330, "n_holdout": 1331, "eb": 0.016478369616506512, "labelled_winner": false } ], "winner": 0, "pool_select": 0.019462465245597776, "pool_holdout": 0.022053372868791696, "lift_select": 0.2522684110571851, "lift_holdout": 0.10433191726875246, "eb_lift": 0.09302820004498757, "p_value": 0.13076553528697094, "held_up": false, "z_path": [ -0.776, -0.489, -1.2, -1.625, -0.8, -0.833, -0.647, -0.656, 0.021, -0.222 ], "first_cross": null, "final_z": -0.222 }, { "id": "51436069220cb800020005c4", "variation": "headline", "week": 201306, "impressions": 10599, "pooled_ctr": 0.011133125766581754, "arms": [ { "headline": "Shooting Victim Gabby Giffords Can Finally Speak Again \u2014 And It's About Gun Violence", "impressions": 2654, "clicks": 34, "ctr": 0.0128108515448379, "ctr_select": 0.012057272042200452, "ctr_holdout": 0.013564431047475508, "n_select": 1327, "n_holdout": 1327, "eb": 0.011110250041202481, "labelled_winner": false }, { "headline": "Optimism About Gun Violence From The Woman Who Knows Far Too Much About Gun Violence", "impressions": 2512, "clicks": 21, "ctr": 0.0083598726114649, "ctr_select": 0.007165605095541401, "ctr_holdout": 0.009554140127388535, "n_select": 1256, "n_holdout": 1256, "eb": 0.009372226078229805, "labelled_winner": false }, { "headline": "2 Years Ago Gabby Giffords Was Shot In The Head. Now She's Got Something Important To Say.", "impressions": 2753, "clicks": 39, "ctr": 0.0141663639665819, "ctr_select": 0.015261627906976744, "ctr_holdout": 0.013071895424836602, "n_select": 1376, "n_holdout": 1377, "eb": 0.01231652031675617, "labelled_winner": false }, { "headline": "90% Of Americans Agree On Gun Control Reform. So Why Hasn't Anything Happened Yet?", "impressions": 2680, "clicks": 24, "ctr": 0.008955223880597, "ctr_select": 0.007462686567164179, "ctr_holdout": 0.010447761194029851, "n_select": 1340, "n_holdout": 1340, "eb": 0.009430386436632078, "labelled_winner": false } ], "winner": 2, "pool_select": 0.010568031704095112, "pool_holdout": 0.01169811320754717, "lift_select": 0.44413154069767424, "lift_holdout": 0.11743622180054825, "eb_lift": 0.16545073497305274, "p_value": 0.047832974893710224, "held_up": false, "z_path": [ -0.43, -0.064, 0.395, 0.463, 0.717, -0.273, -0.691, -0.272, -0.273, 0.432 ], "first_cross": null, "final_z": 0.432 }, { "id": "5349aebefe9c5af4c3000048", "variation": "headline", "week": 201414, "impressions": 10598, "pooled_ctr": 0.02349499905642574, "arms": [ { "headline": "The Most Honest And Evil Company Ad I Have Ever Laid My Eyes On. Also The Rudest.", "impressions": 2063, "clicks": 43, "ctr": 0.0208434318952981, "ctr_select": 0.02422480620155039, "ctr_holdout": 0.01745877788554801, "n_select": 1032, "n_holdout": 1031, "eb": 0.022482223237236587, "labelled_winner": false }, { "headline": "The Most Honest And Awful Company Ad I Have Ever Laid My Eyes On. Did I Mention Awful?", "impressions": 2141, "clicks": 55, "ctr": 0.0256889304063521, "ctr_select": 0.02336448598130841, "ctr_holdout": 0.028011204481792718, "n_select": 1070, "n_holdout": 1071, "eb": 0.02223003884609498, "labelled_winner": false }, { "headline": "The Most Honest And Awful Company Ad I Have Ever Laid My Eyes On. Also The Rudest.", "impressions": 2129, "clicks": 48, "ctr": 0.0225457961484264, "ctr_select": 0.016917293233082706, "ctr_holdout": 0.028169014084507043, "n_select": 1064, "n_holdout": 1065, "eb": 0.02019951442709867, "labelled_winner": false }, { "headline": "The Most Honest And Disturbing Company Ad I Have Ever Laid My Eyes On. Also The Rudest.", "impressions": 2156, "clicks": 49, "ctr": 0.0227272727272727, "ctr_select": 0.017625231910946195, "ctr_holdout": 0.027829313543599257, "n_select": 1078, "n_holdout": 1078, "eb": 0.020410724864659658, "labelled_winner": false }, { "headline": "The Most Honest And Gross Company Ad I Have Ever Laid My Eyes On. Also The Rudest.", "impressions": 2109, "clicks": 54, "ctr": 0.0256045519203413, "ctr_select": 0.026565464895635674, "ctr_holdout": 0.024644549763033177, "n_select": 1054, "n_holdout": 1055, "eb": 0.023225288395580204, "labelled_winner": false } ], "winner": 4, "pool_select": 0.021706304265760666, "pool_holdout": 0.025283018867924528, "lift_select": 0.22385941753980676, "lift_holdout": -0.025252882506896723, "eb_lift": 0.06997893843290348, "p_value": 0.22645326693522005, "held_up": false, "z_path": [ -1.169, -1.553, -1.52, -0.692, -0.37, 0.575, 0.657, 0.588, 0.541, 1.02 ], "first_cross": null, "final_z": 1.02 }, { "id": "520b42103715ea76550065fb", "variation": "image", "week": 201332, "impressions": 10570, "pooled_ctr": 0.025733207190160833, "arms": [ { "headline": "Watch The Daily Show Supersize The Hilarity Of Our Minimum Wage Dispute", "impressions": 2533, "clicks": 76, "ctr": 0.0300039478878799, "ctr_select": 0.030015797788309637, "ctr_holdout": 0.02999210734017364, "n_select": 1266, "n_holdout": 1267, "eb": 0.028592189524651028, "labelled_winner": false }, { "headline": "Watch The Daily Show Supersize The Hilarity Of Our Minimum Wage Dispute", "impressions": 2559, "clicks": 56, "ctr": 0.0218835482610394, "ctr_select": 0.0265625, "ctr_holdout": 0.017200938232994525, "n_select": 1280, "n_holdout": 1279, "eb": 0.02736910437003197, "labelled_winner": false }, { "headline": "Watch The Daily Show Supersize The Hilarity Of Our Minimum Wage Dispute", "impressions": 2772, "clicks": 69, "ctr": 0.0248917748917748, "ctr_select": 0.026695526695526696, "ctr_holdout": 0.023088023088023088, "n_select": 1386, "n_holdout": 1386, "eb": 0.027395800646202912, "labelled_winner": false }, { "headline": "Watch The Daily Show Supersize The Hilarity Of Our Minimum Wage Dispute", "impressions": 2706, "clicks": 71, "ctr": 0.0262379896526238, "ctr_select": 0.028085735402808575, "ctr_holdout": 0.024390243902439025, "n_select": 1353, "n_holdout": 1353, "eb": 0.02791451985833549, "labelled_winner": false } ], "winner": 0, "pool_select": 0.02781456953642384, "pool_holdout": 0.023651844843897825, "lift_select": 0.07913939667494163, "lift_holdout": 0.2680662983425415, "eb_lift": 0.027957290052929906, "p_value": 0.5849431591592602, "held_up": true, "z_path": [ 0.103, -0.444, 0.528, 0.521, -0.087, -0.248, 0.238, 0.274, 0.653, 0.825 ], "first_cross": null, "final_z": 0.825 }, { "id": "51d3a5338505f3a3a1002c75", "variation": "other", "week": 201326, "impressions": 10530, "pooled_ctr": 0.02678062678062678, "arms": [ { "headline": "If A 14-Year Old In Africa Can Build These With Scraps, Imagine What We Could Do In America?", "impressions": 2686, "clicks": 60, "ctr": 0.0223380491437081, "ctr_select": 0.014892032762472078, "ctr_holdout": 0.029784065524944156, "n_select": 1343, "n_holdout": 1343, "eb": 0.02037476429104088, "labelled_winner": false }, { "headline": "If A 14-Year Old In Africa Can Build These With Scraps, Imagine What We Could Do In America?", "impressions": 2620, "clicks": 62, "ctr": 0.0236641221374045, "ctr_select": 0.022900763358778626, "ctr_holdout": 0.024427480916030534, "n_select": 1310, "n_holdout": 1310, "eb": 0.023316676239208734, "labelled_winner": false }, { "headline": "If A 14-Year Old In Africa Can Build These With Scraps, Imagine What We Could Do In America?", "impressions": 2577, "clicks": 88, "ctr": 0.0341482343810632, "ctr_select": 0.031055900621118012, "ctr_holdout": 0.037238169123351435, "n_select": 1288, "n_holdout": 1289, "eb": 0.02623233022311065, "labelled_winner": true }, { "headline": "If A 14-Year Old In Africa Can Build These With Scraps, Imagine What We Could Do In America?", "impressions": 2647, "clicks": 72, "ctr": 0.0272006044578768, "ctr_select": 0.0256797583081571, "ctr_holdout": 0.02872260015117158, "n_select": 1324, "n_holdout": 1323, "eb": 0.024325427330211254, "labelled_winner": false } ], "winner": 2, "pool_select": 0.02355175688509022, "pool_holdout": 0.030009496676163344, "lift_select": 0.3186235223402123, "lift_holdout": 0.24087949642053985, "eb_lift": 0.11381627923127069, "p_value": 0.04101713464094579, "held_up": true, "z_path": [ 1.363, 2.446, 1.426, 1.748, 1.939, 1.513, 1.212, 1.325, 1.273, 1.457 ], "first_cross": 1, "final_z": 1.457 }, { "id": "53208267c532cf5ce700003d", "variation": "image", "week": 201410, "impressions": 10511, "pooled_ctr": 0.01141661116925126, "arms": [ { "headline": "HIV-Positive, Homeless, And Gay In New York At The Age Of 16? Give This Story A Listen.", "impressions": 2665, "clicks": 22, "ctr": 0.0082551594746716, "ctr_select": 0.010510510510510511, "ctr_holdout": 0.006001500375093774, "n_select": 1332, "n_holdout": 1333, "eb": 0.010725386208783449, "labelled_winner": false }, { "headline": "HIV-Positive, Homeless, And Gay In New York At The Age Of 16? Give This Story A Listen.", "impressions": 2592, "clicks": 29, "ctr": 0.0111882716049382, "ctr_select": 0.005401234567901234, "ctr_holdout": 0.016975308641975308, "n_select": 1296, "n_holdout": 1296, "eb": 0.008895159488932967, "labelled_winner": false }, { "headline": "HIV-Positive, Homeless, And Gay In New York At The Age Of 16? Give This Story A Listen.", "impressions": 2657, "clicks": 24, "ctr": 0.0090327436958976, "ctr_select": 0.00677710843373494, "ctr_holdout": 0.011286681715575621, "n_select": 1328, "n_holdout": 1329, "eb": 0.009365674335562653, "labelled_winner": false }, { "headline": "HIV-Positive, Homeless, And Gay In New York At The Age Of 16? Give This Story A Listen.", "impressions": 2597, "clicks": 45, "ctr": 0.0173276857912976, "ctr_select": 0.020801232665639446, "ctr_holdout": 0.013856812933025405, "n_select": 1298, "n_holdout": 1299, "eb": 0.014421675064450069, "labelled_winner": true } ], "winner": 3, "pool_select": 0.010848877046060145, "pool_holdout": 0.011984021304926764, "lift_select": 0.9173627443029762, "lift_holdout": 0.15627405696689767, "eb_lift": 0.329324224361766, "p_value": 6.637202282107459e-05, "held_up": false, "z_path": [ 0.329, 0.557, 0.531, 0.451, 1.087, 1.301, 1.515, 2.132, 2.767, 2.935 ], "first_cross": 7, "final_z": 2.935 }, { "id": "53e4faa3220776c2b3000002", "variation": "headline", "week": 201431, "impressions": 10328, "pooled_ctr": 0.013555383423702556, "arms": [ { "headline": "A Politician Decides To Be Sexist To A Woman Who Wants Him To Learn Facts. Then He Gets Xenophobic.", "impressions": 2506, "clicks": 30, "ctr": 0.0119712689545091, "ctr_select": 0.009577015163607342, "ctr_holdout": 0.014365522745411013, "n_select": 1253, "n_holdout": 1253, "eb": 0.0126621063974509, "labelled_winner": false }, { "headline": "A Politician Decides To Be Awful To A Woman Who Wants Him To Learn Facts. Then He Gets Xenophobic.", "impressions": 2586, "clicks": 36, "ctr": 0.0139211136890951, "ctr_select": 0.014694508894044857, "ctr_holdout": 0.013147718484145398, "n_select": 1293, "n_holdout": 1293, "eb": 0.014460516642586579, "labelled_winner": true }, { "headline": "A Politician Decides To Be Rude To A Woman Who Wants Him To Learn Facts. Then He Gets Xenophobic.", "impressions": 2639, "clicks": 38, "ctr": 0.0143993937097385, "ctr_select": 0.013636363636363636, "ctr_holdout": 0.015163002274450341, "n_select": 1320, "n_holdout": 1319, "eb": 0.014078287534439819, "labelled_winner": false }, { "headline": "A Politician Decides To Be Hostile To A Woman Who Wants Him To Learn Facts. Then He Gets Xenophobic.", "impressions": 2597, "clicks": 36, "ctr": 0.0138621486330381, "ctr_select": 0.01926040061633282, "ctr_holdout": 0.008468052347959968, "n_select": 1298, "n_holdout": 1299, "eb": 0.0160999505502359, "labelled_winner": false } ], "winner": 3, "pool_select": 0.014329976762199844, "pool_holdout": 0.012780790085205267, "lift_select": 0.3440636321992254, "lift_holdout": -0.33743905568385946, "eb_lift": 0.12351546812727299, "p_value": 0.08409416499354987, "held_up": false, "z_path": [ -0.645, -0.007, 1.09, 0.345, 0.944, 1.338, 1.101, 0.234, 0.346, -0.018 ], "first_cross": null, "final_z": -0.018 }, { "id": "538e322d13f17fb43e000064", "variation": "headline", "week": 201424, "impressions": 10282, "pooled_ctr": 0.008947675549503988, "arms": [ { "headline": "There's A Lot To Lose For Girls Like Christine By That Awkward Thing Called Puberty", "impressions": 2614, "clicks": 24, "ctr": 0.0091813312930374, "ctr_select": 0.009946442234123947, "ctr_holdout": 0.008416220351951033, "n_select": 1307, "n_holdout": 1307, "eb": 0.008934683937558223, "labelled_winner": false }, { "headline": "Why Are Girls Dropping Out Of School Because Of Something Half The World Goes Through?", "impressions": 2588, "clicks": 27, "ctr": 0.0104327666151468, "ctr_select": 0.010046367851622875, "ctr_holdout": 0.010819165378670788, "n_select": 1294, "n_holdout": 1294, "eb": 0.008966843575680544, "labelled_winner": false }, { "headline": "Puberty Is So Awkward, And Here It's Making Girls Drop Out Of School", "impressions": 2530, "clicks": 19, "ctr": 0.0075098814229249, "ctr_select": 0.0071146245059288534, "ctr_holdout": 0.007905138339920948, "n_select": 1265, "n_holdout": 1265, "eb": 0.007922948208380177, "labelled_winner": false }, { "headline": "A Girl's Body Starts To Change, So... She Never Goes To School Again?", "impressions": 2550, "clicks": 22, "ctr": 0.0086274509803921, "ctr_select": 0.006274509803921568, "ctr_holdout": 0.010980392156862745, "n_select": 1275, "n_holdout": 1275, "eb": 0.007622552914701078, "labelled_winner": false } ], "winner": 1, "pool_select": 0.008364131491927641, "pool_holdout": 0.009531219607080334, "lift_select": 0.2011250494230976, "lift_holdout": 0.1351291675866637, "eb_lift": 0.07205913540868991, "p_value": 0.4424145827960426, "held_up": false, "z_path": [ -0.569, -0.89, -0.515, 0.267, 0.882, 0.607, 0.539, 0.682, 0.798, 0.458 ], "first_cross": null, "final_z": 0.458 }, { "id": "537ccb78db41d8f8ac0000a1", "variation": "headline", "week": 201420, "impressions": 10275, "pooled_ctr": 0.020632603406326035, "arms": [ { "headline": "Thought I Knew Where This Bullying Video Was Going But It Opened My Eyes To Something Bigger", "impressions": 2557, "clicks": 61, "ctr": 0.0238560813453265, "ctr_select": 0.024256651017214397, "ctr_holdout": 0.023455824863174355, "n_select": 1278, "n_holdout": 1279, "eb": 0.022677572252107932, "labelled_winner": false }, { "headline": "Sticks and Stones May Break Adult Bones But Kids Are So Resilient. Isn't That How The Rhyme Goes?", "impressions": 2516, "clicks": 20, "ctr": 0.0079491255961844, "ctr_select": 0.00794912559618442, "ctr_holdout": 0.00794912559618442, "n_select": 1258, "n_holdout": 1258, "eb": 0.016931435343406364, "labelled_winner": false }, { "headline": "We Tell Kids It Builds Character But As Adults We\u2019d Call The Police. That Sounds Fair?", "impressions": 2557, "clicks": 50, "ctr": 0.0195541650371529, "ctr_select": 0.02112676056338028, "ctr_holdout": 0.017982799061767005, "n_select": 1278, "n_holdout": 1279, "eb": 0.02156513168324566, "labelled_winner": false }, { "headline": "The Most Shocking Type Of Bullying Is The Kind That Goes Unnoticed. But We\u2019d Notice\u2026.Right?", "impressions": 2645, "clicks": 81, "ctr": 0.0306238185255198, "ctr_select": 0.03328290468986384, "ctr_holdout": 0.027966742252456538, "n_select": 1322, "n_holdout": 1323, "eb": 0.02597515018162272, "labelled_winner": false } ], "winner": 3, "pool_select": 0.021806853582554516, "pool_holdout": 0.01945903872348706, "lift_select": 0.5262589150637562, "lift_holdout": 0.43721088435374145, "eb_lift": 0.19114617261441325, "p_value": 0.0009154802431240878, "held_up": true, "z_path": [ -1.061, -1.182, 0.232, 0.463, 0.771, 0.964, 1.235, 1.481, 1.311, 1.498 ], "first_cross": null, "final_z": 1.498 }, { "id": "53e55f16220776c2b3000038", "variation": "headline", "week": 201431, "impressions": 10238, "pooled_ctr": 0.015921078335612425, "arms": [ { "headline": "Hey Big Government Agency: Someone Fixed Your Really Clueless Poster About Rape", "impressions": 2556, "clicks": 34, "ctr": 0.0133020344287949, "ctr_select": 0.013302034428794992, "ctr_holdout": 0.013302034428794992, "n_select": 1278, "n_holdout": 1278, "eb": 0.014295786110415888, "labelled_winner": false }, { "headline": "1 Poster Missed The Mark About Sexual Assault, So Someone Corrected The Problem Like A Boss", "impressions": 2535, "clicks": 65, "ctr": 0.0256410256410256, "ctr_select": 0.01971608832807571, "ctr_holdout": 0.03157063930544594, "n_select": 1268, "n_holdout": 1267, "eb": 0.016566741035193896, "labelled_winner": false }, { "headline": "Boom! That's The Sound Of Someone Giving A Government Agency The Smackdown On One Huge Mistake.", "impressions": 2560, "clicks": 25, "ctr": 0.009765625, "ctr_select": 0.01171875, "ctr_holdout": 0.0078125, "n_select": 1280, "n_holdout": 1280, "eb": 0.013731927849170697, "labelled_winner": false }, { "headline": "Misinformation About Rape Cleared Up Using 30 Characters", "impressions": 2587, "clicks": 39, "ctr": 0.0150753768844221, "ctr_select": 0.014683153013910355, "ctr_holdout": 0.015467904098994586, "n_select": 1294, "n_holdout": 1293, "eb": 0.014786211269742106, "labelled_winner": false } ], "winner": 1, "pool_select": 0.01484375, "pool_holdout": 0.016998827667057445, "lift_select": 0.32824173999667927, "lift_holdout": 0.8572245053479575, "eb_lift": 0.11607518552885199, "p_value": 0.09810505308966028, "held_up": true, "z_path": [ 0.618, 1.701, 2.42, 2.685, 2.052, 1.887, 2.482, 2.563, 2.37, 2.681 ], "first_cross": 2, "final_z": 2.681 }, { "id": "55357665346161001c390000", "variation": "both", "week": 201516, "impressions": 10227, "pooled_ctr": 0.013689253935660506, "arms": [ { "headline": "The man who found the Titanic made a discovery during his latest adventure: a beautiful whale.
", "impressions": 2038, "clicks": 42, "ctr": 0.0206084396467124, "ctr_select": 0.018645731108930325, "ctr_holdout": 0.022571148184494603, "n_select": 1019, "n_holdout": 1019, "eb": 0.015473641256922497, "labelled_winner": false }, { "headline": "The man who found the Titanic made a discovery during his latest adventure: a beautiful whale.
", "impressions": 1979, "clicks": 37, "ctr": 0.0186963112683173, "ctr_select": 0.01717171717171717, "ctr_holdout": 0.020222446916076844, "n_select": 990, "n_holdout": 989, "eb": 0.01500465510943511, "labelled_winner": false }, { "headline": "A whale interrupts a deep sea expedition on the site of the largest oil spill in U.S. history.", "impressions": 2019, "clicks": 21, "ctr": 0.0104011887072808, "ctr_select": 0.011881188118811881, "ctr_holdout": 0.008919722497522299, "n_select": 1010, "n_holdout": 1009, "eb": 0.013411934383989474, "labelled_winner": false }, { "headline": "A whale interrupts a deep sea expedition on the site of the largest oil spill in U.S. history.", "impressions": 2077, "clicks": 28, "ctr": 0.0134809821858449, "ctr_select": 0.016377649325626204, "ctr_holdout": 0.010587102983638113, "n_select": 1038, "n_holdout": 1039, "eb": 0.01479002807546293, "labelled_winner": false }, { "headline": "At the site of the largest oil spill in U.S. history, a whale interrupts a deep sea expedition", "impressions": 2114, "clicks": 12, "ctr": 0.0056764427625354, "ctr_select": 0.006622516556291391, "ctr_holdout": 0.004730368968779565, "n_select": 1057, "n_holdout": 1057, "eb": 0.011743529860432934, "labelled_winner": false } ], "winner": 0, "pool_select": 0.014078998826750098, "pool_holdout": 0.013299432818306279, "lift_select": 0.32436484570930113, "lift_holdout": 0.6971511862841309, "eb_lift": 0.09905835260974505, "p_value": 0.16674598729037737, "held_up": true, "z_path": [ -2.19, -0.733, -1.397, -0.75, -0.251, -0.25, -0.669, -0.247, 0.128, 0.436 ], "first_cross": 0, "final_z": 0.436 }, { "id": "54ecbc63326439000c5a0000", "variation": "both", "week": 201508, "impressions": 10193, "pooled_ctr": 0.009320121652114197, "arms": [ { "headline": "When people say his TV show\u2019s racist, he\u2019s like, \u2018I don\u2019t know if you know any of us.\u2019", "impressions": 2466, "clicks": 28, "ctr": 0.0113544201135442, "ctr_select": 0.0113544201135442, "ctr_holdout": 0.0113544201135442, "n_select": 1233, "n_holdout": 1233, "eb": 0.010346107619141311, "labelled_winner": false }, { "headline": "When people say his TV show is racist, he\u2019s like, \u2018I don\u2019t know if you know any of us.\u2019", "impressions": 2544, "clicks": 22, "ctr": 0.0086477987421383, "ctr_select": 0.008647798742138365, "ctr_holdout": 0.008647798742138365, "n_select": 1272, "n_holdout": 1272, "eb": 0.009397978144965052, "labelled_winner": false }, { "headline": "When people say his TV show is racist, he\u2019s like, \u2018I don\u2019t know if you know any of us.\u2019", "impressions": 2563, "clicks": 17, "ctr": 0.0066328521264143, "ctr_select": 0.00546021840873635, "ctr_holdout": 0.0078064012490242, "n_select": 1282, "n_holdout": 1281, "eb": 0.008260666037845846, "labelled_winner": false }, { "headline": "When people say his TV show is racist, he\u2019s like, \u2018I don\u2019t know if you know any of us.\u2019", "impressions": 2620, "clicks": 28, "ctr": 0.0106870229007633, "ctr_select": 0.013740458015267175, "ctr_holdout": 0.007633587786259542, "n_select": 1310, "n_holdout": 1310, "eb": 0.011229133156035975, "labelled_winner": true } ], "winner": 3, "pool_select": 0.009809691975671964, "pool_holdout": 0.008830455259026688, "lift_select": 0.40070229007633573, "lift_holdout": -0.1355385920271417, "eb_lift": 0.1446978339263072, "p_value": 0.09399426860419993, "held_up": false, "z_path": [ 0.528, 1.059, 0.664, 0.317, 0.027, -0.005, -0.017, -0.2, -0.499, -0.228 ], "first_cross": null, "final_z": -0.228 }, { "id": "53ceacf3d7c9144ec1000011", "variation": "headline", "week": 201429, "impressions": 10187, "pooled_ctr": 0.007165995877098263, "arms": [ { "headline": "A Simple Message That Anyone Who Really Cares About The 'Middle Class' Can Appreciate", "impressions": 2588, "clicks": 20, "ctr": 0.0077279752704791, "ctr_select": 0.008500772797527048, "ctr_holdout": 0.006955177743431221, "n_select": 1294, "n_holdout": 1294, "eb": 0.008210658959453781, "labelled_winner": false }, { "headline": "A Graphic That Even The Opposite Of A 'Progressive' Can Appreciate", "impressions": 2519, "clicks": 20, "ctr": 0.0079396585946804, "ctr_select": 0.007142857142857143, "ctr_holdout": 0.008737092930897538, "n_select": 1260, "n_holdout": 1259, "eb": 0.007729668742435282, "labelled_winner": false }, { "headline": "Any Politician Who Disagrees With The Premise Of This Graphic Has Some Questionable Priorities", "impressions": 2574, "clicks": 17, "ctr": 0.0066045066045066, "ctr_select": 0.009324009324009324, "ctr_holdout": 0.003885003885003885, "n_select": 1287, "n_holdout": 1287, "eb": 0.00850401909652067, "labelled_winner": false }, { "headline": "Something To Think Of Next Time A Bus Or Train Gets You Where You Need To Go", "impressions": 2506, "clicks": 16, "ctr": 0.0063846767757382, "ctr_select": 0.007182761372705506, "ctr_holdout": 0.00558659217877095, "n_select": 1253, "n_holdout": 1253, "eb": 0.007744821994172841, "labelled_winner": false } ], "winner": 2, "pool_select": 0.008048684727129957, "pool_holdout": 0.006283133712939329, "lift_select": 0.15845130479276803, "lift_holdout": -0.3816773504273505, "eb_lift": 0.056572518967714425, "p_value": 0.5536509287153362, "held_up": false, "z_path": [ -0.445, -1.414, -0.901, -1.602, -1.141, -1.334, -0.99, -0.743, -0.672, -0.478 ], "first_cross": null, "final_z": -0.478 }, { "id": "55386c3a653733000c440000", "variation": "headline", "week": 201516, "impressions": 10176, "pooled_ctr": 0.0013757861635220125, "arms": [ { "headline": "4 families are racing the clock to get a promising treatment approved before a son dies.", "impressions": 2037, "clicks": 1, "ctr": 0.0004909180166912, "ctr_select": 0.0009823182711198428, "ctr_holdout": 0.0, "n_select": 1018, "n_holdout": 1019, "eb": 0.0012557062432375877, "labelled_winner": false }, { "headline": "A new drug may be life-saving. That's why these 4 families are racing to get it approved.", "impressions": 1991, "clicks": 4, "ctr": 0.0020090406830738, "ctr_select": 0.001004016064257028, "ctr_holdout": 0.003015075376884422, "n_select": 996, "n_holdout": 995, "eb": 0.001264043013762127, "labelled_winner": false }, { "headline": "Their sons' disease has 4 families racing the clock to get a promising experimental drug approved.", "impressions": 1997, "clicks": 3, "ctr": 0.0015022533800701, "ctr_select": 0.002004008016032064, "ctr_holdout": 0.001001001001001001, "n_select": 998, "n_holdout": 999, "eb": 0.0015648762026799121, "labelled_winner": false }, { "headline": "Getting a promising new drug cleared takes time, and 4 families wonder how much they have left.", "impressions": 2068, "clicks": 2, "ctr": 0.0009671179883945, "ctr_select": 0.0009671179883945841, "ctr_holdout": 0.0009671179883945841, "n_select": 1034, "n_holdout": 1034, "eb": 0.001249711878094209, "labelled_winner": false }, { "headline": "Duchenne muscular dystrophy is merciless, so they're desperate to get this promising drug approved.", "impressions": 2083, "clicks": 4, "ctr": 0.0019203072491598, "ctr_select": 0.0019193857965451055, "ctr_holdout": 0.0019212295869356388, "n_select": 1042, "n_holdout": 1041, "eb": 0.0015443819323701332, "labelled_winner": false } ], "winner": 2, "pool_select": 0.0013757861635220125, "pool_holdout": 0.0013757861635220125, "lift_select": 0.45662754079587753, "lift_holdout": -0.2724152724152724, "eb_lift": 0.1374414456050561, "p_value": 0.5503794650473408, "held_up": false, "z_path": [ 0.0, 1.023, 0.031, 0.03, 0.042, 0.042, -0.402, -0.766, -0.766, -0.323 ], "first_cross": null, "final_z": -0.323 }, { "id": "5519b9fa373836000ca10000", "variation": "headline", "week": 201513, "impressions": 10172, "pooled_ctr": 0.011207235548564688, "arms": [ { "headline": "Justin Timberlake gave the best advice to 'weird' and 'different' kids.", "impressions": 2042, "clicks": 22, "ctr": 0.0107737512242899, "ctr_select": 0.011753183153770812, "ctr_holdout": 0.009794319294809012, "n_select": 1021, "n_holdout": 1021, "eb": 0.011512238230895517, "labelled_winner": false }, { "headline": "My favorite 90s boy band star gave the best advice to 'weird' and 'different' kids.", "impressions": 1993, "clicks": 23, "ctr": 0.0115403913697942, "ctr_select": 0.013052208835341365, "ctr_holdout": 0.010030090270812437, "n_select": 996, "n_holdout": 997, "eb": 0.011900869322371183, "labelled_winner": false }, { "headline": "Justin Timberlake to kids who are weird: 'Your critics don't count.'", "impressions": 2011, "clicks": 15, "ctr": 0.0074589756340129, "ctr_select": 0.005964214711729622, "ctr_holdout": 0.008955223880597015, "n_select": 1006, "n_holdout": 1005, "eb": 0.009758975165361108, "labelled_winner": false }, { "headline": "This fantastic acceptance speech is just what kids who don't fit in need to hear.", "impressions": 2029, "clicks": 24, "ctr": 0.011828486939379, "ctr_select": 0.013806706114398421, "ctr_holdout": 0.009852216748768473, "n_select": 1014, "n_holdout": 1015, "eb": 0.01213672035744689, "labelled_winner": false }, { "headline": "This is what an inspiring acceptance speech looks like.", "impressions": 2097, "clicks": 30, "ctr": 0.0143061516452074, "ctr_select": 0.012404580152671756, "ctr_holdout": 0.016205910390848427, "n_select": 1048, "n_holdout": 1049, "eb": 0.011717000995779872, "labelled_winner": false } ], "winner": 3, "pool_select": 0.011406096361848575, "pool_holdout": 0.01100845291920582, "lift_select": 0.21046725158130974, "lift_holdout": -0.10503166783954965, "eb_lift": 0.06405556926926614, "p_value": 0.4210741118025991, "held_up": false, "z_path": [ -1.009, -1.363, -0.021, 0.507, 1.058, 0.714, 0.853, 0.427, 0.546, 0.085 ], "first_cross": null, "final_z": 0.085 }, { "id": "552589eb313437000c270100", "variation": "other", "week": 201514, "impressions": 10168, "pooled_ctr": 0.012293469708890637, "arms": [ { "headline": "Listen to some boys talk about their plans to be 'manly.'", "impressions": 2051, "clicks": 23, "ctr": 0.0112140419307654, "ctr_select": 0.014619883040935672, "ctr_holdout": 0.007804878048780488, "n_select": 1026, "n_holdout": 1025, "eb": 0.013073801126785543, "labelled_winner": false }, { "headline": "Listen to some boys talk about their plans to be 'manly.'", "impressions": 2067, "clicks": 36, "ctr": 0.0174165457184325, "ctr_select": 0.01160541586073501, "ctr_holdout": 0.023233301064859633, "n_select": 1034, "n_holdout": 1033, "eb": 0.012147527135929573, "labelled_winner": true }, { "headline": "Listen to some boys talk about their plans to be 'manly.'", "impressions": 2045, "clicks": 22, "ctr": 0.0107579462102689, "ctr_select": 0.012720156555772993, "ctr_holdout": 0.008797653958944282, "n_select": 1022, "n_holdout": 1023, "eb": 0.012490603194592392, "labelled_winner": false }, { "headline": "Listen to some boys talk about their plans to be 'manly.'", "impressions": 1987, "clicks": 20, "ctr": 0.0100654252642174, "ctr_select": 0.009054325955734407, "ctr_holdout": 0.011077542799597181, "n_select": 994, "n_holdout": 993, "eb": 0.011388370009044241, "labelled_winner": false }, { "headline": "Listen to some boys talk about their plans to be 'manly.'", "impressions": 2018, "clicks": 24, "ctr": 0.0118929633300297, "ctr_select": 0.013875123885034688, "ctr_holdout": 0.009910802775024777, "n_select": 1009, "n_holdout": 1009, "eb": 0.01284001211140214, "labelled_winner": false } ], "winner": 0, "pool_select": 0.012389380530973451, "pool_holdout": 0.012197521148927798, "lift_select": 0.180033416875522, "lift_holdout": -0.360125885129819, "eb_lift": 0.05524251951911885, "p_value": 0.4697244963614622, "held_up": false, "z_path": [ 0.985, 1.65, 0.699, 1.061, 0.692, 0.13, -0.544, -0.685, -0.206, -0.203 ], "first_cross": null, "final_z": -0.203 }, { "id": "54fdb1f362306300264e0000", "variation": "headline", "week": 201510, "impressions": 10161, "pooled_ctr": 0.006298592658202933, "arms": [ { "headline": "The way President Obama uses the word 'we' here makes me want to stand up and cheer.", "impressions": 2003, "clicks": 14, "ctr": 0.0069895157264103, "ctr_select": 0.00499001996007984, "ctr_holdout": 0.008991008991008992, "n_select": 1002, "n_holdout": 1001, "eb": 0.00480393075371331, "labelled_winner": false }, { "headline": "Was President Obama's speech on the 50th anniversary of the march from Selma one of his best? Yes.", "impressions": 2014, "clicks": 6, "ctr": 0.0029791459781529, "ctr_select": 0.0029791459781529296, "ctr_holdout": 0.0029791459781529296, "n_select": 1007, "n_holdout": 1007, "eb": 0.004195147690234376, "labelled_winner": false }, { "headline": "President Obama just said something kind of revolutionary about what it means to be an American.", "impressions": 1984, "clicks": 11, "ctr": 0.0055443548387096, "ctr_select": 0.0030241935483870967, "ctr_holdout": 0.008064516129032258, "n_select": 992, "n_holdout": 992, "eb": 0.004214160669959203, "labelled_winner": false }, { "headline": "There's something powerful about hearing the first black president say these words in Selma", "impressions": 2079, "clicks": 24, "ctr": 0.0115440115440115, "ctr_select": 0.009615384615384616, "ctr_holdout": 0.013474494706448507, "n_select": 1040, "n_holdout": 1039, "eb": 0.006238678921992437, "labelled_winner": true }, { "headline": "President Obama's powerful speech at Selma is just what America needed to hear.", "impressions": 2081, "clicks": 9, "ctr": 0.004324843825084, "ctr_select": 0.0028846153846153848, "ctr_holdout": 0.005763688760806916, "n_select": 1040, "n_holdout": 1041, "eb": 0.004153917095357814, "labelled_winner": false } ], "winner": 3, "pool_select": 0.004723479629994096, "pool_holdout": 0.007874015748031496, "lift_select": 1.035657051282051, "lift_holdout": 0.7112608277189605, "eb_lift": 0.3207803167768155, "p_value": 0.009879649717886754, "held_up": true, "z_path": [ -0.042, 0.662, 0.521, 0.771, 0.656, 1.199, 1.087, 1.52, 1.314, 1.515 ], "first_cross": null, "final_z": 1.515 }, { "id": "54ecb3693264390024330000", "variation": "headline", "week": 201508, "impressions": 10153, "pooled_ctr": 0.014773958435930266, "arms": [ { "headline": "They held up racist signs at a basketball game, so he went on TV and gave them a piece of his mind.", "impressions": 2491, "clicks": 37, "ctr": 0.0148534725010036, "ctr_select": 0.016051364365971106, "ctr_holdout": 0.013654618473895583, "n_select": 1246, "n_holdout": 1245, "eb": 0.015221533064889072, "labelled_winner": false }, { "headline": "They claimed the racist signs were an accident. He doesn't buy it for 1 second.", "impressions": 2604, "clicks": 35, "ctr": 0.0134408602150537, "ctr_select": 0.01152073732718894, "ctr_holdout": 0.015360983102918587, "n_select": 1302, "n_holdout": 1302, "eb": 0.013604711349600203, "labelled_winner": false }, { "headline": "A white TV news anchor from Texas smacks down a group of racist sports fans so perfectly", "impressions": 2601, "clicks": 42, "ctr": 0.0161476355247981, "ctr_select": 0.01769230769230769, "ctr_holdout": 0.014604150653343582, "n_select": 1300, "n_holdout": 1301, "eb": 0.015823583703809506, "labelled_winner": false }, { "headline": "His story about his dad explains exactly what these racist kids did and why it's so sad.", "impressions": 2457, "clicks": 36, "ctr": 0.0146520146520146, "ctr_select": 0.013843648208469055, "ctr_holdout": 0.015459723352318959, "n_select": 1228, "n_holdout": 1229, "eb": 0.01445271049965201, "labelled_winner": false } ], "winner": 2, "pool_select": 0.014775413711583925, "pool_holdout": 0.014772503446917471, "lift_select": 0.19741538461538455, "lift_holdout": -0.011396361772995078, "eb_lift": 0.0709401450738274, "p_value": 0.3121851920454233, "held_up": false, "z_path": [ 1.085, 0.417, 0.526, 0.444, 0.181, -0.002, 0.376, 0.33, 0.668, 0.374 ], "first_cross": null, "final_z": 0.374 }, { "id": "54664b956c94ae538d00000b", "variation": "headline", "week": 201445, "impressions": 10146, "pooled_ctr": 0.0035481963335304554, "arms": [ { "headline": "One Moment On A Motorcyle And All Of His Dreams Came Crashing Down \u2026 Good Thing He Had More", "impressions": 2102, "clicks": 5, "ctr": 0.0023786869647954, "ctr_select": 0.0019029495718363464, "ctr_holdout": 0.0028544243577545195, "n_select": 1051, "n_holdout": 1051, "eb": 0.0024924081156793157, "labelled_winner": false }, { "headline": "He Took A Traumatic Turn That Threw His Life Way Off Course, But He Still Survived", "impressions": 2010, "clicks": 8, "ctr": 0.0039800995024875, "ctr_select": 0.0029850746268656717, "ctr_holdout": 0.004975124378109453, "n_select": 1005, "n_holdout": 1005, "eb": 0.002827873713939021, "labelled_winner": false }, { "headline": "Sometimes A Hero\u2019s Story Skips The Hollywood Ending And Sometimes That's OK.", "impressions": 2035, "clicks": 4, "ctr": 0.0019656019656019, "ctr_select": 0.0019646365422396855, "ctr_holdout": 0.0019665683382497543, "n_select": 1018, "n_holdout": 1017, "eb": 0.0025170654662849032, "labelled_winner": false }, { "headline": "After Living Through What He Did, Most Would Give Up. But He's Not Like Most.", "impressions": 2001, "clicks": 14, "ctr": 0.0069965017491254, "ctr_select": 0.006, "ctr_holdout": 0.007992007992007992, "n_select": 1000, "n_holdout": 1001, "eb": 0.003736377030669639, "labelled_winner": false }, { "headline": "His Story Makes Me Want To Swim, And I Haven't Used My Sea Legs For Years", "impressions": 1998, "clicks": 5, "ctr": 0.0025025025025025, "ctr_select": 0.001001001001001001, "ctr_holdout": 0.004004004004004004, "n_select": 999, "n_holdout": 999, "eb": 0.0022299799852022626, "labelled_winner": false } ], "winner": 3, "pool_select": 0.0027597082594125765, "pool_holdout": 0.004336684407648335, "lift_select": 1.174142857142857, "lift_holdout": 0.8428843883389336, "eb_lift": 0.3539029054705056, "p_value": 0.029268122693604133, "held_up": true, "z_path": [ -1.001, -0.576, -0.998, -0.444, 0.006, 0.006, 0.913, 1.615, 1.225, 1.293 ], "first_cross": null, "final_z": 1.293 }, { "id": "53663b77d5854b2e3f000037", "variation": "image", "week": 201418, "impressions": 10068, "pooled_ctr": 0.03675009932459277, "arms": [ { "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day", "impressions": 2051, "clicks": 77, "ctr": 0.0375426621160409, "ctr_select": 0.03898635477582846, "ctr_holdout": 0.03609756097560975, "n_select": 1026, "n_holdout": 1025, "eb": 0.038813230120448954, "labelled_winner": false }, { "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day", "impressions": 2028, "clicks": 70, "ctr": 0.034516765285996, "ctr_select": 0.03944773175542406, "ctr_holdout": 0.029585798816568046, "n_select": 1014, "n_holdout": 1014, "eb": 0.03895302634529779, "labelled_winner": false }, { "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day", "impressions": 1917, "clicks": 73, "ctr": 0.0380803338549817, "ctr_select": 0.03966597077244259, "ctr_holdout": 0.0364963503649635, "n_select": 958, "n_holdout": 959, "eb": 0.03900839454214944, "labelled_winner": false }, { "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day", "impressions": 2083, "clicks": 72, "ctr": 0.0345655304848775, "ctr_select": 0.0345489443378119, "ctr_holdout": 0.0345821325648415, "n_select": 1042, "n_holdout": 1041, "eb": 0.03743780527923385, "labelled_winner": false }, { "headline": "A 1 Minute Experiment To Show Men Something Women Experience Every Day", "impressions": 1989, "clicks": 78, "ctr": 0.0392156862745098, "ctr_select": 0.04124748490945674, "ctr_holdout": 0.0371859296482412, "n_select": 994, "n_holdout": 995, "eb": 0.0394902313336687, "labelled_winner": false } ], "winner": 4, "pool_select": 0.03873659117997616, "pool_holdout": 0.034763607469209375, "lift_select": 0.06481968735489874, "lift_holdout": 0.06967982770997838, "eb_lift": 0.019455510429170486, "p_value": 0.6469978497298443, "held_up": true, "z_path": [ -0.057, -0.891, -0.129, 0.974, 0.608, 0.625, 0.724, 0.883, 0.21, 0.184 ], "first_cross": null, "final_z": 0.184 }, { "id": "530f12bdfa892222b800001a", "variation": "headline", "week": 201408, "impressions": 10059, "pooled_ctr": 0.016104980614375188, "arms": [ { "headline": "A Late Night Host Got Left Off The List Of Late Night Hosts, So He Decided Call Them Out For It", "impressions": 2092, "clicks": 32, "ctr": 0.0152963671128107, "ctr_select": 0.01338432122370937, "ctr_holdout": 0.017208413001912046, "n_select": 1046, "n_holdout": 1046, "eb": 0.014300964695494172, "labelled_winner": false }, { "headline": "Why A New List Of All White Late Night Hosts Rubbed Arsenio \u2013 A Late Night Host \u2013 The Wrong Way.", "impressions": 2044, "clicks": 30, "ctr": 0.0146771037181996, "ctr_select": 0.019569471624266144, "ctr_holdout": 0.009784735812133072, "n_select": 1022, "n_holdout": 1022, "eb": 0.016200305208825663, "labelled_winner": false }, { "headline": "If You're Going To Leave Someone Off The List Of Late Night Hosts, Next Time Don't Let It Be Arsenio", "impressions": 1994, "clicks": 23, "ctr": 0.0115346038114343, "ctr_select": 0.00802407221664995, "ctr_holdout": 0.015045135406218655, "n_select": 997, "n_holdout": 997, "eb": 0.012702250965879526, "labelled_winner": false }, { "headline": "A Late Night Host Has Some Choice Words For An Article That Conveniently Left Him Off The List", "impressions": 2000, "clicks": 37, "ctr": 0.0185, "ctr_select": 0.018, "ctr_holdout": 0.019, "n_select": 1000, "n_holdout": 1000, "eb": 0.01570490344780625, "labelled_winner": false }, { "headline": "How A Late Night Host Used His Monologue To Shut Down A Racially Biased Article About Late Night", "impressions": 1929, "clicks": 40, "ctr": 0.0207361327112493, "ctr_select": 0.014522821576763486, "ctr_holdout": 0.02694300518134715, "n_select": 964, "n_holdout": 965, "eb": 0.014658303856677286, "labelled_winner": false } ], "winner": 1, "pool_select": 0.014714655000994234, "pool_holdout": 0.01749502982107356, "lift_select": 0.32993071349235725, "lift_holdout": -0.44071339619284833, "eb_lift": 0.10096398507005744, "p_value": 0.14873319729001686, "held_up": false, "z_path": [ 0.974, 0.539, -0.061, -0.066, -1.056, -0.538, -0.937, -0.906, -0.614, -0.952 ], "first_cross": null, "final_z": -0.952 }, { "id": "54db5928366238000c800000", "variation": "headline", "week": 201506, "impressions": 9755, "pooled_ctr": 0.009841107124551511, "arms": [ { "headline": "Ever Been In A Forest And Thought You Heard Whispering? Maybe It Wasn't Your Head.", "impressions": 2516, "clicks": 35, "ctr": 0.0139109697933227, "ctr_select": 0.014308426073131956, "ctr_holdout": 0.013513513513513514, "n_select": 1258, "n_holdout": 1258, "eb": 0.011810778009823174, "labelled_winner": true }, { "headline": "Ever Feel Like Trees Know Something You Don't? Maybe It's Not Just In Your Head.", "impressions": 2444, "clicks": 13, "ctr": 0.0053191489361702, "ctr_select": 0.0057283142389525366, "ctr_holdout": 0.004909983633387889, "n_select": 1222, "n_holdout": 1222, "eb": 0.008823287869690064, "labelled_winner": false }, { "headline": "Ever Been In A Forest And Felt Like Something Was Watching You? Maybe It Wasn't Your Imagination.", "impressions": 2363, "clicks": 20, "ctr": 0.0084638171815488, "ctr_select": 0.009306260575296108, "ctr_holdout": 0.007620660457239628, "n_select": 1182, "n_holdout": 1181, "eb": 0.01006709001552215, "labelled_winner": false }, { "headline": "Ever Heard Trees Talking? Maybe That Wasn't Just Your Imagination.", "impressions": 2432, "clicks": 28, "ctr": 0.0115131578947368, "ctr_select": 0.012335526315789474, "ctr_holdout": 0.01069078947368421, "n_select": 1216, "n_holdout": 1216, "eb": 0.011102186745691073, "labelled_winner": false } ], "winner": 0, "pool_select": 0.010455104551045511, "pool_holdout": 0.009226983801517326, "lift_select": 0.36855887028897416, "lift_holdout": 0.46456456456456463, "eb_lift": 0.12966617905720468, "p_value": 0.11881451760004201, "held_up": true, "z_path": [ -0.032, -0.599, -0.767, -0.088, 0.748, 0.218, 0.508, 0.885, 0.537, 0.752 ], "first_cross": null, "final_z": 0.752 }, { "id": "5227cdeb6886ffa1810001ca", "variation": "headline", "week": 201335, "impressions": 9644, "pooled_ctr": 0.012339278307756118, "arms": [ { "headline": "All They Wanted To Do Was Play The Game. There's Nothing Fun About What's Happening Here, Though.", "impressions": 2325, "clicks": 28, "ctr": 0.0120430107526881, "ctr_select": 0.00946643717728055, "ctr_holdout": 0.014617368873602751, "n_select": 1162, "n_holdout": 1163, "eb": 0.010482085088237829, "labelled_winner": false }, { "headline": "The Most Important Thing To Happen In Football Has Nothing To Do With Touchdowns", "impressions": 2239, "clicks": 26, "ctr": 0.0116123269316659, "ctr_select": 0.009821428571428571, "ctr_holdout": 0.013404825737265416, "n_select": 1120, "n_holdout": 1119, "eb": 0.010610149744159296, "labelled_winner": false }, { "headline": "It's Time We All Pay Attention To The Most Important Thing In Football \u2014 And It's Not The Score", "impressions": 2423, "clicks": 24, "ctr": 0.0099050763516302, "ctr_select": 0.007425742574257425, "ctr_holdout": 0.012386457473162676, "n_select": 1212, "n_holdout": 1211, "eb": 0.009766980151319866, "labelled_winner": false }, { "headline": "A Sports Commentator Proves That A 'Maybe' Does Not A Job Make", "impressions": 2657, "clicks": 41, "ctr": 0.0154309371471584, "ctr_select": 0.016566265060240965, "ctr_holdout": 0.014296463506395787, "n_select": 1328, "n_holdout": 1329, "eb": 0.013022058126480104, "labelled_winner": false } ], "winner": 3, "pool_select": 0.010991289921194525, "pool_holdout": 0.013687266694317711, "lift_select": 0.5072175494430553, "lift_holdout": 0.044508288300613374, "eb_lift": 0.18476159029975592, "p_value": 0.022072021193859894, "held_up": false, "z_path": [ -1.893, 1.449, 2.172, 1.385, 1.151, 1.31, 1.675, 1.566, 1.527, 1.146 ], "first_cross": 2, "final_z": 1.146 }, { "id": "531d5e30365e2adfdb000020", "variation": "image", "week": 201410, "impressions": 9243, "pooled_ctr": 0.013199177756139782, "arms": [ { "headline": "Have Our Beauty Standards Gotten So Bad That They Affect Even Our Fruits And Vegetables?", "impressions": 2225, "clicks": 37, "ctr": 0.016629213483146, "ctr_select": 0.017985611510791366, "ctr_holdout": 0.015274034141958671, "n_select": 1112, "n_holdout": 1113, "eb": 0.015042518979300038, "labelled_winner": false }, { "headline": "Have Our Beauty Standards Gotten So Bad That They Affect Even Our Fruits And Vegetables?", "impressions": 2319, "clicks": 22, "ctr": 0.0094868477792151, "ctr_select": 0.010344827586206896, "ctr_holdout": 0.008628127696289905, "n_select": 1160, "n_holdout": 1159, "eb": 0.01253452614847292, "labelled_winner": false }, { "headline": "Have Our Beauty Standards Gotten So Bad That They Affect Even Our Fruits And Vegetables?", "impressions": 2276, "clicks": 31, "ctr": 0.0136203866432337, "ctr_select": 0.014059753954305799, "ctr_holdout": 0.013181019332161687, "n_select": 1138, "n_holdout": 1138, "eb": 0.013771833326566804, "labelled_winner": false }, { "headline": "Have Our Beauty Standards Gotten So Bad That They Affect Even Our Fruits And Vegetables?", "impressions": 2423, "clicks": 32, "ctr": 0.0132067684688402, "ctr_select": 0.012376237623762377, "ctr_holdout": 0.014037985136251032, "n_select": 1212, "n_holdout": 1211, "eb": 0.013199796898768741, "labelled_winner": false } ], "winner": 0, "pool_select": 0.013630463003028993, "pool_holdout": 0.012767799177667172, "lift_select": 0.3195158159186935, "lift_holdout": 0.1962934198303561, "eb_lift": 0.10359559876705982, "p_value": 0.15063820243661535, "held_up": true, "z_path": [ 1.027, 1.437, 0.29, 0.44, -0.104, -0.083, 0.788, 1.163, 0.734, 0.827 ], "first_cross": null, "final_z": 0.827 }, { "id": "52b04a2aa3623369cb0046c7", "variation": "headline", "week": 201350, "impressions": 9152, "pooled_ctr": 0.00590034965034965, "arms": [ { "headline": "The Most Important You Can Ask Today: What Will Your Last 10 Years Will Look Like?", "impressions": 2202, "clicks": 8, "ctr": 0.0036330608537693, "ctr_select": 0.0027247956403269754, "ctr_holdout": 0.004541326067211626, "n_select": 1101, "n_holdout": 1101, "eb": 0.004433973254697844, "labelled_winner": false }, { "headline": "We Like To Imagine The Next 10 Years. Maybe We Should Ask What The Last 10 Will Look Like.", "impressions": 2381, "clicks": 9, "ctr": 0.0037799244015119, "ctr_select": 0.0033613445378151263, "ctr_holdout": 0.0041981528127623844, "n_select": 1190, "n_holdout": 1191, "eb": 0.004606558360743928, "labelled_winner": false }, { "headline": "A Scary Ad Shows Two Perspectives On Living Your Last 10 Years. Which Describes Where You're Headed?", "impressions": 2248, "clicks": 23, "ctr": 0.0102313167259786, "ctr_select": 0.008896797153024912, "ctr_holdout": 0.011565836298932384, "n_select": 1124, "n_holdout": 1124, "eb": 0.006438222089757918, "labelled_winner": false }, { "headline": "Which Side Of This Ad Describes What Your Last 10 Years Will Look Like?", "impressions": 2321, "clicks": 14, "ctr": 0.0060318828091339, "ctr_select": 0.00603448275862069, "ctr_holdout": 0.006029285099052541, "n_select": 1160, "n_holdout": 1161, "eb": 0.005508936030499569, "labelled_winner": false } ], "winner": 2, "pool_select": 0.005245901639344262, "pool_holdout": 0.006554511688879178, "lift_select": 0.6959519572953738, "lift_holdout": 0.7645610913404506, "eb_lift": 0.22728608586010313, "p_value": 0.051067510214896146, "held_up": true, "z_path": [ 0.61, 0.86, 0.422, 0.685, 1.875, 2.141, 1.959, 2.286, 1.315, 1.583 ], "first_cross": 5, "final_z": 1.583 }, { "id": "54f8e0603737324288250100", "variation": "other", "week": 201509, "impressions": 9137, "pooled_ctr": 0.011929517347050455, "arms": [ { "headline": "It's a pop song like no other. But not every woman is allowed to hear it.", "impressions": 1796, "clicks": 32, "ctr": 0.0178173719376391, "ctr_select": 0.01670378619153675, "ctr_holdout": 0.01893095768374165, "n_select": 898, "n_holdout": 898, "eb": 0.012867448501116123, "labelled_winner": false }, { "headline": "It's a pop song like no other. But not every woman is allowed to hear it.", "impressions": 1812, "clicks": 15, "ctr": 0.0082781456953642, "ctr_select": 0.009933774834437087, "ctr_holdout": 0.006622516556291391, "n_select": 906, "n_holdout": 906, "eb": 0.010974299655541747, "labelled_winner": false }, { "headline": "It's a pop song like no other. But not every woman is allowed to hear it.", "impressions": 1868, "clicks": 26, "ctr": 0.0139186295503211, "ctr_select": 0.010706638115631691, "ctr_holdout": 0.017130620985010708, "n_select": 934, "n_holdout": 934, "eb": 0.011187332868517726, "labelled_winner": false }, { "headline": "It's a pop song like no other. But not every woman is allowed to hear it.", "impressions": 1835, "clicks": 20, "ctr": 0.0108991825613079, "ctr_select": 0.011982570806100218, "ctr_holdout": 0.009814612868047983, "n_select": 918, "n_holdout": 917, "eb": 0.011551704770196836, "labelled_winner": false }, { "headline": "It's a pop song like no other. But not every woman is allowed to hear it.", "impressions": 1826, "clicks": 16, "ctr": 0.008762322015334, "ctr_select": 0.007667031763417305, "ctr_holdout": 0.009857612267250822, "n_select": 913, "n_holdout": 913, "eb": 0.010331460155389886, "labelled_winner": false } ], "winner": 0, "pool_select": 0.011381046180783541, "pool_holdout": 0.012478108581436076, "lift_select": 0.46768459825252706, "lift_holdout": 0.5171335912163484, "eb_lift": 0.13060331156922245, "p_value": 0.09342684866125113, "held_up": true, "z_path": [ 0.411, 0.948, 1.655, 2.029, 2.344, 2.487, 2.276, 1.979, 2.111, 1.754 ], "first_cross": 3, "final_z": 1.754 }, { "id": "54d3a1753139330019030000", "variation": "both", "week": 201505, "impressions": 9136, "pooled_ctr": 0.011711908931698774, "arms": [ { "headline": "Watch Men Learn What Feminism Means And Then Realize Something Interesting", "impressions": 1566, "clicks": 18, "ctr": 0.0114942528735632, "ctr_select": 0.010217113665389528, "ctr_holdout": 0.01277139208173691, "n_select": 783, "n_holdout": 783, "eb": 0.01109085095490348, "labelled_winner": false }, { "headline": "Watch Men Learn What Feminism Means And Then Realize Something Obvious", "impressions": 1453, "clicks": 22, "ctr": 0.0151410874053682, "ctr_select": 0.01790633608815427, "ctr_holdout": 0.012379642365887207, "n_select": 726, "n_holdout": 727, "eb": 0.012941290469923076, "labelled_winner": true }, { "headline": "Watch Men Learn What Feminism Means And Then Realize Something Obvious", "impressions": 1598, "clicks": 16, "ctr": 0.0100125156445556, "ctr_select": 0.008760951188986232, "ctr_holdout": 0.011264080100125156, "n_select": 799, "n_holdout": 799, "eb": 0.010713062113358878, "labelled_winner": false }, { "headline": "Watch Men Learn What Feminism Means And Then Realize Something", "impressions": 1481, "clicks": 17, "ctr": 0.0114787305874409, "ctr_select": 0.008108108108108109, "ctr_holdout": 0.014844804318488529, "n_select": 740, "n_holdout": 741, "eb": 0.010592733234950068, "labelled_winner": false }, { "headline": "Watch Men Learn What Feminism Means And Then Realize Something Interesting", "impressions": 1522, "clicks": 17, "ctr": 0.0111695137976346, "ctr_select": 0.011826544021024968, "ctr_holdout": 0.010512483574244415, "n_select": 761, "n_holdout": 761, "eb": 0.011494917492236901, "labelled_winner": false }, { "headline": "Watch Men Learn What Feminism Means And Then Realize Something Interesting", "impressions": 1516, "clicks": 17, "ctr": 0.0112137203166226, "ctr_select": 0.011873350923482849, "ctr_holdout": 0.010554089709762533, "n_select": 758, "n_holdout": 758, "eb": 0.011506129501929776, "labelled_winner": false } ], "winner": 1, "pool_select": 0.011386030216772498, "pool_holdout": 0.012037644998905668, "lift_select": 0.572658402203857, "lift_holdout": 0.028410653995248225, "eb_lift": 0.1365937226180518, "p_value": 0.07097528747784294, "held_up": false, "z_path": [ 0.029, 0.87, 0.07, 1.092, 0.787, 1.298, 0.664, 0.298, 0.627, 0.94 ], "first_cross": null, "final_z": 0.94 }, { "id": "52d815dc392cef894b0021c6", "variation": "headline", "week": 201402, "impressions": 8932, "pooled_ctr": 0.03884908195253023, "arms": [ { "headline": "To The Men That Mistreated These Girls (And Their Mothers): You. Just. Got. Burned.", "impressions": 4433, "clicks": 170, "ctr": 0.0383487480261673, "ctr_select": 0.037454873646209384, "ctr_holdout": 0.03924221921515562, "n_select": 2216, "n_holdout": 2217, "eb": 0.03811038071424032, "labelled_winner": false }, { "headline": "To The Men That Mistreated These Girls (And Their Mothers): You. Just. Got. Told.", "impressions": 4499, "clicks": 177, "ctr": 0.0393420760168926, "ctr_select": 0.04, "ctr_holdout": 0.03868385949310805, "n_select": 2250, "n_holdout": 2249, "eb": 0.0393592039561831, "labelled_winner": false } ], "winner": 1, "pool_select": 0.0387371249440215, "pool_holdout": 0.03896103896103896, "lift_select": 0.0326011560693642, "lift_holdout": -0.007114273010226713, "eb_lift": 0.016058987678113867, "p_value": 0.6594314470492292, "held_up": false, "z_path": [ -1.152, 0.3, 0.116, -0.171, 0.284, -0.178, -0.054, -0.065, -0.017, 0.243 ], "first_cross": null, "final_z": 0.243 }, { "id": "550975176133340018390000", "variation": "headline", "week": 201511, "impressions": 8894, "pooled_ctr": 0.008882392624241061, "arms": [ { "headline": "Her name was 'Essence.' This is a little story about making it felt all around her.", "impressions": 2197, "clicks": 27, "ctr": 0.0122894856622667, "ctr_select": 0.012750455373406194, "ctr_holdout": 0.011828935395814377, "n_select": 1098, "n_holdout": 1099, "eb": 0.010049516325297993, "labelled_winner": false }, { "headline": "Her name was 'Essence.' This is a story about making that felt all around her after she was gone.", "impressions": 2246, "clicks": 16, "ctr": 0.0071237756010685, "ctr_select": 0.007123775601068566, "ctr_holdout": 0.007123775601068566, "n_select": 1123, "n_holdout": 1123, "eb": 0.008232664499078892, "labelled_winner": false }, { "headline": "Her name was 'Essence.' This is a story about her impact on the world, through her family.", "impressions": 2251, "clicks": 17, "ctr": 0.0075521990226565, "ctr_select": 0.004440497335701598, "ctr_holdout": 0.010666666666666666, "n_select": 1126, "n_holdout": 1125, "eb": 0.007354335954321053, "labelled_winner": false }, { "headline": "Her name was 'Essence,' and how she's being remembered is remarkable and beautiful.", "impressions": 2200, "clicks": 19, "ctr": 0.0086363636363636, "ctr_select": 0.01090909090909091, "ctr_holdout": 0.006363636363636364, "n_select": 1100, "n_holdout": 1100, "eb": 0.009458446193521423, "labelled_winner": false } ], "winner": 0, "pool_select": 0.008769957274567124, "pool_holdout": 0.008994827973915, "lift_select": 0.4538788473214703, "lift_holdout": 0.31508189262966324, "eb_lift": 0.1459025409897481, "p_value": 0.10306896837806057, "held_up": true, "z_path": [ 0.45, -0.379, -0.503, -0.231, 0.0, -0.201, 0.0, 0.177, 0.655, 1.19 ], "first_cross": null, "final_z": 1.19 }, { "id": "54997b38666230001ea80000", "variation": "image", "week": 201451, "impressions": 8528, "pooled_ctr": 0.014657598499061914, "arms": [ { "headline": "Your Doctor Asks You This Question All The Time. Why Can't You Ever Answer It?", "impressions": 2890, "clicks": 39, "ctr": 0.0134948096885813, "ctr_select": 0.010380622837370242, "ctr_holdout": 0.01660899653979239, "n_select": 1445, "n_holdout": 1445, "eb": 0.012942870025825295, "labelled_winner": false }, { "headline": "Your Doctor Asks You This Question All The Time. Why Can't You Ever Answer It?", "impressions": 2772, "clicks": 41, "ctr": 0.0147907647907647, "ctr_select": 0.016594516594516596, "ctr_holdout": 0.012987012987012988, "n_select": 1386, "n_holdout": 1386, "eb": 0.015309053777959929, "labelled_winner": true }, { "headline": "Your Doctor Asks You This Question All The Time. Why Can't You Ever Answer It?", "impressions": 2866, "clicks": 45, "ctr": 0.0157013258897418, "ctr_select": 0.01674808094905792, "ctr_holdout": 0.01465457083042568, "n_select": 1433, "n_holdout": 1433, "eb": 0.01538383305758285, "labelled_winner": false } ], "winner": 2, "pool_select": 0.014540337711069419, "pool_holdout": 0.01477485928705441, "lift_select": 0.15183576075456418, "lift_holdout": -0.008141428239125448, "eb_lift": 0.05801071221827847, "p_value": 0.3915290449871476, "held_up": false, "z_path": [ 1.099, 1.337, 1.461, 0.703, -0.103, -0.265, -0.666, -0.134, 0.445, 0.279 ], "first_cross": null, "final_z": 0.279 }, { "id": "53062f508fa4ee627e000018", "variation": "other", "week": 201407, "impressions": 8481, "pooled_ctr": 0.010729866760995166, "arms": [ { "headline": "Meet The Woman Who Altered The Course Of Her Life To Work With Malala...And To Change The World", "impressions": 2085, "clicks": 22, "ctr": 0.0105515587529976, "ctr_select": 0.01055662188099808, "ctr_holdout": 0.010546500479386385, "n_select": 1042, "n_holdout": 1043, "eb": 0.010109166145989425, "labelled_winner": false }, { "headline": "Meet The Woman Who Altered The Course Of Her Life To Work With Malala...And To Change The World", "impressions": 2161, "clicks": 22, "ctr": 0.0101804720037019, "ctr_select": 0.012037037037037037, "ctr_holdout": 0.008325624421831638, "n_select": 1080, "n_holdout": 1081, "eb": 0.010584738673895167, "labelled_winner": false }, { "headline": "Meet The Woman Who Altered The Course Of Her Life To Work With Malala...And To Change The World", "impressions": 2154, "clicks": 22, "ctr": 0.0102135561745589, "ctr_select": 0.005571030640668524, "ctr_holdout": 0.014856081708449397, "n_select": 1077, "n_holdout": 1077, "eb": 0.0085320534599906, "labelled_winner": false }, { "headline": "Meet The Woman Who Altered The Course Of Her Life To Work With Malala...And To Change The World", "impressions": 2081, "clicks": 25, "ctr": 0.012013455069678, "ctr_select": 0.011538461538461539, "ctr_holdout": 0.012487992315081652, "n_select": 1040, "n_holdout": 1041, "eb": 0.010413010750845877, "labelled_winner": false } ], "winner": 1, "pool_select": 0.009907997169143666, "pool_holdout": 0.01155115511551155, "lift_select": 0.21488095238095228, "lift_holdout": -0.27923880005286106, "eb_lift": 0.06830255330099066, "p_value": 0.413175899839451, "held_up": false, "z_path": [ 0.967, -0.054, -0.646, 0.148, -0.306, -0.698, -0.471, -0.455, -0.438, -0.57 ], "first_cross": null, "final_z": -0.57 }, { "id": "5342bdf93ce9c40360000024", "variation": "headline", "week": 201417, "impressions": 8465, "pooled_ctr": 0.010277613703484938, "arms": [ { "headline": "Scientists Studying The 'Big Bang' Have Discovered Some Bananas-Crazy Information", "impressions": 2080, "clicks": 26, "ctr": 0.0125, "ctr_select": 0.010576923076923078, "ctr_holdout": 0.014423076923076924, "n_select": 1040, "n_holdout": 1040, "eb": 0.01045101629195058, "labelled_winner": false }, { "headline": "Either Scientists Are Just Messing With Me, Or The 'Big Bang' Is Super Crazy To Think About", "impressions": 2144, "clicks": 14, "ctr": 0.0065298507462686, "ctr_select": 0.0065298507462686565, "ctr_holdout": 0.0065298507462686565, "n_select": 1072, "n_holdout": 1072, "eb": 0.00917230860205343, "labelled_winner": false }, { "headline": "Yeah, But How Much Do We Really Know About The 'Big Bang' Theory? Oh, A Lot? Oh. Crazy.", "impressions": 2099, "clicks": 25, "ctr": 0.0119104335397808, "ctr_select": 0.01619047619047619, "ctr_holdout": 0.0076263107721639654, "n_select": 1050, "n_holdout": 1049, "eb": 0.012201615767956973, "labelled_winner": false }, { "headline": "The Most Bad-Assed Scientists Take On The Most Bad-Assed Problem", "impressions": 2142, "clicks": 22, "ctr": 0.0102707749766573, "ctr_select": 0.008403361344537815, "ctr_holdout": 0.012138188608776844, "n_select": 1071, "n_holdout": 1071, "eb": 0.009765212568038488, "labelled_winner": false } ], "winner": 2, "pool_select": 0.010394519253484526, "pool_holdout": 0.010160680529300567, "lift_select": 0.5575974025974024, "lift_holdout": -0.24942913516749066, "eb_lift": 0.17385089876731508, "p_value": 0.032723083563681477, "held_up": false, "z_path": [ 0.571, 0.991, 0.563, 0.224, -0.68, -0.651, -0.605, -0.027, -0.502, -0.174 ], "first_cross": null, "final_z": -0.174 }, { "id": "5527349366386200182e0000", "variation": "headline", "week": 201514, "impressions": 8148, "pooled_ctr": 0.005031909671084929, "arms": [ { "headline": "Considering the perspective of people who don't believe in climate change can help you reach them.", "impressions": 2050, "clicks": 5, "ctr": 0.0024390243902439, "ctr_select": 0.001951219512195122, "ctr_holdout": 0.002926829268292683, "n_select": 1025, "n_holdout": 1025, "eb": 0.0036625238927699266, "labelled_winner": false }, { "headline": "These five steps can help you through tough discussions. Let's use climate change as an example.", "impressions": 2085, "clicks": 17, "ctr": 0.0081534772182254, "ctr_select": 0.008637236084452975, "ctr_holdout": 0.007670182166826462, "n_select": 1042, "n_holdout": 1043, "eb": 0.00572751238392299, "labelled_winner": false }, { "headline": "If facts don't change peoples minds, what can? A step-by-step guide to talking about tough topics.", "impressions": 2025, "clicks": 12, "ctr": 0.0059259259259259, "ctr_select": 0.005928853754940711, "ctr_holdout": 0.005923000987166831, "n_select": 1012, "n_holdout": 1013, "eb": 0.004878133587599783, "labelled_winner": false }, { "headline": "How to help folks who don't believe in climate change discover why it's good for them if they do.", "impressions": 1988, "clicks": 7, "ctr": 0.0035211267605633, "ctr_select": 0.001006036217303823, "ctr_holdout": 0.006036217303822937, "n_select": 994, "n_holdout": 994, "eb": 0.0033948479638698705, "labelled_winner": false } ], "winner": 1, "pool_select": 0.00441934691873312, "pool_holdout": 0.005644171779141104, "lift_select": 0.954414587332054, "lift_holdout": 0.3589561882529493, "eb_lift": 0.2960087744287967, "p_value": 0.017338648457058986, "held_up": true, "z_path": [ -0.605, -0.029, 1.099, 2.069, 1.555, 1.447, 1.643, 0.787, 0.976, 0.853 ], "first_cross": 3, "final_z": 0.853 }, { "id": "552e7acc346134000cf40000", "variation": "headline", "week": 201515, "impressions": 8119, "pooled_ctr": 0.007390072669047913, "arms": [ { "headline": "Why are people out standing in the street in Germany, Finland, Brazil, Japan, and the U.S. today?", "impressions": 2031, "clicks": 15, "ctr": 0.0073855243722304, "ctr_select": 0.005905511811023622, "ctr_holdout": 0.008866995073891626, "n_select": 1016, "n_holdout": 1015, "eb": 0.006595697029212951, "labelled_winner": false }, { "headline": "A massive, global protest is going down today. You should know why.", "impressions": 2045, "clicks": 10, "ctr": 0.0048899755501222, "ctr_select": 0.0019569471624266144, "ctr_holdout": 0.007820136852394917, "n_select": 1022, "n_holdout": 1023, "eb": 0.005386134206634738, "labelled_winner": false }, { "headline": "Imagine working 60 hours a week and not being able to pay your bills. Now image how pissed you'd be.", "impressions": 2066, "clicks": 14, "ctr": 0.0067763794772507, "ctr_select": 0.007744433688286544, "ctr_holdout": 0.005808325266214908, "n_select": 1033, "n_holdout": 1033, "eb": 0.007159123908305138, "labelled_winner": false }, { "headline": "These photos from around the world should scare the pants off of McDonald's and Walmart.", "impressions": 1977, "clicks": 21, "ctr": 0.0106221547799696, "ctr_select": 0.012145748987854251, "ctr_holdout": 0.00910010111223458, "n_select": 988, "n_holdout": 989, "eb": 0.0084666120581788, "labelled_winner": false } ], "winner": 3, "pool_select": 0.006898250800689825, "pool_holdout": 0.007881773399014778, "lift_select": 0.7606998264893001, "lift_holdout": 0.15457532861476242, "eb_lift": 0.2273563694338483, "p_value": 0.02196068790332485, "held_up": true, "z_path": [ -0.541, -0.399, 0.063, 0.705, 1.156, 0.822, 1.338, 1.052, 0.664, 1.319 ], "first_cross": null, "final_z": 1.319 }, { "id": "5318a8133d3a7922dd00006e", "variation": "image", "week": 201409, "impressions": 8108, "pooled_ctr": 0.051060680809077455, "arms": [ { "headline": "Watch A Powerful Twist Happen As This Girl Grows Up A Second-A-Day", "impressions": 4027, "clicks": 210, "ctr": 0.0521480009932952, "ctr_select": 0.05114200595829196, "ctr_holdout": 0.053154495777446596, "n_select": 2014, "n_holdout": 2013, "eb": 0.05109849250825092, "labelled_winner": true }, { "headline": "Watch A Powerful Twist Happen As This Girl Grows Up A Second-A-Day", "impressions": 4081, "clicks": 204, "ctr": 0.0499877481009556, "ctr_select": 0.050980392156862744, "ctr_holdout": 0.04899559039686428, "n_select": 2040, "n_holdout": 2041, "eb": 0.051023094711642604, "labelled_winner": false } ], "winner": 0, "pool_select": 0.051060680809077455, "pool_holdout": 0.051060680809077455, "lift_select": 0.0015927157242299206, "lift_holdout": 0.04100640522593468, "eb_lift": 0.0007405247751171018, "p_value": 0.9813525526912157, "held_up": true, "z_path": [ -2.452, -2.509, -1.906, -1.731, -1.302, -0.669, -0.426, -0.162, 0.183, 0.442 ], "first_cross": 0, "final_z": 0.442 }, { "id": "54f77c1f3135360026660100", "variation": "image", "week": 201509, "impressions": 7653, "pooled_ctr": 0.007578727296485038, "arms": [ { "headline": "EXCLUSIVE: A massively viral Chinese documentary, summarized in English for the first time.", "impressions": 2547, "clicks": 21, "ctr": 0.0082449941107184, "ctr_select": 0.00706436420722135, "ctr_holdout": 0.009426551453260016, "n_select": 1274, "n_holdout": 1273, "eb": 0.007900078399988042, "labelled_winner": false }, { "headline": "EXCLUSIVE: A massively viral Chinese documentary, summarized in English for the first time.", "impressions": 2543, "clicks": 16, "ctr": 0.0062917813605977, "ctr_select": 0.009433962264150943, "ctr_holdout": 0.003147128245476003, "n_select": 1272, "n_holdout": 1271, "eb": 0.008740204896686274, "labelled_winner": false }, { "headline": "EXCLUSIVE: A massively viral Chinese documentary, summarized in English for the first time.", "impressions": 2563, "clicks": 21, "ctr": 0.0081935232149824, "ctr_select": 0.00858034321372855, "ctr_holdout": 0.0078064012490242, "n_select": 1282, "n_holdout": 1281, "eb": 0.008438123405908042, "labelled_winner": false } ], "winner": 1, "pool_select": 0.008359456635318705, "pool_holdout": 0.0067973856209150325, "lift_select": 0.12853773584905648, "lift_holdout": -0.5370090177328573, "eb_lift": 0.04554701076609535, "p_value": 0.6064830776223172, "held_up": false, "z_path": [ 0.008, -0.29, 0.015, -0.227, -0.639, -0.984, -0.738, -0.738, -0.851, -0.801 ], "first_cross": null, "final_z": -0.801 }, { "id": "54e2b61c633065000c140000", "variation": "headline", "week": 201507, "impressions": 7624, "pooled_ctr": 0.002229800629590766, "arms": [ { "headline": "For once, an education reform idea that all my teacher friends will love.", "impressions": 1546, "clicks": 2, "ctr": 0.001293661060802, "ctr_select": 0.0, "ctr_holdout": 0.00258732212160414, "n_select": 773, "n_holdout": 773, "eb": 0.0019677109947059137, "labelled_winner": false }, { "headline": "Stop thinking of teachers as teachers. Think of them as designers.", "impressions": 1477, "clicks": 1, "ctr": 0.0006770480704129, "ctr_select": 0.0013550135501355014, "ctr_holdout": 0.0, "n_select": 738, "n_holdout": 739, "eb": 0.0023175066839829315, "labelled_winner": false }, { "headline": "Here's the simplest (and best) education idea I've heard in a long time.", "impressions": 1542, "clicks": 7, "ctr": 0.0045395590142671, "ctr_select": 0.005188067444876783, "ctr_holdout": 0.0038910505836575876, "n_select": 771, "n_holdout": 771, "eb": 0.003264029315525094, "labelled_winner": false }, { "headline": "Forget charter schools. Forget merit pay. Here's how to dramatically improve teachers.", "impressions": 1529, "clicks": 4, "ctr": 0.0026160889470241, "ctr_select": 0.003926701570680628, "ctr_holdout": 0.00130718954248366, "n_select": 764, "n_holdout": 765, "eb": 0.0029469470159465484, "labelled_winner": false }, { "headline": "I'm not a teacher because I love creative freedom. What if we tried giving teachers some of that?", "impressions": 1530, "clicks": 3, "ctr": 0.0019607843137254, "ctr_select": 0.00261437908496732, "ctr_holdout": 0.00130718954248366, "n_select": 765, "n_holdout": 765, "eb": 0.0026215998547545073, "labelled_winner": false } ], "winner": 2, "pool_select": 0.0026239832065074785, "pool_holdout": 0.0018358248098610018, "lift_select": 0.977172503242542, "lift_holdout": 1.1195108393551974, "eb_lift": 0.24392157214661325, "p_value": 0.1191782144187907, "held_up": true, "z_path": [ 0.0, -0.009, 0.569, 0.57, 0.438, 0.438, 1.125, 1.405, 0.621, 0.892 ], "first_cross": null, "final_z": 0.892 }, { "id": "531f270a65114a89d8000037", "variation": "image", "week": 201410, "impressions": 7262, "pooled_ctr": 0.03552740291930598, "arms": [ { "headline": "Ellen, Katy Perry, And A Hockey Player Walk Into An Ad And Shatter A Ridiculous Argument", "impressions": 2371, "clicks": 69, "ctr": 0.0291016448755799, "ctr_select": 0.03204047217537943, "ctr_holdout": 0.026160337552742614, "n_select": 1186, "n_holdout": 1185, "eb": 0.03653325471312591, "labelled_winner": false }, { "headline": "Ellen, Katy Perry, And A Hockey Player Walk Into An Ad And Shatter A Ridiculous Argument", "impressions": 2473, "clicks": 105, "ctr": 0.0424585523655479, "ctr_select": 0.04611650485436893, "ctr_holdout": 0.038803556992724336, "n_select": 1236, "n_holdout": 1237, "eb": 0.041365780913075044, "labelled_winner": false }, { "headline": "Ellen, Katy Perry, And A Hockey Player Walk Into An Ad And Shatter A Ridiculous Argument", "impressions": 2418, "clicks": 84, "ctr": 0.0347394540942928, "ctr_select": 0.0380479735318445, "ctr_holdout": 0.03143093465674111, "n_select": 1209, "n_holdout": 1209, "eb": 0.03856340749094694, "labelled_winner": true } ], "winner": 1, "pool_select": 0.03883227760947398, "pool_holdout": 0.03222252822913798, "lift_select": 0.18758176685257855, "lift_holdout": 0.20423688410753904, "eb_lift": 0.06524220209486153, "p_value": 0.10265026803004539, "held_up": true, "z_path": [ 1.788, 2.022, 1.511, 0.939, 1.107, 0.482, 0.765, 0.609, 1.029, 1.4 ], "first_cross": 1, "final_z": 1.4 }, { "id": "54fde77b3233330019640000", "variation": "other", "week": 201510, "impressions": 6920, "pooled_ctr": 0.014884393063583815, "arms": [ { "headline": "Coming out was easy. The hard part? Admitting she still liked men.", "impressions": 1730, "clicks": 33, "ctr": 0.0190751445086705, "ctr_select": 0.020809248554913295, "ctr_holdout": 0.017341040462427744, "n_select": 865, "n_holdout": 865, "eb": 0.017438361121974746, "labelled_winner": true }, { "headline": "Coming out was easy. The hard part? Admitting she still liked men.", "impressions": 1807, "clicks": 30, "ctr": 0.0166021029330381, "ctr_select": 0.018805309734513276, "ctr_holdout": 0.014396456256921373, "n_select": 904, "n_holdout": 903, "eb": 0.016916867103047916, "labelled_winner": false }, { "headline": "Coming out was easy. The hard part? Admitting she still liked men.", "impressions": 1679, "clicks": 18, "ctr": 0.0107206670637284, "ctr_select": 0.008333333333333333, "ctr_holdout": 0.013110846245530394, "n_select": 840, "n_holdout": 839, "eb": 0.014092872744019156, "labelled_winner": false }, { "headline": "Coming out was easy. The hard part? Admitting she still liked men.", "impressions": 1704, "clicks": 22, "ctr": 0.0129107981220657, "ctr_select": 0.01643192488262911, "ctr_holdout": 0.009389671361502348, "n_select": 852, "n_holdout": 852, "eb": 0.01624793173184783, "labelled_winner": false } ], "winner": 0, "pool_select": 0.016180294712510836, "pool_holdout": 0.013587742122000578, "lift_select": 0.2860858794384804, "lift_holdout": 0.2762267863731398, "eb_lift": 0.07775299719918904, "p_value": 0.21279563128668189, "held_up": true, "z_path": [ -0.23, 0.746, 0.663, 1.306, 0.763, 0.305, 0.447, 0.442, 0.836, 0.556 ], "first_cross": null, "final_z": 0.556 }, { "id": "53fb5762842e02dc3100001f", "variation": "headline", "week": 201434, "impressions": 6790, "pooled_ctr": 0.01752577319587629, "arms": [ { "headline": "He Didn't Want To Perform In Their Town. And Sure Enough, Within 10 Seconds, The Heckling Started", "impressions": 3425, "clicks": 63, "ctr": 0.0183941605839416, "ctr_select": 0.018107476635514017, "ctr_holdout": 0.018680677174547577, "n_select": 1712, "n_holdout": 1713, "eb": 0.017860608422402773, "labelled_winner": true }, { "headline": "He Didn't Want To Perform In Their Town. And Sure Enough, Within 10 Seconds, They Started Heckling", "impressions": 3365, "clicks": 56, "ctr": 0.0166419019316493, "ctr_select": 0.017241379310344827, "ctr_holdout": 0.016042780748663103, "n_select": 1682, "n_holdout": 1683, "eb": 0.01749453532002019, "labelled_winner": false } ], "winner": 0, "pool_select": 0.01767825574543312, "pool_holdout": 0.017373380447585393, "lift_select": 0.024279595015576083, "lift_holdout": 0.0752471133010777, "eb_lift": 0.01031508309391671, "p_value": 0.8481821328783327, "held_up": true, "z_path": [ 2.055, 0.891, 0.712, 0.36, 1.093, 0.862, 0.353, 0.431, 0.296, 0.55 ], "first_cross": 0, "final_z": 0.55 }, { "id": "53cd4f31ddae0f8434000038", "variation": "image", "week": 201429, "impressions": 6621, "pooled_ctr": 0.01616070080048331, "arms": [ { "headline": "Which Cities Will Be Completely Underwater In Less Than 100 Years?", "impressions": 3307, "clicks": 48, "ctr": 0.0145146658602963, "ctr_select": 0.013301088270858524, "ctr_holdout": 0.01572897761645493, "n_select": 1654, "n_holdout": 1653, "eb": 0.014175289817459198, "labelled_winner": false }, { "headline": "Which Cities Will Be Completely Underwater In Less Than 100 Years?", "impressions": 3314, "clicks": 59, "ctr": 0.0178032589016294, "ctr_select": 0.016294508147254073, "ctr_holdout": 0.01931200965600483, "n_select": 1657, "n_holdout": 1657, "eb": 0.01542254797366253, "labelled_winner": true } ], "winner": 1, "pool_select": 0.014799154334038054, "pool_holdout": 0.017522658610271902, "lift_select": 0.10104319337873946, "lift_holdout": 0.10211641312717212, "eb_lift": 0.0421235987917683, "p_value": 0.47569672424122555, "held_up": true, "z_path": [ 1.27, 0.894, 1.867, 1.989, 1.912, 1.897, 1.418, 1.692, 1.335, 1.061 ], "first_cross": 3, "final_z": 1.061 }, { "id": "54451b14ecd9a21c0b00002b", "variation": "image", "week": 201442, "impressions": 6274, "pooled_ctr": 0.02693656359579216, "arms": [ { "headline": "These Congressmen Think They\u2019re Smarter Than Scientists. Jon Stewart Disproves That Real Quick.", "impressions": 3175, "clicks": 82, "ctr": 0.0258267716535433, "ctr_select": 0.02707808564231738, "ctr_holdout": 0.024574669187145556, "n_select": 1588, "n_holdout": 1587, "eb": 0.027083563522875723, "labelled_winner": false }, { "headline": "These Congressmen Think They\u2019re Smarter Than Scientists. Jon Stewart Disproves That Real Quick.", "impressions": 3099, "clicks": 87, "ctr": 0.0280735721200387, "ctr_select": 0.027096774193548386, "ctr_holdout": 0.029051000645577793, "n_select": 1550, "n_holdout": 1549, "eb": 0.027091106877113132, "labelled_winner": true } ], "winner": 1, "pool_select": 0.02708731676226896, "pool_holdout": 0.026785714285714284, "lift_select": 0.0003491461100568749, "lift_holdout": 0.08457069076823776, "eb_lift": 0.00013992212212965427, "p_value": 0.9974274747438153, "held_up": true, "z_path": [ 2.057, 1.247, 1.37, 0.836, 1.119, 0.735, 0.045, 0.49, 0.476, 0.55 ], "first_cross": 0, "final_z": 0.55 }, { "id": "5441cdf0275a490d8a000029", "variation": "headline", "week": 201441, "impressions": 6174, "pooled_ctr": 0.025267249757045675, "arms": [ { "headline": "Since When Does Being Popular Mean Having To Put Up With This Kinda Bullying", "impressions": 3085, "clicks": 84, "ctr": 0.0272285251215559, "ctr_select": 0.029831387808041506, "ctr_holdout": 0.02462734931950745, "n_select": 1542, "n_holdout": 1543, "eb": 0.028263104759697795, "labelled_winner": false }, { "headline": "Since When Does Being Popular Mean Having To Put Up With This Kind Of Bullying", "impressions": 3089, "clicks": 72, "ctr": 0.0233085140822272, "ctr_select": 0.02461139896373057, "ctr_holdout": 0.022006472491909384, "n_select": 1544, "n_holdout": 1545, "eb": 0.02617683938479512, "labelled_winner": true } ], "winner": 0, "pool_select": 0.027219701879455604, "pool_holdout": 0.023316062176165803, "lift_select": 0.09594836637638204, "lift_holdout": 0.056239648592208535, "eb_lift": 0.0383326343860404, "p_value": 0.37291741586952387, "held_up": true, "z_path": [ -0.304, 0.694, 1.598, 1.307, 1.321, 1.578, 1.369, 0.74, 0.787, 0.981 ], "first_cross": null, "final_z": 0.981 }, { "id": "54e238c0396166001a8b0000", "variation": "other", "week": 201507, "impressions": 6158, "pooled_ctr": 0.03004222150048717, "arms": [ { "headline": "5 men watch something horrifying that's being done to girls. Here's how they reacted.", "impressions": 1586, "clicks": 46, "ctr": 0.0290037831021437, "ctr_select": 0.025220680958385876, "ctr_holdout": 0.03278688524590164, "n_select": 793, "n_holdout": 793, "eb": 0.028215193845817697, "labelled_winner": false }, { "headline": "5 men watch something horrifying that's being done to girls. Here's how they reacted.", "impressions": 1542, "clicks": 49, "ctr": 0.0317769130998702, "ctr_select": 0.03372243839169909, "ctr_holdout": 0.029831387808041506, "n_select": 771, "n_holdout": 771, "eb": 0.030358729697407993, "labelled_winner": false }, { "headline": "5 men watch something horrifying that's being done to girls. Here's how they reacted.", "impressions": 1493, "clicks": 39, "ctr": 0.0261219022103148, "ctr_select": 0.032171581769437, "ctr_holdout": 0.020080321285140562, "n_select": 746, "n_holdout": 747, "eb": 0.029953653258016598, "labelled_winner": false }, { "headline": "5 men watch something horrifying that's being done to girls. Here's how they reacted.", "impressions": 1537, "clicks": 51, "ctr": 0.033181522446324, "ctr_select": 0.026041666666666668, "ctr_holdout": 0.04031209362808843, "n_select": 768, "n_holdout": 769, "eb": 0.028443790394690936, "labelled_winner": false } ], "winner": 1, "pool_select": 0.029239766081871343, "pool_holdout": 0.030844155844155844, "lift_select": 0.15330739299610885, "lift_holdout": -0.032835005802443806, "eb_lift": 0.03826855565135334, "p_value": 0.3934602692724115, "held_up": false, "z_path": [ 1.642, 0.887, -0.466, -0.265, 0.812, 1.508, 1.161, 0.834, 1.234, 0.928 ], "first_cross": null, "final_z": 0.928 }, { "id": "5526f68d646532002c810100", "variation": "other", "week": 201514, "impressions": 6127, "pooled_ctr": 0.010935204831075566, "arms": [ { "headline": "A Utah town's solution to homelessness is so good, every state will be mad that they weren't first.", "impressions": 2033, "clicks": 25, "ctr": 0.0122970978848991, "ctr_select": 0.010826771653543307, "ctr_holdout": 0.01376597836774828, "n_select": 1016, "n_holdout": 1017, "eb": 0.013059689106881671, "labelled_winner": false }, { "headline": "A Utah town's solution to homelessness is so good, every state will be mad that they weren't first.", "impressions": 2078, "clicks": 17, "ctr": 0.0081809432146294, "ctr_select": 0.015399422521655439, "ctr_holdout": 0.0009624639076034649, "n_select": 1039, "n_holdout": 1039, "eb": 0.01445976369329842, "labelled_winner": false }, { "headline": "A Utah town's solution to homelessness is so good, every state will be mad that they weren't first.", "impressions": 2016, "clicks": 25, "ctr": 0.0124007936507936, "ctr_select": 0.015873015873015872, "ctr_holdout": 0.008928571428571428, "n_select": 1008, "n_holdout": 1008, "eb": 0.014594548245083638, "labelled_winner": false } ], "winner": 2, "pool_select": 0.014038524322559582, "pool_holdout": 0.007832898172323759, "lift_select": 0.13067552602436328, "lift_holdout": 0.13988095238095233, "eb_lift": 0.039607006388167054, "p_value": 0.5455792075042505, "held_up": false, "z_path": [ 0.029, 0.752, 0.051, 0.561, 0.791, 0.482, 0.817, 1.098, 0.9, 1.34 ], "first_cross": null, "final_z": 1.34 }, { "id": "54fa0d6161316400266b0100", "variation": "headline", "week": 201509, "impressions": 6106, "pooled_ctr": 0.025548640681297086, "arms": [ { "headline": "This billboard lets you determine how the women on it look. You just have to see it.", "impressions": 1446, "clicks": 43, "ctr": 0.0297372060857538, "ctr_select": 0.027662517289073305, "ctr_holdout": 0.0318118948824343, "n_select": 723, "n_holdout": 723, "eb": 0.02380428615235581, "labelled_winner": false }, { "headline": "This billboard takes on domestic violence in a way you really have to see to believe", "impressions": 1514, "clicks": 46, "ctr": 0.0303830911492734, "ctr_select": 0.022457067371202115, "ctr_holdout": 0.03830911492734478, "n_select": 757, "n_holdout": 757, "eb": 0.02256535273674273, "labelled_winner": false }, { "headline": "People can control how the women on this billboard look. They just have to look up.", "impressions": 1630, "clicks": 33, "ctr": 0.0202453987730061, "ctr_select": 0.01717791411042945, "ctr_holdout": 0.023312883435582823, "n_select": 815, "n_holdout": 815, "eb": 0.021189927826654173, "labelled_winner": false }, { "headline": "It's hard to ignore the women on this billboard, but that's sort of the point", "impressions": 1516, "clicks": 34, "ctr": 0.0224274406332453, "ctr_select": 0.023746701846965697, "ctr_holdout": 0.021108179419525065, "n_select": 758, "n_holdout": 758, "eb": 0.022883145556820774, "labelled_winner": false } ], "winner": 0, "pool_select": 0.02260072060268588, "pool_holdout": 0.028496560759908286, "lift_select": 0.22396616352957688, "lift_holdout": 0.11634155259852785, "eb_lift": 0.05325341482814916, "p_value": 0.2945264791676031, "held_up": false, "z_path": [ -0.649, -0.452, -0.346, 0.509, -0.021, 0.613, 1.109, 0.952, 0.803, 1.25 ], "first_cross": null, "final_z": 1.25 }, { "id": "54920116623736001e000000", "variation": "other", "week": 201450, "impressions": 5724, "pooled_ctr": 0.009783368273934312, "arms": [ { "headline": "Native American Rap. It's Seriously Good. For My Ears, Your Ears, And For America.", "impressions": 2875, "clicks": 27, "ctr": 0.009391304347826, "ctr_select": 0.009040333796940195, "ctr_holdout": 0.009742519137091163, "n_select": 1438, "n_holdout": 1437, "eb": 0.009930119018841973, "labelled_winner": false }, { "headline": "Native American Rap. It's Seriously Good. For My Ears, Your Ears, And For America.", "impressions": 2849, "clicks": 29, "ctr": 0.0101790101790101, "ctr_select": 0.011938202247191011, "ctr_holdout": 0.008421052631578947, "n_select": 1424, "n_holdout": 1425, "eb": 0.011036307170048719, "labelled_winner": false } ], "winner": 1, "pool_select": 0.010482180293501049, "pool_holdout": 0.009084556254367574, "lift_select": 0.13890449438202235, "lift_holdout": -0.07303643724696351, "eb_lift": 0.05286370402264762, "p_value": 0.4465984444995771, "held_up": false, "z_path": [ 0.461, 0.724, 0.925, 2.028, 1.256, 0.761, 0.537, 0.624, 0.173, 0.303 ], "first_cross": 3, "final_z": 0.303 } ] }