base_rate_test.py

import json, statistics, random
from pathlib import Path
R=Path("/var/lib/octave/sn114/repo")
feats={f["cid"]:f for f in json.load(open(R/"routable_feature.json"))}
per=json.load(open(R/"per_challenge_cap_effect.json"))
# PREDICTION from the base-rate argument: if consequence is question-driven and independent
# of at-risk volume, then P(loser) should be roughly FLAT across at-risk quartiles.
xs=sorted(per, key=lambda p: feats[p['cid']]['uniq_at_risk'])
n=len(xs); q=n//4
out={'quartiles':[]}
for i in range(4):
    grp=xs[i*q:(i+1)*q] if i<3 else xs[3*q:]
    losers=sum(1 for p in grp if p['loss']>1e-9)
    out['quartiles'].append({'quartile':i+1,'n':len(grp),'losers':losers,
      'loser_rate':round(losers/len(grp),4),
      'mean_uniq_at_risk':round(statistics.fmean([feats[p['cid']]['uniq_at_risk'] for p in grp]),1)})
rates=[x['loser_rate'] for x in out['quartiles']]
out['rate_spread']=round(max(rates)-min(rates),4)
out['flat_prediction_holds']=bool(out['rate_spread']<0.20)
# also: is loss magnitude related to at-risk volume among losers only?
L=[p for p in per if p['loss']>1e-9]
def corr(a,b):
    ma,mb=statistics.fmean(a),statistics.fmean(b)
    num=sum((x-ma)*(y-mb) for x,y in zip(a,b))
    da=sum((x-ma)**2 for x in a)**0.5; db=sum((y-mb)**2 for y in b)**0.5
    return num/(da*db) if da and db else 0.0
out['corr_loss_vs_atrisk_among_losers']=round(corr([p['loss'] for p in L],[feats[p['cid']]['uniq_at_risk'] for p in L]),4)
json.dump(out, open(R/"base_rate_test.json","w"), indent=2)
for x in out['quartiles']: print('  Q%d  n=%-4d mean at-risk %-7s losers %-3d rate %.4f'%(x['quartile'],x['n'],x['mean_uniq_at_risk'],x['losers'],x['loser_rate']))
print()
print('  loser-rate spread across quartiles: %.4f  flat prediction holds: %s'%(out['rate_spread'],out['flat_prediction_holds']))
print('  corr(loss magnitude, at-risk volume) among the 21 losers: %+.4f'%out['corr_loss_vs_atrisk_among_losers'])