Buckets:
| """One held-out test evaluation of a frozen package; no fitting or calibration.""" | |
| import argparse | |
| import hashlib | |
| import json | |
| from pathlib import Path | |
| import resource | |
| import time | |
| import numpy as np | |
| from shapely.geometry import Polygon | |
| from inference import Model | |
| from load_dataset import dataset, load_sample | |
| from training.model_factory import ROOT | |
| from training.segmentation_data import CANONICAL_IDS, head_to_canonical | |
| from training.segmentation_loss import segmentation_counts, summarize_counts | |
| from evaluation.quick_eval import detection_target, detection_metrics, semantic_array, _preview | |
| from evaluation.metrics import evaluate_detection | |
| def size_metrics(predictions,targets): | |
| result={} | |
| for name,low,high in [('small',0,32**2),('medium',32**2,96**2),('large',96**2,float('inf'))]: | |
| scoped=[] | |
| for row in targets: | |
| annotations=[];ignored=list(row['ignored_annotations']) | |
| for annotation in row['annotations']: | |
| area=Polygon(np.asarray(annotation['obb_polygon']).reshape(-1,2)).area | |
| if low <= area < high:annotations.append(annotation) | |
| else:ignored.append({'clipped_polygon':annotation['obb_polygon']}) | |
| scoped.append({**row,'annotations':annotations,'ignored_annotations':ignored}) | |
| result[name]=evaluate_detection(predictions,scoped) | |
| return result | |
| def evaluate_full(model,root,split='test',*,output): | |
| if split!='test':raise ValueError('final evaluator accepts only the held-out test split') | |
| output=Path(output).resolve() | |
| if not output.is_relative_to(ROOT/'evaluation'):raise ValueError('evaluation output required') | |
| output.mkdir(parents=True,exist_ok=False) | |
| (output/'INCOMPLETE.json').write_text(json.dumps({'status':'INCOMPLETE','selection_frozen':True})) | |
| start=time.monotonic() | |
| import torch | |
| predictions=[];targets=[];durations={'detection':[],'segmentation':[]} | |
| ids={};all_hashes={} | |
| counts={str(c):dict(intersection=0,union=0,targets=0) for c in CANONICAL_IDS} | |
| for task in ('detection','segmentation'): | |
| data=dataset(root,task,split,license_scope='all') | |
| ids[task]=[r['id'] for r in data.samples] | |
| for index,row in enumerate(data.samples): | |
| sample=load_sample(data.root,row,data.expected_hashes) | |
| t=time.monotonic() | |
| with torch.inference_mode(): | |
| result=model.models[task].predict(sample['image'][:,:,::-1].copy(),imgsz=1024,device='cpu', | |
| **({'conf':.001,'max_det':300} if task=='detection' else {}),save=False,verbose=False)[0] | |
| durations[task].append(time.monotonic()-t) | |
| if task=='detection': | |
| objects=[{'category_id':model.mapping[int(cls)],'confidence':float(score),'obb_polygon':points.reshape(-1).tolist()} | |
| for points,cls,score in zip(result.obb.xyxyxyxy.cpu().numpy(),result.obb.cls.cpu().numpy(),result.obb.conf.cpu().numpy()) if int(cls) in model.mapping] | |
| predictions.append({'sample_id':row['id'],'annotations':objects}) | |
| target=detection_target(sample,'held_out_test');target['split']='test';targets.append(target) | |
| path=output/f"{row['id']}.json";path.write_text(json.dumps(predictions[-1])) | |
| if index<3:_preview(sample,objects,model.mapping,target,output/f"{row['id']}.jpg",model.thresholds,prediction_label='retained pretrained') | |
| else: | |
| raw=semantic_array(result);canonical=head_to_canonical(raw) | |
| target=sample['mask'].copy();target[~sample['valid_mask']]=255 | |
| values=segmentation_counts(canonical,target) | |
| for c,value in values.items(): | |
| for key,n in value.items():counts[c][key]+=n | |
| path=output/f"{row['id']}.npz";np.savez_compressed(path,canonical=canonical) | |
| if index<3:_preview(sample,raw,dict(enumerate(CANONICAL_IDS)),target,output/f"{row['id']}.jpg",prediction_label='fine-tuned') | |
| all_hashes[path.name]=hashlib.sha256(path.read_bytes()).hexdigest() | |
| metrics=detection_metrics(predictions,targets,model.thresholds) | |
| report={'status':'COMPLETED','split':'test','selection_frozen':True,'training_performed':False, | |
| 'package_sha256':hashlib.sha256((model.directory/'model.json').read_bytes()).hexdigest(), | |
| 'detection':metrics,'detection_size_strata':size_metrics(predictions,targets), | |
| 'size_strata_definition':'Oriented polygon area in pixels: small<1024, medium<9216, large>=9216; other-size targets ignored.', | |
| 'segmentation':summarize_counts(counts),'sample_ids':ids,'prediction_sha256':all_hashes, | |
| 'inference_seconds':{task:{'total':sum(v),'mean':float(np.mean(v)),'p95':float(np.percentile(v,95))} for task,v in durations.items()}, | |
| 'timing_scope':'Prediction timings include first-call setup/warmup and exclude checkpoint loading.', | |
| 'elapsed_seconds':time.monotonic()-start,'host_peak_rss_mib':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss/1024, | |
| 'runtime':{'device':'cpu','threads':4,'image_size':1024,'gpu_used':False}, | |
| 'limitations':model.config['limitations']+['Single prepared tiles; large-image seam geometry tested separately.','Internal DOTA split, not an official benchmark result.','Object-size ignored regions suppress intersecting predictions class-agnostically, following the existing evaluator.']} | |
| (output/'RESULTS.json').write_text(json.dumps(report,indent=2)) | |
| (output/'detection_targets.json').write_text(json.dumps(targets)) | |
| (output/'INCOMPLETE.json').unlink() | |
| return report | |
| if __name__=='__main__': | |
| parser=argparse.ArgumentParser() | |
| parser.add_argument('--model-dir',type=Path,default=ROOT/'model') | |
| parser.add_argument('--root',type=Path,default=ROOT/'datasets/corpus') | |
| parser.add_argument('--output',type=Path,required=True) | |
| args=parser.parse_args() | |
| report=evaluate_full(Model(args.model_dir),args.root,output=args.output) | |
| print(json.dumps({'detection':report['detection'],'segmentation':report['segmentation']})) | |
Xet Storage Details
- Size:
- 6.09 kB
- Xet hash:
- 6a2c27f0550184f0284dc3b3ec2c92a1cacd7962fac53b61a28d9e0b95621c1e
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.