Buckets:
| import argparse | |
| import base64 | |
| import json | |
| from pathlib import Path | |
| import cv2 | |
| import numpy as np | |
| PAGE = r"""<!doctype html> | |
| <html lang="en"> | |
| <head> | |
| <meta charset="utf-8"> | |
| <meta name="viewport" content="width=device-width,initial-scale=1"> | |
| <title>Physics mask trace</title> | |
| <style> | |
| :root{color-scheme:dark;font-family:system-ui,sans-serif;background:#111827;color:#e5e7eb} | |
| body{max-width:1320px;margin:auto;padding:24px}h1{margin:0;font-size:26px}h2{font-size:17px} | |
| p{color:#b5c0d1;line-height:1.5}.controls,.panel{background:#192234;border:1px solid #344158;border-radius:12px;padding:16px;margin:16px 0} | |
| .toolbar{display:flex;flex-wrap:wrap;gap:16px;align-items:center}input,select{font:inherit;background:#111827;color:inherit;border:1px solid #52617a;border-radius:5px;padding:6px} | |
| input[type=number]{width:76px}input[type=range]{width:100%;padding:0;cursor:pointer} | |
| .views{display:grid;grid-template-columns:1fr 1fr;gap:16px}.views .panel{margin:0} | |
| canvas{width:100%;height:auto;display:block;image-rendering:pixelated;cursor:crosshair;background:#0b101b} | |
| .details{display:grid;grid-template-columns:1fr 1fr;gap:16px}.legend{display:flex;gap:16px;flex-wrap:wrap;font-size:13px} | |
| .swatch{display:inline-block;width:11px;height:11px;border-radius:3px;margin-right:6px} | |
| table{width:100%;border-collapse:collapse;font-size:14px}th,td{text-align:left;border-bottom:1px solid #344158;padding:9px 5px}th{color:#b5c0d1} | |
| pre{white-space:pre-wrap;overflow-wrap:anywhere;font-size:13px}output{font-variant-numeric:tabular-nums}#error{color:#fda4af} | |
| @media(max-width:760px){body{padding:12px}.views,.details{grid-template-columns:1fr}} | |
| </style> | |
| </head> | |
| <body> | |
| <h1>Physics mask trace</h1> | |
| <p>Inspect real and simulated actor IDs at the same mask pixel. Pixel disagreement and saved sensitivity results do not identify a unique physical cause.</p> | |
| <section class="controls"> | |
| <div class="toolbar"><label>X <input id="x" type="number" min="0" value="50"></label><label>Y <input id="y" type="number" min="0" value="50"></label> | |
| <label><input id="masks" type="checkbox" checked> Actor masks</label><label><input id="fp" type="checkbox" checked> False positives</label><label><input id="fn" type="checkbox" checked> False negatives</label> | |
| <label>Errors for <select id="actor"><option value="0">All actors</option></select></label></div> | |
| <p id="metadata"></p><input id="timeline" aria-label="Frame timeline" type="range" min="0" value="0" step="1"><p><output id="frame"></output></p> | |
| <div class="legend" id="legend"></div> | |
| </section> | |
| <section class="views"><div class="panel"><h2>Real actor IDs</h2><canvas id="real"></canvas></div><div class="panel"><h2>Simulated actor IDs</h2><canvas id="sim"></canvas></div></section> | |
| <section class="details"><div class="panel"><h2>Selected pixel</h2><table id="pixel"></table><p id="error" role="alert"></p></div><div class="panel"><h2>Tracking and pixel errors</h2><table id="tracking"></table><p>False positive: simulated actor without its real mask. False negative: real actor missing from its simulated mask.</p></div></section> | |
| <section id="metrics-panel" class="panel" hidden><h2>Saved refinement results</h2><table id="metrics"></table><details><summary>Saved parameter sensitivity</summary><pre id="sensitivity"></pre></details></section> | |
| <script id="trace-data" type="application/json">__DATA__</script> | |
| <script> | |
| const data=JSON.parse(document.getElementById('trace-data').textContent); | |
| const element=id=>document.getElementById(id); | |
| const palette=[[0,0,0],[65,210,120],[70,145,255],[250,80,165]]; | |
| const actorName=id=>id===0?'background':data.names[id-1]; | |
| const real=element('real'),sim=element('sim'); | |
| const controls=['masks','fp','fn','actor']; | |
| let current=null,ticket=0; | |
| real.width=sim.width=data.width;real.height=sim.height=data.height; | |
| element('x').max=data.width-1;element('y').max=data.height-1; | |
| element('x').value=Math.min(50,data.width-1);element('y').value=Math.min(50,data.height-1); | |
| element('timeline').max=data.frames.length-1; | |
| element('metadata').textContent=`Mask coordinates: ${data.width} × ${data.height}. ${data.frames.length} frames. ${data.video?'Background: first RGB panel of the supplied video.':'No source video supplied.'}`; | |
| function swatch(label,color){const span=document.createElement('span'),square=document.createElement('i');square.className='swatch';square.style.background=`rgb(${color.join(',')})`;span.append(square,document.createTextNode(label));element('legend').append(span)} | |
| data.names.forEach((name,index)=>{const option=document.createElement('option');option.value=index+1;option.textContent=name;element('actor').append(option);swatch(name,palette[index+1])}); | |
| swatch('False positive',[255,105,60]);swatch('False negative',[60,230,245]); | |
| function row(table,key,value){const tr=document.createElement('tr'),th=document.createElement('th'),td=document.createElement('td');th.textContent=key;td.textContent=String(value);tr.append(th,td);table.append(tr)} | |
| function bitmap(url){return new Promise((resolve,reject)=>{const image=new Image();image.onload=()=>resolve(image);image.onerror=()=>reject(new Error('Embedded image could not be decoded'));image.src=url})} | |
| function pixels(image){const canvas=document.createElement('canvas');canvas.width=data.width;canvas.height=data.height;const context=canvas.getContext('2d',{willReadFrequently:true});context.drawImage(image,0,0);return context.getImageData(0,0,data.width,data.height).data} | |
| async function frameData(index){const frame=data.frames[index];if(!frame.decoded){frame.decoded=Promise.all([bitmap(frame.actors),bitmap(frame.errors),frame.geometry?bitmap(frame.geometry):null,frame.rgb?bitmap(frame.rgb):null]).then(([actors,errors,geometry,rgb])=>({actors:pixels(actors),errors:pixels(errors),geometry:geometry?pixels(geometry):null,rgb,index}))}return frame.decoded} | |
| function blend(pixels,index,color,alpha){for(let channel=0;channel<3;channel++)pixels[index+channel]=Math.round(pixels[index+channel]*(1-alpha)+color[channel]*alpha)} | |
| function draw(canvas,channel){const context=canvas.getContext('2d');context.imageSmoothingEnabled=false;context.fillStyle='#0b101b';context.fillRect(0,0,data.width,data.height);if(current.rgb)context.drawImage(current.rgb,0,0,data.width,data.height);const picture=context.getImageData(0,0,data.width,data.height),out=picture.data;const selected=Number(element('actor').value),bits=selected?1<<(selected-1):(1<<data.names.length)-1;for(let index=0;index<out.length;index+=4){const actor=current.actors[index+channel];if(actor&&element('masks').checked)blend(out,index,palette[actor],current.rgb?0.52:1);const fp=element('fp').checked&&(current.errors[index]&bits),fn=element('fn').checked&&(current.errors[index+1]&bits);if(fp&&fn)blend(out,index,[215,160,245],0.82);else if(fp)blend(out,index,[255,105,60],0.82);else if(fn)blend(out,index,[60,230,245],0.82)}context.putImageData(picture,0,0);const x=Number(element('x').value),y=Number(element('y').value);if(Number.isInteger(x)&&Number.isInteger(y)&&x>=0&&y>=0&&x<data.width&&y<data.height){context.strokeStyle='#fff';context.lineWidth=1;context.strokeRect(x-2.5,y-2.5,6,6)}} | |
| function query(){if(!current)return;const x=Number(element('x').value),y=Number(element('y').value);element('pixel').replaceChildren();element('tracking').replaceChildren();if(!Number.isInteger(x)||!Number.isInteger(y)||x<0||y<0||x>=data.width||y>=data.height){element('error').textContent='Enter integer coordinates inside the mask image.';return}element('error').textContent='';const offset=(y*data.width+x)*4,actual=current.actors[offset],predicted=current.actors[offset+1],frame=data.frames[current.index];row(element('pixel'),'Mask pixel',`${x}, ${y}`);row(element('pixel'),'Source frame',frame.frame_index);row(element('pixel'),'Timestamp',`${frame.timestamp.toFixed(4)} s`);row(element('pixel'),'Real actor',actorName(actual));row(element('pixel'),'Simulated actor',actorName(predicted));row(element('pixel'),'Actor disagreement',actual!==predicted);row(element('pixel'),'Trusted disagreement',Boolean(current.errors[offset+2]));if(current.geometry){const encoded=current.geometry[offset]+(current.geometry[offset+1]<<8)+(current.geometry[offset+2]<<16),id=encoded-1;row(element('pixel'),'Simulated geometry ID',id);const info=data.geom_mapping[String(id)];row(element('pixel'),'Simulated geometry',info===undefined?(id<0?'background':'name unavailable'):(typeof info==='string'?info:JSON.stringify(info)))}else row(element('pixel'),'Simulated geometry','not saved');const header=document.createElement('tr');for(const label of ['Actor','Observed','Confidence','FP','FN']){const th=document.createElement('th');th.textContent=label;header.append(th)}element('tracking').append(header);data.names.forEach((name,index)=>{const tr=document.createElement('tr');for(const value of [name,frame.observed[index],frame.confidence[index].toFixed(4),Boolean(current.errors[offset]&(1<<index)),Boolean(current.errors[offset+1]&(1<<index))]){const td=document.createElement('td');td.textContent=String(value);tr.append(td)}element('tracking').append(tr)});draw(real,0);draw(sim,1)} | |
| async function showFrame(){const request=++ticket;try{const index=Number(element('timeline').value),decoded=await frameData(index);if(request!==ticket)return;current=decoded;const frame=data.frames[index];element('frame').textContent=`Frame ${frame.frame_index} · ${frame.timestamp.toFixed(4)} s · ${index+1}/${data.frames.length}`;query()}catch(error){element('error').textContent=error.message}} | |
| element('timeline').addEventListener('input',showFrame); | |
| for(const id of ['x','y',...controls])element(id).addEventListener('input',query); | |
| for(const canvas of [real,sim])canvas.addEventListener('click',event=>{const box=canvas.getBoundingClientRect();element('x').value=Math.min(data.width-1,Math.max(0,Math.floor((event.clientX-box.left)*data.width/box.width)));element('y').value=Math.min(data.height-1,Math.max(0,Math.floor((event.clientY-box.top)*data.height/box.height)));query()}); | |
| if(data.metrics){element('metrics-panel').hidden=false;for(const key of ['selected_block','target_actor','selection_reason','baseline_loss','best_loss','objective_improvement','best_source','physical_cause_uniquely_identified'])if(Object.hasOwn(data.metrics,key))row(element('metrics'),key,data.metrics[key]);element('sensitivity').textContent=data.sensitivity?JSON.stringify(data.sensitivity,null,2):'No saved sensitivity data supplied.'} | |
| showFrame(); | |
| </script> | |
| </body> | |
| </html> | |
| """ | |
| def image_url(image, extension=".png"): | |
| options = [cv2.IMWRITE_JPEG_QUALITY, 75] if extension == ".jpg" else [] | |
| ok, encoded = cv2.imencode( | |
| extension, cv2.cvtColor(image, cv2.COLOR_RGB2BGR), options | |
| ) | |
| if not ok: | |
| raise OSError("Cannot encode an embedded frame") | |
| mime = "jpeg" if extension == ".jpg" else "png" | |
| return f"data:image/{mime};base64,{base64.b64encode(encoded).decode()}" | |
| def read_trace(path): | |
| with np.load(path, allow_pickle=False) as saved: | |
| arrays = {name: saved[name] for name in saved.files} | |
| target, simulated = arrays["target_actor_ids"], arrays["sim_actor_ids"] | |
| names = arrays["names"].tolist() | |
| if target.ndim != 3 or simulated.shape != target.shape or min(target.shape) < 1: | |
| raise ValueError( | |
| "Actor ID images must have matching nonempty (frames, height, width) shapes" | |
| ) | |
| frames = target.shape[0] | |
| if not 1 <= len(names) <= 3 or len(set(names)) != len(names): | |
| raise ValueError("Trace requires one to three unique actor names") | |
| for image in (target, simulated): | |
| if ( | |
| image.dtype.kind not in "iu" | |
| or np.any(image < 0) | |
| or np.any(image > len(names)) | |
| ): | |
| raise ValueError( | |
| "Actor IDs must be integers between zero and the actor count" | |
| ) | |
| for key in ("confidence", "observed"): | |
| if arrays[key].shape != (len(names), frames): | |
| raise ValueError(f"{key} must have shape (actors, frames)") | |
| if not np.isfinite(arrays["confidence"]).all() or np.any(arrays["confidence"] < 0): | |
| raise ValueError("Confidence must be finite and nonnegative") | |
| if ( | |
| arrays["timestamps"].shape != (frames,) | |
| or not np.isfinite(arrays["timestamps"]).all() | |
| ): | |
| raise ValueError("Timestamps must contain one finite value per frame") | |
| indices = arrays["frame_indices"] | |
| if ( | |
| indices.shape != (frames,) | |
| or indices.dtype.kind not in "iu" | |
| or np.any(indices < 0) | |
| or len(np.unique(indices)) != frames | |
| ): | |
| raise ValueError("Source frame indices must be unique nonnegative integers") | |
| geometry = arrays.get("geom_ids") | |
| if geometry is not None and ( | |
| geometry.shape != target.shape | |
| or geometry.dtype.kind not in "iu" | |
| or np.any(geometry < -1) | |
| or np.any(geometry >= 2**24 - 1) | |
| ): | |
| raise ValueError("Geometry IDs must fit unsigned RGB encoding after adding one") | |
| for key in ("false_positive", "false_negative"): | |
| if key in arrays and arrays[key].shape != (len(names), *target.shape): | |
| raise ValueError(f"{key} must have shape (actors, frames, height, width)") | |
| if ( | |
| "trusted_disagreement" in arrays | |
| and arrays["trusted_disagreement"].shape != target.shape | |
| ): | |
| raise ValueError("Trusted disagreement must match the actor ID image shape") | |
| return arrays | |
| def source_frame_map(path): | |
| if path.name != "mujoco_replay.mp4": | |
| return None | |
| trajectory = path.parent / "sim_trajectory.npz" | |
| if not trajectory.is_file(): | |
| return None | |
| with np.load(trajectory, allow_pickle=False) as saved: | |
| indices = saved["frame_indices"] | |
| if len(np.unique(indices)) != len(indices): | |
| raise ValueError("Replay trajectory has duplicate source frame indices") | |
| return {int(source): index for index, source in enumerate(indices)} | |
| def create_viewer(args): | |
| arrays = read_trace(args.trace) | |
| target, simulated = arrays["target_actor_ids"], arrays["sim_actor_ids"] | |
| count, height, width = target.shape | |
| names = arrays["names"].tolist() | |
| mapping = json.loads(str(arrays["geom_mapping_json"].item())) | |
| if not isinstance(mapping, dict): | |
| raise TypeError("Geometry mapping must be a JSON object") | |
| data = { | |
| "width": width, | |
| "height": height, | |
| "names": names, | |
| "geom_mapping": mapping, | |
| "video": args.video is not None, | |
| "frames": [], | |
| } | |
| if args.metrics: | |
| data["metrics"] = json.loads(args.metrics.read_text()) | |
| if not isinstance(data["metrics"], dict): | |
| raise ValueError("Metrics must be a JSON object") | |
| sensitivity = args.metrics.parent / "sensitivity.json" | |
| if sensitivity.is_file(): | |
| data["sensitivity"] = json.loads(sensitivity.read_text()) | |
| frame_map = source_frame_map(args.video) if args.video else None | |
| capture = cv2.VideoCapture(str(args.video)) if args.video else None | |
| try: | |
| if capture is not None and not capture.isOpened(): | |
| raise OSError("Cannot open source video") | |
| for index in range(count): | |
| actors = np.zeros((height, width, 3), dtype=np.uint8) | |
| actors[..., 0], actors[..., 1] = target[index], simulated[index] | |
| errors = np.zeros_like(actors) | |
| for actor in range(len(names)): | |
| real_mask, sim_mask = ( | |
| target[index] == actor + 1, | |
| simulated[index] == actor + 1, | |
| ) | |
| fp = ( | |
| arrays["false_positive"][actor, index] | |
| if "false_positive" in arrays | |
| else sim_mask & ~real_mask | |
| ) | |
| fn = ( | |
| arrays["false_negative"][actor, index] | |
| if "false_negative" in arrays | |
| else real_mask & ~sim_mask | |
| ) | |
| errors[..., 0] |= fp.astype(np.uint8) << actor | |
| errors[..., 1] |= fn.astype(np.uint8) << actor | |
| if "trusted_disagreement" not in arrays: | |
| weight = ( | |
| arrays["observed"][actor, index] | |
| * arrays["confidence"][actor, index] | |
| ) | |
| errors[..., 2] |= ((fp | fn) & (weight > 0)).astype(np.uint8) | |
| if "trusted_disagreement" in arrays: | |
| errors[..., 2] = arrays["trusted_disagreement"][index] | |
| source_index = int(arrays["frame_indices"][index]) | |
| frame = { | |
| "frame_index": source_index, | |
| "timestamp": float(arrays["timestamps"][index]), | |
| "confidence": arrays["confidence"][:, index].astype(float).tolist(), | |
| "observed": arrays["observed"][:, index].astype(bool).tolist(), | |
| "actors": image_url(actors), | |
| "errors": image_url(errors), | |
| } | |
| if "geom_ids" in arrays: | |
| encoded = arrays["geom_ids"][index].astype(np.int64) + 1 | |
| geometry = np.stack( | |
| [ | |
| ((encoded >> shift) & 255).astype(np.uint8) | |
| for shift in (0, 8, 16) | |
| ], | |
| axis=-1, | |
| ) | |
| frame["geometry"] = image_url(geometry) | |
| if capture is not None: | |
| if frame_map is not None and source_index not in frame_map: | |
| raise ValueError( | |
| "Trace source frame is absent from the replay trajectory" | |
| ) | |
| video_index = ( | |
| source_index if frame_map is None else frame_map[source_index] | |
| ) | |
| capture.set(cv2.CAP_PROP_POS_FRAMES, video_index) | |
| ok, rgb = capture.read() | |
| if not ok: | |
| raise OSError(f"Cannot read source video frame {source_index}") | |
| if args.video.name == "mujoco_replay.mp4": | |
| rgb = rgb[:, : rgb.shape[1] // 3] | |
| rgb = cv2.resize(rgb, (width, height), interpolation=cv2.INTER_AREA) | |
| frame["rgb"] = image_url(cv2.cvtColor(rgb, cv2.COLOR_BGR2RGB), ".jpg") | |
| data["frames"].append(frame) | |
| finally: | |
| if capture is not None: | |
| capture.release() | |
| payload = json.dumps(data, separators=(",", ":"), allow_nan=False).replace( | |
| "<", "\\u003c" | |
| ) | |
| output = args.output or args.trace.with_suffix(".html") | |
| if output.resolve() in { | |
| path.resolve() | |
| for path in (args.trace, args.video, args.metrics) | |
| if path is not None | |
| }: | |
| raise ValueError("HTML output must differ from the input files") | |
| output.parent.mkdir(parents=True, exist_ok=True) | |
| output.write_text(PAGE.replace("__DATA__", payload)) | |
| print( | |
| json.dumps( | |
| { | |
| "output": str(output), | |
| "frames": count, | |
| "mask_size": [width, height], | |
| "bytes": output.stat().st_size, | |
| } | |
| ) | |
| ) | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("trace", type=Path) | |
| parser.add_argument("--video", type=Path) | |
| parser.add_argument("--output", type=Path) | |
| parser.add_argument("--metrics", type=Path) | |
| args = parser.parse_args() | |
| try: | |
| create_viewer(args) | |
| except (OSError, ValueError, KeyError, TypeError) as error: | |
| parser.error(str(error)) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 19.5 kB
- Xet hash:
- 9123adc39d8c198fa373ed9dc172a0325ede25e8edd1f6277d5df22bdda2ee59
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.