//! Live DINO feature view in a desktop window. //! //! The same pipeline the headset runs — capture, encode, PCA colour, //! display — pointed at a monitor instead of a passthrough camera. Two //! reasons it exists: //! //! * It is the demo you can actually show someone. Point it at a video, a //! photo, a game, and watch objects resolve into stable colour regions. //! * It de-risks the Quest camera. The capture loop, the downscale, and the //! "new frame arrives asynchronously while the window redraws" structure //! are all the same; the passthrough camera becomes another //! `FrameSource` and nothing around it changes. //! //! ```text //! cargo run --release --features capture --example viewer -- [model.safetensors] [monitor] //! ``` //! //! Without a checkpoint it runs on synthetic weights, which still shows the //! scene's structure but colours it arbitrarily. use std::sync::Arc; use std::time::{Duration, Instant}; use blade_graphics as gpu; use dinovision::dinov3::Config; use dinovision::inference::{self, Weights, Worker}; use dinovision::render::GridView; use dinovision::source::FrameSource; fn surface_config(size: winit::dpi::PhysicalSize) -> gpu::SurfaceConfig { gpu::SurfaceConfig { size: gpu::Extent { width: size.width.max(1), height: size.height.max(1), depth: 1, }, usage: gpu::TextureUsage::TARGET, display_sync: gpu::DisplaySync::Recent, ..Default::default() } } struct Viewer { context: Arc, surface: gpu::Surface, encoder: gpu::CommandEncoder, view: GridView, worker: Worker, source: Box, config: Config, last_shown: u64, frames: u64, encodes: u64, last_report: Instant, window: winit::window::Window, } impl Viewer { fn redraw(&mut self) { // Feed the encoder whenever it is free. Unlike the headset there is // no compositor to starve here, so it runs flat out. if self.worker.is_ready() && let Some(rgb) = self.source.next_frame() { let patches = dinovision::preprocess::patches_from_rgb8(rgb, &self.config); self.worker.submit(patches); } let generation = self.worker.generation(); if generation != self.last_shown && let Some(grid) = self.worker.latest() { self.view.upload(&self.context, grid.patch_rgb()); self.last_shown = generation; self.encodes += 1; } let frame = self.surface.acquire_frame(); self.encoder.start(); self.encoder.init_texture(frame.texture()); { let mut pass = self.encoder.render("view", gpu::RenderTargetSet { colors: &[gpu::RenderTarget { view: frame.texture_view(), init_op: gpu::InitOp::DontCare, finish_op: gpu::FinishOp::Store, }], depth_stencil: None, }); self.view.draw(&mut pass, 0); } self.encoder.present(frame); let _sp = self.context.submit(&mut self.encoder); self.frames += 1; if self.last_report.elapsed() >= Duration::from_secs(3) { let secs = self.last_report.elapsed().as_secs_f64(); let latency = self .worker .latest() .map(|g| g.latency_ms) .unwrap_or(f64::NAN); self.window.set_title(&format!( "dinovision — {:.0} fps display, {:.1} Hz inference ({:.0} ms)", self.frames as f64 / secs, self.encodes as f64 / secs, latency )); self.frames = 0; self.encodes = 0; self.last_report = Instant::now(); } } } #[derive(Default)] struct App { viewer: Option, weights: Option, decoder: Option, monitor: usize, } impl winit::application::ApplicationHandler for App { fn resumed(&mut self, event_loop: &winit::event_loop::ActiveEventLoop) { if self.viewer.is_some() { return; } let window = event_loop .create_window( winit::window::Window::default_attributes() .with_title("dinovision — starting…") .with_inner_size(winit::dpi::LogicalSize::new(720, 720)), ) .expect("failed to create window"); let context = Arc::new(unsafe { gpu::Context::init(gpu::ContextDesc { presentation: true, validation: false, ..Default::default() }) .expect("failed to initialize GPU context") }); let surface = context .create_surface_configured(&window, surface_config(window.inner_size())) .expect("failed to create surface"); let config = Config::vits16().at_resolution(224); // A reconstruction fills the whole image, so the display grid is the // image size; PCA colouring only has one value per patch. let display = match self.decoder.clone() { Some(path) => { log::info!("decoder: {}", path.display()); inference::Display::Reconstruction(path) } None => inference::Display::PcaColour, }; let view_grid = match &display { inference::Display::Reconstruction(_) => config.image_size, inference::Display::PcaColour => config.grid(), }; let view = GridView::new(&context, surface.info().format, view_grid); let weights = match self.weights.clone() { Some(p) => { log::info!("weights: {}", p.display()); Weights::SafeTensors(p) } None => { log::warn!("no checkpoint given — synthetic weights, colours are arbitrary"); Weights::Synthetic } }; // One submission: nothing else is competing for this GPU, so the // chunking that matters on a headset would only cost overhead. let worker = inference::spawn(Arc::clone(&context), config.clone(), weights, None, 1, display); let source: Box = match dinovision::source::ScreenCapture::new( config.image_size, self.monitor, ) { Ok(s) => Box::new(s), Err(e) => { log::warn!("screen capture unavailable ({e}); falling back to the test pattern"); Box::new(dinovision::source::TestPattern::new(config.image_size)) } }; let encoder = context.create_command_encoder(gpu::CommandEncoderDesc { name: "viewer", buffer_count: 2, manual_barriers: false, }); self.viewer = Some(Viewer { context, surface, encoder, view, worker, source, config, last_shown: 0, frames: 0, encodes: 0, last_report: Instant::now(), window, }); } fn about_to_wait(&mut self, _event_loop: &winit::event_loop::ActiveEventLoop) { if let Some(v) = &self.viewer { v.window.request_redraw(); } } fn window_event( &mut self, event_loop: &winit::event_loop::ActiveEventLoop, _id: winit::window::WindowId, event: winit::event::WindowEvent, ) { let Some(viewer) = self.viewer.as_mut() else { return; }; match event { winit::event::WindowEvent::CloseRequested => event_loop.exit(), winit::event::WindowEvent::KeyboardInput { event: winit::event::KeyEvent { physical_key: winit::keyboard::PhysicalKey::Code(code), state: winit::event::ElementState::Pressed, .. }, .. } => match code { winit::keyboard::KeyCode::Escape => event_loop.exit(), // Refit the colour basis on demand: point the capture at // something new and the old principal components will be a // poor fit for it. winit::keyboard::KeyCode::KeyR => { log::info!("refitting colour basis"); viewer.worker.request_refit(); } _ => {} }, winit::event::WindowEvent::Resized(size) => { viewer .context .reconfigure_surface(&mut viewer.surface, surface_config(size)); } winit::event::WindowEvent::RedrawRequested => viewer.redraw(), _ => {} } } } fn main() { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); let mut args = std::env::args().skip(1); let weights = args.next().map(std::path::PathBuf::from).filter(|p| { let ok = p.exists(); if !ok { log::warn!("{} does not exist; using synthetic weights", p.display()); } ok }); // A decoder file switches the view from PCA colour to a real RGB // reconstruction. let decoder = args.next().map(std::path::PathBuf::from).filter(|p| p.exists()); let monitor = args.next().and_then(|s| s.parse().ok()).unwrap_or(0); println!("R refits the colour basis · Esc quits"); let event_loop = winit::event_loop::EventLoop::new().expect("failed to create event loop"); event_loop.set_control_flow(winit::event_loop::ControlFlow::Poll); event_loop .run_app(&mut App { viewer: None, weights, decoder, monitor, }) .expect("event loop failed"); }