//! The DinoVision XR application. //! //! Renders the DINO feature view to both eyes at the headset's refresh //! rate, while inference runs on a worker thread at whatever rate it can //! manage. The renderer never waits for the encoder: it draws the most //! recent completed grid, so a long inference does not become a synchronous //! render-frame wait. //! //! ```text //! cargo apk run --manifest-path android-xr/Cargo.toml --release --no-logcat //! adb logcat -v time | grep -E "dinovision|RustStdoutStderr" //! ``` //! //! Frames come from the passthrough camera when it opens, and from //! `dinovision::source::TestPattern` when it does not — a missing runtime //! permission or an older Horizon OS should degrade to something visible //! rather than a black screen. The camera needs a grant that the manifest //! alone does not provide: //! //! ```text //! adb shell pm grant rust.dinovision_xr horizonos.permission.HEADSET_CAMERA //! ``` #![cfg(target_os = "android")] use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use blade_graphics as gpu; use dinovision::dinov3::Config; use dinovision::inference::{self, Weights, Worker}; use dinovision::render::GridView; use dinovision::source::{FrameSource, TestPattern}; use log::{info, warn}; use openxr as xr; const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO; const MAX_EYES: usize = 2; /// Where the app looks for weights pushed by `adb push`. /// /// Bundling 43 MB of f16 weights as an asset is the eventual answer; for /// bring-up, a path outside the APK avoids a rebuild per weight change. const WEIGHTS_PATH: &str = "/data/local/tmp/dinovision/model.safetensors"; /// Trained RGB decoder. When present the app shows a real reconstruction /// instead of the PCA colouring — the roundtrip the project is for. const DECODER_PATH: &str = "/data/local/tmp/dinovision/decoder.bin"; /// Encoder depth used with the decoder. /// /// The published decoder is trained against this depth and is not valid at /// any other. Depth is an artifact property, not a runtime quality switch. const RECONSTRUCTION_LAYERS: usize = 3; const CAPTURE_SIZE: (i32, i32) = (1280, 960); /// Show the camera frame directly instead of the DINO roundtrip, for /// checking geometry independently of the model: /// /// ```text /// adb shell touch /data/local/tmp/dinovision/raw_camera /// ``` const RAW_CAMERA_PATH: &str = "/data/local/tmp/dinovision/raw_camera"; /// Dump camera frames to disk for retraining, one raw RGB file each: /// /// ```text /// adb shell "echo room-a-01 > /data/local/tmp/dinovision/capture" /// # …wear the headset and look around… /// adb pull /sdcard/Android/data/rust.dinovision_xr/files/captures/room-a-01 /// ``` /// /// The decoder has only ever been trained on clean, well-lit colour /// photographs, while these cameras produce noisy, wide-angle, nearly /// monochrome frames. That mismatch is the most likely single cause of poor /// reconstruction on the headset, and no amount of architecture work /// addresses it — the model has to see the distribution it will be used on. /// The flag is read from `/data/local/tmp` — apps may read there — but the /// frames go to the app's own external directory. SELinux forbids an /// `untrusted_app` writing to `shell_data_file` no matter how permissive /// the mode bits look: /// /// ```text /// avc: denied { write } … tcontext=u:object_r:shell_data_file /// ``` const CAPTURE_FLAG_PATH: &str = "/data/local/tmp/dinovision/capture"; const CAPTURE_DIR_BASE: &str = "/sdcard/Android/data/rust.dinovision_xr/files/captures"; fn capture_directory() -> Option { if !std::path::Path::new(CAPTURE_FLAG_PATH).exists() { return None; } let requested = std::fs::read_to_string(CAPTURE_FLAG_PATH).unwrap_or_default(); let requested = requested.trim(); let session = if requested.is_empty() { format!( "session-{}", SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs() ) } else if requested .chars() .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_')) { requested.to_string() } else { warn!("invalid capture session {requested:?}; use only ASCII letters, digits, '-' and '_'"); return None; }; let directory = format!("{CAPTURE_DIR_BASE}/{session}"); if std::fs::read_dir(&directory) .ok() .and_then(|mut entries| entries.next()) .is_some() { warn!("capture session directory is not empty; refusing to overwrite {directory}"); return None; } match std::fs::create_dir_all(&directory) { Ok(()) => { info!("capturing session {session:?} to {directory}"); Some(directory) } Err(error) => { warn!("could not create capture session directory {directory}: {error}"); None } } } /// Frames to keep, and how many to skip between them. /// /// Consecutive frames of a barely-moving head are near-duplicates and teach /// almost nothing, so sampling sparsely buys far more variety per byte. At /// 224² RGB a frame is 150 KB, so this caps the dump around 225 MB. const CAPTURE_LIMIT: usize = 1500; const CAPTURE_STRIDE: u64 = 8; /// Run one camera and one roundtrip, shown to both eyes: /// /// ```text /// adb shell touch /data/local/tmp/dinovision/mono /// ``` /// /// Halves the work, since encoder and decoder both run once instead of /// twice, and roughly doubles the update rate. The cost is that the right /// eye sees the world from the left camera's position, so there is no /// stereo depth — worth it when the question is latency rather than /// whether the roundtrip fuses. const MONO_PATH: &str = "/data/local/tmp/dinovision/mono"; /// How far each new result moves the displayed image. 1.0 shows the raw /// result and flickers; lower is steadier but lags. At ~12 Hz inference, /// 0.35 settles within a few frames while killing most of the jitter. const SMOOTHING: f32 = 0.35; /// Optional override for the camera image's vertical half-angle, in /// degrees. /// /// Unset — the normal case — the image simply spans each eye's field of /// view, which is what passthrough itself does with these cameras. Only /// worth setting if the camera turns out to see meaningfully more or less /// than the display shows: /// /// ```text /// adb shell "echo 40 > /data/local/tmp/dinovision/camera_half_fov_deg" /// ``` /// /// The full frame is resampled rather than cropped, so the horizontal /// extent follows from the sensor aspect: for a rectilinear lens the /// tangents scale with the sensor dimensions, giving /// `tan(hfov/2) = (w/h) · tan(vfov/2)`. const HALF_FOV_PATH: &str = "/data/local/tmp/dinovision/camera_half_fov_deg"; fn read_half_fov() -> Option { let deg = std::fs::read_to_string(HALF_FOV_PATH) .ok() .and_then(|s| s.trim().parse::().ok())?; info!("camera half-FOV override: {deg} deg"); Some(deg.to_radians()) } /// Minimum gap between inference submissions, in milliseconds, read from a /// pushed file so it can be changed without rebuilding: /// /// ```text /// adb shell "echo 300 > /data/local/tmp/dinovision/inference_interval_ms" /// ``` /// /// Inference and rendering share one Vulkan queue, so a long compute /// submission delays the frame queued behind it. Left unthrottled the /// worker resubmits the instant it finishes, so the GPU is never free and /// the render loop is dragged down to the encoder's cadence. Raising this /// trades feature-update rate for frame rate. A very large value disables /// inference entirely, which is how the renderer's standalone cost is /// measured. const INTERVAL_PATH: &str = "/data/local/tmp/dinovision/inference_interval_ms"; /// How many submissions the encoder is split across, same override /// mechanism as the interval. const CHUNKS_PATH: &str = "/data/local/tmp/dinovision/submission_chunks"; /// Interactive default. The paper harness overrides this and sweeps every /// declared chunk cell in a precommitted non-monotonic order. const DEFAULT_CHUNKS: usize = 12; fn read_chunks() -> usize { let n = std::fs::read_to_string(CHUNKS_PATH) .ok() .and_then(|s| s.trim().parse::().ok()) .unwrap_or(DEFAULT_CHUNKS); info!("submission chunks: {n}"); n } /// Conservative interactive default. Audited co-tenancy runs override it /// explicitly and record the selected value in every JSON window. const DEFAULT_INTERVAL_MS: u64 = 500; fn read_interval() -> Duration { let ms = std::fs::read_to_string(INTERVAL_PATH) .ok() .and_then(|s| s.trim().parse::().ok()) .unwrap_or(DEFAULT_INTERVAL_MS); info!("inference interval: {ms} ms"); Duration::from_millis(ms) } /// One eye's chain: its own camera, its own encode, its own image. /// /// Kept entirely separate per eye because the experiment is whether the /// roundtrip survives stereo fusion. Sharing anything — one camera shown /// twice, or one encode reused — would answer a different and easier /// question. struct EyePipeline { source: Box, worker: Worker, view: GridView, /// Exponentially smoothed image, damping per-frame feature jitter. smoothed: Vec, last_shown: u64, last_submit: Instant, eye: dinovision::camera::Eye, /// Head orientation when the in-flight frame was captured, and when /// the frame currently on screen was. The difference between the /// latter and the live pose is what keeps the image world-locked. pending_orientation: Option<[f32; 4]>, shown_orientation: Option<[f32; 4]>, } struct App { surface: gpu::XrSurface, /// One per eye when both cameras open, otherwise a single shared chain /// drawn to both. eyes: Vec, config: Config, frames: u64, last_report: Instant, /// Completed results per eye pipeline in the current reporting window. /// Keeping these separate avoids calling two asynchronous eye updates a /// single "inference Hz" figure. inference_counts: Vec, /// Wall latency for every completed worker result in the current window. inference_latencies_ms: Vec>, /// Minimum gap between inference submissions. See [`INTERVAL_PATH`]. interval: Duration, submission_chunks: usize, camera_active: bool, /// Bypass inference and show the camera frame. See [`RAW_CAMERA_PATH`]. raw_camera: bool, raw_scratch: Vec, /// Tangent of the image half-angle, per axis. See [`HALF_FOV_PATH`]. camera_half_tan: Option<[f32; 2]>, logged_fov: bool, /// Frames written so far, and the counter that strides between them. captured: usize, capture_tick: u64, capturing: bool, capture_dir: Option, /// Head orientation from the previous frame. Used when submitting, so /// it is one frame stale — around 10 ms against the 100 ms the /// inference itself takes, which is the lag that actually matters. last_head: Option<[f32; 4]>, } impl App { fn new(context: &Arc, config: Config) -> Self { let surface = context .create_xr_surface() .expect("unable to create XR surface"); // A trained decoder turns this into the actual roundtrip, at a // shallower and much cheaper encoder depth. let decoder_path = std::path::PathBuf::from(DECODER_PATH); let (config, display, view_grid) = if decoder_path.exists() { info!("reconstructing with {}", decoder_path.display()); let config = config.with_layers(RECONSTRUCTION_LAYERS); let grid = config.image_size; ( config, inference::Display::Reconstruction(decoder_path), grid, ) } else { let grid = config.grid(); (config, inference::Display::PcaColour, grid) }; // Raw camera frames are full resolution regardless of what the // model would have produced. let raw_camera = std::path::Path::new(RAW_CAMERA_PATH).exists(); if raw_camera { info!("raw camera mode — inference bypassed"); } let view_grid = if raw_camera { config.image_size } else { view_grid }; // Real weights if they have been pushed, synthetic otherwise, so // the render path can be brought up before the checkpoint is on // the device. let weights_path = std::path::PathBuf::from(WEIGHTS_PATH); let have_weights = weights_path.exists(); if have_weights { info!("using weights from {}", weights_path.display()); } else { warn!( "no weights at {WEIGHTS_PATH} — running synthetic; \ adb push the checkpoint there for real features" ); } // One chain per eye if both cameras open. Two encodes cost twice the // GPU, which is the price of the question being asked: whether a // reconstructed world still fuses into a single stereo percept. A // single camera shown to both eyes cannot answer it — that is // monocular, and would look flat and offset no matter how good the // reconstruction is. let chunks = read_chunks(); let mono = std::path::Path::new(MONO_PATH).exists(); if mono { info!("monocular: one camera and one roundtrip for both eyes"); } let wanted: &[dinovision::camera::Eye] = if mono { &[dinovision::camera::Eye::Left] } else { &[ dinovision::camera::Eye::Left, dinovision::camera::Eye::Right, ] }; let mut sources: Vec<(dinovision::camera::Eye, Box)> = Vec::new(); for &eye in wanted { match dinovision::camera::PassthroughCamera::new(config.image_size, eye, CAPTURE_SIZE) { Ok(camera) => { info!("{eye:?} camera open"); sources.push((eye, Box::new(camera))); } Err(e) => warn!("{eye:?} camera unavailable: {e}"), } } if sources.is_empty() { warn!( "no camera opened; falling back to the test pattern. Grant it with: \ adb shell pm grant rust.dinovision_xr horizonos.permission.HEADSET_CAMERA" ); sources.push(( dinovision::camera::Eye::Left, Box::new(TestPattern::new(config.image_size)), )); } let eyes: Vec = sources .into_iter() .map(|(eye, source)| EyePipeline { source, // No plan cache: two sessions would race on the same file. worker: inference::spawn( Arc::clone(context), config.clone(), if have_weights { Weights::SafeTensors(weights_path.clone()) } else { Weights::Synthetic }, None, chunks, display.clone(), ), view: GridView::new(context, surface.format(), view_grid), smoothed: Vec::new(), last_shown: 0, last_submit: Instant::now(), eye, pending_orientation: None, shown_orientation: None, }) .collect(); info!( "{} eye pipeline(s) — {}", eyes.len(), if eyes.len() > 1 { "stereo" } else { "monocular" } ); let eye_count = eyes.len(); let interval = read_interval(); let capture_dir = capture_directory(); Self { surface, eyes, config, frames: 0, last_report: Instant::now(), inference_counts: vec![0; eye_count], inference_latencies_ms: vec![Vec::new(); eye_count], interval, submission_chunks: chunks, camera_active: true, raw_camera, raw_scratch: Vec::new(), last_head: None, captured: 0, capture_tick: 0, capturing: capture_dir.is_some(), capture_dir, camera_half_tan: read_half_fov().map(|v| { let t = v.tan(); let aspect = CAPTURE_SIZE.0 as f32 / CAPTURE_SIZE.1 as f32; [t * aspect, t] }), logged_fov: false, } } /// Acquire or release the camera as focus comes and goes. /// /// Android hands the camera to the foreground app and takes it back /// otherwise; holding it while backgrounded is what produced /// `ACameraDevice` error 3. Releasing on the way out also means the /// headset camera stops streaming to an app the wearer has navigated /// away from, which is the behaviour anyone would expect of it. fn set_camera_active(&mut self, active: bool) { if active == self.camera_active { return; } self.camera_active = active; let size = self.config.image_size; for pipeline in &mut self.eyes { if active { match dinovision::camera::PassthroughCamera::new(size, pipeline.eye, CAPTURE_SIZE) { Ok(camera) => { info!("{:?} camera reacquired on focus", pipeline.eye); pipeline.source = Box::new(camera); } Err(e) => warn!("could not reacquire the {:?} camera: {e}", pipeline.eye), } } else { // Dropping the camera closes the device and stops the stream. info!("releasing the {:?} camera on focus loss", pipeline.eye); pipeline.source = Box::new(TestPattern::new(size)); } } } fn render(&mut self, context: &gpu::Context, encoder: &mut gpu::CommandEncoder) { let interval = self.interval; let raw = self.raw_camera; let config = self.config.clone(); let head = self.last_head; // Snapshot frames for retraining, from the first eye only — the two // cameras see nearly the same scene, so the second would mostly // duplicate the first. if self.capturing && self.captured < CAPTURE_LIMIT { self.capture_tick += 1; if self.capture_tick.is_multiple_of(CAPTURE_STRIDE) && let Some(pipeline) = self.eyes.first_mut() && let Some(rgb) = pipeline.source.next_frame() { let path = format!( "{}/{:05}.rgb", self.capture_dir.as_deref().expect("capture directory"), self.captured ); match std::fs::write(&path, rgb) { Ok(()) => { self.captured += 1; if self.captured.is_multiple_of(100) { info!("captured {} / {CAPTURE_LIMIT} frames", self.captured); } } Err(e) => { warn!("capture failed ({e}); stopping"); self.capturing = false; } } } } for (pipeline_index, pipeline) in self.eyes.iter_mut().enumerate() { if raw { // Straight to the display, no model in the way, for judging // geometry and render submission rate separately from the // network. Keep this before worker submission: "raw" must // not leave hidden inference contending for the same queue. if let Some(rgb) = pipeline.source.next_frame() { self.raw_scratch.clear(); self.raw_scratch .extend(rgb.iter().map(|&b| b as f32 / 255.0)); pipeline.view.upload(context, &self.raw_scratch); } continue; } // Keep each worker fed. They report not-ready while busy, and // frames offered meanwhile are dropped rather than queued — a // queued frame would be stale by the time it ran. if pipeline.worker.is_ready() && pipeline.last_submit.elapsed() >= interval && let Some(rgb) = pipeline.source.next_frame() { let patches = dinovision::preprocess::patches_from_rgb8(rgb, &config); if pipeline.worker.submit(patches) { pipeline.last_submit = Instant::now(); // Remember where the head was pointing, so the result // can be put back in the world where it was seen. pipeline.pending_orientation = head; } } let generation = pipeline.worker.generation(); if generation != pipeline.last_shown && let Some(grid) = pipeline.worker.latest() { // Blend towards the new frame rather than snapping to it. // DINO features are sensitive enough that sensor noise and // auto-exposure make individual patches jump between // consecutive frames, which reads as constant flickering // even with the head still. let new = grid.patch_rgb(); if pipeline.smoothed.len() != new.len() { pipeline.smoothed = new.to_vec(); } else { for (s, &n) in pipeline.smoothed.iter_mut().zip(new) { *s += (n - *s) * SMOOTHING; } } pipeline.view.upload(context, &pipeline.smoothed); pipeline.last_shown = generation; pipeline.shown_orientation = pipeline.pending_orientation; self.inference_counts[pipeline_index] += 1; self.inference_latencies_ms[pipeline_index].push(grid.latency_ms); } } let Some(frame) = self.surface.acquire_frame(context) else { return; }; encoder.start(); encoder.init_texture(frame.texture()); let eyes = frame.xr_view_count().min(MAX_EYES as u32); for eye in 0..eyes { // Each eye's frustum is asymmetric and differs from the other's, // so the overlay has to be placed per eye. Skipping this is what // made the two views refuse to fuse. let xr_view = frame.xr_view(eye); let fov = [ xr_view.fov.angle_left, xr_view.fov.angle_right, xr_view.fov.angle_up, xr_view.fov.angle_down, ]; if !self.logged_fov { info!( "eye {eye} frustum: L {:.1} R {:.1} U {:.1} D {:.1} deg \ (axis offset x {:.2} y {:.2})", fov[0].to_degrees(), fov[1].to_degrees(), fov[2].to_degrees(), fov[3].to_degrees(), -(fov[1].tan() + fov[0].tan()) / (fov[1].tan() - fov[0].tan()), -(fov[2].tan() + fov[3].tan()) / (fov[2].tan() - fov[3].tan()), ); if eye + 1 == eyes { self.logged_fov = true; } } let now = xr_view.pose.orientation; if eye == 0 { self.last_head = Some(now); } // With two cameras each eye shows its own; with one, both show // the same and the view is monocular. let index = (eye as usize).min(self.eyes.len() - 1); // Put the image back where the head was pointing when the // camera saw it, so it holds still in the world while the view // sweeps across it. Without this the picture is glued to the // screen and drags a tenth of a second behind every turn. let centre = self.eyes[index] .shown_orientation .and_then(|then| dinovision::render::reprojection_offset(then, now)) .unwrap_or([0.0, 0.0]); self.eyes[index].view.set_transform( context, eye as usize, // Same size as filling the buffer, but recentred on each // eye's own axis. Still a 2D scale and offset; nothing is // projected. // // The recentring is not optional, and this device says so // with numbers. Its frusta are asymmetric and mirrored — // left eye L -49.0 R +45.0, right eye L -45.0 R +49.0 — so // an image filling the buffer has its centre at -2.0 deg in // the left eye and +2.0 deg in the right. That is 4 deg of // *divergence*, pulling the eyes apart, against a fusion // limit of roughly 1 deg. It reads exactly as "the object // is further left in my left eye" and it cannot converge. match self.camera_half_tan { Some(half_tan) => dinovision::render::EyeTransform::for_eye(fov, half_tan), None => dinovision::render::EyeTransform::filling_at(fov, centre), }, ); let mut pass = encoder.render( "eye", gpu::RenderTargetSet { colors: &[gpu::RenderTarget { view: frame.xr_texture_view(eye), // The view covers every pixel, so clearing would only // be wasted bandwidth on a tiler. init_op: gpu::InitOp::Clear(gpu::TextureColor::OpaqueBlack), finish_op: gpu::FinishOp::Store, }], depth_stencil: None, }, ); self.eyes[index].view.draw(&mut pass, eye as usize); } encoder.present(frame); let _sync_point = context.submit(encoder); self.frames += 1; if self.last_report.elapsed() >= Duration::from_secs(5) { let secs = self.last_report.elapsed().as_secs_f64(); let render_hz = self.frames as f64 / secs; let per_eye_hz: Vec = self .inference_counts .iter() .map(|&count| count as f64 / secs) .collect(); // This is the lower eye update rate, not a synchronization claim: // the two cameras and workers are currently independent. let min_eye_hz = per_eye_hz.iter().copied().fold(f64::INFINITY, f64::min); let latencies: Vec> = self .eyes .iter() .map(|eye| eye.worker.latest().map(|grid| grid.latency_ms)) .collect(); info!( "render {:.1} Hz | per-eye updates {:?} Hz | min-eye {:.1} Hz | worker latency {:?} ms", render_hz, per_eye_hz, min_eye_hz, latencies, ); let counts_json = self .inference_counts .iter() .map(u64::to_string) .collect::>() .join(","); let rates_json = per_eye_hz .iter() .map(|value| format!("{value:.6}")) .collect::>() .join(","); let latency_json = latencies .iter() .map(|value| value.map_or_else(|| "null".into(), |v| format!("{v:.6}"))) .collect::>() .join(","); let latency_samples_json = self .inference_latencies_ms .iter() .map(|samples| { format!( "[{}]", samples .iter() .map(|value| format!("{value:.6}")) .collect::>() .join(",") ) }) .collect::>() .join(","); info!( "DINOVISION_APP_JSON {{\"schema_version\":1,\"kind\":\"dinovision_app_window\",\ \"window_seconds\":{secs:.6},\"render_frames\":{},\"render_hz\":{render_hz:.6},\ \"per_eye_update_counts\":[{counts_json}],\"per_eye_update_hz\":[{rates_json}],\ \"min_eye_update_hz\":{min_eye_hz:.6},\"worker_latest_latency_ms\":[{latency_json}],\ \"worker_latency_samples_ms\":[{latency_samples_json}],\"submission_chunks\":{},\ \"inference_interval_ms\":{},\"raw_camera\":{}}}", self.frames, self.submission_chunks, self.interval.as_millis(), self.raw_camera, ); self.frames = 0; self.inference_counts.fill(0); self.inference_latencies_ms.iter_mut().for_each(Vec::clear); self.last_report = Instant::now(); } } fn destroy(mut self, context: &gpu::Context) { // Drop the workers first: each holds an `Arc` and must // finish any in-flight submission before the surface goes away. for pipeline in self.eyes.drain(..) { drop(pipeline.worker); pipeline.view.destroy(context); } context.destroy_xr_surface(&mut self.surface); } } fn spawn_event_pump() -> Arc { let should_exit = Arc::new(AtomicBool::new(false)); let flag = Arc::clone(&should_exit); std::thread::spawn(move || { while let Some(event) = ndk_glue::poll_events() { if matches!(event, ndk_glue::Event::Destroy) { flag.store(true, Ordering::Relaxed); break; } } }); should_exit } #[ndk_glue::main] pub fn main() { android_logger::init_once( android_logger::Config::default() .with_max_level(log::LevelFilter::Info) .with_tag("dinovision"), ); std::panic::set_hook(Box::new(|info| log::error!("panic: {info}"))); info!("=== dinovision starting ==="); let entry = unsafe { xr::Entry::load().expect("no OpenXR loader") }; entry.initialize_android_loader().unwrap(); let available = entry.enumerate_extensions().unwrap(); assert!( available.khr_vulkan_enable2, "runtime lacks XR_KHR_vulkan_enable2" ); let mut extensions = xr::ExtensionSet::default(); extensions.khr_vulkan_enable2 = true; extensions.khr_android_create_instance = true; let xr_instance = entry .create_instance( &xr::ApplicationInfo { application_name: "DinoVision", application_version: 0, engine_name: "Blade", engine_version: 0, api_version: xr::Version::new(1, 0, 0), }, &extensions, &[], ) .unwrap(); let system = xr_instance .system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) .unwrap(); // One context, shared by the renderer and by meganeura. This is the // whole reason the crate pins meganeura's blade revision. let context = Arc::new(unsafe { gpu::Context::init(gpu::ContextDesc { xr: Some(gpu::XrDesc { instance: xr_instance.clone(), system_id: system, }), ..Default::default() }) .expect("failed to initialize GPU context") }); let info = context.device_information(); info!("GPU: {} ({})", info.device_name, info.driver_name); // The published decoder and correctness manifest are fixed at 224. let config = Config::vits16().at_resolution(224); let mut encoder = context.create_command_encoder(gpu::CommandEncoderDesc { name: "dinovision", buffer_count: 2, manual_barriers: false, }); let mut app: Option = None; let should_exit = spawn_event_pump(); let mut events = xr::EventDataBuffer::new(); 'main: loop { if should_exit.load(Ordering::Relaxed) { break 'main; } while let Some(event) = xr_instance.poll_event(&mut events).unwrap() { use xr::Event::*; match event { SessionStateChanged(e) => { info!("XR session state: {:?}", e.state()); match e.state() { xr::SessionState::READY => { if app.is_none() { app = Some(App::new(&context, config.clone())); } context.xr_session().unwrap().begin(VIEW_TYPE).unwrap(); } xr::SessionState::STOPPING => { context.xr_session().unwrap().end().unwrap(); if let Some(app) = app.take() { app.destroy(&context); } } // Release the camera the moment we stop being the // focused app. Android revokes it from background // apps anyway — that is what error code 3 was — and // holding it is both rude to whatever wants it next // and a privacy question, since a headset camera // should not keep streaming to an app the wearer has // navigated away from. xr::SessionState::VISIBLE | xr::SessionState::SYNCHRONIZED => { if let Some(app) = app.as_mut() { app.set_camera_active(false); } } xr::SessionState::FOCUSED => { if let Some(app) = app.as_mut() { app.set_camera_active(true); } } xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => break 'main, _ => {} } } InstanceLossPending(_) => break 'main, _ => {} } } match &mut app { Some(app) => app.render(&context, &mut encoder), // Not in session yet; idle rather than spin. None => std::thread::sleep(Duration::from_millis(50)), } } if let Some(app) = app.take() { app.destroy(&context); } context.destroy_command_encoder(&mut encoder); info!("=== dinovision stopped ==="); }