Enable production condensation with recently_freed tombstone guard
Browse files- condenser.rs: add recently_freed tombstone set (FREED_RECENCY_NS=5s).
unregister() stamps freed addresses; scan_and_compress() prunes stale
tombstones and skips any address within the recency window. Closes the
secondary address-reuse race: free(X) processed β malloc(X) re-registers
X β scan now skips X for 5s before treating it as a fresh allocation.
- membrane.rs: flip PIPELINE from test_mode:true β test_mode:false.
72h VPS stability check passed (no SIGBUS/SIGSEGV). Three-layer guard:
(1) 64K ring + deferred scan (#260) minimises unprocessed-free window;
(2) 60s burst gate blocks scan during startup coldload (original crash
window); (3) recently_freed tombstone closes the reuse race.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
- rust_core/src/condenser.rs +39 -2
- rust_core/src/membrane.rs +16 -6
|
@@ -13,6 +13,23 @@
|
|
| 13 |
//! the membrane's tracked allocations and demoting idle ones.
|
| 14 |
//! When the predictor fires a spike ("this region is about to be
|
| 15 |
//! accessed"), the condenser pre-promotes it.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
use std::collections::HashMap;
|
| 18 |
use std::fs;
|
|
@@ -24,6 +41,10 @@ use crate::membrane::{MembraneState, MembraneSummary};
|
|
| 24 |
|
| 25 |
const PAGE_SIZE: usize = 4096;
|
| 26 |
const COLD_DIR: &str = "/tmp/condensate_cold";
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
|
| 28 |
/// Tier state for a managed memory region
|
| 29 |
#[derive(Clone, Debug, PartialEq)]
|
|
@@ -163,6 +184,10 @@ pub struct Condenser {
|
|
| 163 |
config: CondenserConfig,
|
| 164 |
/// Managed regions: address β ManagedRegion
|
| 165 |
regions: HashMap<usize, ManagedRegion>,
|
|
|
|
|
|
|
|
|
|
|
|
|
| 166 |
/// Start time
|
| 167 |
start: Instant,
|
| 168 |
/// Stats
|
|
@@ -181,6 +206,7 @@ impl Condenser {
|
|
| 181 |
Self {
|
| 182 |
config,
|
| 183 |
regions: HashMap::with_capacity(1000),
|
|
|
|
| 184 |
start: Instant::now(),
|
| 185 |
total_compressed: 0,
|
| 186 |
total_decompressed: 0,
|
|
@@ -220,6 +246,10 @@ impl Condenser {
|
|
| 220 |
/// Remove a region (freed by the application)
|
| 221 |
pub fn unregister(&mut self, address: usize) {
|
| 222 |
if let Some(region) = self.regions.remove(&address) {
|
|
|
|
|
|
|
|
|
|
|
|
|
| 223 |
// Reclaim any saved bytes
|
| 224 |
let usage = region.ram_usage();
|
| 225 |
if usage < region.size {
|
|
@@ -348,6 +378,8 @@ impl Condenser {
|
|
| 348 |
/// Guards applied per region before compression:
|
| 349 |
/// 1. Skip regions smaller than PAGE_SIZE (4096 bytes) β not worth it.
|
| 350 |
/// 2. Skip if compressed_size > original_size * 0.9 β less than 10% savings.
|
|
|
|
|
|
|
| 351 |
///
|
| 352 |
/// Returns (regions_compressed, bytes_saved)
|
| 353 |
pub fn scan_and_compress(&mut self) -> (u32, u64) {
|
|
@@ -355,16 +387,21 @@ impl Condenser {
|
|
| 355 |
let threshold = self.config.idle_threshold_ns;
|
| 356 |
self.scan_count += 1;
|
| 357 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 358 |
let mut compressed_count = 0u32;
|
| 359 |
let mut bytes_saved = 0u64;
|
| 360 |
|
| 361 |
// Collect addresses to compress (can't mutate while iterating)
|
| 362 |
let to_compress: Vec<usize> = self.regions.iter()
|
| 363 |
-
.filter(|(
|
| 364 |
r.is_hot() &&
|
| 365 |
r.size >= self.config.min_manage_size &&
|
| 366 |
r.size >= PAGE_SIZE && // minimum page size guard
|
| 367 |
-
now - r.last_access_ns > threshold
|
|
|
|
| 368 |
})
|
| 369 |
.map(|(&addr, _)| addr)
|
| 370 |
.collect();
|
|
|
|
| 13 |
//! the membrane's tracked allocations and demoting idle ones.
|
| 14 |
//! When the predictor fires a spike ("this region is about to be
|
| 15 |
//! accessed"), the condenser pre-promotes it.
|
| 16 |
+
//!
|
| 17 |
+
//! ---- Changelog ----
|
| 18 |
+
//! [2026-06-21] CC β recently_freed tombstone guard (Option B safety)
|
| 19 |
+
//! What: Added `recently_freed: HashMap<usize, u64>` tombstone set.
|
| 20 |
+
//! unregister() stamps freed addresses with their freed_at_ns.
|
| 21 |
+
//! scan_and_compress() prunes stale tombstones and skips any
|
| 22 |
+
//! address still within FREED_RECENCY_NS (5s) of its free event.
|
| 23 |
+
//! Why: Closes the address-reuse race: free(X) processed β X removed
|
| 24 |
+
//! from regions and added to tombstones β malloc(X) re-registers X
|
| 25 |
+
//! β scan would otherwise operate on the new live object at X.
|
| 26 |
+
//! The primary race (free not yet processed) is mitigated by #260
|
| 27 |
+
//! (64K ring + deferred scan + burst gate). This guard closes the
|
| 28 |
+
//! secondary race (processed free β immediate reuse β re-register).
|
| 29 |
+
//! Together these make test_mode: false materially safe to enable.
|
| 30 |
+
//! How: FREED_RECENCY_NS const; recently_freed field on Condenser;
|
| 31 |
+
//! unregister stamps; scan_and_compress prunes + filters.
|
| 32 |
+
//! -------------------
|
| 33 |
|
| 34 |
use std::collections::HashMap;
|
| 35 |
use std::fs;
|
|
|
|
| 41 |
|
| 42 |
const PAGE_SIZE: usize = 4096;
|
| 43 |
const COLD_DIR: &str = "/tmp/condensate_cold";
|
| 44 |
+
/// How long (ns) after a free event to block that address from compression.
|
| 45 |
+
/// Closes the address-reuse race: free(X) processed β malloc(X) re-registered
|
| 46 |
+
/// β scan skips X for this window before treating it as a fresh allocation.
|
| 47 |
+
const FREED_RECENCY_NS: u64 = 5_000_000_000;
|
| 48 |
|
| 49 |
/// Tier state for a managed memory region
|
| 50 |
#[derive(Clone, Debug, PartialEq)]
|
|
|
|
| 184 |
config: CondenserConfig,
|
| 185 |
/// Managed regions: address β ManagedRegion
|
| 186 |
regions: HashMap<usize, ManagedRegion>,
|
| 187 |
+
/// Tombstone set: addresses freed within the last FREED_RECENCY_NS.
|
| 188 |
+
/// Prevents scan_and_compress from operating on a reused address whose
|
| 189 |
+
/// free event was processed but a new malloc has already re-registered it.
|
| 190 |
+
recently_freed: HashMap<usize, u64>,
|
| 191 |
/// Start time
|
| 192 |
start: Instant,
|
| 193 |
/// Stats
|
|
|
|
| 206 |
Self {
|
| 207 |
config,
|
| 208 |
regions: HashMap::with_capacity(1000),
|
| 209 |
+
recently_freed: HashMap::new(),
|
| 210 |
start: Instant::now(),
|
| 211 |
total_compressed: 0,
|
| 212 |
total_decompressed: 0,
|
|
|
|
| 246 |
/// Remove a region (freed by the application)
|
| 247 |
pub fn unregister(&mut self, address: usize) {
|
| 248 |
if let Some(region) = self.regions.remove(&address) {
|
| 249 |
+
// Stamp tombstone so scan_and_compress skips this address if it
|
| 250 |
+
// is reused by a new malloc before the recency window expires.
|
| 251 |
+
self.recently_freed.insert(address, self.elapsed_ns());
|
| 252 |
+
|
| 253 |
// Reclaim any saved bytes
|
| 254 |
let usage = region.ram_usage();
|
| 255 |
if usage < region.size {
|
|
|
|
| 378 |
/// Guards applied per region before compression:
|
| 379 |
/// 1. Skip regions smaller than PAGE_SIZE (4096 bytes) β not worth it.
|
| 380 |
/// 2. Skip if compressed_size > original_size * 0.9 β less than 10% savings.
|
| 381 |
+
/// 3. Skip addresses in the recently_freed tombstone set β the address may
|
| 382 |
+
/// have been reused by a new malloc before the recency window expires.
|
| 383 |
///
|
| 384 |
/// Returns (regions_compressed, bytes_saved)
|
| 385 |
pub fn scan_and_compress(&mut self) -> (u32, u64) {
|
|
|
|
| 387 |
let threshold = self.config.idle_threshold_ns;
|
| 388 |
self.scan_count += 1;
|
| 389 |
|
| 390 |
+
// Prune stale tombstones first β entries older than FREED_RECENCY_NS
|
| 391 |
+
// are safe to forget; the address has been "clean" long enough.
|
| 392 |
+
self.recently_freed.retain(|_, freed_at| now - *freed_at < FREED_RECENCY_NS);
|
| 393 |
+
|
| 394 |
let mut compressed_count = 0u32;
|
| 395 |
let mut bytes_saved = 0u64;
|
| 396 |
|
| 397 |
// Collect addresses to compress (can't mutate while iterating)
|
| 398 |
let to_compress: Vec<usize> = self.regions.iter()
|
| 399 |
+
.filter(|(addr, r)| {
|
| 400 |
r.is_hot() &&
|
| 401 |
r.size >= self.config.min_manage_size &&
|
| 402 |
r.size >= PAGE_SIZE && // minimum page size guard
|
| 403 |
+
now - r.last_access_ns > threshold &&
|
| 404 |
+
!self.recently_freed.contains_key(addr) // tombstone guard
|
| 405 |
})
|
| 406 |
.map(|(&addr, _)| addr)
|
| 407 |
.collect();
|
|
@@ -38,6 +38,15 @@
|
|
| 38 |
//! chunk metadata, crashing in libc+0x17934d (svcfd_create region).
|
| 39 |
//! How: PipelineConfig { test_mode: true } on global PIPELINE β condenser
|
| 40 |
//! learns allocation patterns but never dereferences observed addresses.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 41 |
//! -------------------
|
| 42 |
|
| 43 |
use libc::{c_void, size_t};
|
|
@@ -475,14 +484,15 @@ static WRITE_POS: AtomicUsize = AtomicUsize::new(0);
|
|
| 475 |
static MEMBRANE: std::sync::LazyLock<Mutex<MembraneState>> =
|
| 476 |
std::sync::LazyLock::new(|| Mutex::new(MembraneState::new()));
|
| 477 |
|
| 478 |
-
/// Global pipeline β only accessed by drain thread
|
| 479 |
-
///
|
| 480 |
-
///
|
| 481 |
-
///
|
| 482 |
-
///
|
|
|
|
| 483 |
static PIPELINE: std::sync::LazyLock<Mutex<Pipeline>> =
|
| 484 |
std::sync::LazyLock::new(|| Mutex::new(Pipeline::new(PipelineConfig {
|
| 485 |
-
test_mode:
|
| 486 |
..PipelineConfig::default()
|
| 487 |
})));
|
| 488 |
|
|
|
|
| 38 |
//! chunk metadata, crashing in libc+0x17934d (svcfd_create region).
|
| 39 |
//! How: PipelineConfig { test_mode: true } on global PIPELINE β condenser
|
| 40 |
//! learns allocation patterns but never dereferences observed addresses.
|
| 41 |
+
//! [2026-06-21] CC β Flip PIPELINE to test_mode: false (production condensation)
|
| 42 |
+
//! What: Removed test_mode: true safety hold. PIPELINE now runs in production
|
| 43 |
+
//! mode β condenser reads idle allocations and writes compressed bytes back.
|
| 44 |
+
//! Why: 72h stability check passed on VPS TID (inode 1844695, no SIGBUS/SIGSEGV).
|
| 45 |
+
//! #260 improvements (64K ring, deferred scan, 60s burst gate) dramatically
|
| 46 |
+
//! reduce the primary race (unprocessed free event). Option B guard added
|
| 47 |
+
//! to condenser (recently_freed tombstone set, FREED_RECENCY_NS=5s) closes
|
| 48 |
+
//! the secondary race (processed free β malloc reuse β re-register β scan).
|
| 49 |
+
//! How: PipelineConfig { test_mode: false } β see condenser.rs changelog.
|
| 50 |
//! -------------------
|
| 51 |
|
| 52 |
use libc::{c_void, size_t};
|
|
|
|
| 484 |
static MEMBRANE: std::sync::LazyLock<Mutex<MembraneState>> =
|
| 485 |
std::sync::LazyLock::new(|| Mutex::new(MembraneState::new()));
|
| 486 |
|
| 487 |
+
/// Global pipeline β only accessed by drain thread.
|
| 488 |
+
/// Production mode: condenser reads idle allocations and writes compressed
|
| 489 |
+
/// bytes back. Protected by three layered guards:
|
| 490 |
+
/// 1. 64K ring + deferred scan (#260): minimises unprocessed-free window
|
| 491 |
+
/// 2. 60s burst gate (#260): no scan during startup coldload (original crash window)
|
| 492 |
+
/// 3. recently_freed tombstone (5s): blocks reused addresses after free processing
|
| 493 |
static PIPELINE: std::sync::LazyLock<Mutex<Pipeline>> =
|
| 494 |
std::sync::LazyLock::new(|| Mutex::new(Pipeline::new(PipelineConfig {
|
| 495 |
+
test_mode: false,
|
| 496 |
..PipelineConfig::default()
|
| 497 |
})));
|
| 498 |
|