From c5fd270cff119bd07c07e5c183177fa9af63ed6c Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 13 Jul 2026 14:44:44 +0200 Subject: [PATCH 01/16] feat(memtrack): track RSS via kmem:rss_stat tracepoint Sample the kernel's per-mm resident counter through the kmem:rss_stat tracepoint, emitting absolute byte values per mm member. Adds the EVENT_TYPE_RSS contract, MemtrackEventKind::Rss, the parser arm, and a writer bench case. An rss_stat update from reclaim or another process's madvise fires in the actor's context; track (mm_id, member) -> owning pid so those updates reach the owner. External events may only lower a counter, so stale reads and mm_id collisions cannot invent peaks. --- crates/memtrack/src/ebpf/c/event.h | 6 ++ crates/memtrack/src/ebpf/c/main.bpf.c | 1 + crates/memtrack/src/ebpf/c/rss.bpf.h | 74 +++++++++++++++++++ .../memtrack/src/ebpf/c/utils/event_helpers.h | 19 ++++- crates/memtrack/src/ebpf/events.rs | 37 ++++++++++ crates/memtrack/src/ebpf/memtrack/tracking.rs | 4 + .../runner-shared/benches/memtrack_writer.rs | 6 +- .../src/artifacts/memtrack/mod.rs | 14 ++++ 8 files changed, 156 insertions(+), 5 deletions(-) create mode 100644 crates/memtrack/src/ebpf/c/rss.bpf.h diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index ff41f0be..89974526 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -9,6 +9,7 @@ #define EVENT_TYPE_MMAP 6 #define EVENT_TYPE_MUNMAP 7 #define EVENT_TYPE_BRK 8 +#define EVENT_TYPE_RSS 9 /* Common header shared by all event types */ struct event_header { @@ -45,6 +46,11 @@ struct event { uint64_t addr; /* address of mapping */ uint64_t size; /* size of mapping */ } mmap; + + struct { + int32_t member; + uint64_t size; + } rss; } data; }; diff --git a/crates/memtrack/src/ebpf/c/main.bpf.c b/crates/memtrack/src/ebpf/c/main.bpf.c index 1c5ad8d1..e46a144b 100644 --- a/crates/memtrack/src/ebpf/c/main.bpf.c +++ b/crates/memtrack/src/ebpf/c/main.bpf.c @@ -8,6 +8,7 @@ #include "allocator.h" #include "attach.h" #include "event.h" +#include "rss.bpf.h" #include "utils/event_helpers.h" #include "utils/map_helpers.h" #include "utils/process_tracking.h" diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h new file mode 100644 index 00000000..822af655 --- /dev/null +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -0,0 +1,74 @@ +#ifndef __RSS_BPF_H__ +#define __RSS_BPF_H__ + +#include "event.h" +#include "utils/event_helpers.h" +#include "utils/process_tracking.h" + +/* (rss_stat mm_id << 32 | member) -> {owning tgid, last in-context size}. Keyed per + * counter so an external (curr==0) update is attributed only once that mm/member was + * established in-context. An external event may only lower a counter: any size above + * the last in-context value is dropped, so neither a stale/racing reclaim read nor an + * mm_id hash collision with another task can invent a peak. LRU eviction + re-seeding + * on the owner's next in-context event covers hash reuse, so no teardown hook needed. */ +struct rss_owner { + __u32 pid; + __u64 size; +}; +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 40960); + __type(key, __u64); + __type(value, struct rss_owner); +} mm_to_pid SEC(".maps"); + +static __always_inline int submit_rss_event(__u32 owner_pid, __s32 member, __u64 size) { + SUBMIT_EVENT_AS(owner_pid, EVENT_TYPE_RSS, { + e->data.rss.member = member; + e->data.rss.size = size; + }); +} + +SEC("tracepoint/kmem/rss_stat") +int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { + if (ctx->member == MM_SWAPENTS) { + return 0; + } + + __u32 cur = bpf_get_current_pid_tgid() >> 32; + __u64 key = ((__u64)ctx->mm_id << 32) | (__u32)ctx->member; + __u64 size = ctx->size; + __u32 owner; + + if (ctx->curr) { + if (!is_tracked(cur)) { + return 0; + } + owner = cur; + struct rss_owner state = {.pid = cur, .size = size}; + bpf_map_update_elem(&mm_to_pid, &key, &state, BPF_ANY); + } else { + struct rss_owner* found = bpf_map_lookup_elem(&mm_to_pid, &key); + if (!found) { + return 0; + } + owner = found->pid; + /* The owner's own teardown also presents as curr==0 (current->mm is cleared + * on exit), so drop it. Genuine external actors (reclaim, another process's + * madvise) run in a different task, so cur != owner. */ + if (cur == owner) { + return 0; + } + /* An external actor may only lower a counter. A larger value is a stale + * reclaim read or an mm_id hash collision with another task; dropping it + * keeps the reconstructed peak identical to the in-context timeline. */ + if (size > found->size) { + return 0; + } + found->size = size; + } + + return submit_rss_event(owner, ctx->member, size); +} + +#endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index b5d79b37..1be39e7e 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -38,12 +38,11 @@ static __always_inline __u64* take_param(void* map) { return value; } -#define SUBMIT_EVENT(evt_type, fill_data) \ +#define SUBMIT_EVENT_AS(owner_pid, evt_type, fill_data) \ { \ __u64 tid = bpf_get_current_pid_tgid(); \ - __u32 pid = tid >> 32; \ \ - if (!is_tracked(pid) || !is_enabled()) { \ + if (!is_enabled()) { \ return 0; \ } \ \ @@ -58,7 +57,7 @@ static __always_inline __u64* take_param(void* map) { } \ \ e->header.timestamp = bpf_ktime_get_ns(); \ - e->header.pid = pid; \ + e->header.pid = owner_pid; \ e->header.tid = tid & 0xFFFFFFFF; \ e->header.event_type = evt_type; \ \ @@ -68,6 +67,18 @@ static __always_inline __u64* take_param(void* map) { return 0; \ } +#define SUBMIT_EVENT(evt_type, fill_data) \ + { \ + __u64 tid = bpf_get_current_pid_tgid(); \ + __u32 pid = tid >> 32; \ + \ + if (!is_tracked(pid)) { \ + return 0; \ + } \ + \ + SUBMIT_EVENT_AS(pid, evt_type, fill_data); \ + } + static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { SUBMIT_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index f18216c6..cfe56ec8 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -74,6 +74,13 @@ pub fn parse_event(data: &[u8]) -> Option { size: event.data.mmap.size, }, ), + EVENT_TYPE_RSS => ( + 0, + MemtrackEventKind::Rss { + member: event.data.rss.member, + size: event.data.rss.size, + }, + ), unknown => { panic!("Unknown event type: {unknown}"); } @@ -185,4 +192,34 @@ mod tests { _ => panic!("Expected Malloc event kind"), } } + + #[test] + fn test_parse_rss_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_RSS as u8; + event.header.timestamp = 12345678; + event.header.pid = 1000; + event.header.tid = 2000; + event.data.rss.member = 1; + event.data.rss.size = 4096 * 10; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1000); + assert_eq!(parsed.addr, 0); + + match parsed.kind { + MemtrackEventKind::Rss { member, size } => { + assert_eq!(member, 1); + assert_eq!(size, 4096 * 10); + } + _ => panic!("Expected Rss event kind"), + } + } } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index dd2a5604..966d4eaa 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -4,9 +4,13 @@ use paste::paste; impl MemtrackBpf { attach_tracepoint!(sched_fork); + attach_tracepoint!(rss_stat); pub fn attach_tracepoints(&mut self) -> Result<()> { self.attach_sched_fork()?; + if let Err(e) = self.attach_rss_stat() { + warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); + } Ok(()) } diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index 62aa2451..17b4f172 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -13,7 +13,7 @@ fn generate_events(n: usize) -> Vec { let mut events = Vec::with_capacity(n); for _ in 0..n { let size = rng.gen_range(8..8192); - let kind = match rng.gen_range(0..8) { + let kind = match rng.gen_range(0..9) { 0 => MemtrackEventKind::Malloc { size }, 1 => MemtrackEventKind::Free, 2 => MemtrackEventKind::Realloc { @@ -25,6 +25,10 @@ fn generate_events(n: usize) -> Vec { 5 => MemtrackEventKind::Mmap { size }, 6 => MemtrackEventKind::Munmap { size }, 7 => MemtrackEventKind::Brk { size }, + 8 => MemtrackEventKind::Rss { + member: rng.gen_range(0..4), + size, + }, _ => unreachable!(), }; diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index ad843b28..d9a00923 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -78,6 +78,10 @@ pub enum MemtrackEventKind { Brk { size: u64, }, + Rss { + member: i32, + size: u64, + }, } pub struct MemtrackEventStream { @@ -116,6 +120,16 @@ mod tests { addr: 0x20, kind: MemtrackEventKind::Free, }, + MemtrackEvent { + pid: 1, + tid: 11, + timestamp: 300, + addr: 0, + kind: MemtrackEventKind::Rss { + member: 1, + size: 40960, + }, + }, ]; let artifact = MemtrackArtifact { From 4c27558b2fd55707348e4d178942b15d2f7ec695 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 17 Jul 2026 15:38:40 +0200 Subject: [PATCH 02/16] feat(memtrack): reconstruct RSS from gated folio rmap fentry hooks Attach fentry hooks on the folio-rmap add/remove functions, emitting signed page-count deltas per MM_* bucket so anon, file, and shmem RSS can be reconstructed over time. Gated behind CODSPEED_MEMTRACK_TRACK_RMAP; the programs stay autoload-off by default so the skeleton loads on any kernel. Adds the EVENT_TYPE_RMAP contract, MemtrackEventKind::Rmap, parser arm, and bench case. --- crates/memtrack/src/ebpf/c/event.h | 7 + crates/memtrack/src/ebpf/c/rss.bpf.h | 124 ++++++++++++++++++ crates/memtrack/src/ebpf/events.rs | 38 ++++++ crates/memtrack/src/ebpf/memtrack/macros.rs | 15 +++ crates/memtrack/src/ebpf/memtrack/mod.rs | 109 ++++++++++++++- crates/memtrack/src/ebpf/memtrack/tracking.rs | 69 ++++++++++ .../runner-shared/benches/memtrack_writer.rs | 6 +- .../src/artifacts/memtrack/mod.rs | 4 + 8 files changed, 369 insertions(+), 3 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 89974526..6739c525 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -10,6 +10,7 @@ #define EVENT_TYPE_MUNMAP 7 #define EVENT_TYPE_BRK 8 #define EVENT_TYPE_RSS 9 +#define EVENT_TYPE_RMAP 10 /* Common header shared by all event types */ struct event_header { @@ -51,6 +52,12 @@ struct event { int32_t member; uint64_t size; } rss; + + struct { + int32_t member; /* MM_* counter index, same values as rss.member */ + int64_t delta; + uint64_t addr; + } rmap; } data; }; diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index 822af655..da7d476a 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -22,6 +22,10 @@ struct { __type(value, struct rss_owner); } mm_to_pid SEC(".maps"); +#define FOLIO_MAPPING_ANON 0x1UL + +const volatile __u32 page_shift = 12; + static __always_inline int submit_rss_event(__u32 owner_pid, __s32 member, __u64 size) { SUBMIT_EVENT_AS(owner_pid, EVENT_TYPE_RSS, { e->data.rss.member = member; @@ -71,4 +75,124 @@ int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { return submit_rss_event(owner, ctx->member, size); } +static __always_inline __u64 folio_nr_pages_est(struct folio* folio) { + unsigned long flags = BPF_CORE_READ(folio, flags).f; + if (!(flags & (1UL << PG_head))) { + return 1; + } + unsigned long order = BPF_CORE_READ(folio, _flags_1) & 0xff; + return 1UL << order; +} + +static __always_inline int folio_is_anon(struct folio* folio) { + unsigned long mapping = (unsigned long)BPF_CORE_READ(folio, mapping); + return (mapping & FOLIO_MAPPING_ANON) != 0; +} + +/* Mirrors the kernel's mm_counter(): anon folios are also swapbacked, so the + * anon check must come first. */ +static __always_inline __s32 folio_mm_counter(struct folio* folio) { + if (folio_is_anon(folio)) { + return MM_ANONPAGES; + } + unsigned long flags = BPF_CORE_READ(folio, flags).f; + if (flags & (1UL << PG_swapbacked)) { + return MM_SHMEMPAGES; + } + return MM_FILEPAGES; +} + +static __always_inline __u64 folio_page_address(struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + __u64 page_idx = ((__u64)page - (__u64)folio) / bpf_core_type_size(struct page); + __u64 pgoff = BPF_CORE_READ(folio, index) + page_idx; + __u64 vm_pgoff = BPF_CORE_READ(vma, vm_pgoff); + __u64 vm_start = BPF_CORE_READ(vma, vm_start); + if (pgoff < vm_pgoff) { + return vm_start; + } + return vm_start + ((pgoff - vm_pgoff) << page_shift); +} + +static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, __s64 delta, + __u64 addr) { + __u64 mm = (__u64)BPF_CORE_READ(vma, vm_mm); + struct task_struct* task = bpf_get_current_task_btf(); + if ((__u64)BPF_CORE_READ(task, mm) != mm) { + return 0; + } + + __u64 tid = bpf_get_current_pid_tgid(); + __u32 pid = tid >> 32; + if (!is_tracked(pid)) { + return 0; + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_RMAP, { + e->data.rmap.member = member; + e->data.rmap.delta = delta; + e->data.rmap.addr = addr; + }); +} + +SEC("fentry/folio_add_new_anon_rmap") +int BPF_PROG(fentry_folio_add_new_anon_rmap, struct folio* folio, struct vm_area_struct* vma, + unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); +} + +SEC("fentry/folio_add_anon_rmap_ptes") +int BPF_PROG(fentry_folio_add_anon_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma, unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)nr_pages, address); +} + +SEC("fentry/folio_add_anon_rmap_pmd") +int BPF_PROG(fentry_folio_add_anon_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma, unsigned long address) { + return submit_rmap(vma, MM_ANONPAGES, (__s64)folio_nr_pages_est(folio), address); +} + +SEC("fentry/folio_add_file_rmap_ptes") +int BPF_PROG(fentry_folio_add_file_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)nr_pages, + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_add_file_rmap_pmd") +int BPF_PROG(fentry_folio_add_file_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_add_file_rmap_pud") +int BPF_PROG(fentry_folio_add_file_rmap_pud, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), (__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_ptes") +int BPF_PROG(fentry_folio_remove_rmap_ptes, struct folio* folio, struct page* page, int nr_pages, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)nr_pages, + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_pmd") +int BPF_PROG(fentry_folio_remove_rmap_pmd, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + +SEC("fentry/folio_remove_rmap_pud") +int BPF_PROG(fentry_folio_remove_rmap_pud, struct folio* folio, struct page* page, + struct vm_area_struct* vma) { + return submit_rmap(vma, folio_mm_counter(folio), -(__s64)folio_nr_pages_est(folio), + folio_page_address(folio, page, vma)); +} + #endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index cfe56ec8..8fe3a63d 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -81,6 +81,13 @@ pub fn parse_event(data: &[u8]) -> Option { size: event.data.rss.size, }, ), + EVENT_TYPE_RMAP => ( + event.data.rmap.addr, + MemtrackEventKind::Rmap { + member: event.data.rmap.member, + delta: event.data.rmap.delta, + }, + ), unknown => { panic!("Unknown event type: {unknown}"); } @@ -222,4 +229,35 @@ mod tests { _ => panic!("Expected Rss event kind"), } } + + #[test] + fn test_parse_rmap_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_RMAP as u8; + event.header.timestamp = 12345678; + event.header.pid = 1000; + event.header.tid = 2000; + event.data.rmap.member = 3; + event.data.rmap.delta = 8; + event.data.rmap.addr = 0x7f00; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1000); + assert_eq!(parsed.addr, 0x7f00); + + match parsed.kind { + MemtrackEventKind::Rmap { member, delta } => { + assert_eq!(member, 3); + assert_eq!(delta, 8); + } + _ => panic!("Expected Rmap event kind"), + } + } } diff --git a/crates/memtrack/src/ebpf/memtrack/macros.rs b/crates/memtrack/src/ebpf/memtrack/macros.rs index 88da64a6..f4eca485 100644 --- a/crates/memtrack/src/ebpf/memtrack/macros.rs +++ b/crates/memtrack/src/ebpf/memtrack/macros.rs @@ -134,3 +134,18 @@ macro_rules! attach_tracepoint { } }; } + +macro_rules! attach_fentry { + ($func:ident, $prog:ident) => { + fn $func(&mut self) -> Result<()> { + let link = self + .skel + .progs + .$prog + .attach() + .context(format!("Failed to attach {} fentry", stringify!($prog)))?; + self.probes.push(link); + Ok(()) + } + }; +} diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index 2faba0ec..fbcd4b91 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -1,8 +1,9 @@ use crate::prelude::*; -use libbpf_rs::Link; use libbpf_rs::skel::OpenSkel; use libbpf_rs::skel::SkelBuilder; +use libbpf_rs::{AsRawLibbpf, Link}; use std::collections::HashMap; +use std::ffi::CString; use std::mem::MaybeUninit; use std::path::Path; @@ -62,6 +63,27 @@ fn symbol_file_offset<'a>( Some((address - section.address() + sh_offset) as usize) } +fn page_shift() -> Result { + let page_size = unsafe { libc::sysconf(libc::_SC_PAGESIZE) }; + ensure!(page_size > 0, "Failed to read system page size"); + Ok((page_size as u32).trailing_zeros()) +} + +fn kernel_func_exists(btf: &libbpf_rs::btf::Btf, name: &str) -> bool { + let Ok(name) = CString::new(name) else { + return false; + }; + + let id = unsafe { + libbpf_rs::libbpf_sys::btf__find_by_name_kind( + btf.as_libbpf_object().as_ptr(), + name.as_ptr(), + libbpf_rs::libbpf_sys::BTF_KIND_FUNC, + ) + }; + id >= 0 +} + /// Attach targets resolved from a library's symbol tables. pub struct ResolvedSymbols { offsets: HashMap, @@ -76,16 +98,97 @@ impl ResolvedSymbols { pub struct MemtrackBpf { skel: Box>, probes: Vec, + track_rmap: bool, + btf_disabled_rmap_targets: Vec<&'static str>, } impl MemtrackBpf { pub fn new() -> Result { + let track_rmap = std::env::var("CODSPEED_MEMTRACK_TRACK_RMAP").is_ok_and(|v| v == "1"); + Self::new_with_rmap(track_rmap) + } + + pub fn new_with_rmap(track_rmap: bool) -> Result { let builder = MainSkelBuilder::default(); let open_object = Box::leak(Box::new(MaybeUninit::uninit())); - let open_skel = builder + let mut open_skel = builder .open(open_object) .context("Failed to open syscalls BPF skeleton")?; + open_skel + .maps + .rodata_data + .as_deref_mut() + .context("rodata map missing")? + .page_shift = page_shift()?; + let mut btf_disabled_rmap_targets = Vec::new(); + + if !track_rmap { + open_skel + .progs + .fentry_folio_add_new_anon_rmap + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_anon_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_anon_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_add_file_rmap_pud + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_ptes + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_pmd + .set_autoload(false); + open_skel + .progs + .fentry_folio_remove_rmap_pud + .set_autoload(false); + } else { + let btf = libbpf_rs::btf::Btf::from_vmlinux().context("Failed to load vmlinux BTF")?; + macro_rules! disable_missing_kernel_func { + ($prog:ident, $target:literal) => { + if !kernel_func_exists(&btf, $target) { + open_skel.progs.$prog.set_autoload(false); + btf_disabled_rmap_targets.push($target); + warn!( + "Kernel function {} not present in BTF, disabling rmap fentry", + $target + ); + } + }; + } + + disable_missing_kernel_func!(fentry_folio_add_new_anon_rmap, "folio_add_new_anon_rmap"); + disable_missing_kernel_func!( + fentry_folio_add_anon_rmap_ptes, + "folio_add_anon_rmap_ptes" + ); + disable_missing_kernel_func!(fentry_folio_add_anon_rmap_pmd, "folio_add_anon_rmap_pmd"); + disable_missing_kernel_func!(fentry_folio_add_file_rmap_ptes, "folio_add_file_rmap_ptes"); + disable_missing_kernel_func!(fentry_folio_add_file_rmap_pmd, "folio_add_file_rmap_pmd"); + disable_missing_kernel_func!(fentry_folio_add_file_rmap_pud, "folio_add_file_rmap_pud"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_ptes, "folio_remove_rmap_ptes"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_pmd, "folio_remove_rmap_pmd"); + disable_missing_kernel_func!(fentry_folio_remove_rmap_pud, "folio_remove_rmap_pud"); + } + let skel = Box::new( open_skel .load() @@ -95,6 +198,8 @@ impl MemtrackBpf { Ok(Self { skel, probes: Vec::new(), + track_rmap, + btf_disabled_rmap_targets, }) } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 966d4eaa..70fc7077 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -5,12 +5,81 @@ use paste::paste; impl MemtrackBpf { attach_tracepoint!(sched_fork); attach_tracepoint!(rss_stat); + attach_fentry!( + attach_fentry_folio_add_new_anon_rmap, + fentry_folio_add_new_anon_rmap + ); + attach_fentry!( + attach_fentry_folio_add_anon_rmap_ptes, + fentry_folio_add_anon_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_add_anon_rmap_pmd, + fentry_folio_add_anon_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_ptes, + fentry_folio_add_file_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_pmd, + fentry_folio_add_file_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_add_file_rmap_pud, + fentry_folio_add_file_rmap_pud + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_ptes, + fentry_folio_remove_rmap_ptes + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_pmd, + fentry_folio_remove_rmap_pmd + ); + attach_fentry!( + attach_fentry_folio_remove_rmap_pud, + fentry_folio_remove_rmap_pud + ); + + fn rmap_target_enabled(&self, target: &str) -> bool { + !self.btf_disabled_rmap_targets.contains(&target) + } pub fn attach_tracepoints(&mut self) -> Result<()> { self.attach_sched_fork()?; if let Err(e) = self.attach_rss_stat() { warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); } + if self.track_rmap { + if self.rmap_target_enabled("folio_add_new_anon_rmap") { + self.attach_fentry_folio_add_new_anon_rmap()?; + } + if self.rmap_target_enabled("folio_add_anon_rmap_ptes") { + self.attach_fentry_folio_add_anon_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_add_anon_rmap_pmd") { + self.attach_fentry_folio_add_anon_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_ptes") { + self.attach_fentry_folio_add_file_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_pmd") { + self.attach_fentry_folio_add_file_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_add_file_rmap_pud") { + self.attach_fentry_folio_add_file_rmap_pud()?; + } + if self.rmap_target_enabled("folio_remove_rmap_ptes") { + self.attach_fentry_folio_remove_rmap_ptes()?; + } + if self.rmap_target_enabled("folio_remove_rmap_pmd") { + self.attach_fentry_folio_remove_rmap_pmd()?; + } + if self.rmap_target_enabled("folio_remove_rmap_pud") { + self.attach_fentry_folio_remove_rmap_pud()?; + } + } Ok(()) } diff --git a/crates/runner-shared/benches/memtrack_writer.rs b/crates/runner-shared/benches/memtrack_writer.rs index 17b4f172..a6c610e8 100644 --- a/crates/runner-shared/benches/memtrack_writer.rs +++ b/crates/runner-shared/benches/memtrack_writer.rs @@ -13,7 +13,7 @@ fn generate_events(n: usize) -> Vec { let mut events = Vec::with_capacity(n); for _ in 0..n { let size = rng.gen_range(8..8192); - let kind = match rng.gen_range(0..9) { + let kind = match rng.gen_range(0..10) { 0 => MemtrackEventKind::Malloc { size }, 1 => MemtrackEventKind::Free, 2 => MemtrackEventKind::Realloc { @@ -29,6 +29,10 @@ fn generate_events(n: usize) -> Vec { member: rng.gen_range(0..4), size, }, + 9 => MemtrackEventKind::Rmap { + member: rng.gen_range(0..4), + delta: rng.gen_range(-1024..1024), + }, _ => unreachable!(), }; diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index d9a00923..aba8b8cc 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -82,6 +82,10 @@ pub enum MemtrackEventKind { member: i32, size: u64, }, + Rmap { + member: i32, + delta: i64, + }, } pub struct MemtrackEventStream { From bba02c1e2299d19e443c1057a6ae75ef7971abdf Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 14 Jul 2026 18:24:39 +0200 Subject: [PATCH 03/16] feat(memtrack): emit fork/exec/exit lifecycle events A forked child's inherited RSS is invisible to rss_stat: the fork-time counter copies fire outside the child's context, and anon COW faults are counter-neutral, so a child that only touches inherited memory never reports anything on its own. A fork event carrying the parent pid lets consumers seed the child from the parent's last absolutes; exec and exit mark where the address space is replaced or torn down. --- crates/memtrack/src/ebpf/c/event.h | 8 ++++ crates/memtrack/src/ebpf/c/rss.bpf.h | 43 +++++++++++++++++++ crates/memtrack/src/ebpf/events.rs | 35 +++++++++++++++ crates/memtrack/src/ebpf/memtrack/tracking.rs | 6 +++ .../src/artifacts/memtrack/mod.rs | 5 +++ 5 files changed, 97 insertions(+) diff --git a/crates/memtrack/src/ebpf/c/event.h b/crates/memtrack/src/ebpf/c/event.h index 6739c525..5dabffbe 100644 --- a/crates/memtrack/src/ebpf/c/event.h +++ b/crates/memtrack/src/ebpf/c/event.h @@ -11,6 +11,9 @@ #define EVENT_TYPE_BRK 8 #define EVENT_TYPE_RSS 9 #define EVENT_TYPE_RMAP 10 +#define EVENT_TYPE_FORK 11 +#define EVENT_TYPE_EXEC 12 +#define EVENT_TYPE_EXIT 13 /* Common header shared by all event types */ struct event_header { @@ -58,6 +61,11 @@ struct event { int64_t delta; uint64_t addr; } rmap; + + /* Process lifecycle events (fork carries the parent; exec/exit have no payload) */ + struct { + uint32_t parent_pid; + } fork; } data; }; diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index da7d476a..12eb12cb 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -195,4 +195,47 @@ int BPF_PROG(fentry_folio_remove_rmap_pud, struct folio* folio, struct page* pag folio_page_address(folio, page, vma)); } +/* == Process lifecycle events == + * + * FORK lets userland seed a child's RSS from its parent at fork time: the + * kernel copies the mm counters during dup_mmap, but those updates fire + * rss_stat out of the child's context and anon COW faults are + * counter-neutral, so a child that only touches inherited memory never + * reports its RSS on its own. EXEC and EXIT mark the points where the + * address space is replaced or torn down, so userland resets to zero. + */ + +#define CLONE_THREAD 0x00010000 + +SEC("tracepoint/task/task_newtask") +int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { + if (ctx->clone_flags & CLONE_THREAD) { + return 0; + } + + __u64 tid = bpf_get_current_pid_tgid(); + __u32 parent_pid = tid >> 32; + if (!is_tracked(parent_pid)) { + return 0; + } + + __u32 child_pid = ctx->pid; + SUBMIT_EVENT_AS(child_pid, EVENT_TYPE_FORK, { e->data.fork.parent_pid = parent_pid; }); +} + +SEC("tracepoint/sched/sched_process_exec") +int tracepoint_sched_process_exec(void* ctx) { + SUBMIT_EVENT(EVENT_TYPE_EXEC, {}); +} + +SEC("tracepoint/sched/sched_process_exit") +int tracepoint_sched_process_exit(void* ctx) { + __u64 tid = bpf_get_current_pid_tgid(); + if ((tid >> 32) != (tid & 0xFFFFFFFF)) { + return 0; + } + + SUBMIT_EVENT(EVENT_TYPE_EXIT, {}); +} + #endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/events.rs b/crates/memtrack/src/ebpf/events.rs index 8fe3a63d..ccab7f5f 100644 --- a/crates/memtrack/src/ebpf/events.rs +++ b/crates/memtrack/src/ebpf/events.rs @@ -88,6 +88,14 @@ pub fn parse_event(data: &[u8]) -> Option { delta: event.data.rmap.delta, }, ), + EVENT_TYPE_FORK => ( + 0, + MemtrackEventKind::Fork { + parent_pid: event.data.fork.parent_pid as i32, + }, + ), + EVENT_TYPE_EXEC => (0, MemtrackEventKind::Exec), + EVENT_TYPE_EXIT => (0, MemtrackEventKind::Exit), unknown => { panic!("Unknown event type: {unknown}"); } @@ -260,4 +268,31 @@ mod tests { _ => panic!("Expected Rmap event kind"), } } + + #[test] + fn test_parse_fork_event() { + let mut event: bindings::event = unsafe { std::mem::zeroed() }; + event.header.event_type = bindings::EVENT_TYPE_FORK as u8; + event.header.timestamp = 12345678; + event.header.pid = 1001; + event.header.tid = 2000; + event.data.fork.parent_pid = 1000; + + let bytes = unsafe { + std::slice::from_raw_parts( + &event as *const _ as *const u8, + std::mem::size_of_val(&event), + ) + }; + + let parsed = parse_event(bytes).unwrap(); + assert_eq!(parsed.pid, 1001); + + match parsed.kind { + MemtrackEventKind::Fork { parent_pid } => { + assert_eq!(parent_pid, 1000); + } + _ => panic!("Expected Fork event kind"), + } + } } diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index 70fc7077..bc8156c4 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -5,6 +5,9 @@ use paste::paste; impl MemtrackBpf { attach_tracepoint!(sched_fork); attach_tracepoint!(rss_stat); + attach_tracepoint!(task_newtask); + attach_tracepoint!(sched_process_exec); + attach_tracepoint!(sched_process_exit); attach_fentry!( attach_fentry_folio_add_new_anon_rmap, fentry_folio_add_new_anon_rmap @@ -48,6 +51,9 @@ impl MemtrackBpf { pub fn attach_tracepoints(&mut self) -> Result<()> { self.attach_sched_fork()?; + self.attach_task_newtask()?; + self.attach_sched_process_exec()?; + self.attach_sched_process_exit()?; if let Err(e) = self.attach_rss_stat() { warn!("Failed to attach rss_stat tracepoint, RSS collection disabled: {e:#}"); } diff --git a/crates/runner-shared/src/artifacts/memtrack/mod.rs b/crates/runner-shared/src/artifacts/memtrack/mod.rs index aba8b8cc..94c911c4 100644 --- a/crates/runner-shared/src/artifacts/memtrack/mod.rs +++ b/crates/runner-shared/src/artifacts/memtrack/mod.rs @@ -86,6 +86,11 @@ pub enum MemtrackEventKind { member: i32, delta: i64, }, + Fork { + parent_pid: pid_t, + }, + Exec, + Exit, } pub struct MemtrackEventStream { From 86ea67e348af75dd8d1c4194fdca14d0042b51ba Mon Sep 17 00:00:00 2001 From: not-matthias Date: Fri, 17 Jul 2026 16:22:09 +0200 Subject: [PATCH 04/16] test(memtrack): validate RSS and rmap reconstruction against /proc Nine fixtures compare three per-process views - the fixture's own /proc report, rss_stat peaks, and rmap-reconstructed totals - plus an external-reclaim fixture proving out-of-context decrements reach the owner. Fork-seeded children are validated via fork_idle, whose 32 MiB is observable only through the fork-event seed. Pids are redacted and rows keep first-activity order so snapshots are stable across runs; fixtures report VmHWM instead of ru_maxrss, which survives execve and would leak the harness's peak RSS. --- crates/memtrack/.clang-format | 1 + crates/memtrack/Cargo.toml | 2 +- crates/memtrack/src/ebpf/memtrack/mod.rs | 5 +- crates/memtrack/src/ebpf/tracker.rs | 45 ++- crates/memtrack/testdata/rss/anon.c | 17 + crates/memtrack/testdata/rss/file.c | 30 ++ crates/memtrack/testdata/rss/fork.c | 39 ++ crates/memtrack/testdata/rss/fork_idle.c | 49 +++ crates/memtrack/testdata/rss/madvise.c | 64 ++++ crates/memtrack/testdata/rss/madvise_extern.c | 47 +++ crates/memtrack/testdata/rss/mremap_move.c | 31 ++ crates/memtrack/testdata/rss/munmap_hole.c | 40 ++ crates/memtrack/testdata/rss/rss_report.h | 50 +++ crates/memtrack/testdata/rss/shmem.c | 22 ++ crates/memtrack/testdata/rss/triangle.c | 24 ++ crates/memtrack/tests/rss_tests.rs | 359 ++++++++++++++++++ crates/memtrack/tests/shared.rs | 50 ++- .../tests/snapshots/rss_tests__rss_anon.snap | 30 ++ .../tests/snapshots/rss_tests__rss_file.snap | 30 ++ .../tests/snapshots/rss_tests__rss_fork.snap | 42 ++ .../snapshots/rss_tests__rss_fork_idle.snap | 45 +++ .../snapshots/rss_tests__rss_madvise.snap | 30 ++ .../snapshots/rss_tests__rss_mremap_move.snap | 30 ++ .../snapshots/rss_tests__rss_munmap_hole.snap | 30 ++ .../tests/snapshots/rss_tests__rss_shmem.snap | 30 ++ .../snapshots/rss_tests__rss_triangle.snap | 30 ++ 26 files changed, 1152 insertions(+), 20 deletions(-) create mode 100644 crates/memtrack/testdata/rss/anon.c create mode 100644 crates/memtrack/testdata/rss/file.c create mode 100644 crates/memtrack/testdata/rss/fork.c create mode 100644 crates/memtrack/testdata/rss/fork_idle.c create mode 100644 crates/memtrack/testdata/rss/madvise.c create mode 100644 crates/memtrack/testdata/rss/madvise_extern.c create mode 100644 crates/memtrack/testdata/rss/mremap_move.c create mode 100644 crates/memtrack/testdata/rss/munmap_hole.c create mode 100644 crates/memtrack/testdata/rss/rss_report.h create mode 100644 crates/memtrack/testdata/rss/shmem.c create mode 100644 crates/memtrack/testdata/rss/triangle.c create mode 100644 crates/memtrack/tests/rss_tests.rs create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_file.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap create mode 100644 crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap diff --git a/crates/memtrack/.clang-format b/crates/memtrack/.clang-format index b687ea40..4755a340 100644 --- a/crates/memtrack/.clang-format +++ b/crates/memtrack/.clang-format @@ -4,3 +4,4 @@ BasedOnStyle: Google PointerAlignment: Left ColumnLimit: 100 IndentWidth: 4 +AllowShortFunctionsOnASingleLine: None diff --git a/crates/memtrack/Cargo.toml b/crates/memtrack/Cargo.toml index c4311d9e..b3bd218e 100644 --- a/crates/memtrack/Cargo.toml +++ b/crates/memtrack/Cargo.toml @@ -46,7 +46,7 @@ bindgen = "0.72" tempfile = { workspace = true } rstest = { workspace = true } test-log = { workspace = true } -insta = { workspace = true } +insta = { workspace = true, features = ["json", "redactions"] } test-with = { workspace = true } [package.metadata.dist] diff --git a/crates/memtrack/src/ebpf/memtrack/mod.rs b/crates/memtrack/src/ebpf/memtrack/mod.rs index fbcd4b91..f40f4c1b 100644 --- a/crates/memtrack/src/ebpf/memtrack/mod.rs +++ b/crates/memtrack/src/ebpf/memtrack/mod.rs @@ -181,7 +181,10 @@ impl MemtrackBpf { "folio_add_anon_rmap_ptes" ); disable_missing_kernel_func!(fentry_folio_add_anon_rmap_pmd, "folio_add_anon_rmap_pmd"); - disable_missing_kernel_func!(fentry_folio_add_file_rmap_ptes, "folio_add_file_rmap_ptes"); + disable_missing_kernel_func!( + fentry_folio_add_file_rmap_ptes, + "folio_add_file_rmap_ptes" + ); disable_missing_kernel_func!(fentry_folio_add_file_rmap_pmd, "folio_add_file_rmap_pmd"); disable_missing_kernel_func!(fentry_folio_add_file_rmap_pud, "folio_add_file_rmap_pud"); disable_missing_kernel_func!(fentry_folio_remove_rmap_ptes, "folio_remove_rmap_ptes"); diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index 11ecaf04..ff6c0881 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -33,10 +33,26 @@ impl Tracker { }) } + /// Track per-process RSS via the rss_stat tracepoint and folio rmap fentry + /// hooks without attaching any allocator probes. No exec-mapping watcher + /// runs, so `spawn` arms no on-demand allocator attachment — the tracker + /// observes only tracepoints and (when `track_rmap`) the rmap fentries. + pub fn new_without_allocators_with_rmap(track_rmap: bool) -> Result { + Self::bump_memlock_rlimit()?; + + let mut bpf = MemtrackBpf::new_with_rmap(track_rmap)?; + bpf.attach_tracepoints()?; + + Ok(Self { + bpf: Arc::new(Mutex::new(bpf)), + worker: Mutex::new(None), + }) + } + /// Spawn `cmd` under tracking: the target is wrapped so it stops itself - /// before exec'ing, its pid is armed while stopped, then it is resumed. - /// The watcher observes the target's own `execve` mappings — no allocation - /// escapes untracked. + /// before exec'ing, its pid is armed while stopped, then it is resumed. When + /// the tracker runs an exec-mapping watcher, arming the pid before resume + /// ensures no allocation mapping escapes untracked. /// /// `uid_gid` drops the child's privileges (a `Command`'s uid/gid cannot be /// read back, so it cannot be preserved through the wrap). @@ -48,11 +64,9 @@ impl Tracker { let child = spawn_stopped(&mut wrapped)?; let pid = child.id() as i32; - self.worker - .lock() - .as_ref() - .context("tracker already finished")? - .set_root_pid(pid); + if let Some(worker) = self.worker.lock().as_ref() { + worker.set_root_pid(pid); + } let (tx, rx) = mpsc::channel(); let poller = { @@ -81,15 +95,14 @@ impl Tracker { self.bpf.lock().dropped_events_count() } - /// Stop the attach worker and surface any fatal error it recorded, - /// including missed exec mappings (incomplete allocator coverage). + /// Stop the attach worker, if any, and surface any fatal error it recorded, + /// including missed exec mappings (incomplete allocator coverage). A tracker + /// without an allocator watcher has no worker, so this is a no-op. pub fn finish(&self) -> Result<()> { - let worker = self - .worker - .lock() - .take() - .context("tracker already finished")?; - worker.finish() + match self.worker.lock().take() { + Some(worker) => worker.finish(), + None => Ok(()), + } } /// Detach all attached probes. Called explicitly at teardown because the diff --git a/crates/memtrack/testdata/rss/anon.c b/crates/memtrack/testdata/rss/anon.c new file mode 100644 index 00000000..8aa0974e --- /dev/null +++ b/crates/memtrack/testdata/rss/anon.c @@ -0,0 +1,17 @@ +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + int ret = write_rss_report(argv[1]); + munmap(mem, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/file.c b/crates/memtrack/testdata/rss/file.c new file mode 100644 index 00000000..5d45c293 --- /dev/null +++ b/crates/memtrack/testdata/rss/file.c @@ -0,0 +1,30 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + char template[] = "/tmp/memtrack-rss-file-XXXXXX"; + int fd = mkstemp(template); + if (fd < 0) return 1; + unlink(template); + char chunk[65536]; + memset(chunk, 0x42, sizeof(chunk)); + for (size_t off = 0; off < len; off += sizeof(chunk)) { + if (write(fd, chunk, sizeof(chunk)) != (ssize_t)sizeof(chunk)) return 1; + } + void* mem = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem == MAP_FAILED) return 1; + volatile char sink = 0; + for (size_t i = 0; i < len; i += 4096) sink ^= ((volatile char*)mem)[i]; + int ret = write_rss_report(argv[1]); + munmap(mem, len); + close(fd); + return ret + (sink & 0); +} diff --git a/crates/memtrack/testdata/rss/fork.c b/crates/memtrack/testdata/rss/fork.c new file mode 100644 index 00000000..f6fcfac0 --- /dev/null +++ b/crates/memtrack/testdata/rss/fork.c @@ -0,0 +1,39 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +static int append_rss(const char* path, const char* label) { + long kb = rss_status_kb("RssAnon:"); + if (kb < 0) return 1; + FILE* report = fopen(path, "a"); + if (!report) return 1; + fprintf(report, "%s: %ld\n", label, kb); + fclose(report); + return 0; +} + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 0x20 * 1024 * 1024; + void* parent = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (parent == MAP_FAILED) return 1; + memset(parent, 0x42, len); + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + void* child = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (child == MAP_FAILED) _exit(1); + memset(child, 0x24, len); + _exit(append_rss(argv[1], "ChildRssAnonKb")); + } + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + int ret = append_rss(argv[1], "ParentRssAnonKb"); + munmap(parent, len); + return ret || !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} diff --git a/crates/memtrack/testdata/rss/fork_idle.c b/crates/memtrack/testdata/rss/fork_idle.c new file mode 100644 index 00000000..0c5e5643 --- /dev/null +++ b/crates/memtrack/testdata/rss/fork_idle.c @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* The child only touches memory inherited from the parent: COW faults of anon + * pages are counter-neutral, so the child never reports its own RSS. Its + * footprint is only observable through fork-event seeding. The child must not + * allocate (no stdio/malloc), so the parent samples /proc//status while + * the child blocks on a pipe. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 0x20 * 1024 * 1024; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + + int ready[2]; + int release[2]; + if (pipe(ready) || pipe(release)) return 1; + + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + memset(mem, 0x24, len); + char b = 1; + if (write(ready[1], &b, 1) != 1) _exit(1); + if (read(release[0], &b, 1) != 1) _exit(1); + _exit(0); + } + + char b; + if (read(ready[0], &b, 1) != 1) return 1; + long child_anon = rss_status_kb_pid(pid, "RssAnon:"); + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "ChildRssAnon: %ld\n", child_anon); + fclose(report); + if (write(release[1], &b, 1) != 1) return 1; + + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + return ret || child_anon < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0; +} diff --git a/crates/memtrack/testdata/rss/madvise.c b/crates/memtrack/testdata/rss/madvise.c new file mode 100644 index 00000000..e5f77972 --- /dev/null +++ b/crates/memtrack/testdata/rss/madvise.c @@ -0,0 +1,64 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* AnonHugePages for the whole process, from smaps_rollup (not in /proc/status). */ +static long anon_huge_pages_kb(void) { + FILE* rollup = fopen("/proc/self/smaps_rollup", "r"); + if (!rollup) return 0; + char line[256]; + long kb = 0; + while (fgets(line, sizeof(line), rollup)) { + if (sscanf(line, "AnonHugePages: %ld", &kb) == 1) break; + } + fclose(rollup); + return kb; +} + +/* Two 32 MiB anon regions: one advised MADV_HUGEPAGE (2 MiB-aligned so the + * kernel *may* fault it as PMD folios), one MADV_NOHUGEPAGE (guaranteed pte + * path). MADV_HUGEPAGE is only advisory, so nothing here depends on THP + * actually materializing: the accounted totals are identical either way. + * + * Both regions are dropped with MADV_DONTNEED and faulted a second time. If + * the in-context removes were missed, the reconstructed running total reaches + * 128 MiB instead of 64 MiB, so the snapshot peak is the assertion. The report + * also carries the observed AnonHugePages (hex, so it stays out of the + * snapshot) letting the test require PMD-sized rmap deltas exactly when THP + * actually materialized. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 32UL * 1024 * 1024; + const size_t align = 2UL * 1024 * 1024; + + void* raw = mmap(NULL, len + align, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (raw == MAP_FAILED) return 1; + char* huge = (char*)(((uintptr_t)raw + align - 1) & ~(uintptr_t)(align - 1)); + if (madvise(huge, len, MADV_HUGEPAGE) != 0) return 1; + + char* base = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (base == MAP_FAILED) return 1; + if (madvise(base, len, MADV_NOHUGEPAGE) != 0) return 1; + + memset(huge, 0x42, len); + memset(base, 0x42, len); + + if (madvise(huge, len, MADV_DONTNEED) != 0) return 1; + if (madvise(base, len, MADV_DONTNEED) != 0) return 1; + + memset(huge, 0x43, len); + memset(base, 0x43, len); + + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "ThpKb: 0x%lx\n", anon_huge_pages_kb()); + fclose(report); + munmap(raw, len + align); + munmap(base, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/madvise_extern.c b/crates/memtrack/testdata/rss/madvise_extern.c new file mode 100644 index 00000000..4be4c6b2 --- /dev/null +++ b/crates/memtrack/testdata/rss/madvise_extern.c @@ -0,0 +1,47 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char** argv) { + (void)argc; (void)argv; + sleep(1); /* let the tracker attach + enable + add root pid */ + + size_t len = 64UL * 1024 * 1024; + + char path[] = "/tmp/memtrack_madv_XXXXXX"; + int fd = mkstemp(path); + if (fd < 0) return 1; + unlink(path); + if (ftruncate(fd, len) != 0) return 1; + + void* mem = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + if (mem == MAP_FAILED) return 1; + + /* Fault the file pages into A's RSS (MM_FILEPAGES), in-context -> seeds ownership. */ + volatile char sink = 0; + for (size_t i = 0; i < len; i += 4096) sink ^= ((volatile char*)mem)[i]; + (void)sink; + + pid_t pid = fork(); + if (pid < 0) return 1; + if (pid == 0) { + /* B: page out A's (parent's) region from B's context. */ + int pidfd = syscall(SYS_pidfd_open, getppid(), 0); + if (pidfd < 0) _exit(1); + struct iovec iov = {.iov_base = mem, .iov_len = len}; + syscall(SYS_process_madvise, pidfd, &iov, 1UL, MADV_PAGEOUT, 0UL); + _exit(0); + } + + int status; + if (waitpid(pid, &status, 0) < 0) return 1; + sleep(1); /* let the external decrement flush to the ring buffer */ + /* No munmap: an in-context decrement would mask the external-path signal. */ + return 0; +} diff --git a/crates/memtrack/testdata/rss/mremap_move.c b/crates/memtrack/testdata/rss/mremap_move.c new file mode 100644 index 00000000..3b86fc03 --- /dev/null +++ b/crates/memtrack/testdata/rss/mremap_move.c @@ -0,0 +1,31 @@ +#define _GNU_SOURCE +#include +#include +#include + +#include "rss_report.h" + +/* Fault 32 MiB, then force mremap to move it to a reserved destination. The + * move relocates page tables without rmap remove/add, so no events should + * fire and the second memset must not refault: a peak of 64 MiB instead of + * 32 MiB means the move was double-counted or the pages were dropped. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 32UL * 1024 * 1024; + + char* src = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (src == MAP_FAILED) return 1; + memset(src, 0x42, len); + + void* reserved = mmap(NULL, len, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) return 1; + + char* dst = mremap(src, len, len, MREMAP_MAYMOVE | MREMAP_FIXED, reserved); + if (dst == MAP_FAILED) return 1; + memset(dst, 0x43, len); + + int ret = write_rss_report(argv[1]); + munmap(dst, len); + return ret; +} diff --git a/crates/memtrack/testdata/rss/munmap_hole.c b/crates/memtrack/testdata/rss/munmap_hole.c new file mode 100644 index 00000000..e769ff5b --- /dev/null +++ b/crates/memtrack/testdata/rss/munmap_hole.c @@ -0,0 +1,40 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* Fault 64 MiB, punch a 16 MiB hole with munmap, then map and fault the hole + * again. If the partial-unmap removes were missed (or covered the wrong + * range), the reconstructed running total peaks at 80 MiB instead of 64 MiB. + * MADV_NOHUGEPAGE keeps every folio a single page: the region is only + * page-aligned, so a straddling PMD folio would blur the hole boundaries. */ +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t len = 64UL * 1024 * 1024; + const size_t hole_off = 24UL * 1024 * 1024; + const size_t hole_len = 16UL * 1024 * 1024; + + char* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) return 1; + if (madvise(mem, len, MADV_NOHUGEPAGE) != 0) return 1; + memset(mem, 0x42, len); + + if (munmap(mem + hole_off, hole_len) != 0) return 1; + + void* refill = mmap(mem + hole_off, hole_len, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); + if (refill == MAP_FAILED) return 1; + if (madvise(refill, hole_len, MADV_NOHUGEPAGE) != 0) return 1; + memset(refill, 0x43, hole_len); + + int ret = write_rss_report(argv[1]); + FILE* report = fopen(argv[1], "a"); + if (!report) return 1; + fprintf(report, "Layout: 0x%lx 0x%lx 0x%lx 0x%lx\n", (unsigned long)mem, hole_off, hole_len, + len); + fclose(report); + return ret; +} diff --git a/crates/memtrack/testdata/rss/rss_report.h b/crates/memtrack/testdata/rss/rss_report.h new file mode 100644 index 00000000..e144d145 --- /dev/null +++ b/crates/memtrack/testdata/rss/rss_report.h @@ -0,0 +1,50 @@ +#ifndef RSS_REPORT_H +#define RSS_REPORT_H + +#include +#include +#include + +static long rss_status_kb_pid(int pid, const char* key) { + char status_path[64]; + snprintf(status_path, sizeof(status_path), "/proc/%d/status", pid); + FILE* status = fopen(status_path, "r"); + if (!status) return -1; + char line[256]; + long kb = -1; + size_t key_len = strlen(key); + while (fgets(line, sizeof(line), status)) { + if (strncmp(line, key, key_len) == 0 && sscanf(line + key_len, " %ld", &kb) == 1) { + break; + } + } + fclose(status); + return kb; +} + +static long rss_status_kb(const char* key) { + return rss_status_kb_pid(getpid(), key); +} + +static int write_rss_report(const char* path) { + long anon = rss_status_kb("RssAnon:"); + long file = rss_status_kb("RssFile:"); + long shmem = rss_status_kb("RssShmem:"); + /* VmHWM instead of getrusage(): ru_maxrss includes signal->maxrss, which + * survives execve and so reports the peak of the pre-exec parent image. + * VmHWM belongs to the mm and starts fresh at exec. */ + long max_rss = rss_status_kb("VmHWM:"); + if (anon < 0 || file < 0 || shmem < 0 || max_rss < 0) { + return 1; + } + FILE* report = fopen(path, "w"); + if (!report) { + return 1; + } + fprintf(report, "RssAnon: %ld\nRssFile: %ld\nRssShmem: %ld\nMaxRssKb: %ld\n", anon, file, + shmem, max_rss); + fclose(report); + return 0; +} + +#endif diff --git a/crates/memtrack/testdata/rss/shmem.c b/crates/memtrack/testdata/rss/shmem.c new file mode 100644 index 00000000..63b53e01 --- /dev/null +++ b/crates/memtrack/testdata/rss/shmem.c @@ -0,0 +1,22 @@ +#define _GNU_SOURCE +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + size_t len = 64UL * 1024 * 1024; + int fd = memfd_create("memtrack-rss-shmem", 0); + if (fd < 0) return 1; + if (ftruncate(fd, len) != 0) return 1; + void* mem = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (mem == MAP_FAILED) return 1; + memset(mem, 0x42, len); + int ret = write_rss_report(argv[1]); + munmap(mem, len); + close(fd); + return ret; +} diff --git a/crates/memtrack/testdata/rss/triangle.c b/crates/memtrack/testdata/rss/triangle.c new file mode 100644 index 00000000..69bf7bf6 --- /dev/null +++ b/crates/memtrack/testdata/rss/triangle.c @@ -0,0 +1,24 @@ +#include +#include +#include + +#include "rss_report.h" + +int main(int argc, char** argv) { + if (argc != 2) return 1; + sleep(1); + const size_t chunk = 16UL * 1024 * 1024; + void* bufs[4]; + for (int i = 0; i < 4; i++) { + bufs[i] = mmap(NULL, chunk, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (bufs[i] == MAP_FAILED) return 1; + memset(bufs[i], 0x42, chunk); + usleep(50 * 1000); + } + int ret = write_rss_report(argv[1]); + for (int i = 0; i < 4; i++) { + munmap(bufs[i], chunk); + usleep(50 * 1000); + } + return ret; +} diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs new file mode 100644 index 00000000..d1099348 --- /dev/null +++ b/crates/memtrack/tests/rss_tests.rs @@ -0,0 +1,359 @@ +#[macro_use] +mod shared; + +use itertools::Itertools; +use rstest::rstest; +use runner_shared::artifacts::{MemtrackEvent, MemtrackEventKind}; +use serde::Serialize; +use std::collections::BTreeMap; +use std::process::Command; +use tempfile::TempDir; + +const MIB: u64 = 1024 * 1024; + +fn mib_16(bytes: u64) -> u64 { + (bytes + 8 * MIB) / (16 * MIB) * 16 +} + +fn parse_report(report: &str) -> BTreeMap { + report + .lines() + .filter_map(|line| { + let mut parts = line.split_whitespace(); + let key = parts.next()?.to_string(); + let kb: u64 = parts.next()?.parse().ok()?; + Some((key, mib_16(kb * 1024))) + }) + .collect() +} + +#[derive(Debug, Serialize)] +struct PidRss { + pid: i32, + file_mib: u64, + anon_mib: u64, + shmem_mib: u64, + max_rss_mib: u64, +} + +#[derive(Serialize)] +struct RssSummary { + report: BTreeMap, + rss_stat: Vec, + rmap: Vec, +} + +#[derive(Default)] +struct RssAccum { + latest: [u64; 4], + peaks: [u64; 4], + max_rss: u64, +} + +impl RssAccum { + fn update_peaks(&mut self) { + for (peak, latest) in self.peaks.iter_mut().zip(self.latest) { + *peak = (*peak).max(latest); + } + self.max_rss = self + .max_rss + .max(self.latest[0] + self.latest[1] + self.latest[3]); + } +} + +#[derive(Default)] +struct RmapAccum { + totals: [i64; 4], + peaks: [i64; 4], + max_rss: i64, +} + +impl RmapAccum { + fn update_peaks(&mut self) { + for (peak, total) in self.peaks.iter_mut().zip(self.totals) { + *peak = (*peak).max(total); + } + self.max_rss = self + .max_rss + .max(self.totals[0] + self.totals[1] + self.totals[3]); + } +} + +fn per_pid_peaks(events: &[MemtrackEvent]) -> (Vec, Vec) { + // Pid values can wrap, so numeric order is not stable; both views emit + // their rows in one shared first-activity order, keeping the relative + // order of processes consistent between `rss_stat` and `rmap` after + // the pids are redacted. + let mut order: Vec = Vec::new(); + let mut rss: BTreeMap = BTreeMap::new(); + let mut rmap: BTreeMap = BTreeMap::new(); + + fn seen(order: &mut Vec, pid: i32) { + if !order.contains(&pid) { + order.push(pid); + } + } + + for event in events.iter().sorted_by_key(|event| event.timestamp) { + match event.kind { + MemtrackEventKind::Rss { member, size } => { + let Ok(index @ 0..4) = usize::try_from(member) else { + continue; + }; + seen(&mut order, event.pid); + let acc = rss.entry(event.pid).or_default(); + acc.latest[index] = size; + acc.update_peaks(); + } + MemtrackEventKind::Rmap { member, delta } => { + let Ok(index @ 0..4) = usize::try_from(member) else { + continue; + }; + seen(&mut order, event.pid); + let acc = rmap.entry(event.pid).or_default(); + acc.totals[index] += delta * 4096; + acc.update_peaks(); + } + MemtrackEventKind::Fork { parent_pid } => { + seen(&mut order, event.pid); + + let seed = rss.get(&parent_pid).map(|p| p.latest).unwrap_or_default(); + let acc = rss.entry(event.pid).or_default(); + acc.latest = seed; + acc.update_peaks(); + + let seed = rmap.get(&parent_pid).map(|p| p.totals).unwrap_or_default(); + let acc = rmap.entry(event.pid).or_default(); + acc.totals = seed; + acc.update_peaks(); + } + MemtrackEventKind::Exec | MemtrackEventKind::Exit => { + if let Some(acc) = rss.get_mut(&event.pid) { + acc.latest = [0; 4]; + } + if let Some(acc) = rmap.get_mut(&event.pid) { + acc.totals = [0; 4]; + } + } + _ => {} + } + } + + let rss_stat = order + .iter() + .filter_map(|pid| { + let acc = rss.remove(pid)?; + Some(PidRss { + pid: *pid, + file_mib: mib_16(acc.peaks[0]), + anon_mib: mib_16(acc.peaks[1]), + shmem_mib: mib_16(acc.peaks[3]), + max_rss_mib: mib_16(acc.max_rss), + }) + }) + .collect(); + let rmap = order + .iter() + .filter_map(|pid| { + let acc = rmap.remove(pid)?; + Some(PidRss { + pid: *pid, + file_mib: mib_16(acc.peaks[0].max(0) as u64), + anon_mib: mib_16(acc.peaks[1].max(0) as u64), + shmem_mib: mib_16(acc.peaks[3].max(0) as u64), + max_rss_mib: mib_16(acc.max_rss.max(0) as u64), + }) + }) + .collect(); + (rss_stat, rmap) +} + +/// Compile a fixture that writes a `/proc` RSS report to its argv[1], run it under +/// `track`, and return the raw report text alongside the collected events. +fn track_fixture( + source: &str, + name: &str, + track: impl FnOnce(Command) -> shared::TrackResult, +) -> Result<(String, Vec), Box> { + let temp_dir = TempDir::new()?; + std::fs::write( + temp_dir.path().join("rss_report.h"), + include_str!("../testdata/rss/rss_report.h"), + )?; + let binary = shared::compile_c_source(source, name, temp_dir.path())?; + let report_path = temp_dir.path().join(format!("{name}.report")); + let mut command = Command::new(&binary); + command.arg(&report_path); + + let (events, thread_handle) = track(command)?; + let raw_report = std::fs::read_to_string(&report_path)?; + thread_handle.join().unwrap(); + Ok((raw_report, events)) +} + +/// Pins reconstructed rmap addresses to the exact punched range: hole pages are +/// the only ones removed and later re-added; every other page in the region is +/// added first and only removed afterwards. +fn assert_rmap_hole_addresses( + events: &[MemtrackEvent], + base: u64, + hole_off: u64, + hole_len: u64, + len: u64, +) { + const PAGE: u64 = 4096; + let n_pages = (len / PAGE) as usize; + let mut first_remove = vec![u64::MAX; n_pages]; + let mut added = vec![false; n_pages]; + let mut readded = vec![false; n_pages]; + + for event in events.iter().sorted_by_key(|e| e.timestamp) { + let MemtrackEventKind::Rmap { delta, .. } = event.kind else { + continue; + }; + if event.addr < base || event.addr >= base + len { + continue; + } + let first = ((event.addr - base) / PAGE) as usize; + let last = (first + delta.unsigned_abs() as usize).min(n_pages); + for page in first..last { + if delta > 0 { + added[page] = true; + if event.timestamp > first_remove[page] { + readded[page] = true; + } + } else { + first_remove[page] = first_remove[page].min(event.timestamp); + } + } + } + + let hole = (hole_off / PAGE) as usize..((hole_off + hole_len) / PAGE) as usize; + for page in 0..n_pages { + assert!(added[page], "page {page} never saw an rmap add"); + assert_eq!( + readded[page], + hole.contains(&page), + "page {page}: remove-then-add pattern does not match the hole range" + ); + } +} + +#[test_with::env(GITHUB_ACTIONS)] +#[rstest] +#[case::anon(include_str!("../testdata/rss/anon.c"), "anon")] +#[case::file(include_str!("../testdata/rss/file.c"), "file")] +#[case::shmem(include_str!("../testdata/rss/shmem.c"), "shmem")] +#[case::fork(include_str!("../testdata/rss/fork.c"), "fork")] +#[case::fork_idle(include_str!("../testdata/rss/fork_idle.c"), "fork_idle")] +#[case::triangle(include_str!("../testdata/rss/triangle.c"), "triangle")] +#[case::madvise(include_str!("../testdata/rss/madvise.c"), "madvise")] +#[case::munmap_hole(include_str!("../testdata/rss/munmap_hole.c"), "munmap_hole")] +#[case::mremap_move(include_str!("../testdata/rss/mremap_move.c"), "mremap_move")] +fn test_rss_rmap_tracking( + #[case] source: &str, + #[case] name: &str, +) -> Result<(), Box> { + let (raw_report, events) = track_fixture(source, name, shared::track_command_with_rmap)?; + let (rss_stat, rmap) = per_pid_peaks(&events); + let summary = RssSummary { + report: parse_report(&raw_report), + rss_stat, + rmap, + }; + insta::assert_json_snapshot!(format!("rss_{name}"), summary, { + ".rss_stat[].pid" => "[pid]", + ".rmap[].pid" => "[pid]", + }); + + if let Some(layout) = raw_report + .lines() + .find_map(|line| line.strip_prefix("Layout:")) + { + let values: Vec = layout + .split_whitespace() + .map(|token| u64::from_str_radix(token.trim_start_matches("0x"), 16)) + .collect::>()?; + let [base, hole_off, hole_len, len] = values[..] else { + panic!("malformed Layout line: {layout}"); + }; + assert_rmap_hole_addresses(&events, base, hole_off, hole_len, len); + } + + // ThpKb > 0 means the MADV_HUGEPAGE region really faulted as PMD folios, so + // huge-folio accounting must be visible: a +512-page delta from the new-anon + // fault path and a -512-page delta that can only come from the + // folio_remove_rmap_pmd hook (MADV_DONTNEED / munmap of a pmd-mapped THP). + if let Some(thp) = raw_report + .lines() + .find_map(|line| line.strip_prefix("ThpKb:")) + { + let thp_kb = u64::from_str_radix(thp.trim().trim_start_matches("0x"), 16)?; + if thp_kb > 0 { + let deltas = events.iter().filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { delta, .. } => Some(delta), + _ => None, + }); + let (mut huge_add, mut huge_remove) = (false, false); + for delta in deltas { + huge_add |= delta >= 512; + huge_remove |= delta <= -512; + } + assert!( + huge_add, + "THP present ({thp_kb} kB) but no huge-folio rmap add" + ); + assert!( + huge_remove, + "THP present ({thp_kb} kB) but no pmd-sized rmap remove" + ); + } + } + Ok(()) +} + +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_external_reclaim() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/madvise_extern.c"), + "madvise_extern", + temp_dir.path(), + )?; + let (events, handle) = shared::track_command(Command::new(&binary))?; + handle.join().unwrap(); + + // Single fork: parent_pid == A (owner), event.pid == B (external caller, single-threaded + // so its tid == its pid). + let (a, b) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .expect("expected a fork event"); + + let peak = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rss { member: 0, size } if e.pid == a => Some(size), + _ => None, + }) + .max() + .unwrap_or(0); + assert!(peak >= 32 * MIB, "peak file RSS too small: {peak}"); + + // A file decrement owned by A but emitted from B's context: only present when + // out-of-context rss_stat updates are attributed to the owning process. + let external_decrement = events.iter().any(|e| { + e.pid == a + && e.tid == b + && matches!(e.kind, MemtrackEventKind::Rss { member: 0, size } if size < peak) + }); + assert!( + external_decrement, + "external file-RSS decrement not attributed to A (tid=B)" + ); + Ok(()) +} diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index d31bce09..f38edb7c 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -6,7 +6,7 @@ use runner_shared::artifacts::{MemtrackEvent as Event, MemtrackEventKind}; use std::path::Path; use std::process::Command; -type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; +pub type TrackResult = anyhow::Result<(Vec, std::thread::JoinHandle<()>)>; /// Asserts memory events using snapshot testing without marker filtering. /// @@ -27,6 +27,17 @@ macro_rules! assert_events_snapshot { // Dedup events by address and type to remove duplicates let events = $events .iter() + .filter(|e| { + // Allocation snapshots track only allocator events; RSS and + // process-lifecycle events are asserted by dedicated tests. + !matches!( + e.kind, + MemtrackEventKind::Rss { .. } + | MemtrackEventKind::Fork { .. } + | MemtrackEventKind::Exec + | MemtrackEventKind::Exit + ) + }) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) .collect::>(); @@ -76,6 +87,7 @@ macro_rules! assert_events_with_marker { // Remove events outside our 0xC0D59EED marker allocations let filtered_events = $events .iter() + .filter(|e| !matches!(e.kind, MemtrackEventKind::Rss { .. })) .sorted_by_key(|e| e.timestamp) .dedup_by(|a, b| a.addr == b.addr && discriminant(&a.kind) == discriminant(&b.kind)) .skip_while(|e| { @@ -141,12 +153,46 @@ pub fn track_binary(binary: &Path) -> TrackResult { track_command(Command::new(binary)) } +pub fn compile_c_source( + source_code: &str, + name: &str, + output_dir: &Path, +) -> Result> { + let source_path = output_dir.join(format!("{name}.c")); + let binary_path = output_dir.join(name); + std::fs::write(&source_path, source_code)?; + + let output = Command::new("gcc") + .args(["-O0", "-o", binary_path.to_str().unwrap()]) + .arg(&source_path) + .output()?; + if !output.status.success() { + error!("gcc stderr: {}", String::from_utf8_lossy(&output.stderr)); + return Err("Failed to compile C fixture".into()); + } + + Ok(binary_path) +} + /// Track a command, collecting all memory events. /// /// No allocators are pre-attached: the exec-mapping watcher discovers and /// attaches them as the tracked tree maps executable files. pub fn track_command(command: Command) -> TrackResult { - let tracker = Tracker::new()?; + track_command_impl(command, false) +} + +/// Track a command with folio rmap hooks enabled, reconstructing per-process RSS. +pub fn track_command_with_rmap(command: Command) -> TrackResult { + track_command_impl(command, true) +} + +fn track_command_impl(command: Command, track_rmap: bool) -> TrackResult { + let tracker = if track_rmap { + Tracker::new_without_allocators_with_rmap(true)? + } else { + Tracker::new()? + }; tracker.enable_tracking()?; let mut session = tracker.spawn(&command, None)?; diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_anon.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap new file mode 100644 index 00000000..2745b74d --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_file.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 0, + "RssFile:": 64, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 64, + "anon_mib": 0, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 64, + "anon_mib": 0, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap new file mode 100644 index 00000000..bc24de5b --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_fork.snap @@ -0,0 +1,42 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "ChildRssAnonKb:": 64, + "ParentRssAnonKb:": 32 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap new file mode 100644 index 00000000..3a97ec7d --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_fork_idle.snap @@ -0,0 +1,45 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "ChildRssAnon:": 32, + "MaxRssKb:": 32, + "RssAnon:": 32, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + }, + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_madvise.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap new file mode 100644 index 00000000..6e2a2202 --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_mremap_move.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 32, + "RssAnon:": 32, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 32, + "shmem_mib": 0, + "max_rss_mib": 32 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_munmap_hole.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap new file mode 100644 index 00000000..19bb441e --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_shmem.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 0, + "RssFile:": 0, + "RssShmem:": 64 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 0, + "shmem_mib": 64, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 0, + "shmem_mib": 64, + "max_rss_mib": 64 + } + ] +} diff --git a/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap b/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap new file mode 100644 index 00000000..bd39e6cc --- /dev/null +++ b/crates/memtrack/tests/snapshots/rss_tests__rss_triangle.snap @@ -0,0 +1,30 @@ +--- +source: crates/memtrack/tests/rss_tests.rs +expression: summary +--- +{ + "report": { + "MaxRssKb:": 64, + "RssAnon:": 64, + "RssFile:": 0, + "RssShmem:": 0 + }, + "rss_stat": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ], + "rmap": [ + { + "pid": "[pid]", + "file_mib": 0, + "anon_mib": 64, + "shmem_mib": 0, + "max_rss_mib": 64 + } + ] +} From e2b479d180dcedc50d430bd1b15cc5ccd7a65349 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 15:10:48 +0200 Subject: [PATCH 05/16] chore: wip [skip ci] --- .../memtrack/testdata/rss/rmap_late_enable.c | 59 +++++++ crates/memtrack/tests/rss_tests.rs | 147 ++++++++++++++++++ crates/memtrack/tests/shared.rs | 47 ++++++ justfile | 39 +++++ 4 files changed, 292 insertions(+) create mode 100644 crates/memtrack/testdata/rss/rmap_late_enable.c create mode 100644 justfile diff --git a/crates/memtrack/testdata/rss/rmap_late_enable.c b/crates/memtrack/testdata/rss/rmap_late_enable.c new file mode 100644 index 00000000..0bdae37b --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_late_enable.c @@ -0,0 +1,59 @@ +#include +#include +#include +#include + +#include "rss_report.h" + +/* Two-phase fixture that faults anonymous memory before and after a handshake: + * + * 1. Fault a baseline region, then signal `ready` and block on `go`. + * 2. After `go` appears, fault a second region of the same (anon) member. + * + * The handshake lets the caller act between the two phases (e.g. start + * observing only phase 2). Both regions touch MM_ANONPAGES, so an absolute + * counter read after phase 2 covers baseline + growth, while a delta observed + * only from phase 2 covers growth alone. + * + * argv: [1]=report path, [2]=ready path, [3]=go path. */ +static void touch(const char* path) { + FILE* f = fopen(path, "w"); + if (f) { + fclose(f); + } +} + +int main(int argc, char** argv) { + if (argc != 4) { + return 1; + } + const char* report_path = argv[1]; + const char* ready_path = argv[2]; + const char* go_path = argv[3]; + + size_t region = 64UL * 1024 * 1024; + + void* baseline = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (baseline == MAP_FAILED) { + return 1; + } + memset(baseline, 0x42, region); + + touch(ready_path); + while (access(go_path, F_OK) != 0) { + usleep(1000); + } + + void* growth = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (growth == MAP_FAILED) { + return 1; + } + memset(growth, 0x42, region); + + int ret = write_rss_report(report_path); + munmap(baseline, region); + munmap(growth, region); + return ret; +} diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs index d1099348..d12487ff 100644 --- a/crates/memtrack/tests/rss_tests.rs +++ b/crates/memtrack/tests/rss_tests.rs @@ -357,3 +357,150 @@ fn test_rss_external_reclaim() -> Result<(), Box> { ); Ok(()) } + +/// TEMPORARY diagnostic: track a real-world workload (`ls /nix/store`, ~50 MiB +/// peak on a populated store) and cross-check the reconstructed rss_stat and +/// rmap peaks against the kernel's own accounting (`wait4` ru_maxrss) from an +/// identical untracked run. `ls` is single-process, so raw byte peaks are +/// accumulated directly without per-pid splitting. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_ls_nix_store() -> Result<(), Box> { + if !std::path::Path::new("/nix/store").is_dir() { + eprintln!("skipping: /nix/store not available"); + return Ok(()); + } + + let ls_command = || { + let mut cmd = Command::new("ls"); + cmd.arg("/nix/store").stdout(std::process::Stdio::null()); + cmd + }; + + // Ground truth: identical untracked run, reaped via wait4 for ru_maxrss. + let child = ls_command().spawn()?; + let pid = child.id() as i32; + let mut status = 0i32; + let mut rusage: libc::rusage = unsafe { std::mem::zeroed() }; + let reaped = unsafe { libc::wait4(pid, &mut status, 0, &mut rusage) }; + assert_eq!(reaped, pid, "wait4 failed"); + assert!( + libc::WIFEXITED(status) && libc::WEXITSTATUS(status) == 0, + "untracked ls failed: status {status}" + ); + let truth_bytes = rusage.ru_maxrss as u64 * 1024; + + let (events, handle) = shared::track_command_with_rmap(ls_command())?; + handle.join().unwrap(); + + let mut rss = RssAccum::default(); + let mut rmap = RmapAccum::default(); + for event in events.iter().sorted_by_key(|event| event.timestamp) { + match event.kind { + MemtrackEventKind::Rss { member, size } => { + if let Ok(index @ 0..4) = usize::try_from(member) { + rss.latest[index] = size; + rss.update_peaks(); + } + } + MemtrackEventKind::Rmap { member, delta } => { + if let Ok(index @ 0..4) = usize::try_from(member) { + rmap.totals[index] += delta * 4096; + rmap.update_peaks(); + } + } + _ => {} + } + } + + let rss_bytes = rss.max_rss; + let rmap_bytes = rmap.max_rss.max(0) as u64; + eprintln!( + "ls /nix/store max RSS: wait4={:.1} MiB rss_stat={:.1} MiB rmap={:.1} MiB", + truth_bytes as f64 / MIB as f64, + rss_bytes as f64 / MIB as f64, + rmap_bytes as f64 / MIB as f64, + ); + + let within = |measured: u64| { + (truth_bytes as f64 * 0.8..=truth_bytes as f64 * 1.2).contains(&(measured as f64)) + }; + assert!( + within(rss_bytes), + "rss_stat peak {rss_bytes} outside 20% of wait4 {truth_bytes}" + ); + assert!( + within(rmap_bytes), + "rmap peak {rmap_bytes} outside 20% of wait4 {truth_bytes}" + ); + Ok(()) +} + +/// rss_stat is an absolute kernel counter; the rmap estimate is reconstructed +/// from zero by summing folio add/remove deltas. When tracking is enabled only +/// after a process has already faulted a resident region, rss_stat's first +/// reading includes that region but the rmap accumulator never saw its adds, so +/// rmap sits a fixed offset (the pre-enable resident set) below rss_stat. +/// +/// The fixture faults a 64 MiB anon baseline before `enable_tracking`, then a +/// 64 MiB anon region after. Reduced with the same Exec-reset lifecycle the +/// production parser uses, rss_stat peaks at ~128 MiB (absolute) while rmap +/// peaks at ~64 MiB (post-enable growth only). +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_late_enable_baseline_loss() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let temp_dir = TempDir::new()?; + std::fs::write( + temp_dir.path().join("rss_report.h"), + include_str!("../testdata/rss/rss_report.h"), + )?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/rmap_late_enable.c"), + "rmap_late_enable", + temp_dir.path(), + )?; + let report_path = temp_dir.path().join("rmap_late_enable.report"); + let ready_path = temp_dir.path().join("ready"); + let go_path = temp_dir.path().join("go"); + + let mut command = Command::new(&binary); + command.arg(&report_path).arg(&ready_path).arg(&go_path); + + let (events, handle) = + shared::track_command_with_rmap_late_enable(command, &ready_path, &go_path)?; + handle.join().unwrap(); + + let (rss_stat, rmap) = per_pid_peaks(&events); + let rss = rss_stat.first().ok_or("no rss_stat pid observed")?; + let rmap = rmap.first().ok_or("no rmap pid observed")?; + eprintln!( + "late-enable anon peaks: rss_stat={} MiB rmap={} MiB (region={} MiB each)", + rss.anon_mib, rmap.anon_mib, REGION_MIB + ); + + // rss_stat's absolute counter covers baseline + growth. + assert!( + rss.anon_mib >= 2 * REGION_MIB - 16, + "rss_stat anon peak {} MiB below the expected ~{} MiB baseline+growth", + rss.anon_mib, + 2 * REGION_MIB + ); + // Post-enable growth is visible to rmap. + assert!( + rmap.anon_mib >= REGION_MIB - 16, + "rmap anon peak {} MiB too small; post-enable growth should be tracked", + rmap.anon_mib + ); + // The reproduction: rmap misses the pre-enable baseline, undercounting + // rss_stat by roughly one region. + let gap = rss.anon_mib.saturating_sub(rmap.anon_mib); + assert!( + gap >= REGION_MIB - 16, + "expected rmap to undercount rss_stat by ~{} MiB (pre-enable baseline loss); gap was {} MiB", + REGION_MIB, + gap + ); + Ok(()) +} diff --git a/crates/memtrack/tests/shared.rs b/crates/memtrack/tests/shared.rs index f38edb7c..9d5a040a 100644 --- a/crates/memtrack/tests/shared.rs +++ b/crates/memtrack/tests/shared.rs @@ -214,3 +214,50 @@ fn track_command_impl(command: Command, track_rmap: bool) -> TrackResult { Ok((events, thread_handle)) } + +/// Track a command with rmap, enabling tracking only after the target creates +/// `ready_path`. The target is spawned and resumed first, so any memory it +/// faults before signalling `ready` is already resident when tracking turns on. +/// The caller enables tracking, then creates `go_path` to release the target. +pub fn track_command_with_rmap_late_enable( + command: Command, + ready_path: &Path, + go_path: &Path, +) -> TrackResult { + let tracker = Tracker::new_without_allocators_with_rmap(true)?; + + let mut session = tracker.spawn(&command, None)?; + let rx = session.take_events()?; + + let handshake = (|| -> anyhow::Result<()> { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while !ready_path.exists() { + if std::time::Instant::now() > deadline { + anyhow::bail!("target never signalled baseline-ready"); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } + tracker.enable_tracking()?; + std::fs::write(go_path, b"go")?; + Ok(()) + })(); + + // A failed handshake leaves the target blocked on `go_path`; Session has no + // Drop kill, so reap it explicitly before propagating or the test hangs. + if let Err(e) = handshake { + unsafe { libc::kill(session.pid(), libc::SIGKILL) }; + let _ = session.wait(); + let _ = tracker.finish(); + return Err(e); + } + + session.wait()?; + drop(session); + let events: Vec = rx.iter().collect(); + + tracker.finish()?; + let thread_handle = std::thread::spawn(move || drop(tracker)); + + info!("Tracked {} events (late enable)", events.len()); + Ok((events, thread_handle)) +} diff --git a/justfile b/justfile new file mode 100644 index 00000000..0b6c28f0 --- /dev/null +++ b/justfile @@ -0,0 +1,39 @@ +# memtrack's eBPF tests need the host kernel and root, so run them in a privileged +# container. --pid=host is required: eBPF filters on host-namespace PIDs, and without +# it the PID filter matches nothing (valid artifact, 0 events). The tracefs/debugfs +# mounts let libbpf read tracepoint IDs. GITHUB_ACTIONS gates the tests at BUILD time. + +image := "codspeed-memtrack-test" +root := justfile_directory() + +# Build the memtrack test container image. +memtrack-image: + docker build -t {{image}} {{root}}/.agents/docker + +# Run the memtrack RSS tests in docker. Pass any value for `update` to rewrite snapshots, +# e.g. `just memtrack-rss 1`. +memtrack-rss update="": memtrack-image + docker run --rm --privileged --pid=host \ + -v {{root}}:/work \ + -v codspeed-memtrack-target:/tmp/target \ + -v /sys/kernel/tracing:/sys/kernel/tracing:ro \ + -v /sys/kernel/debug:/sys/kernel/debug:ro \ + -e GITHUB_ACTIONS=1 \ + -e CARGO_TARGET_DIR=/tmp/target \ + {{ if update != "" { "-e INSTA_UPDATE=always" } else { "" } }} \ + -w /work {{image}} \ + cargo test -p memtrack --test rss_tests + +# TEMPORARY: cross-check tracked RSS of a real-world `ls /nix/store` run against +# wait4 rusage. Needs the host nix store mounted into the container. +memtrack-rss-ls: memtrack-image + docker run --rm --privileged --pid=host \ + -v {{root}}:/work \ + -v codspeed-memtrack-target:/tmp/target \ + -v /nix/store:/nix/store:ro \ + -v /sys/kernel/tracing:/sys/kernel/tracing:ro \ + -v /sys/kernel/debug:/sys/kernel/debug:ro \ + -e GITHUB_ACTIONS=1 \ + -e CARGO_TARGET_DIR=/tmp/target \ + -w /work {{image}} \ + cargo test -p memtrack --test rss_tests test_rss_ls_nix_store -- --nocapture From 3037b25f80ed367292bd8c564fc2a275380e08b0 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:06:53 +0200 Subject: [PATCH 06/16] fix(memtrack): support pre-memdesc folio->flags layout in rmap hooks --- crates/memtrack/src/ebpf/c/rss.bpf.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index 12eb12cb..1f02ea8a 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -75,8 +75,21 @@ int tracepoint_rss_stat(struct trace_event_raw_rss_stat* ctx) { return submit_rss_event(owner, ctx->member, size); } +/* Kernels < 6.18 store folio->flags as a bare unsigned long instead of + * memdesc_flags_t; probe which layout the running kernel has. */ +struct folio___legacy { + unsigned long flags; +} __attribute__((preserve_access_index)); + +static __always_inline unsigned long folio_read_flags(struct folio* folio) { + if (bpf_core_field_exists(folio->flags.f)) { + return BPF_CORE_READ(folio, flags).f; + } + return BPF_CORE_READ((struct folio___legacy*)folio, flags); +} + static __always_inline __u64 folio_nr_pages_est(struct folio* folio) { - unsigned long flags = BPF_CORE_READ(folio, flags).f; + unsigned long flags = folio_read_flags(folio); if (!(flags & (1UL << PG_head))) { return 1; } @@ -95,7 +108,7 @@ static __always_inline __s32 folio_mm_counter(struct folio* folio) { if (folio_is_anon(folio)) { return MM_ANONPAGES; } - unsigned long flags = BPF_CORE_READ(folio, flags).f; + unsigned long flags = folio_read_flags(folio); if (flags & (1UL << PG_swapbacked)) { return MM_SHMEMPAGES; } From a68424a886a989456a41a6b91c31b94a7492709a Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:07:42 +0200 Subject: [PATCH 07/16] ci: run memtrack rss_tests shard on old- and new-layout kernels --- .github/workflows/ci.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6935beb..e6aeb673 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,14 +88,22 @@ jobs: run: cargo run -- exec -m walltime --skip-upload --warmup-time 0s --max-rounds 5 -- ls -la bpf-tests: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: [ubuntu-latest] # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. - test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests] + test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] + include: + # The folio CO-RE relocations in the rmap hooks depend on kernel + # struct layouts that changed in 6.18; keep an older-kernel image in + # the matrix so the pre-6.18 layout stays covered once ubuntu-latest + # moves past it. + - os: ubuntu-22.04 + test: rss_tests steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: @@ -103,7 +111,7 @@ jobs: submodules: true - uses: ./.github/actions/install-rust with: - cache-key: ${{ matrix.test }} + cache-key: ${{ matrix.os }}-${{ matrix.test }} - uses: ./.github/actions/install-bpf-deps - name: Install additional allocators From 4e0fd914dc244d2757d081ee60e27ed79b934ba0 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:12:40 +0200 Subject: [PATCH 08/16] style(memtrack): reformat eBPF C for clang-format 21 --- crates/memtrack/src/ebpf/c/allocator.h | 34 ++++++++++++++------------ 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/allocator.h b/crates/memtrack/src/ebpf/c/allocator.h index 440d7a87..88566b36 100644 --- a/crates/memtrack/src/ebpf/c/allocator.h +++ b/crates/memtrack/src/ebpf/c/allocator.h @@ -5,22 +5,24 @@ #include "utils/map_helpers.h" #include "utils/process_tracking.h" -#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ - BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ - SEC("uprobe") \ - int uprobe_##name(struct pt_regs* ctx) { return store_param(&name##_arg, arg_expr); } \ - SEC("uretprobe") \ - int uretprobe_##name(struct pt_regs* ctx) { \ - __u64* arg_ptr = take_param(&name##_arg); \ - if (!arg_ptr) { \ - return 0; \ - } \ - __u64 ret_val = PT_REGS_RC(ctx); \ - if (ret_val == 0) { \ - return 0; \ - } \ - __u64 arg0 = *arg_ptr; \ - submit_block; \ +#define UPROBE_ARG_RET(name, arg_expr, submit_block) \ + BPF_HASH_MAP(name##_arg, __u64, __u64, 10000); \ + SEC("uprobe") \ + int uprobe_##name(struct pt_regs* ctx) { \ + return store_param(&name##_arg, arg_expr); \ + } \ + SEC("uretprobe") \ + int uretprobe_##name(struct pt_regs* ctx) { \ + __u64* arg_ptr = take_param(&name##_arg); \ + if (!arg_ptr) { \ + return 0; \ + } \ + __u64 ret_val = PT_REGS_RC(ctx); \ + if (ret_val == 0) { \ + return 0; \ + } \ + __u64 arg0 = *arg_ptr; \ + submit_block; \ } #define UPROBE_RET(name, arg_expr, submit_block) \ From 892b4191e2326aa085a7cb345cc0cbbfcdb08fa2 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:19:46 +0200 Subject: [PATCH 09/16] fix(memtrack): relocate pageflags values and folio order via CO-RE --- crates/memtrack/src/ebpf/c/rss.bpf.h | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index 1f02ea8a..ca208bf3 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -88,12 +88,25 @@ static __always_inline unsigned long folio_read_flags(struct folio* folio) { return BPF_CORE_READ((struct folio___legacy*)folio, flags); } +/* Kernels < 6.6 store the large-folio order in a dedicated byte instead of + * the low byte of _flags_1. */ +struct folio___order_byte { + unsigned char _folio_order; +} __attribute__((preserve_access_index)); + +static __always_inline unsigned long folio_order(struct folio* folio) { + if (bpf_core_field_exists(((struct folio___order_byte*)folio)->_folio_order)) { + return BPF_CORE_READ((struct folio___order_byte*)folio, _folio_order); + } + return BPF_CORE_READ(folio, _flags_1) & 0xff; +} + static __always_inline __u64 folio_nr_pages_est(struct folio* folio) { unsigned long flags = folio_read_flags(folio); - if (!(flags & (1UL << PG_head))) { + if (!(flags & (1UL << bpf_core_enum_value(enum pageflags, PG_head)))) { return 1; } - unsigned long order = BPF_CORE_READ(folio, _flags_1) & 0xff; + unsigned long order = folio_order(folio); return 1UL << order; } @@ -109,7 +122,7 @@ static __always_inline __s32 folio_mm_counter(struct folio* folio) { return MM_ANONPAGES; } unsigned long flags = folio_read_flags(folio); - if (flags & (1UL << PG_swapbacked)) { + if (flags & (1UL << bpf_core_enum_value(enum pageflags, PG_swapbacked))) { return MM_SHMEMPAGES; } return MM_FILEPAGES; From d29f54d23de7e2f6fa53511b0afd647242250b4f Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:19:47 +0200 Subject: [PATCH 10/16] ci: run memtrack bpf tests on arm runners --- .github/workflows/ci.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6aeb673..b06591ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,18 +92,21 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest] + os: [ubuntu-latest, ubuntu-24.04-arm] # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. test: [c_tests, cpp_tests, rust_tests, spawn_tests, dlopen_tests, rss_tests] include: # The folio CO-RE relocations in the rmap hooks depend on kernel - # struct layouts that changed in 6.18; keep an older-kernel image in - # the matrix so the pre-6.18 layout stays covered once ubuntu-latest - # moves past it. + # struct layouts that changed across versions (folio.flags in 6.18, + # pageflags values in 6.10/6.12); keep older-kernel images in the + # matrix so the legacy layouts stay covered once the latest images + # move past them. - os: ubuntu-22.04 test: rss_tests + - os: ubuntu-22.04-arm + test: rss_tests steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: From f3053581f28df0cfbcdecee35ac86ecd59a3b211 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:27:05 +0200 Subject: [PATCH 11/16] ci: run memtrack bpf tests on codspeed-macro runners --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b06591ca..6e85bebd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -92,7 +92,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, ubuntu-24.04-arm] + os: [ubuntu-latest, ubuntu-24.04-arm, codspeed-macro] # Each memtrack integration test binary runs its cases serially # (eBPF tracker can't overlap with itself in one process), so we # shard at the test-binary level to parallelize across jobs. @@ -126,8 +126,11 @@ jobs: run: sudo -E $(which cargo) test --lib --test ${{ matrix.test }} -- --test-threads 1 --nocapture working-directory: crates/memtrack - # Since we ran the tests with sudo, the build artifacts will have root ownership + # Since we ran the tests with sudo, the build artifacts will have root + # ownership; always restore it so a failed run cannot poison the + # workspace of later jobs on persistent self-hosted runners. - name: Clean up + if: always() run: sudo chown -R $USER:$USER . ~/.cargo benchmarks: From 960fb929c716cd18fff145ed3e252053b145b36b Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:35:47 +0200 Subject: [PATCH 12/16] ci: print kernel version in bpf-tests jobs --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e85bebd..d9474bcc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,6 +112,8 @@ jobs: with: lfs: true submodules: true + - name: Show kernel version + run: uname -a - uses: ./.github/actions/install-rust with: cache-key: ${{ matrix.os }}-${{ matrix.test }} From d20ebd552078a354b61953c425559f09267c8fef Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 18:54:25 +0200 Subject: [PATCH 13/16] ci: probe rmap BTF symbols in bpf-tests --- .github/workflows/ci.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9474bcc..8ebce530 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,8 +112,18 @@ jobs: with: lfs: true submodules: true - - name: Show kernel version - run: uname -a + - name: Show kernel version and rmap BTF symbols + run: | + uname -rm + for sym in folio_add_file_rmap_ptes folio_remove_rmap_ptes \ + folio_add_new_anon_rmap page_add_file_rmap \ + page_remove_rmap page_add_anon_rmap; do + if strings /sys/kernel/btf/vmlinux | grep -qx "$sym"; then + echo "btf: $sym present" + else + echo "btf: $sym ABSENT" + fi + done - uses: ./.github/actions/install-rust with: cache-key: ${{ matrix.os }}-${{ matrix.test }} From 62aa408f763a0ceb2c0d59516f2bb8cecff62b80 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 19:06:37 +0200 Subject: [PATCH 14/16] ci: verify rmap symbols are BTF FUNC kind --- .github/workflows/ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ebce530..5ceef720 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,13 +115,18 @@ jobs: - name: Show kernel version and rmap BTF symbols run: | uname -rm + sudo apt-get update -qq + sudo apt-get install -y -qq linux-tools-common "linux-tools-$(uname -r)" || true for sym in folio_add_file_rmap_ptes folio_remove_rmap_ptes \ folio_add_new_anon_rmap page_add_file_rmap \ page_remove_rmap page_add_anon_rmap; do - if strings /sys/kernel/btf/vmlinux | grep -qx "$sym"; then - echo "btf: $sym present" + if sudo bpftool btf dump file /sys/kernel/btf/vmlinux format raw 2>/dev/null \ + | grep -q "FUNC '$sym'"; then + echo "btf-func: $sym present" + elif strings /sys/kernel/btf/vmlinux | grep -qx "$sym"; then + echo "btf-str: $sym present (FUNC kind unverified)" else - echo "btf: $sym ABSENT" + echo "btf: $sym ABSENT" fi done - uses: ./.github/actions/install-rust From fad337a04300e9a5fd84eef1a453397c706d7efa Mon Sep 17 00:00:00 2001 From: not-matthias Date: Mon, 20 Jul 2026 19:14:26 +0200 Subject: [PATCH 15/16] ci: slim bpf-tests kernel diagnostic to uname --- .github/workflows/ci.yml | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ceef720..20572ef9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -112,23 +112,8 @@ jobs: with: lfs: true submodules: true - - name: Show kernel version and rmap BTF symbols - run: | - uname -rm - sudo apt-get update -qq - sudo apt-get install -y -qq linux-tools-common "linux-tools-$(uname -r)" || true - for sym in folio_add_file_rmap_ptes folio_remove_rmap_ptes \ - folio_add_new_anon_rmap page_add_file_rmap \ - page_remove_rmap page_add_anon_rmap; do - if sudo bpftool btf dump file /sys/kernel/btf/vmlinux format raw 2>/dev/null \ - | grep -q "FUNC '$sym'"; then - echo "btf-func: $sym present" - elif strings /sys/kernel/btf/vmlinux | grep -qx "$sym"; then - echo "btf-str: $sym present (FUNC kind unverified)" - else - echo "btf: $sym ABSENT" - fi - done + - name: Show kernel version + run: uname -rm - uses: ./.github/actions/install-rust with: cache-key: ${{ matrix.os }}-${{ matrix.test }} From b8c2c4fe9b1d12e826dc393ea4e4df95fa92cd79 Mon Sep 17 00:00:00 2001 From: not-matthias Date: Tue, 21 Jul 2026 16:36:04 +0200 Subject: [PATCH 16/16] chore: wip [skip ci] --- crates/memtrack/src/ebpf/c/rss.bpf.h | 141 ++++++++++++- .../memtrack/src/ebpf/c/utils/event_helpers.h | 34 ++- .../src/ebpf/c/utils/process_tracking.h | 12 -- crates/memtrack/src/ebpf/memtrack/tracking.rs | 2 - crates/memtrack/src/ebpf/tracker.rs | 6 +- crates/memtrack/src/main.rs | 9 +- .../memtrack/testdata/rss/rmap_leader_exit.c | 74 +++++++ .../memtrack/testdata/rss/rmap_thread_fork.c | 60 ++++++ crates/memtrack/testdata/rss/rss_report.h | 18 +- crates/memtrack/tests/rss_tests.rs | 197 ++++++++++++++++-- 10 files changed, 491 insertions(+), 62 deletions(-) create mode 100644 crates/memtrack/testdata/rss/rmap_leader_exit.c create mode 100644 crates/memtrack/testdata/rss/rmap_thread_fork.c diff --git a/crates/memtrack/src/ebpf/c/rss.bpf.h b/crates/memtrack/src/ebpf/c/rss.bpf.h index ca208bf3..2850a634 100644 --- a/crates/memtrack/src/ebpf/c/rss.bpf.h +++ b/crates/memtrack/src/ebpf/c/rss.bpf.h @@ -22,6 +22,30 @@ struct { __type(value, struct rss_owner); } mm_to_pid SEC(".maps"); +/* Foreign-actor rmap attribution: rmap events run by a task other than the mm's + * owner (kswapd reclaim, another process's process_madvise, khugepaged, KSM, + * uffd) carry no owning-pid context, so mm_owner recovers it from the mm_struct + * pointer. pid_mm is the inverse, letting the exec and exit hooks remove an entry + * by value. + * + * Lifecycle invariant: every mm_owner entry is removed when its process execs + * (the old mm is freed mid-life) or when its thread group dies, whichever comes + * first; LRU eviction is only a backstop. A stale entry surviving mm-pointer + * reuse would misattribute another process's events, so ownership is only ever + * registered from an in-context (task->mm == mm) event. + * + * pid_mm is a plain hash on purpose: an LRU inverse could be evicted while its + * forward twin stays lookup-hot, leaving exec/exit unable to remove the live + * mm_owner entry. Like tracked_pids, its entries are bound to the process + * lifecycle and removed at group death. */ +struct { + __uint(type, BPF_MAP_TYPE_LRU_HASH); + __uint(max_entries, 10240); + __type(key, __u64); + __type(value, __u32); +} mm_owner SEC(".maps"); +BPF_HASH_MAP(pid_mm, __u32, __u64, 10240); + #define FOLIO_MAPPING_ANON 0x1UL const volatile __u32 page_shift = 12; @@ -144,17 +168,50 @@ static __always_inline int submit_rmap(struct vm_area_struct* vma, __s32 member, __u64 addr) { __u64 mm = (__u64)BPF_CORE_READ(vma, vm_mm); struct task_struct* task = bpf_get_current_task_btf(); - if ((__u64)BPF_CORE_READ(task, mm) != mm) { - return 0; - } - __u64 tid = bpf_get_current_pid_tgid(); __u32 pid = tid >> 32; - if (!is_tracked(pid)) { + + if ((__u64)BPF_CORE_READ(task, mm) == mm) { + if (!is_tracked(pid)) { + return 0; + } + + /* Register ownership so foreign actors can later attribute to this pid. + * Both maps are validated (not just written) on every in-context event: + * the lookups keep the hot path cheap AND keep both entries LRU-fresh, + * since pid_mm is otherwise never read until exec/exit and could be + * evicted independently of its still-hot mm_owner twin. */ + __u32* owner = bpf_map_lookup_elem(&mm_owner, &mm); + if (!owner || *owner != pid) { + bpf_map_update_elem(&mm_owner, &mm, &pid, BPF_ANY); + } + __u64* cur_mm = bpf_map_lookup_elem(&pid_mm, &pid); + if (!cur_mm || *cur_mm != mm) { + bpf_map_update_elem(&pid_mm, &pid, &mm, BPF_ANY); + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_RMAP, { + e->data.rmap.member = member; + e->data.rmap.delta = delta; + e->data.rmap.addr = addr; + }); + } + + /* Foreign actor (task->mm != mm, including kthreads whose task->mm is NULL): + * recover the owner from the in-context registration. Fail toward dropping + * the event on any uncertainty about ownership. */ + __u32* found = bpf_map_lookup_elem(&mm_owner, &mm); + if (!found) { + return 0; + } + __u32 owner = *found; + if (!is_tracked(owner)) { return 0; } - SUBMIT_EVENT_AS(pid, EVENT_TYPE_RMAP, { + /* SUBMIT_EVENT_AS stamps header.tid from the current task, identifying the + * foreign actor that performed the rmap change. */ + SUBMIT_EVENT_AS(owner, EVENT_TYPE_RMAP, { e->data.rmap.member = member; e->data.rmap.delta = delta; e->data.rmap.addr = addr; @@ -245,23 +302,87 @@ int tracepoint_task_newtask(struct trace_event_raw_task_newtask* ctx) { return 0; } + /* Register the child here rather than on sched_process_fork: that + * tracepoint fires for CLONE_THREAD too and carries only raw task pids, + * which would fill the tracking maps with thread tids that no exit path + * removes (group death deletes only the tgid). task_newtask fires before + * wake_up_new_task, so registration precedes any event from the child. */ __u32 child_pid = ctx->pid; + track_child(child_pid, parent_pid); + SUBMIT_EVENT_AS(child_pid, EVENT_TYPE_FORK, { e->data.fork.parent_pid = parent_pid; }); } +/* Remove pid's ownership registration. The mm_owner value is verified against + * pid before deleting: a stale pid_mm entry (LRU eviction skew) could otherwise + * point at an mm since re-registered by another process, and deleting that + * would silence a live owner's foreign attribution. */ +static __always_inline void drop_mm_ownership(__u32 pid) { + __u64* mm = bpf_map_lookup_elem(&pid_mm, &pid); + if (mm) { + __u32* owner = bpf_map_lookup_elem(&mm_owner, mm); + if (owner && *owner == pid) { + bpf_map_delete_elem(&mm_owner, mm); + } + } + bpf_map_delete_elem(&pid_mm, &pid); +} + SEC("tracepoint/sched/sched_process_exec") int tracepoint_sched_process_exec(void* ctx) { - SUBMIT_EVENT(EVENT_TYPE_EXEC, {}); + __u32 pid = bpf_get_current_pid_tgid() >> 32; + if (!is_tracked(pid)) { + return 0; + } + + /* Maintain ownership before submitting (SUBMIT_EVENT_AS returns from the + * function). Exec frees the old mm long before group death, so the stale + * pointer must be dropped here or a reused mm_struct would be misattributed. */ + drop_mm_ownership(pid); + + struct task_struct* task = bpf_get_current_task_btf(); + __u64 new_mm = (__u64)BPF_CORE_READ(task, mm); + if (new_mm) { + bpf_map_update_elem(&mm_owner, &new_mm, &pid, BPF_ANY); + bpf_map_update_elem(&pid_mm, &pid, &new_mm, BPF_ANY); + } + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXEC, {}); } SEC("tracepoint/sched/sched_process_exit") int tracepoint_sched_process_exit(void* ctx) { - __u64 tid = bpf_get_current_pid_tgid(); - if ((tid >> 32) != (tid & 0xFFFFFFFF)) { + __u32 pid = bpf_get_current_pid_tgid() >> 32; + if (!is_tracked(pid)) { return 0; } - SUBMIT_EVENT(EVENT_TYPE_EXIT, {}); + /* EXIT marks the death of the whole thread group, not of one thread: the + * leader can pthread_exit while workers keep running, and the last thread + * to exit need not be the leader. do_exit decrements signal->live before + * this tracepoint fires, so live == 0 identifies the dying thread group's + * final exit — but concurrently exiting threads can BOTH read 0, so the + * tracked_pids delete below arbitrates: only the task that wins it emits. */ + struct task_struct* task = bpf_get_current_task_btf(); + if (BPF_CORE_READ(task, signal, live.counter) != 0) { + return 0; + } + + /* Untrack the pid before submitting: lifetime events are gated only on + * is_tracked, so a stale entry would keep streaming events if the kernel + * reuses the pid for an unrelated process. Untracking here also keeps the + * fixed-size tracking maps from filling up over long sessions. The delete + * doubles as the exactly-once claim on EXIT. */ + if (bpf_map_delete_elem(&tracked_pids, &pid) != 0) { + return 0; + } + bpf_map_delete_elem(&pids_ppid, &pid); + + /* Drop the ownership mapping so foreign actors stop attributing to a pid + * the kernel may reuse. */ + drop_mm_ownership(pid); + + SUBMIT_EVENT_AS(pid, EVENT_TYPE_EXIT, {}); } #endif /* __RSS_BPF_H__ */ diff --git a/crates/memtrack/src/ebpf/c/utils/event_helpers.h b/crates/memtrack/src/ebpf/c/utils/event_helpers.h index 1be39e7e..fbbc0536 100644 --- a/crates/memtrack/src/ebpf/c/utils/event_helpers.h +++ b/crates/memtrack/src/ebpf/c/utils/event_helpers.h @@ -38,14 +38,19 @@ static __always_inline __u64* take_param(void* map) { return value; } +/* Submission is split into two classes: + * - lifetime events (rss_stat, rmap, fork/exec/exit): emitted whenever the + * pid is tracked, ignoring the enable toggle. The parser reconstructs + * absolute per-process state from these, so it needs every event from + * process birth — a delta stream that starts mid-life can never recover + * the resident baseline faulted before enable. + * - allocator events (malloc/free/mmap/...): high-volume and only meaningful + * inside a measurement window, so they stay behind is_enabled(). + */ #define SUBMIT_EVENT_AS(owner_pid, evt_type, fill_data) \ { \ __u64 tid = bpf_get_current_pid_tgid(); \ \ - if (!is_enabled()) { \ - return 0; \ - } \ - \ struct event* e = bpf_ringbuf_reserve(&events, sizeof(*e), 0); \ if (!e) { \ __u32 zero = 0; \ @@ -79,33 +84,42 @@ static __always_inline __u64* take_param(void* map) { SUBMIT_EVENT_AS(pid, evt_type, fill_data); \ } +#define SUBMIT_GATED_EVENT(evt_type, fill_data) \ + { \ + if (!is_enabled()) { \ + return 0; \ + } \ + \ + SUBMIT_EVENT(evt_type, fill_data); \ + } + static __always_inline int submit_alloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_MALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_MALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_aligned_alloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_ALIGNED_ALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_calloc_event(__u64 size, __u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_CALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_CALLOC, { e->data.alloc.addr = addr; e->data.alloc.size = size; }); } static __always_inline int submit_free_event(__u64 addr) { - SUBMIT_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); + SUBMIT_GATED_EVENT(EVENT_TYPE_FREE, { e->data.free.addr = addr; }); } static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, __u64 size) { - SUBMIT_EVENT(EVENT_TYPE_REALLOC, { + SUBMIT_GATED_EVENT(EVENT_TYPE_REALLOC, { e->data.realloc.old_addr = old_addr; e->data.realloc.new_addr = new_addr; e->data.realloc.size = size; @@ -113,7 +127,7 @@ static __always_inline int submit_realloc_event(__u64 old_addr, __u64 new_addr, } static __always_inline int submit_mmap_event(__u64 addr, __u64 size, __u8 event_type) { - SUBMIT_EVENT(event_type, { + SUBMIT_GATED_EVENT(event_type, { e->data.mmap.addr = addr; e->data.mmap.size = size; }); diff --git a/crates/memtrack/src/ebpf/c/utils/process_tracking.h b/crates/memtrack/src/ebpf/c/utils/process_tracking.h index 19cff338..69dd2a73 100644 --- a/crates/memtrack/src/ebpf/c/utils/process_tracking.h +++ b/crates/memtrack/src/ebpf/c/utils/process_tracking.h @@ -43,16 +43,4 @@ static __always_inline void track_child(__u32 child_pid, __u32 parent_pid) { bpf_map_update_elem(&pids_ppid, &child_pid, &parent_pid, BPF_ANY); } -SEC("tracepoint/sched/sched_process_fork") -int tracepoint_sched_fork(struct trace_event_raw_sched_process_fork* ctx) { - __u32 parent_pid = ctx->parent_pid; - __u32 child_pid = ctx->child_pid; - - if (is_tracked(parent_pid)) { - track_child(child_pid, parent_pid); - } - - return 0; -} - #endif /* __PROCESS_TRACKING_H__ */ diff --git a/crates/memtrack/src/ebpf/memtrack/tracking.rs b/crates/memtrack/src/ebpf/memtrack/tracking.rs index bc8156c4..38e11a93 100644 --- a/crates/memtrack/src/ebpf/memtrack/tracking.rs +++ b/crates/memtrack/src/ebpf/memtrack/tracking.rs @@ -3,7 +3,6 @@ use crate::prelude::*; use paste::paste; impl MemtrackBpf { - attach_tracepoint!(sched_fork); attach_tracepoint!(rss_stat); attach_tracepoint!(task_newtask); attach_tracepoint!(sched_process_exec); @@ -50,7 +49,6 @@ impl MemtrackBpf { } pub fn attach_tracepoints(&mut self) -> Result<()> { - self.attach_sched_fork()?; self.attach_task_newtask()?; self.attach_sched_process_exec()?; self.attach_sched_process_exit()?; diff --git a/crates/memtrack/src/ebpf/tracker.rs b/crates/memtrack/src/ebpf/tracker.rs index ff6c0881..83b34853 100644 --- a/crates/memtrack/src/ebpf/tracker.rs +++ b/crates/memtrack/src/ebpf/tracker.rs @@ -79,12 +79,14 @@ impl Tracker { Ok(Session::new(child, rx, poller)) } - /// Enable event tracking in the BPF program + /// Enable allocator-event tracking in the BPF program. Lifetime events + /// (rss_stat, rmap, fork/exec/exit) are emitted for tracked pids + /// regardless of this toggle. pub fn enable_tracking(&self) -> Result<()> { self.bpf.lock().enable_tracking() } - /// Disable event tracking in the BPF program + /// Disable allocator-event tracking in the BPF program pub fn disable_tracking(&self) -> Result<()> { self.bpf.lock().disable_tracking() } diff --git a/crates/memtrack/src/main.rs b/crates/memtrack/src/main.rs index 8ad48791..283cff19 100644 --- a/crates/memtrack/src/main.rs +++ b/crates/memtrack/src/main.rs @@ -95,8 +95,9 @@ fn track_command( } })) } else { - // Without IPC, nothing toggles the tracking_enabled map, so events would - // be dropped by the eBPF is_enabled() check. Enable it up front. + // Without IPC, nothing toggles the tracking_enabled map, so allocator + // events would be dropped by the eBPF is_enabled() check. Enable it up + // front. tracker.enable_tracking()?; None }; @@ -135,8 +136,8 @@ fn track_command( let status = session.wait().context("Failed to wait for command")?; debug!("Command exited with status: {status}"); - // Stop event production before draining: the child has exited, so anything - // still arriving is already in the ring buffer. + // Stop allocator-event production before draining: the child has exited, + // so anything still arriving is already in the ring buffer. if let Err(e) = tracker.disable_tracking() { warn!("Failed to disable tracking: {e:#}"); } diff --git a/crates/memtrack/testdata/rss/rmap_leader_exit.c b/crates/memtrack/testdata/rss/rmap_leader_exit.c new file mode 100644 index 00000000..51e7b23c --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_leader_exit.c @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* The main thread publishes its tid and leaves via pthread_exit while a worker + * thread keeps the process alive. The worker waits until the leader is a + * zombie — its exit path, including the sched_process_exit tracepoint, has + * fully run — then faults an anon region, writes the report, and exits the + * whole group. A tracker keyed on leader exit instead of thread-group death + * would stop watching before the worker's region is faulted. + * + * argv: [1]=report path. */ + +static const char* report_path; +static pid_t leader_tid; + +static int leader_is_zombie(void) { + char path[64]; + char buf[256]; + snprintf(path, sizeof(path), "/proc/self/task/%d/stat", leader_tid); + FILE* f = fopen(path, "r"); + if (!f) { + return 1; + } + size_t n = fread(buf, 1, sizeof(buf) - 1, f); + fclose(f); + buf[n] = '\0'; + /* The state field follows the parenthesized comm. */ + const char* p = strrchr(buf, ')'); + if (!p || p[1] == '\0' || p[2] == '\0') { + return 0; + } + return p[2] == 'Z'; +} + +static void* worker(void* arg) { + (void)arg; + while (!leader_is_zombie()) { + usleep(1000); + } + + size_t region = 64UL * 1024 * 1024; + void* mem = + mmap(NULL, region, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + _exit(1); + } + memset(mem, 0x42, region); + + /* The leader is a zombie by now, so /proc//status has no Rss lines; + * report through this worker's own task instead. */ + int ret = write_rss_report_pid((int)syscall(SYS_gettid), report_path); + munmap(mem, region); + _exit(ret); +} + +int main(int argc, char** argv) { + if (argc != 2) { + return 1; + } + report_path = argv[1]; + leader_tid = (pid_t)syscall(SYS_gettid); + + pthread_t thread; + if (pthread_create(&thread, NULL, worker, NULL) != 0) { + return 1; + } + pthread_exit(NULL); +} diff --git a/crates/memtrack/testdata/rss/rmap_thread_fork.c b/crates/memtrack/testdata/rss/rmap_thread_fork.c new file mode 100644 index 00000000..831fd110 --- /dev/null +++ b/crates/memtrack/testdata/rss/rmap_thread_fork.c @@ -0,0 +1,60 @@ +#include +#include +#include +#include +#include + +#include "rss_report.h" + +/* A worker thread (not the group leader) calls fork(); the child faults an + * anon region and writes the report. Child tracking must key on the parent's + * TGID: a scheme keyed on the raw creator tid would only cover forks issued + * by the leader. + * + * argv: [1]=report path. */ + +static const char* report_path; + +static void* worker(void* arg) { + (void)arg; + + pid_t pid = fork(); + if (pid < 0) { + _exit(1); + } + if (pid == 0) { + size_t region = 64UL * 1024 * 1024; + void* mem = mmap(NULL, region, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mem == MAP_FAILED) { + _exit(1); + } + memset(mem, 0x42, region); + + int ret = write_rss_report(report_path); + munmap(mem, region); + _exit(ret); + } + + int status; + if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + _exit(1); + } + return NULL; +} + +int main(int argc, char** argv) { + if (argc != 2) { + return 1; + } + report_path = argv[1]; + + pthread_t thread; + if (pthread_create(&thread, NULL, worker, NULL) != 0) { + return 1; + } + if (pthread_join(thread, NULL) != 0) { + return 1; + } + return 0; +} diff --git a/crates/memtrack/testdata/rss/rss_report.h b/crates/memtrack/testdata/rss/rss_report.h index e144d145..169bdc07 100644 --- a/crates/memtrack/testdata/rss/rss_report.h +++ b/crates/memtrack/testdata/rss/rss_report.h @@ -26,14 +26,18 @@ static long rss_status_kb(const char* key) { return rss_status_kb_pid(getpid(), key); } -static int write_rss_report(const char* path) { - long anon = rss_status_kb("RssAnon:"); - long file = rss_status_kb("RssFile:"); - long shmem = rss_status_kb("RssShmem:"); +/* Reads Rss* through /proc//status of the given task. For the + * thread-group dir this is the leader's task: once the leader is a zombie its + * mm pointer is gone and the Rss lines disappear, so callers whose leader has + * exited must pass a live thread's tid instead. */ +static int write_rss_report_pid(int pid, const char* path) { + long anon = rss_status_kb_pid(pid, "RssAnon:"); + long file = rss_status_kb_pid(pid, "RssFile:"); + long shmem = rss_status_kb_pid(pid, "RssShmem:"); /* VmHWM instead of getrusage(): ru_maxrss includes signal->maxrss, which * survives execve and so reports the peak of the pre-exec parent image. * VmHWM belongs to the mm and starts fresh at exec. */ - long max_rss = rss_status_kb("VmHWM:"); + long max_rss = rss_status_kb_pid(pid, "VmHWM:"); if (anon < 0 || file < 0 || shmem < 0 || max_rss < 0) { return 1; } @@ -47,4 +51,8 @@ static int write_rss_report(const char* path) { return 0; } +static int write_rss_report(const char* path) { + return write_rss_report_pid(getpid(), path); +} + #endif diff --git a/crates/memtrack/tests/rss_tests.rs b/crates/memtrack/tests/rss_tests.rs index d12487ff..be834fd2 100644 --- a/crates/memtrack/tests/rss_tests.rs +++ b/crates/memtrack/tests/rss_tests.rs @@ -194,6 +194,11 @@ fn track_fixture( /// Pins reconstructed rmap addresses to the exact punched range: hole pages are /// the only ones removed and later re-added; every other page in the region is /// added first and only removed afterwards. +/// +/// Only own-context events (tid == pid; the fixture is single-threaded) are +/// considered: foreign actors like kcompactd migrating a page produce a +/// remove-then-add on arbitrary pages, which the attribution of foreign rmap +/// events makes visible here. fn assert_rmap_hole_addresses( events: &[MemtrackEvent], base: u64, @@ -211,6 +216,9 @@ fn assert_rmap_hole_addresses( let MemtrackEventKind::Rmap { delta, .. } = event.kind else { continue; }; + if event.tid != event.pid { + continue; + } if event.addr < base || event.addr >= base + len { continue; } @@ -358,6 +366,68 @@ fn test_rss_external_reclaim() -> Result<(), Box> { Ok(()) } +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_external_reclaim() -> Result<(), Box> { + let temp_dir = TempDir::new()?; + let binary = shared::compile_c_source( + include_str!("../testdata/rss/madvise_extern.c"), + "madvise_extern", + temp_dir.path(), + )?; + let (events, handle) = shared::track_command_with_rmap(Command::new(&binary))?; + handle.join().unwrap(); + + // Single fork: parent_pid == A (owner that faulted the file region), event.pid == B + // (external caller of process_madvise(MADV_PAGEOUT) against A from its own context). + let (a, b) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .expect("expected a fork event"); + + // Sanity: A's own in-context file-page faults are reconstructed by rmap. + let in_context_add: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 0, delta } if e.pid == a && delta > 0 => Some(delta), + _ => None, + }) + .sum(); + let in_context_bytes = in_context_add as u64 * 4096; + assert!( + in_context_bytes >= 32 * MIB, + "in-context file rmap adds too small: {in_context_bytes}" + ); + + // The point of the test: a file-page remove owned by A but emitted from B's + // context (tid == B), only present when the foreign reclaim's rmap events are + // attributed to the owning process via the mm_owner map. + let external_removed: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 0, delta } + if e.pid == a && e.tid == b && delta < 0 => + { + Some(-delta) + } + _ => None, + }) + .sum(); + assert!( + external_removed > 0, + "external MADV_PAGEOUT remove not attributed to the owner (pid=A, tid=B)" + ); + let external_bytes = external_removed as u64 * 4096; + assert!( + external_bytes >= 8 * MIB, + "external MADV_PAGEOUT remove not attributed to the owner: only {external_bytes} bytes" + ); + Ok(()) +} + /// TEMPORARY diagnostic: track a real-world workload (`ls /nix/store`, ~50 MiB /// peak on a populated store) and cross-check the reconstructed rss_stat and /// rmap peaks against the kernel's own accounting (`wait4` ru_maxrss) from an @@ -437,18 +507,18 @@ fn test_rss_ls_nix_store() -> Result<(), Box> { } /// rss_stat is an absolute kernel counter; the rmap estimate is reconstructed -/// from zero by summing folio add/remove deltas. When tracking is enabled only -/// after a process has already faulted a resident region, rss_stat's first -/// reading includes that region but the rmap accumulator never saw its adds, so -/// rmap sits a fixed offset (the pre-enable resident set) below rss_stat. +/// from zero by summing folio add/remove deltas. Both are emitted for the +/// whole lifetime of a tracked pid, independent of the enable toggle (which +/// only gates allocator events): a delta stream that starts mid-life could +/// never recover the resident baseline faulted before enable. /// /// The fixture faults a 64 MiB anon baseline before `enable_tracking`, then a /// 64 MiB anon region after. Reduced with the same Exec-reset lifecycle the -/// production parser uses, rss_stat peaks at ~128 MiB (absolute) while rmap -/// peaks at ~64 MiB (post-enable growth only). +/// production parser uses, both series peak at ~128 MiB: the pre-enable +/// baseline is visible to rmap because the pid is tracked from spawn. #[test_with::env(GITHUB_ACTIONS)] #[test] -fn test_rss_rmap_late_enable_baseline_loss() -> Result<(), Box> { +fn test_rss_rmap_late_enable_covers_baseline() -> Result<(), Box> { const REGION_MIB: u64 = 64; let temp_dir = TempDir::new()?; @@ -487,20 +557,113 @@ fn test_rss_rmap_late_enable_baseline_loss() -> Result<(), Box= REGION_MIB - 16, - "rmap anon peak {} MiB too small; post-enable growth should be tracked", - rmap.anon_mib + rmap.anon_mib >= 2 * REGION_MIB - 16, + "rmap anon peak {} MiB misses the pre-enable baseline; expected ~{} MiB", + rmap.anon_mib, + 2 * REGION_MIB ); - // The reproduction: rmap misses the pre-enable baseline, undercounting - // rss_stat by roughly one region. - let gap = rss.anon_mib.saturating_sub(rmap.anon_mib); + let gap = rss.anon_mib.abs_diff(rmap.anon_mib); assert!( - gap >= REGION_MIB - 16, - "expected rmap to undercount rss_stat by ~{} MiB (pre-enable baseline loss); gap was {} MiB", - REGION_MIB, + gap <= 16, + "rss_stat ({} MiB) and rmap ({} MiB) diverged by {} MiB despite lifetime tracking", + rss.anon_mib, + rmap.anon_mib, gap ); Ok(()) } + +/// The leader thread can pthread_exit while worker threads keep the process +/// alive; EXIT must mark thread-group death, not leader exit. The fixture's +/// worker faults a 64 MiB anon region only after the leader is a zombie, so +/// an exit path keyed on the leader would untrack the pid before the region +/// is faulted and emit EXIT ahead of the worker's rmap events. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_leader_exit_keeps_tracking() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let (_raw_report, events) = track_fixture( + include_str!("../testdata/rss/rmap_leader_exit.c"), + "rmap_leader_exit", + shared::track_command_with_rmap, + )?; + + // The fixture process is the pid with the largest anon rmap peak. + let (_rss_stat, rmap) = per_pid_peaks(&events); + let rmap_peak = rmap + .iter() + .max_by_key(|p| p.anon_mib) + .ok_or("no rmap pid observed")?; + assert!( + rmap_peak.anon_mib >= REGION_MIB - 16, + "rmap anon peak {} MiB misses the worker's post-leader-exit region (~{} MiB)", + rmap_peak.anon_mib, + REGION_MIB + ); + + let pid = rmap_peak.pid; + let exits: Vec<&MemtrackEvent> = events + .iter() + .filter(|e| e.pid == pid && matches!(e.kind, MemtrackEventKind::Exit)) + .collect(); + assert_eq!(exits.len(), 1, "expected exactly one EXIT for pid {pid}"); + + let last_rmap_ts = events + .iter() + .filter(|e| e.pid == pid && matches!(e.kind, MemtrackEventKind::Rmap { .. })) + .map(|e| e.timestamp) + .max() + .ok_or("no rmap events for fixture pid")?; + assert!( + exits[0].timestamp > last_rmap_ts, + "EXIT fired before the worker's rmap events: leader exit was treated as process death" + ); + Ok(()) +} + +/// A fork issued by a worker thread must still track the child: registration +/// keys on the parent's tgid (task_newtask fires in the cloning task, whose +/// pid_tgid upper half is the tgid), not on the raw creator tid. +#[test_with::env(GITHUB_ACTIONS)] +#[test] +fn test_rss_rmap_thread_fork_tracks_child() -> Result<(), Box> { + const REGION_MIB: u64 = 64; + + let (_raw_report, events) = track_fixture( + include_str!("../testdata/rss/rmap_thread_fork.c"), + "rmap_thread_fork", + shared::track_command_with_rmap, + )?; + + // Single fork in the fixture: parent = the fixture process (tgid), child = + // the region-faulting process. + let (parent, child) = events + .iter() + .find_map(|e| match e.kind { + MemtrackEventKind::Fork { parent_pid } => Some((parent_pid, e.pid)), + _ => None, + }) + .ok_or("no fork event: worker-thread fork was not tracked")?; + assert_ne!(parent, child); + + let child_anon: i64 = events + .iter() + .filter_map(|e| match e.kind { + MemtrackEventKind::Rmap { member: 1, delta } if e.pid == child && delta > 0 => { + Some(delta) + } + _ => None, + }) + .sum(); + assert!( + child_anon * 4096 >= ((REGION_MIB - 16) * MIB) as i64, + "child of a worker-thread fork missed rmap tracking: anon adds {} bytes, expected ~{} MiB", + child_anon * 4096, + REGION_MIB + ); + Ok(()) +}