diff --git a/Cargo.lock b/Cargo.lock index bafdf95..1664215 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,7 +95,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -156,11 +156,12 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "midden" -version = "0.1.1" +version = "0.2.0" dependencies = [ "clap", "flate2", "serde_json", + "zerocopy", ] [[package]] @@ -223,7 +224,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.3", ] [[package]] @@ -251,6 +252,17 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "3.0.3" @@ -289,6 +301,26 @@ dependencies = [ "windows-link", ] +[[package]] +name = "zerocopy" +version = "0.8.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879fb705ce98c32e41ebdb970fbe1204f8492423b314c6ab0354c3e7b5542866" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708882a28301d604fa039cc7727607a98d04c4b86dc76ec9cc683f805d709759" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "zlib-rs" version = "0.6.8" diff --git a/Cargo.toml b/Cargo.toml index 5ff1bf9..68e55c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "midden" -version = "0.1.1" +version = "0.2.0" edition = "2024" rust-version = "1.88" description = "Find what is holding a JVM heap dump's memory: retained sizes, owners, leak suspects." @@ -13,6 +13,7 @@ categories = ["command-line-utilities", "development-tools::debugging"] [dependencies] clap = { version = "4.6.4", features = ["derive"] } flate2 = "1" +zerocopy = { version = "0.8", features = ["alloc"] } serde_json = "1.0.151" [lints.rust] diff --git a/README.md b/README.md index 4a591d7..c0286f5 100644 --- a/README.md +++ b/README.md @@ -61,9 +61,10 @@ boxed, references, garbage, system, direct and baseline. Pick them with `--only` ## Benchmark -midden 0.1.0 compared with Eclipse MAT 1.17, JProfiler 16.2.1 and the VisualVM 2.2.2 heap engine, +midden 0.2.0 compared with Eclipse MAT 1.17, JProfiler 16.2.1 and the VisualVM 2.2.2 heap engine, run on 8 CPUs with 4 GiB of memory. Each dump was in the page cache before its runs, -and the tables show the best of 2 or 3 runs. +and the tables show the best of 2 or 3 runs. The other tools were measured alongside midden 0.1.0 on the +same machine. ### Cold first run @@ -72,14 +73,14 @@ Suspects. JProfiler is `jpanalyze -retained=true`, which builds the index its GU | Dump | Objects | midden | midden, 1 thread | MAT | JProfiler | VisualVM | |---|---:|---:|---:|---:|---:|---:| -| Paper server, 938 MB | 11.2 M | **1.95 seconds** | 3.62 seconds | 20.95 seconds | 12.05 seconds | 55.8 seconds | -| mixed-small, 414 MB | 4.1 M | **0.68 seconds** | 1.21 seconds | 7.09 seconds | 4.11 seconds | 772 seconds | -| mixed-1g, 1.2 GB | 11.8 M | **1.79 seconds** | 3.41 seconds | 17.15 seconds | 10.25 seconds | over 1,800 seconds, stopped | -| arrays-1g, 1.2 GB | 37 k | **0.10 seconds** | 0.33 seconds | 1.84 seconds | 0.97 seconds | 0.37 seconds | -| chain-8m, 429 MB | 8.0 M | **0.89 seconds** | 1.36 seconds | 12.26 seconds | 7.01 seconds | 23.8 seconds | -| small-20m, 945 MB | 19.2 M | **6.92 seconds** | 14.95 seconds | 45.71 seconds | 33.46 seconds | 334 seconds | - -midden was 7 to 18 times faster than MAT and 5 to 10 times faster than JProfiler, and it was faster on a +| Paper server, 938 MB | 11.2 M | **1.30 seconds** | 3.53 seconds | 20.95 seconds | 12.05 seconds | 55.8 seconds | +| mixed-small, 414 MB | 4.1 M | **0.40 seconds** | 1.11 seconds | 7.09 seconds | 4.11 seconds | 772 seconds | +| mixed-1g, 1.2 GB | 11.8 M | **1.10 seconds** | 3.78 seconds | 17.15 seconds | 10.25 seconds | over 1,800 seconds, stopped | +| arrays-1g, 1.2 GB | 37 k | **0.08 seconds** | 0.26 seconds | 1.84 seconds | 0.97 seconds | 0.37 seconds | +| chain-8m, 429 MB | 8.0 M | **0.85 seconds** | 1.64 seconds | 12.26 seconds | 7.01 seconds | 23.8 seconds | +| small-20m, 945 MB | 19.2 M | **5.85 seconds** | 14.71 seconds | 45.71 seconds | 33.46 seconds | 334 seconds | + +midden was 8 to 23 times faster than MAT and 6 to 12 times faster than JProfiler, and it was faster on a single thread than either of them on eight. MAT's full three-report run (suspects, overview and top components) took 26 to 38 seconds on Paper. @@ -87,15 +88,15 @@ components) took 26 to 38 seconds on Paper. | Dump | midden | MAT | JProfiler | |---|---:|---:|---:| -| Paper | 1.66 seconds | 30.59 seconds | **1.54 seconds** | -| small-20m | 6.74 seconds | 18.80 seconds | **3.94 seconds** | +| Paper | **1.06 seconds** | 30.59 seconds | 1.54 seconds | +| small-20m | 5.49 seconds | 18.80 seconds | **3.94 seconds** | -JProfiler reopens faster on the large dumps because its analysis stores the dominator tree. midden's -cache only stores the parse, so it saves at most 0.3 seconds. +JProfiler reopens small-20m faster because its analysis stores the dominator tree. midden's cache only +stores the parse, so it saves 0.2 to 0.4 seconds. ### Memory -Peak RSS on Paper: midden 1.0 GB, JProfiler 1.3 GB, MAT 2.1 to 3.3 GB (at its 3 GB heap), VisualVM 3.5 GB. +Peak RSS on Paper: midden 0.6 GB, JProfiler 1.3 GB, MAT 2.1 to 3.3 GB (at its 3 GB heap), VisualVM 3.5 GB. ### Features diff --git a/src/analysis/baseline.rs b/src/analysis/baseline.rs index 1f92658..5c6621e 100644 --- a/src/analysis/baseline.rs +++ b/src/analysis/baseline.rs @@ -96,7 +96,7 @@ impl Heap<'_> { if loader == crate::dump::NONE { return "bootstrap".to_string(); } - let id = dump.objects[loader as usize].id; + let id = dump.objects.id(loader as usize); let first = dump .classes .iter() diff --git a/src/analysis/collections.rs b/src/analysis/collections.rs index 9d37ffb..acda33b 100644 --- a/src/analysis/collections.rs +++ b/src/analysis/collections.rs @@ -167,13 +167,13 @@ impl Heap<'_> { Shape::Array(field) => { let array = graph.field(collection, field)?; stats.backing = array; - stats.capacity = u64::from(self.dump.objects[array as usize].len); + stats.capacity = u64::from(self.dump.objects.get(array as usize).len); stats.entries = graph.edges(array).len() as u64; } Shape::Identity(field) => { let array = graph.field(collection, field)?; stats.backing = array; - stats.capacity = u64::from(self.dump.objects[array as usize].len) / 2; + stats.capacity = u64::from(self.dump.objects.get(array as usize).len) / 2; stats.entries = graph .edges(array) .labels() @@ -183,7 +183,7 @@ impl Heap<'_> { Shape::Hash { table, next } => { let Some(buckets) = graph.field(collection, table) else { return Some(stats) }; stats.backing = buckets; - stats.capacity = u64::from(self.dump.objects[buckets as usize].len); + stats.capacity = u64::from(self.dump.objects.get(buckets as usize).len); stats.used_buckets = graph.edges(buckets).len() as u64; self.hash_nodes(buckets, next, &mut count); stats.entries = nodes; @@ -353,7 +353,7 @@ impl Heap<'_> { if !self.reachable(object) { continue; } - let record = &dump.objects[object as usize]; + let record = dump.objects.get(object as usize); if record.kind == Kind::ObjectArray { part.sparse.extend(self.sparse_array(object, ref_size)); continue; @@ -399,7 +399,7 @@ impl Heap<'_> { /// A big plain object array, owned by no collection, under half filled. fn sparse_array(&self, object: u32, ref_size: u64) -> Option { - let record = &self.dump.objects[object as usize]; + let record = self.dump.objects.get(object as usize); let filled = self.graph.edges(object).len() as u64; let idom = self.dominators.idom[object as usize]; let backs_collection = idom != self.graph.root && self.shape_of(idom).is_some(); diff --git a/src/analysis/duplicates.rs b/src/analysis/duplicates.rs index f0be8f5..ffea4b2 100644 --- a/src/analysis/duplicates.rs +++ b/src/analysis/duplicates.rs @@ -2,9 +2,10 @@ //! contents, and boxed primitives holding the same value. use super::Heap; -use crate::dump::{FastMap, Kind}; +use crate::dump::{self, Class, FastMap, Kind}; use crate::hprof::Ty; use crate::parallel; +use crate::store::Bits; /// Strings sharing one content. pub struct DuplicateGroup { @@ -60,73 +61,140 @@ const BOXED: [(&str, Ty); 8] = [ ("java.lang.Double", Ty::Double), ]; +/// The boxed classes among `classes`: `(class, class id, value offset, type)`. The index pass tallies +/// their values with this too. +pub fn boxed_classes(classes: &[Class], names: &[String], id_size: u32) -> Vec<(u32, u64, u32, Ty)> { + BOXED + .iter() + .filter_map(|&(name, ty)| { + let class = dump::class_named(classes, name)?; + let (offset, field_ty) = + dump::field_offset(classes, names, id_size, class, "value").filter(|field| field.1 == ty)?; + Some((class, classes[class as usize].id, offset, field_ty)) + }) + .collect() +} + +/// Items sharing one hash: the lowest-ranked of them, how many, and the bytes of all but that one. +struct HashGroup { + hash: u64, + first: T, + count: u64, + wasted: u64, +} + +/// Group `items` by hash over the workers, each taking one share of the hashes and reading every item +/// for it, so nothing is copied out. `hash` gets an item's place too; `None` leaves the item out. +fn group_by_hash( + items: &[T], + hash: impl Fn(usize, T) -> Option + Sync, + rank: impl Fn(T) -> u32 + Sync, + bytes: impl Fn(T) -> u64 + Sync, +) -> Vec> { + let shares = parallel::threads(); + let share_ids: Vec = (0..shares).collect(); + parallel::items(&share_ids, |&share| { + let mut groups: FastMap = FastMap::default(); + let mine = items.iter().enumerate().filter_map(|(at, &item)| Some((hash(at, item)?, item))); + for (hash, item) in mine.filter(|&(hash, _)| (hash >> 32) as usize % shares == share) { + let group = groups.entry(hash).or_insert((item, 0, 0)); + if rank(item) < rank(group.0) { + group.0 = item; + } + group.1 += 1; + group.2 += bytes(item); + } + groups + .into_iter() + .map(|(hash, (first, count, total))| HashGroup { + hash, + first, + count, + wasted: total - bytes(first), + }) + .collect::>() + }) + .into_iter() + .flatten() + .collect() +} + impl Heap<'_> { /// Group Strings by the content of their backing arrays. - pub fn duplicates(&self, pairs: &[(u32, u32)], hash_of: impl Fn(u32) -> Option) -> Duplicates { - let mut seen: Vec<(u32, u32)> = pairs.to_vec(); - seen.sort_unstable_by_key(|pair| (pair.1, pair.0)); - seen.dedup_by_key(|pair| pair.1); - let mut groups: FastMap = FastMap::default(); - for &(string, array) in &seen { - let Some(hash) = hash_of(array) else { continue }; - let group = groups.entry(hash).or_insert((0, string, array, 0)); - group.0 += 1; - if group.0 > 1 { - group.3 += self.shallow(array) + self.shallow(string); + pub fn duplicates( + &self, + pairs: &[(u32, u32)], + hash_of: impl Fn(u32) -> Option + Sync, + ) -> Duplicates { + // Each array once, with its first String: the pairs come in String order. + let (mut shared, mut first) = (Bits::new(self.dump.objects.len()), Bits::new(pairs.len())); + let mut seen = 0; + for (at, &(_, array)) in pairs.iter().enumerate() { + if !shared.get(array as usize) { + shared.set(array as usize); + first.set(at); + seen += 1; } } + drop(shared); + let groups = group_by_hash( + pairs, + |at, (_, array)| first.get(at).then(|| hash_of(array)).flatten(), + |(_, array)| array, + |(string, array)| self.shallow(array) + self.shallow(string), + ); let distinct = groups.len() as u64; let mut out: Vec = groups .into_iter() - .filter(|(_, group)| group.0 > 1) - .map(|(hash, (count, string, array, wasted))| DuplicateGroup { - string, - array, - hash, - count, - wasted, + .filter(|group| group.count > 1) + .map(|group| { + let (string, array) = group.first; + DuplicateGroup { string, array, hash: group.hash, count: group.count, wasted: group.wasted } }) .collect(); out.sort_by(|a, b| b.wasted.cmp(&a.wasted).then(a.array.cmp(&b.array))); Duplicates { wasted: out.iter().map(|group| group.wasted).sum(), groups: out, - strings: seen.len() as u64, + strings: seen, distinct, } } /// Reachable primitive arrays of at least `MIN_ARRAY_DATA_BYTES` that no String owns. pub fn hashable_arrays(&self, strings: &[(u32, u32)]) -> Vec { - let mut string_arrays: Vec = strings.iter().map(|&(_, array)| array).collect(); - string_arrays.sort_unstable(); let dump = self.dump; + let mut string_arrays = Bits::new(dump.objects.len()); + for &(_, array) in strings { + string_arrays.set(array as usize); + } parallel::ranges(dump.objects.len(), |lo, hi| { (lo as u32..hi as u32) .filter(|&object| { - let record = &dump.objects[object as usize]; + let record = dump.objects.get(object as usize); record.kind == Kind::PrimitiveArray && record.shallow >= dump.sizing.array_header + MIN_ARRAY_DATA_BYTES }) - .filter(|&object| self.reachable(object) && string_arrays.binary_search(&object).is_err()) + .filter(|&object| self.reachable(object) && !string_arrays.get(object as usize)) .collect::>() }) .concat() } /// Group primitive arrays by content. - pub fn array_duplicates(&self, arrays: &[u32], hash_of: impl Fn(u32) -> Option) -> ArrayDuplicates { - let mut groups: FastMap = FastMap::default(); - for &array in arrays { - let Some(hash) = hash_of(array) else { continue }; - let group = groups.entry(hash).or_insert(ArrayGroup { array, count: 0, wasted: 0 }); - group.count += 1; - if group.count > 1 { - group.wasted += self.shallow(array); - } - } + pub fn array_duplicates( + &self, + arrays: &[u32], + hash_of: impl Fn(u32) -> Option + Sync, + ) -> ArrayDuplicates { + let groups = + group_by_hash(arrays, |_, array| hash_of(array), |array| array, |array| self.shallow(array)); let distinct = groups.len() as u64; - let mut out: Vec = groups.into_values().filter(|group| group.count > 1).collect(); + let mut out: Vec = groups + .into_iter() + .filter(|group| group.count > 1) + .map(|group| ArrayGroup { array: group.first, count: group.count, wasted: group.wasted }) + .collect(); out.sort_by(|a, b| b.wasted.cmp(&a.wasted).then(a.array.cmp(&b.array))); ArrayDuplicates { wasted: out.iter().map(|group| group.wasted).sum(), @@ -138,15 +206,7 @@ impl Heap<'_> { /// The boxed classes present: `(class, class id, value offset, type)`. pub fn boxed_classes(&self) -> Vec<(u32, u64, u32, Ty)> { - let dump = self.dump; - BOXED - .iter() - .filter_map(|&(name, ty)| { - let class = dump.class_named(name)?; - let (offset, field_ty) = dump.field_offset(class, "value").filter(|field| field.1 == ty)?; - Some((class, dump.classes[class as usize].id, offset, field_ty)) - }) - .collect() + boxed_classes(&self.dump.classes, &self.dump.names, self.dump.header.id_size) } /// Boxed duplicates from the tallies the detail pass made. diff --git a/src/analysis/mod.rs b/src/analysis/mod.rs index d3a2a2d..1e1173d 100644 --- a/src/analysis/mod.rs +++ b/src/analysis/mod.rs @@ -22,6 +22,7 @@ use super::graph::{ARRAY_ELEMENT, Graph, LOADER}; use super::hprof::RootKind; use super::parallel; use super::pattern::{Pattern, short}; +use super::store::Bits; /// Referrer label flag: the source is a class object, the field a static. pub const STATIC: u32 = 1 << 30; @@ -85,7 +86,7 @@ pub struct Heap<'a> { /// Classes `--exclude` hides from suspects and tables. excluded: Vec, /// Per object: reachable only through weak referents. - weakly_reachable: Vec, + weakly_reachable: Bits, } /// A number of objects and their bytes. @@ -112,16 +113,36 @@ pub struct Referrer { } impl<'a> Heap<'a> { - pub fn new(dump: &'a Dump, graph: &'a Graph) -> Heap<'a> { - // The BFS and the dominators only read the graph: run them side by side. - let (strong, dominators) = match parallel::threads() { - 1 => (dom::reach(graph, dump), dom::dominators(graph, dump)), + /// The heap of a dump. The BFS and the dominators only read the graph, so they run side by side, and + /// `meanwhile` runs on the heap once the BFS is done while the dominator tree is still being built: it + /// must not read the tree. + pub fn build( + dump: &'a Dump, + graph: &'a Graph, + excludes: &[Pattern], + meanwhile: impl FnOnce(&Heap<'a>) -> T + Send, + ) -> (Heap<'a>, T) { + let live = || { + let mut heap = Heap::live(dump, graph, dom::reach(graph, dump)); + heap.set_excludes(excludes); + let during = meanwhile(&heap); + (heap, during) + }; + let ((mut heap, during), dominators) = match parallel::threads() { + 1 => (live(), dom::dominators(graph, dump)), _ => std::thread::scope(|scope| { - let bfs = scope.spawn(|| dom::reach(graph, dump)); + let bfs = scope.spawn(live); let tree = dom::dominators(graph, dump); (bfs.join().expect("reachability"), tree) }), }; + heap.dominators = dominators; + heap.reachability.keep_differing(&heap.dominators.idom); + (heap, during) + } + + /// A heap with liveness and its tables, and an empty dominator tree. + fn live(dump: &'a Dump, graph: &'a Graph, strong: Reachability) -> Heap<'a> { let total = dump.total_shallow(); let weak = dom::weak_only(graph, dump, &strong); let unreachable = Objects { @@ -140,7 +161,7 @@ impl<'a> Heap<'a> { graph, live: strong.bytes, reachability: strong, - dominators, + dominators: DominatorTree::empty(), total, weak_only: Objects { count: weak.count, bytes: weak.bytes }, unreachable, @@ -153,7 +174,7 @@ impl<'a> Heap<'a> { } } - pub fn set_excludes(&mut self, patterns: &[Pattern]) { + fn set_excludes(&mut self, patterns: &[Pattern]) { for (i, class) in self.dump.classes.iter().enumerate() { self.excluded[i] = patterns.iter().any(|pattern| pattern.matches(&class.name)); } @@ -178,24 +199,24 @@ impl<'a> Heap<'a> { } pub fn retained(&self, object: u32) -> u64 { - self.dominators.retained[object as usize] + self.dominators.retained(object) } pub fn shallow(&self, object: u32) -> u64 { - u64::from(self.dump.objects[object as usize].shallow) + u64::from(self.dump.objects.shallow(object as usize)) } pub fn reachable(&self, object: u32) -> bool { - self.reachability.parent[object as usize] != NONE + self.reachability.reached(object, &self.dominators.idom) } /// Alive only through soft, weak or phantom references. pub fn weakly_reachable(&self, object: u32) -> bool { - self.weakly_reachable[object as usize] + self.weakly_reachable.get(object as usize) } pub fn class(&self, object: u32) -> u32 { - self.dump.objects[object as usize].class + self.dump.objects.class(object as usize) } pub fn is_string(&self, object: u32) -> bool { @@ -251,7 +272,7 @@ impl<'a> Heap<'a> { let object = tree.preorder[pos]; let group = group_of(object); if !inside[group as usize] { - retained[group as usize] += tree.retained[object as usize]; + retained[group as usize] += tree.retained(object); inside[group as usize] = true; open.push((tree.subtree_end[pos], group)); } @@ -265,10 +286,10 @@ impl<'a> Heap<'a> { pub fn histogram(&self) -> Vec { let dump = self.dump; let retained = - self.grouped_retained(|object| dump.objects[object as usize].class, dump.classes.len()); + self.grouped_retained(|object| dump.objects.class(object as usize), dump.classes.len()); let shallow = parallel::ranges(dump.objects.len(), |lo, hi| { let mut sums = vec![0u64; dump.classes.len()]; - for record in &dump.objects[lo..hi] { + for record in dump.objects.range(lo, hi) { sums[record.class as usize] += u64::from(record.shallow); } sums @@ -293,7 +314,7 @@ impl<'a> Heap<'a> { let dump = self.dump; let (package_of_class, names) = dense_ids(dump.classes.iter().map(|class| package_of(&class.name))); let retained = self.grouped_retained( - |object| package_of_class[dump.objects[object as usize].class as usize], + |object| package_of_class[dump.objects.class(object as usize) as usize], names.len(), ); let mut rows: Vec<(String, ClassRow)> = names @@ -322,7 +343,7 @@ impl<'a> Heap<'a> { let parts = parallel::ranges(dominated.len(), |lo, hi| { let mut tally = Tally::new(self.dump.classes.len()); for &member in &dominated[lo..hi] { - tally.add(&self.dump.objects[member as usize]); + tally.add(self.dump.objects.get(member as usize)); } tally }); @@ -339,7 +360,7 @@ impl<'a> Heap<'a> { if !self.reachable(source) { continue; } - let record = &self.dump.objects[source as usize]; + let record = self.dump.objects.get(source as usize); // Only a class object can be in the map; the rest skip the lookup. let class_object = if record.kind == Kind::Class { self.class_objects.get(&source) } else { None }; @@ -347,8 +368,10 @@ impl<'a> Heap<'a> { Some(&class) => (class, STATIC), None => (record.class, 0), }; - for (target, label) in graph.edges(source).iter() { + let edges = graph.edges(source); + for (k, target) in edges.targets().enumerate() { if let Some(index) = slot(target) { + let label = edges.coarse(k); while counts.len() <= index { counts.push(FastMap::default()); } @@ -405,8 +428,14 @@ impl<'a> Heap<'a> { let dominator = self.dominators.idom[object as usize]; dominator != root && self.class(dominator) == self.class(object) }; - let mut current: Vec = - members.iter().copied().filter(|&object| self.reachable(object) && !same_class(object)).collect(); + let mut current: Vec = parallel::ranges(members.len(), |lo, hi| { + members[lo..hi] + .iter() + .copied() + .filter(|&object| self.reachable(object) && !same_class(object)) + .collect::>() + }) + .concat(); let mut out = Vec::new(); for _ in 0..6 { // Bytes per owning class, `NONE` standing for the roots. @@ -415,7 +444,7 @@ impl<'a> Heap<'a> { for &object in ¤t[lo..hi] { let dominator = self.dominators.idom[object as usize]; let class = if dominator == root { NONE } else { self.class(dominator) }; - *bytes.entry(class).or_default() += self.dominators.retained[object as usize]; + *bytes.entry(class).or_default() += self.dominators.retained(object); } bytes }); @@ -432,29 +461,37 @@ impl<'a> Heap<'a> { if class == NONE { break; } - let mut next: Vec = parallel::ranges(current.len(), |lo, hi| { + let next = parallel::ranges(current.len(), |lo, hi| { current[lo..hi] .iter() .map(|&object| self.dominators.idom[object as usize]) .filter(|&dominator| dominator != root && self.class(dominator) == class) .collect::>() - }) - .concat(); - next.sort_unstable(); - next.dedup(); - current = next; + }); + // Each dominator once; the order does not matter to the sums. + let mut listed = Bits::new(self.dump.objects.len()); + current = next + .into_iter() + .flatten() + .filter(|&dominator| { + !listed.get(dominator as usize) && { + listed.set(dominator as usize); + true + } + }) + .collect(); } out } /// Root-to-object path with the label of the edge into each hop. pub fn root_path(&self, object: u32) -> Vec<(u32, Option)> { - self.labelled(&self.reachability.path(self.graph.root, object)) + self.labelled(&self.reachability.path(self.graph.root, object, &self.dominators.idom)) } /// Up to `limit` root paths arriving through different referrers. pub fn root_paths(&self, object: u32, limit: usize) -> Vec)>> { - dom::paths(self.graph, self.dump, &self.reachability, object, limit) + dom::paths(self.graph, self.dump, (&self.reachability, &self.dominators.idom), object, limit) .into_iter() .map(|path| self.labelled(&path)) .collect() @@ -488,7 +525,7 @@ impl<'a> Heap<'a> { /// The object's shape without its address: class name, `class X`, `T[]`. pub fn kind_text(&self, object: u32) -> String { let dump = self.dump; - let record = &dump.objects[object as usize]; + let record = dump.objects.get(object as usize); match self.class_objects.get(&object) { Some(&class) if record.kind == Kind::Class => { format!("class {}", dump.classes[class as usize].name) @@ -525,7 +562,7 @@ impl<'a> Heap<'a> { let mut roots: Vec<_> = dump.roots.iter().filter(|(rooted, _)| *rooted == object).map(|(_, root)| *root).collect(); // A class object is a root because it is loaded; a frame holding it says less. - let is_class = dump.objects[object as usize].kind == Kind::Class; + let is_class = dump.objects.kind(object as usize) == Kind::Class; roots.sort_by_key(|root| (is_class && root.kind != RootKind::StickyClass, root.kind)); let Some(root) = roots.first() else { return String::new() }; let thread = |serial: u32| self.thread_name(serial); @@ -596,7 +633,7 @@ impl<'a> Heap<'a> { let parts = parallel::ranges(self.dump.objects.len(), |lo, hi| { let mut best = vec![(0u64, NONE); n]; for object in lo as u32..hi as u32 { - let retained = self.dominators.retained[object as usize]; + let retained = self.dominators.retained(object); let class = self.class(object) as usize; if self.reachable(object) && retained > best[class].0 { best[class] = (retained, object); @@ -654,7 +691,7 @@ impl Tally { Tally { instances: vec![0; classes], shallow: vec![0; classes], objects: 0 } } - pub fn add(&mut self, record: &super::dump::Object) { + pub fn add(&mut self, record: super::dump::Object) { self.instances[record.class as usize] += 1; self.shallow[record.class as usize] += u64::from(record.shallow); self.objects += 1; diff --git a/src/analysis/native.rs b/src/analysis/native.rs index 6dbf31c..aac5205 100644 --- a/src/analysis/native.rs +++ b/src/analysis/native.rs @@ -39,7 +39,7 @@ impl Heap<'_> { /// Ids the detail pass must fetch: the buffers, and the Bits counters. pub fn direct_wants(&self, buffers: &[u32]) -> Vec { let mut want: Vec = - buffers.iter().map(|&buffer| self.dump.objects[buffer as usize].id).collect(); + buffers.iter().map(|&buffer| self.dump.objects.id(buffer as usize)).collect(); for (_, value) in self.bits_statics() { if let Value::Ref(id) = value { want.push(id); @@ -106,7 +106,7 @@ impl Heap<'_> { pub fn int_field(&self, object: u32, name: &str, fetched: &Fetched) -> Option { let dump = self.dump; let (offset, ty) = dump.field_offset(self.class(object), name)?; - let raw = fetched.raw.get(&dump.objects[object as usize].id)?; + let raw = fetched.raw.get(&dump.objects.id(object as usize))?; match Value::decode(ty, raw.data.get(offset as usize..)?, dump.header.id_size)? { Value::Int(int) => Some(int as u64), Value::Long(long) => Some(long as u64), diff --git a/src/analysis/references.rs b/src/analysis/references.rs index 76f3929..f863c4f 100644 --- a/src/analysis/references.rs +++ b/src/analysis/references.rs @@ -61,9 +61,12 @@ impl Heap<'_> { let mut top: Vec<(u32, u64)> = by_class.into_iter().collect(); top.sort_by(|a, b| b.1.cmp(&a.1).then(a.0.cmp(&b.0))); top.truncate(5); - let (count, bytes) = weak_only_from(self.graph, self.dump, &self.reachability, |object| { - self.ref_kind(object) == Some(kind) - }); + let (count, bytes) = weak_only_from( + self.graph, + self.dump, + (&self.reachability, &self.dominators.idom), + |object| self.ref_kind(object) == Some(kind), + ); ReferenceRow { kind, references, referents, only: Objects { count, bytes }, top } }) .filter(|row| row.references > 0) diff --git a/src/analysis/suspects.rs b/src/analysis/suspects.rs index a80335a..b1698ea 100644 --- a/src/analysis/suspects.rs +++ b/src/analysis/suspects.rs @@ -60,7 +60,7 @@ impl Heap<'_> { let mut current = top; for _ in 0..MAX_DESCENT { let Some(child) = self.dominators.biggest_child(current) else { break }; - if self.dominators.retained[child as usize] * 4 < self.dominators.retained[current as usize] * 3 { + if self.dominators.retained(child) * 4 < self.dominators.retained(current) * 3 { break; } let label = self.graph.label_of(current, child); @@ -85,7 +85,7 @@ impl Heap<'_> { let mut out = Vec::new(); let mut inside: FastMap = FastMap::default(); for &object in self.top_level() { - if self.dominators.retained[object as usize] < floor { + if self.dominators.retained(object) < floor { break; } if self.excluded(self.class(object)) { @@ -122,17 +122,21 @@ impl Heap<'_> { |object: u32| bits[object as usize / u64::BITS as usize] >> (object % u64::BITS) & 1 != 0; let mut referrers = self.referrers(|target| is_member(target).then(|| slot[self.class(target) as usize] as usize)); - let mut covered: Vec = Vec::new(); + let mut covered = vec![false; self.dump.classes.len()]; for (i, (&row, class_members)) in rows.iter().zip(&members).enumerate() { - let owned = class_members - .iter() - .filter(|&&member| { - let dominator = self.dominators.idom[member as usize]; - dominator != self.graph.root && covered.contains(&self.class(dominator)) - }) - .count() as u64; - let seen = owned + inside.get(&row.class).copied().unwrap_or(0); - covered.push(row.class); + let owned: usize = parallel::ranges(class_members.len(), |lo, hi| { + class_members[lo..hi] + .iter() + .filter(|&&member| { + let dominator = self.dominators.idom[member as usize]; + dominator != self.graph.root && covered[self.class(dominator) as usize] + }) + .count() + }) + .into_iter() + .sum(); + let seen = owned as u64 + inside.get(&row.class).copied().unwrap_or(0); + covered[row.class as usize] = true; if seen * 2 >= row.instances { continue; } @@ -175,7 +179,7 @@ impl Heap<'_> { until = tree.subtree_end[pos as usize]; } if pos < until { - tally.add(&dump.objects[object as usize]); + tally.add(dump.objects.get(object as usize)); } } tally diff --git a/src/analysis/threads.rs b/src/analysis/threads.rs index 6602a21..61408be 100644 --- a/src/analysis/threads.rs +++ b/src/analysis/threads.rs @@ -54,7 +54,7 @@ impl Heap<'_> { .filter(|(_, root)| { root.thread_serial == thread.serial && stack_kinds.contains(&root.kind) }) - .filter(|(object, _)| dump.objects[*object as usize].kind != Kind::Class) + .filter(|(object, _)| dump.objects.kind(*object as usize) != Kind::Class) .map(|(object, root)| StackLocal { object: *object, frame: root.frame, kind: root.kind }) .collect(); locals.sort_by(|a, b| { diff --git a/src/cache.rs b/src/cache.rs index 6457fdf..4e4485a 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -6,14 +6,18 @@ use std::fs::{self, File}; use std::io::{self, BufReader, BufWriter, Read, Write}; use std::path::{Path, PathBuf}; -use super::dump::{Buckets, Class, Dump, FastMap, Field, Kind, Object, RefKind, Slot, Static, Thread, Trace}; +use super::dump::{ + ArrayHashes, Class, Dump, FastMap, Field, ObjectTable, Packing, RefKind, Shapes, Slot, Static, Symbols, + Thread, Trace, +}; use super::graph::{Graph, ReferencePolicy}; use super::hash::{FNV_OFFSET_BASIS, FNV_PRIME}; use super::hprof::{Frame, Header, IO_BUFFER_SIZE, Piece, Root, RootKind, Ty, Value}; use super::sizes::{SizeMode, Sizing}; +use super::store::Column; const MAGIC: &[u8; 8] = b"CTDLHEAP"; -const VERSION: u32 = 5; +const VERSION: u32 = 9; /// Spreads each key field before the next is folded in. Part of the file name: never change. const KEY_ROTATION: u32 = 17; /// Written for a class that is no Reference; reads back as None. @@ -159,22 +163,23 @@ impl Reader { self.0.read_exact(&mut buf)?; String::from_utf8(buf).map_err(|_| io::ErrorKind::InvalidData.into()) } - fn u32s(&mut self) -> io::Result> { + fn u32s(&mut self) -> io::Result> { self.column(u32::from_le_bytes) } - fn u64s(&mut self) -> io::Result> { + fn u64s(&mut self) -> io::Result> { self.column(u64::from_le_bytes) } - fn column(&mut self, decode: impl Fn([u8; N]) -> T) -> io::Result> { + /// A column read in batches, so the file is never held whole. + fn column(&mut self, decode: impl Fn([u8; N]) -> T) -> io::Result> { let len = self.u64().and_then(|len| self.fits(len, N as u64))?; - let mut out = Vec::with_capacity(len); + let mut out = Column::with_capacity(len); let mut buf = vec![0u8; IO_BUFFER_SIZE]; let mut remaining = len; while remaining > 0 { let batch = remaining.min(IO_BUFFER_SIZE / N); let bytes = &mut buf[..batch * N]; self.0.read_exact(bytes)?; - out.extend(bytes.as_chunks::().0.iter().map(|chunk| decode(*chunk))); + bytes.as_chunks::().0.iter().for_each(|chunk| out.push(decode(*chunk))); remaining -= batch; } Ok(out) @@ -209,8 +214,8 @@ pub fn save(path: &Path, key: &Key, dump: &Dump, graph: &Graph) -> io::Result<() writer.str(name)?; } writer.count(dump.symbols.len())?; - for (id, symbol) in &dump.symbols { - writer.u64(*id)?; + for (id, symbol) in dump.symbols.iter() { + writer.u64(id)?; writer.str(symbol)?; } writer.count(dump.classes.len())?; @@ -244,12 +249,10 @@ pub fn save(path: &Path, key: &Key, dump: &Dump, graph: &Graph) -> io::Result<() writer.u64(class.instances)?; writer.u8(class.ref_kind.map_or(NO_REF_KIND, |kind| kind as u8))?; } - let (objects, count) = (&dump.objects, dump.objects.len()); - writer.u64s(count, objects.iter().map(|object| object.id))?; - writer.u32s(count, objects.iter().map(|object| object.class))?; - writer.u32s(count, objects.iter().map(|object| object.len))?; - writer.u32s(count, objects.iter().map(|object| object.shallow))?; - writer.bytes(count, objects.iter().map(|object| [object.kind as u8]))?; + let (objects, shapes) = (&dump.objects, dump.objects.shapes()); + writer.u64s(objects.len(), (0..objects.len()).map(|object| objects.id(object)))?; + writer.u32s(shapes.words().len(), shapes.words().iter().copied())?; + writer.u32s(2 * shapes.long().len(), shapes.long().iter().flat_map(|&(object, len)| [object, len]))?; writer.count(dump.roots.len())?; for (idx, root) in &dump.roots { writer.u32(*idx)?; @@ -288,13 +291,34 @@ pub fn save(path: &Path, key: &Key, dump: &Dump, graph: &Graph) -> io::Result<() for value in [dump.class_class, dump.string_class, dump.value_label, dump.name_label] { writer.u32(value)?; } - let (offsets, targets, labels, weak) = graph.parts(); - writer.u64s(offsets.len(), offsets.iter().copied())?; + let (targets, weak) = graph.parts(); + let offsets = graph.offsets(); + writer.u64s(offsets.len(), offsets)?; writer.u32s(targets.len(), targets.iter().copied())?; - writer.u32s(labels.len(), labels.iter().copied())?; + writer.u32s(targets.len(), graph.all_labels())?; writer.u32s(2 * weak.len(), weak.iter().flat_map(|&(referrer, referent)| [referrer, referent]))?; writer.u64(graph.dangling)?; writer.u32s(graph.roots.len(), graph.roots.iter().copied())?; + // What the index pass took for the detail read, so a cached load need not read the dump for it. + writer.u8(u8::from(dump.array_hashes.is_some()))?; + if let Some(hashes) = &dump.array_hashes { + let pair = |&[string, content]: &[u64; 2]| { + let mut bytes = [0u8; 16]; + bytes[..8].copy_from_slice(&string.to_le_bytes()); + bytes[8..].copy_from_slice(&content.to_le_bytes()); + bytes + }; + writer.bytes(hashes.values().len(), hashes.values().iter().map(pair))?; + } + writer.u8(u8::from(dump.boxed_values.is_some()))?; + if let Some(boxed) = &dump.boxed_values { + writer.count(boxed.len())?; + for (&(class, bits), &count) in boxed { + writer.u32(class)?; + writer.u64(bits)?; + writer.u64(count)?; + } + } writer.0.flush()?; drop(writer); fs::rename(&part_path, path) @@ -336,8 +360,11 @@ fn read(mut reader: Reader, key: &Key, dump_path: &str) -> io::Result<(Dump, Gra .map(|piece| Piece { start: piece[0], end: piece[1], min: piece[2], max: piece[3] }) .collect(); let names: Vec = (0..reader.u32()?).map(|_| reader.str()).collect::>()?; - let symbols: FastMap = - (0..reader.u32()?).map(|_| Ok((reader.u64()?, reader.str()?))).collect::>()?; + let mut symbols = Symbols::default(); + for _ in 0..reader.u32()? { + let id = reader.u64()?; + symbols.insert(id, reader.str()?.as_bytes()); + } let ty_from_tag = |tag: u8| Ty::from_tag(tag).ok_or_else(invalid); let mut classes = Vec::new(); for _ in 0..reader.u32()? { @@ -365,19 +392,14 @@ fn read(mut reader: Reader, key: &Key, dump_path: &str) -> io::Result<(Dump, Gra classes.push(class); } let ids = reader.u64s()?; - let class_idxs = reader.u32s()?; - let lens = reader.u32s()?; - let shallow_sizes = reader.u32s()?; - let count = reader.u64().and_then(|count| reader.fits(count, 1))?; - if [class_idxs.len(), lens.len(), shallow_sizes.len(), count].iter().any(|&len| len != ids.len()) { + let words = reader.u32s()?; + let long: Vec<(u32, u32)> = + reader.u32s()?.as_chunks::<2>().0.iter().map(|pair| (pair[0], pair[1])).collect(); + let count = ids.len(); + if words.len() != count || !long.is_sorted_by_key(|&(object, _)| object) { return Err(invalid()); } - let mut objects = Vec::with_capacity(count); - for i in 0..count { - let kind = *Kind::ALL.get(reader.u8()? as usize).ok_or_else(invalid)?; - let (id, class, len, shallow) = (ids[i], class_idxs[i], lens[i], shallow_sizes[i]); - objects.push(Object { id, class, len, shallow, kind }); - } + let shapes = Shapes::new(words, Packing::new(classes.len()), long); let mut roots = Vec::new(); for _ in 0..reader.u32()? { let (idx, id) = (reader.u32()?, reader.u64()?); @@ -391,7 +413,7 @@ fn read(mut reader: Reader, key: &Key, dump_path: &str) -> io::Result<(Dump, Gra .map(|_| Ok(Thread { object: reader.u32()?, serial: reader.u32()?, trace_serial: reader.u32()? })) .collect::>()?; let traces: HashMap = (0..reader.u32()?) - .map(|_| Ok((reader.u32()?, Trace { frames: reader.u64s()? }))) + .map(|_| Ok((reader.u32()?, Trace { frames: reader.u64s()?.to_vec() }))) .collect::>()?; let frames: FastMap = (0..reader.u32()?) .map(|_| { @@ -410,11 +432,39 @@ fn read(mut reader: Reader, key: &Key, dump_path: &str) -> io::Result<(Dump, Gra let weak: Vec<(u32, u32)> = reader.u32s()?.as_chunks::<2>().0.iter().map(|edge| (edge[0], edge[1])).collect(); let dangling = reader.u64()?; - let graph_roots = reader.u32s()?; + let graph_roots = reader.u32s()?.to_vec(); + let array_hashes = if reader.u8()? == 0 { + None + } else { + let values = reader.column(|bytes: [u8; 16]| { + let (string, content) = bytes.split_at(8); + [ + u64::from_le_bytes(string.try_into().unwrap_or_default()), + u64::from_le_bytes(content.try_into().unwrap_or_default()), + ] + })?; + Some(values) + }; + let boxed_values = if reader.u8()? == 0 { + None + } else { + let entries = reader.u32()?; + let mut boxed = FastMap::default(); + for _ in 0..entries { + let (class, bits, count) = (reader.u32()?, reader.u64()?, reader.u64()?); + boxed.insert((class, bits), count); + } + Some(boxed) + }; if offsets.len() != count + 1 || targets.len() != labels.len() { return Err(invalid()); } - let lookup = Buckets::build(&objects); + let objects = ObjectTable::from_columns(&classes, sizing, &ids, shapes); + drop(ids); + let array_hashes = match array_hashes { + Some(values) => Some(ArrayHashes::new(&objects, values).ok_or_else(invalid)?), + None => None, + }; let dump = Dump { path: dump_path.to_string(), file_size, @@ -429,6 +479,8 @@ fn read(mut reader: Reader, key: &Key, dump_path: &str) -> io::Result<(Dump, Gra symbols, classes, objects, + array_hashes, + boxed_values, roots, dangling_roots, marked_unreachable, @@ -436,11 +488,11 @@ fn read(mut reader: Reader, key: &Key, dump_path: &str) -> io::Result<(Dump, Gra traces, frames, class_by_serial, - lookup, class_class, string_class, value_label, name_label, }; - Ok((dump, Graph::from_parts(offsets, targets, labels, weak, dangling, graph_roots))) + let graph = Graph::from_parts(offsets, targets, labels, &dump, weak, dangling, graph_roots); + Ok((dump, graph)) } diff --git a/src/detail.rs b/src/detail.rs index 4c369d3..06348d6 100644 --- a/src/detail.rs +++ b/src/detail.rs @@ -5,7 +5,7 @@ use std::io; use super::dump::{Dump, FastMap}; use super::hash::{FNV_OFFSET_BASIS, FNV_PRIME, GOLDEN_RATIO}; -use super::hprof::{Body, IO_BUFFER_SIZE, Sink, Ty, Value}; +use super::hprof::{Body, IO_BUFFER_SIZE, Piece, Sink, Ty, Value}; use super::source::Source; use super::strings::Needle; use crate::error::Result; @@ -81,11 +81,12 @@ impl Request { /// Whether the read only keeps record bodies. fn records_only(&self) -> bool { - self.hash.is_empty() - && self.content.is_empty() - && self.boxed.is_empty() - && self.search.is_none() - && self.filter.is_none() + self.hash.is_empty() && self.content.is_empty() && self.by_id() + } + + /// Whether every record the read needs is named by id, so only the pieces holding those ids are read. + fn by_id(&self) -> bool { + self.boxed.is_empty() && self.search.is_none() && self.filter.is_none() } } @@ -137,16 +138,19 @@ pub fn fetch(source: &Source, dump: &Dump, mut request: Request) -> Result Option> { - if !request.records_only() || !source.is_seekable() || dump.pieces.is_empty() { + if !request.by_id() || !source.is_seekable() || dump.pieces.is_empty() { return None; } + let holds = |ids: &[u64], piece: &Piece| { + let i = ids.partition_point(|&id| id < piece.min); + ids.get(i).is_some_and(|&id| id <= piece.max) + }; let mut ranges: Vec<(u64, u64)> = Vec::new(); for piece in &dump.pieces { - let i = request.want.partition_point(|&id| id < piece.min); - if request.want.get(i).is_some_and(|&id| id <= piece.max) { + if [&request.want, &request.hash, &request.content].into_iter().any(|ids| holds(ids, piece)) { match ranges.last_mut() { Some((_, end)) if *end == piece.start => *end = piece.end, _ => ranges.push((piece.start, piece.end)), @@ -254,16 +258,13 @@ impl Sink for Fetcher<'_> { if !wanted && hashed.is_none() && content.is_none() && !searched { return Ok(()); } - let (mut string_hash, mut content_hash, mut data) = (FNV_OFFSET_BASIS, GOLDEN_RATIO, Vec::new()); - let mut whole = Vec::new(); + let (mut hash, mut data, mut whole) = (ArrayHash::new(), Vec::new(), Vec::new()); body.chunks(IO_BUFFER_SIZE, |chunk| { if hashed.is_some() { - for &byte in chunk { - string_hash = (string_hash ^ u64::from(byte)).wrapping_mul(FNV_PRIME); - } + hash.string(chunk); } if content.is_some() { - content_hash = mix_words(content_hash, chunk); + hash.content(chunk); } if wanted { keep(&mut data, chunk); @@ -273,12 +274,12 @@ impl Sink for Fetcher<'_> { } Ok(()) })?; - let len_bits = u64::from(len).rotate_left(u32::BITS); + let [string_hash, content_hash] = hash.finish(len); if let Some(i) = hashed { - self.hashes.push((i, (string_hash ^ len_bits) | 1)); + self.hashes.push((i, string_hash)); } if let Some(i) = content { - self.hashes.push((self.request.hash.len() + i, (content_hash ^ len_bits) | 1)); + self.hashes.push((self.request.hash.len() + i, content_hash)); } if searched && self.request.search.as_ref().is_some_and(|(_, needle)| needle.matches(ty, &whole)) { self.found.push(id); @@ -290,6 +291,35 @@ impl Sink for Fetcher<'_> { } } +/// An array's two hashes, fed its body in `IO_BUFFER_SIZE` chunks: FNV-1a over the bytes, the String hash +/// the report shows, and the grouping hash for equal arrays. Both fold in the length and are never zero. +pub struct ArrayHash { + string: u64, + content: u64, +} + +impl ArrayHash { + pub fn new() -> ArrayHash { + ArrayHash { string: FNV_OFFSET_BASIS, content: GOLDEN_RATIO } + } + + pub fn string(&mut self, chunk: &[u8]) { + for &byte in chunk { + self.string = (self.string ^ u64::from(byte)).wrapping_mul(FNV_PRIME); + } + } + + pub fn content(&mut self, chunk: &[u8]) { + self.content = mix_words(self.content, chunk); + } + + /// The String hash and the grouping hash. + pub fn finish(&self, len: u32) -> [u64; 2] { + let len_bits = u64::from(len).rotate_left(u32::BITS); + [(self.string ^ len_bits) | 1, (self.content ^ len_bits) | 1] + } +} + /// Groups arrays by content, eight bytes a step through a folded 128-bit multiply. Only compared, so /// unlike the shown String hash it need not be FNV-1a. fn mix_words(mut hash: u64, bytes: &[u8]) -> u64 { diff --git a/src/dom.rs b/src/dom.rs index 0afa0e2..afd01ab 100644 --- a/src/dom.rs +++ b/src/dom.rs @@ -1,19 +1,26 @@ //! Reachability, root paths, the weakly-held set and the dominator tree. `d` dominates `o` when every //! root path to `o` passes through `d`; the bytes under `d` are its retained size. -use std::sync::atomic::{AtomicU32, Ordering::Relaxed}; +use std::sync::atomic::Ordering::Relaxed; use super::dump::{Dump, NONE}; use super::graph::Graph; use super::parallel; +use super::store::{Bits, Column}; /// Parent and dominator chains are cut at this length, so a corrupt tree cannot loop. const MAX_CHAIN_LEN: usize = 50_000_000; -/// Breadth-first reachability. `parent` is the previous hop on a shortest -/// path from a root (`graph.root` for roots, `NONE` when unreachable). +/// A level this wide or wider is split over the workers; narrower ones run on the calling thread. +const PARALLEL_LEVEL: usize = 1 << 14; + +/// Breadth-first reachability. An object's parent is the previous hop on a shortest path from a root +/// (`graph.root` for roots, `NONE` when unreachable). pub struct Reachability { - pub parent: Vec, + /// Every object's parent, until [`Reachability::keep_differing`] leaves only those in `differing`. + parent: Column, + /// Objects whose parent is not their idom, and the parent, by object. + differing: Vec<(u32, u32)>, pub count: u64, pub bytes: u64, } @@ -22,41 +29,135 @@ pub fn reach(graph: &Graph, dump: &Dump) -> Reachability { reach_blocking(graph, dump, &[]) } -/// Reachability that ignores the edges in `blocked`. +/// Reachability that ignores the edges in `blocked`. Wide levels run over the workers and give the +/// same parents and order as one queue: a target goes to the first frontier object, in queue order, +/// that has an edge to it. fn reach_blocking(graph: &Graph, dump: &Dump, blocked: &[(u32, u32)]) -> Reachability { - let mut parent = vec![NONE; dump.objects.len()]; - let mut queue: Vec = Vec::new(); + let object_count = dump.objects.len(); + // Parents plus one, so zeroed pages read back `NONE` for the unreached. + let mut parent_column = Column::::zeroed(object_count); + let mut seen_bits = Bits::new(object_count); + let (parent, seen) = (parent_column.atomics(), seen_bits.atomics()); + let visit = |object: u32| { + let bit = 1u64 << (object % 64); + seen[object as usize / 64].fetch_or(bit, Relaxed) & bit == 0 + }; + let is_seen = |object: u32| seen[object as usize / 64].load(Relaxed) >> (object % 64) & 1 != 0; + let allowed = |from: u32, to: u32| blocked.is_empty() || !blocked.contains(&(from, to)); + let mut order = Column::::with_capacity(object_count); for &root in &graph.roots { - if parent[root as usize] == NONE { - parent[root as usize] = graph.root; - queue.push(root); + if visit(root) { + parent[root as usize].store(graph.root + 1, Relaxed); + order.push(root); } } - let mut head = 0; - while head < queue.len() { - let object = queue[head]; - head += 1; - for target in graph.edges(object).targets() { - if parent[target as usize] == NONE && !blocked.contains(&(object, target)) { - parent[target as usize] = object; - queue.push(target); + // Claims hold `u32::MAX - position`, so the largest is the earliest frontier object. + let mut claim_column: Option> = None; + let mut lo = 0; + while lo < order.len() { + let hi = order.len(); + if hi - lo < PARALLEL_LEVEL || parallel::threads() == 1 { + for pos in lo..hi { + let from = order[pos]; + for target in graph.edges(from).targets() { + if !is_seen(target) && allowed(from, target) && visit(target) { + parent[target as usize].store(from + 1, Relaxed); + order.push(target); + } + } + } + } else { + let claim = claim_column.get_or_insert_with(|| Column::zeroed(object_count)).atomics(); + let frontier = &order[lo..hi]; + let found = parallel::ranges(frontier.len(), |start, end| { + let mut found: Vec<(u32, u32)> = Vec::new(); + for (pos, &from) in (lo + start..).zip(&frontier[start..end]) { + let key = u32::MAX - pos as u32; + for target in graph.edges(from).targets() { + if !is_seen(target) + && allowed(from, target) + && claim[target as usize].fetch_max(key, Relaxed) < key + { + found.push((target, key)); + } + } + } + found + }); + let kept = parallel::items(&found, |found| { + let mut kept = Vec::new(); + for &(target, key) in found { + if claim[target as usize].load(Relaxed) == key && visit(target) { + let from = order[(u32::MAX - key) as usize]; + parent[target as usize].store(from + 1, Relaxed); + kept.push(target); + } + } + kept + }); + drop(found); + for part in kept { + order.extend_from_slice(&part); } } + lo = hi; } - let bytes = queue.iter().map(|&object| u64::from(dump.objects[object as usize].shallow)).sum(); - Reachability { parent, count: queue.len() as u64, bytes } + drop(claim_column); + drop(seen_bits); + parallel::chunks(&mut parent_column, |_, part| { + for stored in part { + *stored = stored.wrapping_sub(1); + } + }); + let bytes = parallel::ranges(order.len(), |lo, hi| { + order[lo..hi].iter().map(|&object| u64::from(dump.objects.shallow(object as usize))).sum::() + }) + .into_iter() + .sum(); + Reachability { parent: parent_column, differing: Vec::new(), count: order.len() as u64, bytes } } impl Reachability { + /// An object's parent; `idom` answers for most once only the differing parents are kept. + pub fn parent(&self, object: u32, idom: &[u32]) -> u32 { + if !self.parent.is_empty() { + return self.parent[object as usize]; + } + match self.differing.binary_search_by_key(&object, |&(differs, _)| differs) { + Ok(i) => self.differing[i].1, + Err(_) => idom[object as usize], + } + } + + /// Whether a root reaches `object`: it has a parent, or once only the differing ones are kept, an idom. + pub fn reached(&self, object: u32, idom: &[u32]) -> bool { + let source = if self.parent.is_empty() { idom } else { &self.parent }; + source[object as usize] != NONE + } + + /// Drop every parent that equals the object's idom. An object with one referrer has it as both, so + /// few parents stay. + pub fn keep_differing(&mut self, idom: &[u32]) { + let parent = &self.parent; + self.differing = parallel::ranges(parent.len(), |lo, hi| { + (lo as u32..hi as u32) + .filter(|&object| parent[object as usize] != idom[object as usize]) + .map(|object| (object, parent[object as usize])) + .collect::>() + }) + .concat(); + self.parent = Column::with_capacity(0); + } + /// Root-to-object path, the root end first. Empty when unreachable. - pub fn path(&self, root: u32, mut object: u32) -> Vec { + pub fn path(&self, root: u32, mut object: u32, idom: &[u32]) -> Vec { let mut path = Vec::new(); - if self.parent[object as usize] == NONE { + if !self.reached(object, idom) { return path; } while object != NONE && object != root { path.push(object); - object = self.parent[object as usize]; + object = self.parent(object, idom); if path.len() > MAX_CHAIN_LEN { break; } @@ -68,12 +169,18 @@ impl Reachability { /// Up to `limit` root paths to `target` that arrive through different referrers: each /// search blocks the last edge of the paths found so far. -pub fn paths(graph: &Graph, dump: &Dump, first: &Reachability, target: u32, limit: usize) -> Vec> { +pub fn paths( + graph: &Graph, + dump: &Dump, + (first, idom): (&Reachability, &[u32]), + target: u32, + limit: usize, +) -> Vec> { let mut out = Vec::new(); let mut blocked = Vec::new(); let mut reach = None; for _ in 0..limit.max(1) { - let path = reach.as_ref().unwrap_or(first).path(graph.root, target); + let path = reach.as_ref().unwrap_or(first).path(graph.root, target, idom); if path.is_empty() { break; } @@ -93,36 +200,41 @@ pub fn paths(graph: &Graph, dump: &Dump, first: &Reachability, target: u32, limi pub struct WeakSet { pub count: u64, pub bytes: u64, - pub seen: Vec, + pub seen: Bits, } /// The weakly-held set: a search seeded by weak edges leaving the strongly reachable set. pub fn weak_only(graph: &Graph, dump: &Dump, strong: &Reachability) -> WeakSet { - weak_set(graph, dump, strong, |_| true) + weak_set(graph, dump, (strong, &[]), |_| true) } /// Count and bytes of the weakly-held set reached through the weak edges `take` accepts. pub fn weak_only_from( graph: &Graph, dump: &Dump, - strong: &Reachability, + strong: (&Reachability, &[u32]), take: impl Fn(u32) -> bool, ) -> (u64, u64) { let set = weak_set(graph, dump, strong, take); (set.count, set.bytes) } -fn weak_set(graph: &Graph, dump: &Dump, strong: &Reachability, take: impl Fn(u32) -> bool) -> WeakSet { - let mut seen = vec![false; dump.objects.len()]; +fn weak_set( + graph: &Graph, + dump: &Dump, + (strong, idom): (&Reachability, &[u32]), + take: impl Fn(u32) -> bool, +) -> WeakSet { + let mut seen = Bits::new(dump.objects.len()); let mut queue: Vec = Vec::new(); - let visit = |object: u32, queue: &mut Vec, seen: &mut [bool]| { - if strong.parent[object as usize] == NONE && !seen[object as usize] { - seen[object as usize] = true; + let visit = |object: u32, queue: &mut Vec, seen: &mut Bits| { + if !strong.reached(object, idom) && !seen.get(object as usize) { + seen.set(object as usize); queue.push(object); } }; for &(referrer, referent) in graph.weak_edges() { - if strong.parent[referrer as usize] != NONE && take(referrer) { + if strong.reached(referrer, idom) && take(referrer) { visit(referent, &mut queue, &mut seen); } } @@ -137,26 +249,49 @@ fn weak_set(graph: &Graph, dump: &Dump, strong: &Reachability, take: impl Fn(u32 visit(referent, &mut queue, &mut seen); } } - let bytes = queue.iter().map(|&object| u64::from(dump.objects[object as usize].shallow)).sum(); + let bytes = queue.iter().map(|&object| u64::from(dump.objects.shallow(object as usize))).sum(); WeakSet { count: queue.len() as u64, bytes, seen } } /// The dominator tree over the reachable objects, plus retained sizes. pub struct DominatorTree { /// Immediate dominator: `graph.root` for top-level objects, `NONE` if unreachable. - pub idom: Vec, - /// Shallow size plus everything dominated; unreachable objects keep their shallow size. - pub retained: Vec, + pub idom: Column, + /// Shallow size plus everything dominated; unreachable objects keep their shallow size. Four bytes: an + /// object retaining 4 GiB or more reads `u32::MAX` here and has its size in `big`. + retained: Column, + big: Vec<(u32, u64)>, /// Objects dominated by the roots alone, biggest retained first. top_level: Vec, /// Reachable objects in tree preorder: what an object dominates runs from its place to its `subtree_end`. - pub preorder: Vec, - pub subtree_end: Vec, + pub preorder: Column, + pub subtree_end: Column, /// Each object's place in `preorder`, `NONE` if unreachable. - pub preorder_index: Vec, + pub preorder_index: Column, } impl DominatorTree { + /// No tree yet. + pub fn empty() -> DominatorTree { + let none = || Column::with_capacity(0); + DominatorTree { + idom: none(), + retained: Column::with_capacity(0), + big: Vec::new(), + top_level: Vec::new(), + preorder: none(), + subtree_end: none(), + preorder_index: none(), + } + } + + pub fn retained(&self, object: u32) -> u64 { + match self.retained[object as usize] { + u32::MAX => self.big.binary_search_by_key(&object, |&(big, _)| big).map_or(0, |i| self.big[i].1), + bytes => u64::from(bytes), + } + } + pub fn top_level(&self) -> &[u32] { &self.top_level } @@ -174,7 +309,7 @@ impl DominatorTree { } fn bigger(&self, a: u32, b: u32) -> std::cmp::Ordering { - self.retained[b as usize].cmp(&self.retained[a as usize]).then(a.cmp(&b)) + self.retained(b).cmp(&self.retained(a)).then(a.cmp(&b)) } /// Children in preorder: each child's subtree ends where the next begins. @@ -215,117 +350,180 @@ impl DominatorTree { } } +/// Lengauer-Tarjan `eval` with path compression over `ancestor`, keeping the smallest `semi` in `label`. +struct Forest { + semi: Column, + label: Column, + ancestor: Column, + path: Vec, +} + +impl Forest { + fn new(vertex_count: usize) -> Forest { + Forest { + semi: parallel::iota(vertex_count), + label: parallel::iota(vertex_count), + ancestor: parallel::filled(vertex_count, NONE), + path: Vec::new(), + } + } + + fn eval(&mut self, vertex: u32) -> u32 { + let (ancestor, label, semi) = (&mut self.ancestor, &mut self.label, &self.semi); + if ancestor[vertex as usize] == NONE { + return vertex; + } + self.path.clear(); + let mut current = vertex; + while ancestor[ancestor[current as usize] as usize] != NONE { + self.path.push(current); + current = ancestor[current as usize]; + } + for &node in self.path.iter().rev() { + let above = ancestor[node as usize]; + if semi[label[above as usize] as usize] < semi[label[node as usize] as usize] { + label[node as usize] = label[above as usize]; + } + ancestor[node as usize] = ancestor[above as usize]; + } + label[vertex as usize] + } +} + +/// A leaf in the DFS numbering: reachable or not, an object with no references out. +const LEAF: u32 = NONE - 1; +/// In-degree marker for a vertex with one referrer. +const SINGLE: u32 = NONE; + /// SEMI-NCA: Lengauer-Tarjan semidominators with simple link/eval over a DFS numbering, then idoms by /// nearest common ancestor. Predecessor lists use DFS numbers so the hot loops walk contiguous memory. +/// Leaves never enter it: they dominate nothing, so a leaf's idom is the nearest common ancestor of its +/// referrers, met once the rest of the tree is known. pub fn dominators(graph: &Graph, dump: &Dump) -> DominatorTree { let object_count = dump.objects.len(); let root = graph.root; - let mut dfs_number = vec![NONE; object_count + 1]; - let mut vertex: Vec = vec![root]; - let mut parent: Vec = vec![NONE]; + let mut dfs_number = Column::::zeroed(object_count + 1); + parallel::chunks(&mut dfs_number[..object_count], |start, part| { + for (number, object) in part.iter_mut().zip(start as u32..) { + *number = if graph.edges(object).len() == 0 { LEAF } else { NONE }; + } + }); + let mut vertex = Column::::with_capacity(object_count + 1); + let mut parent = Column::::with_capacity(object_count + 1); + vertex.push(root); + parent.push(NONE); dfs_number[object_count] = 0; - let mut stack: Vec<(u32, u32)> = vec![(root, 0)]; - while let Some(&mut (node, ref mut cursor)) = stack.last_mut() { - let next = if node == root { - graph.roots.get(*cursor as usize).copied() - } else { - graph.edges(node).get(*cursor as usize) - }; - let Some(successor) = next else { + // Vertices not yet numbered, a bit each: the walk tests these, which stay in cache, not the numbers. + let mut unvisited = parallel::bits(object_count, |object| graph.edges(object as u32).len() != 0); + // Each frame: its number, then its next and end edge as places in the target array (the root's + // edges are its GC roots). + let targets = graph.all_targets(); + let mut stack: Vec<(u32, usize, usize)> = vec![(0, 0, graph.roots.len())]; + while let Some(frame) = stack.last_mut() { + let (number, next, end) = *frame; + if next == end { stack.pop(); continue; - }; - *cursor += 1; - if dfs_number[successor as usize] == NONE { - dfs_number[successor as usize] = vertex.len() as u32; - parent.push(dfs_number[node as usize]); + } + frame.1 += 1; + let successor = if number == 0 { graph.roots[next] } else { targets[next] }; + if unvisited.take(successor as usize) { + let successor_number = vertex.len() as u32; + parent.push(number); vertex.push(successor); - stack.push((successor, 0)); + let span = graph.span(successor); + stack.push((successor_number, span.start, span.end)); } } - drop(stack); + drop((stack, unvisited)); let vertex_count = vertex.len(); + // Numbers go in after the walk, in parallel, keeping a random write out of it. + { + let numbers = dfs_number.atomics(); + parallel::ranges(vertex_count - 1, |lo, hi| { + for number in (lo + 1)..=hi { + numbers[vertex[number] as usize].store(number as u32, Relaxed); + } + }); + } // Predecessors in DFS numbers, in parallel: in-degrees first, then a scatter with a cursor per vertex. - let in_degree = parallel::zeroed(vertex_count); - for &gc_root in &graph.roots { - in_degree[dfs_number[gc_root as usize] as usize].fetch_add(1, Relaxed); - } - parallel::ranges(vertex_count - 1, |lo, hi| { - for &node in &vertex[(lo + 1)..=hi] { - for target in graph.edges(node).targets() { - in_degree[dfs_number[target as usize] as usize].fetch_add(1, Relaxed); + let number_of = |object: u32| Some(dfs_number[object as usize]).filter(|&number| number != LEAF); + let mut in_degree_column = Column::::zeroed(vertex_count); + let mut pred_offsets = Column::::zeroed(vertex_count + 1); + { + let in_degree = in_degree_column.atomics(); + for number in graph.roots.iter().filter_map(|&gc_root| number_of(gc_root)) { + in_degree[number as usize].fetch_add(1, Relaxed); + } + parallel::ranges(vertex_count - 1, |lo, hi| { + for &node in &vertex[(lo + 1)..=hi] { + for number in graph.edges(node).targets().filter_map(number_of) { + in_degree[number as usize].fetch_add(1, Relaxed); + } } + }); + // A vertex with one referrer has it as its DFS parent and needs no list: it is marked instead. + for i in 0..vertex_count { + let degree = in_degree[i].load(Relaxed); + let listed = if degree == 1 { 0 } else { degree }; + pred_offsets[i + 1] = pred_offsets[i] + u64::from(listed); + in_degree[i].store(if degree == 1 { SINGLE } else { 0 }, Relaxed); } - }); - let mut pred_offsets = vec![0u64; vertex_count + 1]; - for i in 0..vertex_count { - pred_offsets[i + 1] = pred_offsets[i] + u64::from(in_degree[i].load(Relaxed)); - in_degree[i].store(0, Relaxed); - } - let predecessors = parallel::zeroed(pred_offsets[vertex_count] as usize); - let add_predecessor = |number: u32, from: u32| { - let filled = in_degree[number as usize].fetch_add(1, Relaxed) as usize; - predecessors[pred_offsets[number as usize] as usize + filled].store(from, Relaxed); - }; - for &gc_root in &graph.roots { - add_predecessor(dfs_number[gc_root as usize], 0); - } - parallel::ranges(vertex_count - 1, |lo, hi| { - for &node in &vertex[(lo + 1)..=hi] { - let node_number = dfs_number[node as usize]; - for target in graph.edges(node).targets() { - add_predecessor(dfs_number[target as usize], node_number); + } + let mut predecessor_column = Column::::zeroed(pred_offsets[vertex_count] as usize); + { + let (in_degree, predecessors) = (in_degree_column.atomics(), predecessor_column.atomics()); + let add_predecessor = |number: u32, from: u32| { + if in_degree[number as usize].load(Relaxed) == SINGLE { + return; } + let filled = in_degree[number as usize].fetch_add(1, Relaxed) as usize; + predecessors[pred_offsets[number as usize] as usize + filled].store(from, Relaxed); + }; + for number in graph.roots.iter().filter_map(|&gc_root| number_of(gc_root)) { + add_predecessor(number, 0); } - }); + parallel::ranges(vertex_count - 1, |lo, hi| { + for &node in &vertex[(lo + 1)..=hi] { + let node_number = dfs_number[node as usize]; + for number in graph.edges(node).targets().filter_map(number_of) { + add_predecessor(number, node_number); + } + } + }); + } // The fill counted every in-degree back up, so the offsets can go: walking // down from the top, each vertex's list ends where the next one's starts. drop(pred_offsets); // Semidominators, as in Lengauer-Tarjan: process vertices in reverse // preorder, taking the smallest semi reachable through each predecessor. - let mut semi: Vec = (0..vertex_count as u32).collect(); - let mut label: Vec = (0..vertex_count as u32).collect(); - let mut ancestor = vec![NONE; vertex_count]; - let mut path: Vec = Vec::new(); - let mut eval = |vertex: u32, ancestor: &mut [u32], label: &mut [u32], semi: &[u32]| -> u32 { - if ancestor[vertex as usize] == NONE { - return vertex; - } - path.clear(); - let mut current = vertex; - while ancestor[ancestor[current as usize] as usize] != NONE { - path.push(current); - current = ancestor[current as usize]; - } - for &node in path.iter().rev() { - let above = ancestor[node as usize]; - if semi[label[above as usize] as usize] < semi[label[node as usize] as usize] { - label[node as usize] = label[above as usize]; - } - ancestor[node as usize] = ancestor[above as usize]; - } - label[vertex as usize] - }; - let mut hi = predecessors.len(); + let mut forest = Forest::new(vertex_count); + let mut hi = predecessor_column.len(); for vertex in (1..vertex_count).rev() { - let lo = hi - in_degree[vertex].load(Relaxed) as usize; - for pred in predecessors[lo..hi].iter().map(|pred| pred.load(Relaxed)) { - let best = eval(pred, &mut ancestor, &mut label, &semi); - if semi[best as usize] < semi[vertex] { - semi[vertex] = semi[best as usize]; + if in_degree_column[vertex] == SINGLE { + forest.semi[vertex] = parent[vertex]; + forest.ancestor[vertex] = parent[vertex]; + continue; + } + let lo = hi - in_degree_column[vertex] as usize; + for &pred in &predecessor_column[lo..hi] { + let best = forest.eval(pred); + if forest.semi[best as usize] < forest.semi[vertex] { + forest.semi[vertex] = forest.semi[best as usize]; } } hi = lo; - ancestor[vertex] = parent[vertex]; + forest.ancestor[vertex] = parent[vertex]; } - drop((label, ancestor, predecessors, in_degree)); + let semi = forest.semi; + drop((forest.label, forest.ancestor, predecessor_column, in_degree_column)); // Idoms (SEMI-NCA): nearest common ancestor of parent and semi in the tree so far, climbing idom // links, which are final for every smaller number. - let mut dom = vec![0u32; vertex_count]; + let mut dom = Column::::zeroed(vertex_count); for vertex in 1..vertex_count { let mut candidate = parent[vertex]; while candidate > semi[vertex] { @@ -335,69 +533,171 @@ pub fn dominators(graph: &Graph, dump: &Dump) -> DominatorTree { } drop((semi, parent)); - // Retained and subtree sizes: reverse preorder visits children before parents. - let mut subtree_bytes = vec![0u64; vertex_count]; + // Each leaf meets its referrers at their nearest common ancestor, by DFS number plus one (zero: not + // reached), kept where its idom goes. The meet does not depend on the order referrers arrive in. + let mut idom = Column::::zeroed(object_count); + { + let leaf = idom.atomics(); + let ancestor = |mut a: u32, mut b: u32| { + while a != b { + if a > b { + a = dom[a as usize]; + } else { + b = dom[b as usize]; + } + } + a + }; + let meet = |object: u32, number: u32| { + let slot = &leaf[object as usize]; + let mut current = slot.load(Relaxed); + loop { + let next = if current == 0 { number } else { ancestor(current - 1, number) }; + if next + 1 == current { + return; + } + match slot.compare_exchange_weak(current, next + 1, Relaxed, Relaxed) { + Ok(_) => return, + Err(seen) => current = seen, + } + } + }; + for &gc_root in graph.roots.iter().filter(|&&gc_root| dfs_number[gc_root as usize] == LEAF) { + meet(gc_root, 0); + } + parallel::ranges(vertex_count - 1, |lo, hi| { + for number in lo + 1..=hi { + for target in graph.edges(vertex[number]).targets() { + if dfs_number[target as usize] == LEAF { + meet(target, number as u32); + } + } + } + }); + } + + // Retained and subtree sizes: leaves first, then reverse preorder visits children before parents. + let mut subtree_bytes = Column::::zeroed(vertex_count); parallel::chunks(&mut subtree_bytes[1..], |start, part| { for (bytes, &object) in part.iter_mut().zip(&vertex[start + 1..]) { - *bytes = u64::from(dump.objects[object as usize].shallow); + *bytes = u64::from(dump.objects.shallow(object as usize)); } }); - let mut size = vec![1u32; vertex_count]; + let mut size = parallel::filled(vertex_count, 1u32); + { + let (bytes, counts) = (subtree_bytes.atomics(), size.atomics()); + parallel::ranges(object_count, |lo, hi| { + for (object, (&leaf, &number)) in (lo..hi).zip(idom[lo..hi].iter().zip(&dfs_number[lo..hi])) { + if number == LEAF && leaf != 0 { + bytes[leaf as usize - 1].fetch_add(u64::from(dump.objects.shallow(object)), Relaxed); + counts[leaf as usize - 1].fetch_add(1, Relaxed); + } + } + }); + } for number in (1..vertex_count).rev() { let dominator = dom[number] as usize; subtree_bytes[dominator] += subtree_bytes[number]; size[dominator] += size[number]; } // Map each result back to object indices once final, so fewer arrays stand at once. - // Unreachable objects keep their shallow size. - let mut retained = vec![0u64; object_count]; - parallel::chunks(&mut retained, |start, part| { - for ((bytes, &number), object) in - part.iter_mut().zip(&dfs_number[start..]).zip(&dump.objects[start..]) + // Leaves and unreachable objects keep their shallow size. + let mut retained = Column::::zeroed(object_count); + let big = parallel::chunks(&mut retained, |start, part| { + let mut big = Vec::new(); + for (((bytes, &number), shallow), object) in part + .iter_mut() + .zip(&dfs_number[start..]) + .zip(dump.objects.shallow_range(start, dump.objects.len())) + .zip(start as u32..) { - *bytes = if number == NONE { u64::from(object.shallow) } else { subtree_bytes[number as usize] }; + let total = if number >= LEAF { u64::from(shallow) } else { subtree_bytes[number as usize] }; + *bytes = u32::try_from(total).unwrap_or(u32::MAX); + if *bytes == u32::MAX { + big.push((object, total)); + } } - }); + big + }) + .concat(); drop(subtree_bytes); - // A preorder in which each subtree is one range: an object takes its - // dominator's next free place, and DFS numbers put dominators first. - let (mut place, mut next_free) = (vec![0u32; vertex_count], vec![0u32; vertex_count]); - for number in 1..vertex_count { - let dominator = dom[number] as usize; - place[number] = next_free[dominator]; - next_free[dominator] += size[number]; - next_free[number] = place[number] + 1; - } - drop(next_free); - let (preorder, subtree_end) = (parallel::zeroed(vertex_count - 1), parallel::zeroed(vertex_count - 1)); - parallel::ranges(vertex_count - 1, |lo, hi| { - for number in (lo + 1)..=hi { - preorder[place[number] as usize].store(vertex[number], Relaxed); - subtree_end[place[number] as usize].store(place[number] + size[number], Relaxed); - } - }); - drop(size); - let mut idom = vec![0u32; object_count]; + // Idoms as objects, in place: a leaf's slot held its dominator's number plus one. parallel::chunks(&mut idom, |start, part| { for (dominator, &number) in part.iter_mut().zip(&dfs_number[start..]) { - *dominator = if number == NONE { NONE } else { vertex[dom[number as usize] as usize] }; + *dominator = match number { + NONE => NONE, + LEAF if *dominator == 0 => NONE, + LEAF => vertex[*dominator as usize - 1], + number => vertex[dom[number as usize] as usize], + }; } }); - drop((dom, vertex)); - // Last, turn DFS numbers into preorder places. + // A preorder in which each subtree is one range: an object takes its dominator's next free place, + // and DFS numbers put dominators first. Places go where the dominators were, and next free places + // where the sizes were, each read before it is overwritten. + let reachable = size[0] as usize - 1; + let mut place = dom; + let mut free = size; + free[0] = 0; + for number in 1..vertex_count { + let (dominator, size) = (place[number] as usize, free[number]); + place[number] = free[dominator]; + free[dominator] += size; + free[number] = place[number] + 1; + } + let (mut preorder, mut subtree_end) = + (Column::::zeroed(reachable), Column::::zeroed(reachable)); + { + let preorder = preorder.atomics(); + parallel::ranges(vertex_count - 1, |lo, hi| { + for number in (lo + 1)..=hi { + preorder[place[number] as usize].store(vertex[number], Relaxed); + } + }); + } + drop((place, vertex)); + // Leaves take what is left of their dominator's range. A leaf's dominator is a vertex, whose entry + // still holds its number. + dfs_number.truncate(object_count + 1); + { + let (preorder, subtree_end) = (preorder.atomics(), subtree_end.atomics()); + let (free, numbers) = (free.atomics(), dfs_number.atomics()); + parallel::ranges(object_count, |lo, hi| { + for object in lo..hi { + if numbers[object].load(Relaxed) != LEAF { + continue; + } + match idom[object] { + NONE => numbers[object].store(NONE, Relaxed), + dominator => { + let at = + free[numbers[dominator as usize].load(Relaxed) as usize].fetch_add(1, Relaxed); + preorder[at as usize].store(object as u32, Relaxed); + subtree_end[at as usize].store(at + 1, Relaxed); + } + } + } + }); + } + // Last, every placed object's number becomes its place; a vertex's range ends where its next free + // place stopped. + { + let numbers = dfs_number.atomics(); + parallel::chunks(&mut subtree_end, |start, part| { + for (end, (pos, &object)) in part.iter_mut().zip((start as u32..).zip(&preorder[start..])) { + let number = numbers[object as usize].load(Relaxed); + if number != LEAF { + *end = free[number as usize]; + } + numbers[object as usize].store(pos, Relaxed); + } + }); + } + drop(free); dfs_number.truncate(object_count); - parallel::chunks(&mut dfs_number, |_, part| { - for number in part.iter_mut().filter(|number| **number != NONE) { - *number = place[*number as usize]; - } - }); - drop(place); let preorder_index = dfs_number; - - let preorder = preorder.into_iter().map(AtomicU32::into_inner).collect(); - let subtree_end = subtree_end.into_iter().map(AtomicU32::into_inner).collect(); let mut tree = - DominatorTree { idom, retained, top_level: Vec::new(), preorder, subtree_end, preorder_index }; + DominatorTree { idom, retained, big, top_level: Vec::new(), preorder, subtree_end, preorder_index }; tree.top_level = tree.children(object_count as u32); tree } diff --git a/src/dump.rs b/src/dump.rs index 0822ccf..c518e08 100644 --- a/src/dump.rs +++ b/src/dump.rs @@ -7,6 +7,7 @@ use std::hash::{BuildHasherDefault, Hasher}; use super::hash::GOLDEN_RATIO; use super::hprof::{Frame, Header, Piece, Root, Ty, Value}; use super::sizes::Sizing; +use super::store::{BLOCK, Bits, Blocked, Column}; pub const NONE: u32 = u32::MAX; @@ -43,7 +44,46 @@ impl Hasher for FastHash { pub type FastMap = HashMap>; +/// Symbol texts back to back in one string, found by id: no allocation per symbol. +#[derive(Default)] +pub struct Symbols { + text: String, + spans: FastMap, +} + +impl Symbols { + pub fn insert(&mut self, id: u64, text: &[u8]) { + let start = self.text.len(); + self.text.push_str(&String::from_utf8_lossy(text)); + self.spans.insert(id, (start, self.text.len())); + } + + pub fn get(&self, id: u64) -> Option<&str> { + self.spans.get(&id).map(|&(start, end)| &self.text[start..end]) + } + + pub fn len(&self) -> usize { + self.spans.len() + } + + pub fn iter(&self) -> impl Iterator { + self.spans.iter().map(|(&id, &(start, end))| (id, &self.text[start..end])) + } + + /// Only the symbols `keep` names, in a fresh string. + pub fn only(&self, keep: impl IntoIterator) -> Symbols { + let mut kept = Symbols::default(); + for id in keep { + if let Some(text) = self.get(id) { + kept.insert(id, text.as_bytes()); + } + } + kept + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] pub enum Kind { Instance, ObjectArray, @@ -55,16 +95,290 @@ impl Kind { pub const ALL: [Kind; 4] = [Kind::Instance, Kind::ObjectArray, Kind::PrimitiveArray, Kind::Class]; } -/// One heap object. `len` is field bytes for instances, elements for arrays. +/// One heap object, but for its id, which takes a lookup of its own. `len` is the elements of an array, +/// zero for anything else. #[derive(Clone, Copy, Debug)] pub struct Object { - pub id: u64, pub class: u32, pub len: u32, pub shallow: u32, pub kind: Kind, } +/// Class index and kind in one word: the kind in the low bits. +pub const KIND_BITS: u32 = 2; +/// Classes the packed word leaves room for. +pub const MAX_CLASSES: usize = 1 << (u32::BITS - KIND_BITS); + +/// Object ids, ascending: 8-byte steps above the first (the lowest), blocked, while every id has one in +/// four bytes; else whole. +pub enum TableIds { + Steps(u64, Blocked), + Wide(Column), +} + +impl TableIds { + pub fn steps(base: u64, steps: &[u32]) -> TableIds { + TableIds::Steps(base, Blocked::new(steps.len(), |object| u64::from(steps[object]))) + } +} + +/// How a shape word splits: the kind in the low bits, `class_bits` of class, then the length. +#[derive(Clone, Copy)] +pub struct Packing { + class_mask: u32, + len_shift: u32, + /// All ones in the length bits: the length is in the long list. + marker: u32, +} + +impl Packing { + /// Room for classes `0..classes`. + pub fn new(classes: usize) -> Packing { + let class_bits = usize::BITS - classes.saturating_sub(1).leading_zeros(); + let len_shift = KIND_BITS + class_bits; + Packing { + class_mask: ((1u64 << class_bits) - 1) as u32, + len_shift, + marker: u32::MAX.checked_shr(len_shift).unwrap_or(0), + } + } + + /// One object's word, and its length again when that goes in the long list. + pub fn pack(self, class: u32, kind: Kind, len: u32) -> (u32, Option) { + let inline = len.min(self.marker).checked_shl(self.len_shift).unwrap_or(0); + (inline | class << KIND_BITS | kind as u32, (len >= self.marker && len != 0).then_some(len)) + } +} + +/// Class, kind and length of each object in one word. The length is an array's elements or a class +/// object's shallow size; an instance's size comes from its class. A length too long for its bits is +/// kept in `long`, by object. +pub struct Shapes { + words: Column, + packing: Packing, + long: Vec<(u32, u32)>, +} + +impl Shapes { + pub fn new(words: Column, packing: Packing, long: Vec<(u32, u32)>) -> Shapes { + Shapes { words, packing, long } + } + + pub fn words(&self) -> &[u32] { + &self.words + } + + pub fn long(&self) -> &[(u32, u32)] { + &self.long + } + + fn class(&self, word: u32) -> u32 { + (word >> KIND_BITS) & self.packing.class_mask + } + + fn kind(word: u32) -> Kind { + Kind::ALL[(word & ((1 << KIND_BITS) - 1)) as usize] + } + + fn len(&self, object: usize, word: u32) -> u32 { + let inline = word.checked_shr(self.packing.len_shift).unwrap_or(0); + if inline != self.packing.marker { + return inline; + } + let found = self.long.binary_search_by_key(&(object as u32), |&(object, _)| object); + found.map_or(0, |at| self.long[at].1) + } +} + +/// The object table, sorted by id, as columns: ids, and the shape words. Shallow sizes are worked out on +/// read from the class and length. +pub struct ObjectTable { + ids: TableIds, + lookup: Buckets, + shapes: Shapes, + /// Per class: an instance's shallow size, and an array element's footprint. + instance_shallow: Vec, + element_size: Vec, + sizing: Sizing, +} + +impl ObjectTable { + /// An empty table for these classes and sizes. + fn new(classes: &[Class], sizing: Sizing) -> ObjectTable { + ObjectTable { + ids: TableIds::Wide(Column::with_capacity(0)), + lookup: Buckets::empty(), + shapes: Shapes::new(Column::with_capacity(0), Packing::new(classes.len()), Vec::new()), + instance_shallow: classes.iter().map(|class| class.shallow).collect(), + element_size: classes + .iter() + .map(|class| class.element_type.map_or(1, |ty| sizing.field_size(ty))) + .collect(), + sizing, + } + } + + /// A table from merged columns. + pub fn from_steps(classes: &[Class], sizing: Sizing, ids: TableIds, shapes: Shapes) -> ObjectTable { + let mut table = ObjectTable::new(classes, sizing); + (table.ids, table.shapes) = (ids, shapes); + table.index(); + table + } + + /// A table from cached columns. + pub fn from_columns(classes: &[Class], sizing: Sizing, ids: &[u64], shapes: Shapes) -> ObjectTable { + let mut table = ObjectTable::new(classes, sizing); + let base = ids.first().copied().unwrap_or(0); + let steps: Option> = ids.iter().map(|&id| step_of(base, id)).collect(); + table.ids = match steps { + Some(steps) => TableIds::steps(base, &steps), + None => TableIds::Wide(ids.iter().copied().collect()), + }; + table.shapes = shapes; + table.index(); + table + } + + /// The shape words, as the cache stores them. + pub fn shapes(&self) -> &Shapes { + &self.shapes + } + + /// Bucket the ids for lookups, once they are all in. + fn index(&mut self) { + self.lookup = Buckets::build(self); + } + + pub fn len(&self) -> usize { + self.shapes.words.len() + } + + pub fn is_empty(&self) -> bool { + self.shapes.words.is_empty() + } + + pub fn id(&self, object: usize) -> u64 { + match &self.ids { + TableIds::Steps(base, steps) => base + (steps.get(object) << 3), + TableIds::Wide(wide) => wide[object], + } + } + + /// The index of the object with this id. + pub fn lookup(&self, id: u64) -> Option { + let (lo, hi) = self.lookup.range(id)?; + let at = match &self.ids { + TableIds::Steps(base, steps) => { + let step = u64::from(step_of(*base, id)?); + // Within one block the two-byte parts ascend too, the long ones last. + if hi > lo && (hi - 1) / BLOCK == lo / BLOCK { + let wanted = + step.checked_sub(steps.base(lo)).and_then(|wanted| u16::try_from(wanted).ok()); + if let Some(wanted) = wanted.filter(|&wanted| wanted != u16::MAX) { + return steps.above(lo, hi).binary_search(&wanted).ok().map(|at| (lo + at) as u32); + } + } + let (mut at, mut end) = (lo, hi); + while at < end { + let mid = at + (end - at) / 2; + if steps.get(mid) < step { + at = mid + 1; + } else { + end = mid; + } + } + (at < hi && steps.get(at) == step).then_some(at) + } + TableIds::Wide(wide) => wide[lo..hi].binary_search(&id).ok().map(|i| lo + i), + }; + at.map(|at| at as u32) + } + + pub fn class(&self, object: usize) -> u32 { + self.shapes.class(self.shapes.words[object]) + } + + pub fn kind(&self, object: usize) -> Kind { + Shapes::kind(self.shapes.words[object]) + } + + pub fn shallow(&self, object: usize) -> u32 { + self.shallow_of(object, self.shapes.words[object]) + } + + fn shallow_of(&self, object: usize, word: u32) -> u32 { + let class = self.shapes.class(word) as usize; + match Shapes::kind(word) { + Kind::Instance => self.instance_shallow[class], + Kind::ObjectArray => { + self.sizing.array_size(u64::from(self.shapes.len(object, word)), self.sizing.ref_size) + } + Kind::PrimitiveArray => { + self.sizing.array_size(u64::from(self.shapes.len(object, word)), self.element_size[class]) + } + Kind::Class => self.shapes.len(object, word), + } + } + + /// Everything about one object. + pub fn get(&self, object: usize) -> Object { + let word = self.shapes.words[object]; + let kind = Shapes::kind(word); + let len = match kind { + Kind::ObjectArray | Kind::PrimitiveArray => self.shapes.len(object, word), + Kind::Instance | Kind::Class => 0, + }; + Object { class: self.shapes.class(word), len, shallow: self.shallow_of(object, word), kind } + } + + /// Objects `start..end`, in order. + pub fn range(&self, start: usize, end: usize) -> impl Iterator + '_ { + (start..end).map(|object| self.get(object)) + } + + /// Shallow sizes of objects `start..end`, in order. + pub fn shallow_range(&self, start: usize, end: usize) -> impl Iterator + '_ { + (start..).zip(&self.shapes.words[start..end]).map(|(object, &word)| self.shallow_of(object, word)) + } +} + +/// Both hashes of every primitive array, in table order, as the index pass took them: the String hash +/// (zero when not taken) and the grouping hash. A bit per object marks the arrays and a count per 64 +/// objects finds an array's pair. +pub struct ArrayHashes { + arrays: Bits, + before: Column, + values: Column<[u64; 2]>, +} + +impl ArrayHashes { + /// One pair per primitive array in `objects`, in order; `None` when the counts disagree. + pub fn new(objects: &ObjectTable, values: Column<[u64; 2]>) -> Option { + let mut arrays = + super::parallel::bits(objects.len(), |object| objects.kind(object) == Kind::PrimitiveArray); + let mut before = Column::with_capacity(arrays.words().len()); + let mut count = 0; + for &word in arrays.words().iter() { + before.push(count); + count += word.count_ones(); + } + (count as usize == values.len()).then_some(ArrayHashes { arrays, before, values }) + } + + pub fn get(&self, object: u32) -> Option<[u64; 2]> { + let (word, bit) = (object as usize / 64, object % 64); + let bits = self.arrays.word(word); + (bits >> bit & 1 != 0) + .then(|| self.values[(self.before[word] + (bits & ((1 << bit) - 1)).count_ones()) as usize]) + } + + pub fn values(&self) -> &[[u64; 2]] { + &self.values + } +} + pub struct Field { pub name: u32, pub ty: Ty, @@ -177,10 +491,12 @@ pub struct Dump { pub pieces: Vec, /// Interned field names, referenced by `Slot::label` and `Field::name`. pub names: Vec, - pub symbols: FastMap, + pub symbols: Symbols, pub classes: Vec, - /// Sorted by id. - pub objects: Vec, + pub objects: ObjectTable, + /// Primitive array hashes and boxed value tallies from the index pass; `None` when it did not take them. + pub array_hashes: Option, + pub boxed_values: Option>, pub roots: Vec<(u32, Root)>, pub dangling_roots: u64, /// Objects ART marked unreachable: counted, never roots. @@ -189,7 +505,6 @@ pub struct Dump { pub traces: HashMap, pub frames: FastMap, pub class_by_serial: HashMap, - pub lookup: Buckets, /// `java.lang.Class`, `java.lang.String`, and the `value` / `name` labels. pub class_class: u32, pub string_class: u32, @@ -197,11 +512,17 @@ pub struct Dump { pub name_label: u32, } -/// Buckets widen until the id span needs at most this many. +/// An id's 8-byte steps above `base`, if it is aligned and within four bytes of them. +fn step_of(base: u64, id: u64) -> Option { + let distance = id.checked_sub(base).filter(|distance| distance.trailing_zeros() >= 3)?; + u32::try_from(distance >> 3).ok() +} + +/// Buckets widen until the id span needs at most this many, or as many as there are objects. const MAX_BUCKETS: u64 = 1 << 22; /// The object table bucketed by address range: a short binary search per id, not a hash lookup. -pub struct Buckets { +struct Buckets { min: u64, max: u64, shift: u32, @@ -209,20 +530,23 @@ pub struct Buckets { } impl Buckets { - pub fn build(objects: &[Object]) -> Buckets { - let (min, max) = match (objects.first(), objects.last()) { - (Some(first), Some(last)) => (first.id, last.id), - _ => return Buckets { min: 1, max: 0, shift: 0, starts: vec![0, 0] }, - }; + fn empty() -> Buckets { + Buckets { min: 1, max: 0, shift: 0, starts: vec![0, 0] } + } + + fn build(table: &ObjectTable) -> Buckets { + let Some(last) = table.len().checked_sub(1) else { return Buckets::empty() }; + let (min, max) = (table.id(0), table.id(last)); let span = max - min + 1; + let limit = (table.len() as u64).clamp(1, MAX_BUCKETS); let mut shift = 0; - while (span >> shift) > MAX_BUCKETS { + while (span >> shift) > limit { shift += 1; } let bucket_count = ((span - 1) >> shift) as usize + 1; let mut starts = vec![0u32; bucket_count + 1]; - for object in objects { - starts[((object.id - min) >> shift) as usize + 1] += 1; + for object in 0..table.len() { + starts[((table.id(object) - min) >> shift) as usize + 1] += 1; } for i in 0..bucket_count { starts[i + 1] += starts[i]; @@ -230,16 +554,58 @@ impl Buckets { Buckets { min, max, shift, starts } } - pub fn find(&self, objects: &[Object], id: u64) -> Option { + /// The table range an id would be in. + fn range(&self, id: u64) -> Option<(usize, usize)> { if id < self.min || id > self.max { return None; } let bucket = ((id - self.min) >> self.shift) as usize; - let (lo, hi) = (self.starts[bucket] as usize, self.starts[bucket + 1] as usize); - objects[lo..hi].binary_search_by_key(&id, |o| o.id).ok().map(|i| (lo + i) as u32) + Some((self.starts[bucket] as usize, self.starts[bucket + 1] as usize)) } } +/// The class with exactly this name that has a layout, else any with the name. +pub fn class_named(classes: &[Class], name: &str) -> Option { + let named = |class: &Class| class.name == name; + let dumped = classes.iter().position(|class| named(class) && class.dumped); + dumped.or_else(|| classes.iter().position(named)).map(|i| i as u32) +} + +/// A class and its superclasses, nearest first. +pub fn ancestry(classes: &[Class], class: u32) -> impl Iterator + '_ { + let known = |class: u32| (class != NONE).then_some(class); + std::iter::successors(known(class), move |¤t| known(classes[current as usize].superclass)) + .take(MAX_CLASS_DEPTH) +} + +/// Instance field layout for `class`: `(name, type, offset)` up the chain. +pub fn field_layout(classes: &[Class], id_size: u32, class: u32) -> Vec<(u32, Ty, u32)> { + let mut out = Vec::new(); + let mut offset = 0u32; + for ancestor in ancestry(classes, class) { + for field in &classes[ancestor as usize].fields { + out.push((field.name, field.ty, offset)); + offset += field.ty.size(id_size); + } + } + out +} + +/// Offset and type of the instance field called `name`, if `class` has one. +pub fn field_offset( + classes: &[Class], + names: &[String], + id_size: u32, + class: u32, + name: &str, +) -> Option<(u32, Ty)> { + let id = names.iter().position(|interned| interned == name)? as u32; + field_layout(classes, id_size, class) + .into_iter() + .find(|&(field_id, _, _)| field_id == id) + .map(|(_, ty, offset)| (offset, ty)) +} + /// `java/util/Map$Entry`, `[I`, `[Ljava/lang/String;` as source spells them. pub fn pretty_class_name(raw: &str) -> String { match raw.strip_prefix('[') { @@ -287,11 +653,11 @@ pub fn package_of(name: &str) -> &str { impl Dump { pub fn lookup(&self, id: u64) -> Option { - self.lookup.find(&self.objects, id) + self.objects.lookup(id) } pub fn class_of(&self, object: u32) -> &Class { - &self.classes[self.objects[object as usize].class as usize] + &self.classes[self.objects.class(object as usize) as usize] } pub fn class_name(&self, object: u32) -> &str { @@ -304,9 +670,7 @@ impl Dump { /// The class with exactly this name that has a layout, else any with the name. pub fn class_named(&self, name: &str) -> Option { - let named = |class: &Class| class.name == name; - let dumped = self.classes.iter().position(|class| named(class) && class.dumped); - dumped.or_else(|| self.classes.iter().position(named)).map(|i| i as u32) + class_named(&self.classes, name) } /// Whether `class` is `base` or extends it. @@ -316,9 +680,7 @@ impl Dump { /// A class and its superclasses, nearest first. pub fn ancestry(&self, class: u32) -> impl Iterator + '_ { - let known = |class: u32| (class != NONE).then_some(class); - std::iter::successors(known(class), move |¤t| known(self.classes[current as usize].superclass)) - .take(MAX_CLASS_DEPTH) + ancestry(&self.classes, class) } /// Index of an interned field name. @@ -328,24 +690,12 @@ impl Dump { /// Instance field layout for `class`: `(name, type, offset)` up the chain. pub fn field_layout(&self, class: u32) -> Vec<(u32, Ty, u32)> { - let mut out = Vec::new(); - let mut offset = 0u32; - for ancestor in self.ancestry(class) { - for field in &self.classes[ancestor as usize].fields { - out.push((field.name, field.ty, offset)); - offset += field.ty.size(self.header.id_size); - } - } - out + field_layout(&self.classes, self.header.id_size, class) } /// Offset and type of the instance field called `name`, if `class` has one. pub fn field_offset(&self, class: u32, name: &str) -> Option<(u32, Ty)> { - let id = self.name_id(name)?; - self.field_layout(class) - .into_iter() - .find(|&(field_id, _, _)| field_id == id) - .map(|(_, ty, offset)| (offset, ty)) + field_offset(&self.classes, &self.names, self.header.id_size, class, name) } /// The label a class-object edge carries for its `k`-th static field. @@ -370,8 +720,8 @@ impl Dump { let class = self .class_by_serial(frame.class_serial) .map_or("?", |idx| self.classes[idx as usize].name.as_str()); - let method = self.symbols.get(&frame.method_id).map_or("?", String::as_str); - let source = self.symbols.get(&frame.source_id).map_or("", String::as_str); + let method = self.symbols.get(frame.method_id).unwrap_or("?"); + let source = self.symbols.get(frame.source_id).unwrap_or(""); let line = match frame.line { n if n > 0 => format!(":{n}"), -2 => " compiled".to_string(), @@ -393,6 +743,10 @@ impl Dump { } pub fn total_shallow(&self) -> u64 { - self.objects.iter().map(|object| u64::from(object.shallow)).sum() + super::parallel::ranges(self.objects.len(), |lo, hi| { + self.objects.shallow_range(lo, hi).map(u64::from).sum::() + }) + .into_iter() + .sum() } } diff --git a/src/graph.rs b/src/graph.rs index 025f8f8..0cbc448 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -4,10 +4,12 @@ use std::io; use std::sync::atomic::{AtomicU32, Ordering::Relaxed}; -use super::dump::{Dump, Kind, RefKind}; +use super::dump::{Dump, Kind, RefKind, Slot}; use super::hprof::{self, Body, IO_BUFFER_SIZE, Sink, Ty, be_uint}; +use super::index::References; use super::parallel; use super::source::Source; +use super::store::{Blocked, Column}; use crate::error::{Error, Result}; /// Label flag: the edge is an array element, the low bits are its index. @@ -31,16 +33,199 @@ impl ReferencePolicy { } } -/// Offsets, targets, labels and weak edges, as the cache stores them. -pub type Parts<'a> = (&'a [u64], &'a [u32], &'a [u32], &'a [(u32, u32)]); +/// Targets and weak edges, as the cache stores them. +pub type Parts<'a> = (&'a [u32], &'a [(u32, u32)]); + +/// Where each object's edges start, then the total: two bytes each above a base every block of objects. +struct Offsets(Blocked); + +impl Offsets { + // Takes the plain offsets so they are freed once packed. + #[allow(clippy::needless_pass_by_value)] + fn new(offsets: Column) -> Offsets { + Offsets(Blocked::new(offsets.len(), |i| offsets[i])) + } + + fn get(&self, i: usize) -> u64 { + self.0.get(i) + } + + fn len(&self) -> usize { + self.0.len() + } +} + +/// A label code with this bit is an array element; the rest is the nulls since the previous element. +const ELEMENT: u16 = 1 << 15; +/// The low bits of a code whose label is kept whole in the escapes. +const ESCAPED: u16 = ELEMENT - 1; + +/// Edge labels in two bytes each: a code for the field (the most used fields get one), or for an array +/// element the nulls since the previous one. A field without a code and a gap past fifteen bits leave +/// the low bits all ones, and the label is kept whole in `escapes`, by edge. +struct Labels { + codes: Column, + fields: Vec, + escapes: Vec<(u64, u32)>, + /// Per field name, then the loader: its code, or all ones. + code_of: Vec, + names: usize, +} + +impl Labels { + /// Code the labels of the edges `offsets` lays out, each object's from its offset to the next. + // Takes the plain labels so they are freed once coded. + #[allow(clippy::needless_pass_by_value)] + fn new(offsets: &[u64], labels: Column, dump: &Dump) -> Labels { + let names = dump.names.len(); + let slot = |label: u32| match label { + LOADER => names, + label if (label as usize) < names => label as usize, + _ => names + 1, + }; + // The fields the classes name: when they fit the codes, each gets one; else the most used do. + let mut listed = vec![false; names + 2]; + listed[names] = true; + for class in &dump.classes { + for slot in &class.slots { + listed[(slot.label as usize).min(names + 1)] = true; + } + for field in class.statics.iter().filter(|field| field.ty == Ty::Object) { + listed[(field.name as usize).min(names + 1)] = true; + } + } + listed[names + 1] = false; + let label_at = |at: usize| if at == names { LOADER } else { at as u32 }; + let mut fields: Vec = (0..=names).filter(|&at| listed[at]).map(label_at).collect(); + if fields.len() > usize::from(ESCAPED) { + let counts = parallel::ranges(labels.len(), |lo, hi| { + let mut counts = vec![0u64; names + 2]; + for &label in labels[lo..hi].iter().filter(|&&label| label & ARRAY_ELEMENT == 0) { + counts[slot(label)] += 1; + } + counts + }); + let mut used: Vec<(u64, u32)> = (0..=names) + .map(|at| (counts.iter().map(|part| part[at]).sum::(), label_at(at))) + .filter(|&(count, _)| count > 0) + .collect(); + used.sort_unstable_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + used.truncate(usize::from(ESCAPED)); + fields = used.iter().map(|&(_, label)| label).collect(); + } + let mut code_of = vec![ESCAPED; names + 2]; + for (code, &label) in fields.iter().enumerate() { + code_of[slot(label)] = code as u16; + } + // Workers take ranges of objects, each coding its own run of edges. + let objects = offsets.len() - 1; + let per = objects.div_ceil(parallel::threads()).max(1); + let mut codes = Column::::zeroed(labels.len()); + let mut rest = &mut codes[..]; + let mut tasks = Vec::new(); + for lo in (0..objects).step_by(per) { + let hi = (lo + per).min(objects); + let (start, end) = (offsets[lo], offsets[hi]); + let (part, tail) = std::mem::take(&mut rest).split_at_mut((end - start) as usize); + rest = tail; + let (labels, code_of) = (&labels, &code_of); + tasks.push(move || { + let mut escapes = Vec::new(); + for object in lo..hi { + let mut previous: Option = None; + for edge in offsets[object]..offsets[object + 1] { + let label = labels[edge as usize]; + let code = if label & ARRAY_ELEMENT == 0 { + code_of[slot(label)] + } else { + let index = label & !ARRAY_ELEMENT; + let gap = index - previous.map_or(0, |previous| previous + 1); + previous = Some(index); + ELEMENT | u16::try_from(gap).unwrap_or(ESCAPED).min(ESCAPED) + }; + if code & ESCAPED == ESCAPED { + escapes.push((edge, label)); + } + part[(edge - start) as usize] = code; + } + } + escapes + }); + } + let escapes = parallel::run_all(tasks).concat(); + Labels { codes, fields, escapes, code_of, names } + } + + /// The code of a field label, when it has one. + fn code(&self, label: u32) -> Option { + let at = match label { + LOADER => self.names, + label => (label as usize).min(self.names + 1), + }; + self.code_of.get(at).copied().filter(|&code| code != ESCAPED) + } +} + +/// One object's labels, decoded in order. +#[derive(Clone, Copy)] +struct EdgeLabels<'a> { + codes: &'a [u16], + /// The first edge's place among all edges. + start: u64, + fields: &'a [u32], + escapes: &'a [(u64, u32)], +} + +impl<'a> EdgeLabels<'a> { + /// Edge `k`'s field label, or just `ARRAY_ELEMENT` for an element. + fn coarse(self, k: usize) -> u32 { + let code = self.codes[k]; + if code & ELEMENT != 0 { + ARRAY_ELEMENT + } else if code == ESCAPED { + self.escape(self.start + k as u64) + } else { + self.fields[usize::from(code)] + } + } + + /// Edge `k`'s label; an element's index adds up the gaps before it. + fn exact(self, k: usize) -> u32 { + match self.codes[k] & ELEMENT { + 0 => self.coarse(k), + _ => self.iter().nth(k).expect("k is an edge"), + } + } + + fn escape(self, edge: u64) -> u32 { + self.escapes[self.escapes.partition_point(|&(at, _)| at < edge)].1 + } + + fn iter(self) -> impl Iterator + 'a { + let mut previous: Option = None; + (self.start..).zip(self.codes).map(move |(edge, &code)| { + let label = if code & ESCAPED == ESCAPED { + self.escape(edge) + } else if code & ELEMENT != 0 { + ARRAY_ELEMENT | (previous.map_or(0, |previous| previous + 1) + u32::from(code & ESCAPED)) + } else { + self.fields[usize::from(code)] + }; + if label & ARRAY_ELEMENT != 0 { + previous = Some(label & !ARRAY_ELEMENT); + } + label + }) + } +} /// The object graph: out-edges per object, and the GC roots. pub struct Graph { /// The virtual root every GC root hangs off; equals the object count. pub root: u32, - offsets: Vec, - targets: Vec, - labels: Vec, + offsets: Offsets, + targets: Column, + labels: Labels, /// Referent edges left out of the graph proper, sorted by source. weak: Vec<(u32, u32)>, /// References to ids the dump does not contain. @@ -53,7 +238,7 @@ pub struct Graph { #[derive(Clone, Copy)] pub struct Edges<'a> { targets: &'a [u32], - labels: &'a [u32], + labels: EdgeLabels<'a>, } impl<'a> Edges<'a> { @@ -61,32 +246,56 @@ impl<'a> Edges<'a> { self.targets.len() } - pub fn get(&self, i: usize) -> Option { - self.targets.get(i).copied() - } - - pub fn targets(&self) -> impl Iterator + 'a { + pub fn targets(&self) -> impl Iterator + use<'a> { self.targets.iter().copied() } - pub fn labels(&self) -> impl Iterator + 'a { - self.labels.iter().copied() + pub fn labels(&self) -> impl Iterator + use<'a> { + self.labels.iter() } /// `(target, label)` pairs. - pub fn iter(&self) -> impl Iterator + 'a { + pub fn iter(&self) -> impl Iterator + use<'a> { self.targets().zip(self.labels()) } + /// Edge `k`'s field label, or just `ARRAY_ELEMENT` for an element: no gaps to add up. + pub fn coarse(&self, k: usize) -> u32 { + self.labels.coarse(k) + } + + /// `(target, label)` pairs with every element's label just `ARRAY_ELEMENT`. + pub fn iter_coarse(&self) -> impl Iterator + use<'a> { + let labels = self.labels; + self.targets().enumerate().map(move |(k, target)| (target, labels.coarse(k))) + } + pub fn label_of(&self, target: u32) -> Option { - self.iter().find(|&(to, _)| to == target).map(|(_, label)| label) + let k = self.targets.iter().position(|&to| to == target)?; + Some(self.labels.exact(k)) } } impl Graph { + /// Where `object`'s edges sit in [`Graph::all_targets`]. + pub fn span(&self, object: u32) -> std::ops::Range { + self.offsets.get(object as usize) as usize..self.offsets.get(object as usize + 1) as usize + } + + /// Every edge's target, object by object. + pub fn all_targets(&self) -> &[u32] { + &self.targets + } + pub fn edges(&self, object: u32) -> Edges<'_> { - let (lo, hi) = (self.offsets[object as usize] as usize, self.offsets[object as usize + 1] as usize); - Edges { targets: &self.targets[lo..hi], labels: &self.labels[lo..hi] } + let (lo, hi) = + (self.offsets.get(object as usize) as usize, self.offsets.get(object as usize + 1) as usize); + let labels = &self.labels; + let (codes, start) = (&labels.codes[lo..hi], lo as u64); + Edges { + targets: &self.targets[lo..hi], + labels: EdgeLabels { codes, start, fields: &labels.fields, escapes: &labels.escapes }, + } } pub fn edge_count(&self) -> u64 { @@ -108,7 +317,13 @@ impl Graph { /// The target of `object`'s edge labelled `label`. pub fn field(&self, object: u32, label: u32) -> Option { - self.edges(object).iter().find(|&(_, edge_label)| edge_label == label).map(|(target, _)| target) + let edges = self.edges(object); + match self.labels.code(label) { + Some(code) => edges.labels.codes.iter().position(|&at| at == code).map(|k| edges.targets[k]), + None => { + edges.iter_coarse().find(|&(_, edge_label)| edge_label == label).map(|(target, _)| target) + } + } } /// The referent of a Reference object, weak or strong. @@ -118,43 +333,82 @@ impl Graph { /// Reassemble a graph from cached parts. pub fn from_parts( - offsets: Vec, - targets: Vec, - labels: Vec, + offsets: Column, + targets: Column, + labels: Column, + dump: &Dump, weak: Vec<(u32, u32)>, dangling: u64, roots: Vec, ) -> Graph { - Graph { root: offsets.len() as u32 - 1, offsets, targets, labels, weak, dangling, roots } + let labels = Labels::new(&offsets, labels, dump); + let offsets = Offsets::new(offsets); + Graph { root: offsets.len() as u32 - 1, labels, offsets, targets, weak, dangling, roots } } /// The parts the cache writes. pub fn parts(&self) -> Parts<'_> { - (&self.offsets, &self.targets, &self.labels, &self.weak) + (&self.targets, &self.weak) + } + + /// Every edge's label, object by object. + pub fn all_labels(&self) -> impl Iterator + '_ { + (0..self.root).flat_map(|object| self.edges(object).labels()) + } + + /// Where each object's edges start, then the total, as the cache stores them. + pub fn offsets(&self) -> impl ExactSizeIterator + '_ { + (0..self.offsets.len()).map(|i| self.offsets.get(i)) } } -/// Read every reference in the dump into a graph over object indices. -pub fn build(dump: &Dump, source: &Source, reference_policy: ReferencePolicy) -> Result { +/// Every reference in the dump as a graph over object indices: from the ids the index pass copied, or +/// read from the dump when it could not copy them. +pub fn build( + dump: &Dump, + source: &Source, + reference_policy: ReferencePolicy, + references: Option, +) -> Result { let object_count = dump.objects.len(); - // offsets[i + 1] holds object i's out-degree until the prefix sum below. - let mut offsets = vec![0u64; object_count + 1]; + // offsets[i + 1] holds object i's out-degree until the prefix sum below. Read from the dump, every slot + // gets a place and the holes are squeezed out after; copied ids are counted first, so only dangling + // ones leave holes. + let mut offsets = Column::::zeroed(object_count + 1); parallel::chunks(&mut offsets[1..], |start, degrees| { - for (degree, object) in degrees.iter_mut().zip(&dump.objects[start..]) { - *degree = match object.kind { - Kind::Instance => dump.classes[object.class as usize].slots.len() as u64, - Kind::ObjectArray => u64::from(object.len), - Kind::PrimitiveArray | Kind::Class => 0, + for ((degree, object), idx) in + degrees.iter_mut().zip(dump.objects.range(start, dump.objects.len())).zip(start..) + { + *degree = match (object.kind, &references) { + (Kind::Instance, None) => dump.classes[object.class as usize].slots.len() as u64, + (Kind::ObjectArray, None) => u64::from(object.len), + (Kind::Instance, Some(references)) => { + let class = &dump.classes[object.class as usize]; + let kept_referent = reference_policy.keeps(class.ref_kind.unwrap_or(RefKind::Weak)); + let start = references.starts[idx] as usize; + let copied = &references.copied.ids[start..start + class.slots.len()]; + let edge = |(slot, &id): (&Slot, &u32)| id != 0 && (kept_referent || !slot.weak); + class.slots.iter().zip(copied).filter(|&pair| edge(pair)).count() as u64 + } + (Kind::ObjectArray, Some(references)) => { + let start = references.starts[idx] as usize; + let copied = &references.copied.ids[start..start + object.len as usize]; + copied.iter().filter(|&&id| id != 0).count() as u64 + } + (Kind::PrimitiveArray | Kind::Class, _) => 0, }; } }); // Class objects: one edge per reference-typed static, plus the loader. + let exact = references.is_some(); + let static_edge = |id: u64| !exact || (id != 0 && dump.lookup(id).is_some()); let mut class_slots: Vec<(u32, u32)> = Vec::new(); for (class_idx, class) in dump.classes.iter().enumerate() { if let Some(idx) = class.dumped.then(|| dump.lookup(class.id)).flatten() { - let ref_count = class.statics.iter().filter(|field| field.ty == Ty::Object).count() as u64 + 1; + let statics = class.statics.iter().filter(|field| field.ty == Ty::Object); + let edges = statics.filter(|field| static_edge(field.value.bits())).count() as u64; class_slots.push((idx, class_idx as u32)); - offsets[idx as usize + 1] += ref_count; + offsets[idx as usize + 1] += edges + u64::from(static_edge(class.loader)); } } for i in 0..object_count { @@ -163,7 +417,8 @@ pub fn build(dump: &Dump, source: &Source, reference_policy: ReferencePolicy) -> let total = usize::try_from(offsets[object_count]) .map_err(|_| Error::Dump("too many references for this machine".into()))?; // A target is stored plus one, leaving zero for a hole. - let (targets, labels) = (parallel::zeroed(total), parallel::zeroed(total)); + let (mut target_column, mut label_column) = (Column::::zeroed(total), Column::::zeroed(total)); + let (targets, labels) = (target_column.atomics(), label_column.atomics()); let mut dangling = 0u64; for (idx, class_idx) in class_slots { let class = &dump.classes[class_idx as usize]; @@ -172,36 +427,57 @@ pub fn build(dump: &Dump, source: &Source, reference_policy: ReferencePolicy) -> if let hprof::Value::Ref(id) = field.value { match (id, dump.lookup(id)) { (0, _) => {} - (_, Some(target)) => set_edge(&targets, &labels, slot, target, field.name), + (_, Some(target)) => set_edge(targets, labels, slot, target, field.name), (_, None) => dangling += 1, } } - slot += 1; + slot += usize::from(static_edge(field.value.bits())); } if let Some(target) = (class.loader != 0).then(|| dump.lookup(class.loader)).flatten() { - set_edge(&targets, &labels, slot, target, LOADER); + set_edge(targets, labels, slot, target, LOADER); } } - let fillers = source.scan(&dump.chunks, "reading references", || Filler { - dump, - offsets: &offsets, - targets: &targets, - labels: &labels, - weak: Vec::new(), - reference_policy, - dangling: 0, - next: 0, - })?; + let found = match &references { + Some(references) => resolve(dump, references, &offsets, targets, labels, reference_policy), + None => source + .scan(&dump.chunks, "reading references", || Filler { + dump, + offsets: &offsets, + targets, + labels, + weak: Vec::new(), + reference_policy, + dangling: 0, + next: 0, + })? + .into_iter() + .map(|filler| (filler.weak, filler.dangling)) + .collect(), + }; + drop(references); let mut weak: Vec<(u32, u32)> = Vec::new(); - for filler in fillers { - dangling += filler.dangling; - weak.extend(filler.weak); + // Read from the dump every null is a hole; counted from copied ids only dangling ones are. + let mut holes = !exact; + for (part_weak, part_dangling) in found { + dangling += part_dangling; + holes |= part_dangling > 0; + weak.extend(part_weak); } weak.sort_unstable(); + let mut roots: Vec = dump.roots.iter().map(|&(object, _)| object).collect(); + roots.sort_unstable(); + roots.dedup(); + if !holes { + let (mut targets, labels) = (target_column, label_column); + parallel::chunks(&mut targets, |_, part| part.iter_mut().for_each(|target| *target -= 1)); + let labels = Labels::new(&offsets, labels, dump); + let offsets = Offsets::new(offsets); + return Ok(Graph { root: object_count as u32, offsets, targets, labels, weak, dangling, roots }); + } // Squeeze out null and dangling holes: each worker packs its range to the front, then ranges close up. - let mut packed = vec![0u64; object_count + 1]; + let mut packed = Column::::zeroed(object_count + 1); let blocks = parallel::chunks(&mut packed[1..], |start, ends| { let (from, mut dst) = (offsets[start] as usize, offsets[start] as usize); for (i, end) in (start..).zip(ends.iter_mut()) { @@ -217,8 +493,7 @@ pub fn build(dump: &Dump, source: &Source, reference_policy: ReferencePolicy) -> } (start + ends.len(), from, dst) }); - let mut targets: Vec = targets.into_iter().map(AtomicU32::into_inner).collect(); - let mut labels: Vec = labels.into_iter().map(AtomicU32::into_inner).collect(); + let (mut targets, mut labels) = (target_column, label_column); let (mut dst, mut start) = (0usize, 0usize); for (end, from, upto) in blocks { // Skip blocks already in place; a self-copy still costs. @@ -235,11 +510,67 @@ pub fn build(dump: &Dump, source: &Source, reference_policy: ReferencePolicy) -> labels.truncate(dst); targets.shrink_to_fit(); labels.shrink_to_fit(); + drop(offsets); + let labels = Labels::new(&packed, labels, dump); + let offsets = Offsets::new(packed); + Ok(Graph { root: object_count as u32, offsets, targets, labels, weak, dangling, roots }) +} - let mut roots: Vec = dump.roots.iter().map(|&(object, _)| object).collect(); - roots.sort_unstable(); - roots.dedup(); - Ok(Graph { root: object_count as u32, offsets: packed, targets, labels, weak, dangling, roots }) +/// Turn the copied ids into edges, a range of objects per worker, packed as counted. The same rules as +/// [`Filler`]: a referent the policy does not keep goes to the weak list, an id the dump lacks is dangling. +fn resolve( + dump: &Dump, + references: &References, + offsets: &[u64], + targets: &[AtomicU32], + labels: &[AtomicU32], + reference_policy: ReferencePolicy, +) -> Vec<(Vec<(u32, u32)>, u64)> { + parallel::ranges(dump.objects.len(), |lo, hi| { + let (mut weak, mut dangling) = (Vec::new(), 0u64); + for (idx, object) in (lo as u32..).zip(dump.objects.range(lo, hi)) { + let (start, base) = (references.starts[idx as usize] as usize, offsets[idx as usize] as usize); + match object.kind { + Kind::Instance => { + let class = &dump.classes[object.class as usize]; + let kept_referent = reference_policy.keeps(class.ref_kind.unwrap_or(RefKind::Weak)); + let mut place = base; + for (k, slot) in class.slots.iter().enumerate() { + let target = references.copied.id(start + k); + if target == 0 { + continue; + } + let kept = kept_referent || !slot.weak; + match dump.lookup(target) { + Some(target_idx) if !kept => weak.push((idx, target_idx)), + Some(target_idx) => set_edge(targets, labels, place, target_idx, slot.label), + None => dangling += 1, + } + place += usize::from(kept); + } + } + Kind::ObjectArray => { + let mut place = base; + for element in 0..object.len as usize { + let target = references.copied.id(start + element); + if target == 0 { + continue; + } + match dump.lookup(target) { + Some(target_idx) => { + let label = ARRAY_ELEMENT | element as u32; + set_edge(targets, labels, place, target_idx, label); + } + None => dangling += 1, + } + place += 1; + } + } + Kind::PrimitiveArray | Kind::Class => {} + } + } + (weak, dangling) + }) } fn set_edge(targets: &[AtomicU32], labels: &[AtomicU32], slot: usize, target: u32, label: u32) { @@ -264,12 +595,9 @@ struct Filler<'a> { impl Filler<'_> { fn index_of(&mut self, id: u64) -> Option { - let near = self - .dump - .objects - .get(self.next..) - .and_then(|rest| rest.iter().take(2).position(|object| object.id == id)); - let idx = near.map(|ahead| (self.next + ahead) as u32).or_else(|| self.dump.lookup(id)); + let objects = &self.dump.objects; + let near = (self.next..objects.len().min(self.next + 2)).find(|&next| objects.id(next) == id); + let idx = near.map(|next| next as u32).or_else(|| self.dump.lookup(id)); if let Some(i) = idx { self.next = i as usize + 1; } diff --git a/src/hprof/mod.rs b/src/hprof/mod.rs index aa15b5f..5f01d32 100644 --- a/src/hprof/mod.rs +++ b/src/hprof/mod.rs @@ -416,6 +416,18 @@ pub fn walk_chunk(reader: &mut Reader, end: u64, sink: &mut impl Sink) -> Result Ok(()) } +/// Read the heap sub-records before the first object, `[reader position, end)`: the class dumps and roots +/// `HotSpot` writes first. Returns where the objects start, or `end`. +pub fn prelude(reader: &mut Reader, end: u64, sink: &mut impl Sink) -> io::Result { + while reader.position() < end && !reader.at_eof() { + if matches!(reader.peek(), INSTANCE_DUMP | OBJ_ARRAY_DUMP | PRIM_ARRAY_DUMP | PRIM_ARRAY_NODATA) { + break; + } + sub_record(reader, sink)?; + } + Ok(reader.position()) +} + /// End the open chunk at `at`, dropping it when empty. fn close_chunk(walked: &mut Walked, at: u64) { if let Some((start, _)) = walked.chunks.pop_if(|&mut (_, end)| end == u64::MAX) diff --git a/src/index.rs b/src/index.rs index 9963795..6147b59 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1,32 +1,163 @@ -//! Pass one: everything but the references. Symbols, classes and their -//! layouts, the object table with shallow sizes, GC roots and thread stacks. +//! Pass one: everything in one read of the heap. Symbols, classes and their layouts, the object table +//! with shallow sizes, GC roots and thread stacks, and each object's references as raw ids for the graph. -use std::collections::{HashMap, HashSet}; +use std::cmp::Reverse; +use std::collections::{BinaryHeap, HashMap, HashSet}; use std::io; use std::sync::Arc; +use super::analysis::duplicates::boxed_classes; +use super::detail::ArrayHash; use super::dump::{ - self, Buckets, Class, Dump, FastMap, Field, Kind, MAX_CLASS_DEPTH, NONE, Object, RefKind, Slot, Static, + self, ArrayHashes, Class, Dump, FastMap, Field, Kind, MAX_CLASS_DEPTH, MAX_CLASSES, NONE, ObjectTable, + Packing, RefKind, Shapes, Slot, Static, Symbols, TableIds, }; use super::graph::LOADER; -use super::hprof::{self, Body, ClassDump, Frame, Header, Reader, Root, RootKind, Sink, Ty, View, Walked}; +use super::hprof::{ + self, Body, ClassDump, Frame, Header, IO_BUFFER_SIZE, Reader, Root, RootKind, Sink, Ty, Value, View, + Walked, be_uint, +}; use super::parallel; use super::sizes::{SizeMode, Sizing}; use super::source::Progress; +use super::store::Column; use crate::error::{Error, Result}; +/// An object as the walk meets it: id, then class and kind above the length, then where its references +/// start in the copied ids, or for a primitive array where its hashes are (`raw`). +type Raw = [u64; 2]; + +fn raw(id: u64, class: u32, kind: Kind, len: u32) -> Raw { + [id, u64::from(class) << 34 | (kind as u64) << 32 | u64::from(len)] +} + +fn raw_class(record: Raw) -> u32 { + (record[1] >> 34) as u32 +} + +fn raw_kind(record: Raw) -> Kind { + Kind::ALL[(record[1] >> 32 & 3) as usize] +} + +/// Byte and char arrays up to this long get the String hash as the walk reads them; a String's longer +/// array is hashed by the detail read. +const STRING_HASH_MAX_BYTES: u64 = 64 << 10; + /// Primitive array classes by `Ty as usize`; room for every HPROF basic-type tag (long, the largest, is 11). const BASIC_TYPE_SLOTS: usize = 12; -#[derive(Default)] +/// Every object's reference slots as the walk copied them, and where each object's start, by object index. +/// The graph resolves them without reading the dump again. +pub struct References { + pub copied: Copied>, + pub starts: Column, +} + +/// A copied id too far from the base to fit in four bytes; the whole id is in `escapes`. +const ESCAPED: u32 = u32::MAX; + +/// Reference ids as the walk copies them, four bytes each: the id's distance above `base` in 8-byte steps +/// plus one, zero for null, and [`ESCAPED`] for one that does not fit, kept whole in `escapes` by place. +pub struct Copied { + pub ids: C, + pub escapes: Vec<(u64, u64)>, + pub base: u64, +} + +impl Copied { + fn new(ids: C, base: u64) -> Copied { + Copied { ids, escapes: Vec::new(), base } + } + + pub fn len(&self) -> usize { + self.ids.count() + } +} + +impl Copied> { + /// The id copied at `at`, zero for null. + pub fn id(&self, at: usize) -> u64 { + match self.ids[at] { + 0 => 0, + ESCAPED => self + .escapes + .binary_search_by_key(&(at as u64), |&(place, _)| place) + .map_or(0, |i| self.escapes[i].1), + steps => self.base + (u64::from(steps - 1) << 3), + } + } +} + +impl Extend for Copied { + fn extend>(&mut self, ids: I) { + for id in ids { + let steps = id + .checked_sub(self.base) + .filter(|distance| distance.trailing_zeros() >= 3) + .and_then(|distance| u32::try_from((distance >> 3) + 1).ok()) + .filter(|&steps| steps != ESCAPED) + .unwrap_or(ESCAPED); + let value = if id == 0 { 0 } else { steps }; + if value == ESCAPED { + self.escapes.push((self.ids.count() as u64, id)); + } + self.ids.put(value); + } + } +} + +/// Where copied ids go: a column for the whole walk, a `Vec` for a worker's part. +pub trait Ids { + fn count(&self) -> usize; + fn put(&mut self, value: u32); +} + +impl Ids for Column { + fn count(&self) -> usize { + self.len() + } + + fn put(&mut self, value: u32) { + self.push(value); + } +} + +impl Ids for Vec { + fn count(&self) -> usize { + self.len() + } + + fn put(&mut self, value: u32) { + self.push(value); + } +} + pub struct Indexer { id_size: u32, - symbols: FastMap, + symbols: Symbols, classes: Vec, class_by_id: FastMap, class_by_serial: HashMap, class_dumps: Vec, - objects: Vec, + /// Field names, interned by text: one `value` label whatever symbol ids the file used. + names: Vec, + name_by_text: HashMap, + name_by_id: FastMap, + objects: Column, + /// Per record, where its copied references start, or a primitive array's hash slot. Four bytes: the + /// references are only kept while they fit. + starts: Column, + /// The highest object id met, which picks the size convention. + max_id: u64, + references: Copied>, + hashes: Column<[u64; 2]>, + /// Per class: where a boxed class keeps its value. + boxed: Vec>, + boxed_values: FastMap<(u32, u64), u64>, + /// Layouts are fixed and the walk copies references from here on. + frozen: bool, + /// A class dump came after the layouts were fixed: the references are read in a second pass. + late: bool, roots: Vec, traces: HashMap, frames: FastMap, @@ -38,9 +169,28 @@ impl Indexer { pub fn new(id_size: u32) -> Indexer { Indexer { id_size, + symbols: Symbols::default(), + classes: Vec::new(), + class_by_id: FastMap::default(), + class_by_serial: HashMap::new(), + class_dumps: Vec::new(), + names: Vec::new(), + name_by_text: HashMap::new(), + name_by_id: FastMap::default(), + objects: Column::with_capacity(0), + starts: Column::with_capacity(0), + max_id: 0, + references: Copied::new(Column::with_capacity(0), 0), + hashes: Column::with_capacity(0), + boxed: Vec::new(), + boxed_values: FastMap::default(), + frozen: false, + late: false, + roots: Vec::new(), + traces: HashMap::new(), + frames: FastMap::default(), prim_array_classes: [NONE; BASIC_TYPE_SLOTS], last_class: (0, NONE), - ..Indexer::default() } } @@ -72,6 +222,85 @@ impl Indexer { } self.prim_array_classes[ty_idx] } + + /// Fix the layouts once the class dumps before the first object are in (`HotSpot` writes them all + /// first), so the walk can copy references as it goes. A dump naming a class the file never loaded + /// would mint placeholders out of their usual order, so then the references wait for a second pass. + fn freeze(&mut self) { + self.frozen = true; + let known = |id: u64| id == 0 || self.class_by_id.contains_key(&id); + if !self.class_dumps.iter().all(|class_dump| known(class_dump.id) && known(class_dump.super_id)) { + self.late = true; + return; + } + // Class mirrors sit in the heap, so the lowest is a good base for the copied ids. + self.references.base = self.class_by_id.keys().copied().filter(|&id| id != 0).min().unwrap_or(0) & !7; + self.resolve_class_dumps(); + // Reference slots do not depend on the size convention; `finish` lays out again with the real one. + self.lay_out_classes(&Sizing::resolve(SizeMode::Mat, self.id_size, 0)); + self.boxed = vec![None; self.classes.len()]; + for (class, _, offset, ty) in boxed_classes(&self.classes, &self.names, self.id_size) { + self.boxed[class as usize] = Some((offset, ty)); + } + } + + /// Whether the walk copies references: the layouts are fixed and no class dump came after them. + fn copies(&mut self) -> bool { + if !self.frozen { + self.freeze(); + } + !self.late + } +} + +/// What the walk takes from an instance's body: its reference slots, an id each and zero for null or past +/// a short body, and a boxed class's value. +fn copy_instance( + (class, slots, boxed): (u32, &[Slot], Option<(u32, Ty)>), + id_size: u32, + body: Body, + references: &mut impl Extend, + boxed_values: &mut FastMap<(u32, u64), u64>, +) -> io::Result<()> { + if slots.is_empty() && boxed.is_none() { + return Ok(()); + } + let bytes = body.bytes()?; + let width = id_size as usize; + references.extend( + slots + .iter() + .map(|slot| bytes.get(slot.offset as usize..slot.offset as usize + width).map_or(0, be_uint)), + ); + let value = boxed.and_then(|(offset, ty)| Value::decode(ty, bytes.get(offset as usize..)?, id_size)); + if let Some(value) = value { + *boxed_values.entry((class, value.bits())).or_default() += 1; + } + Ok(()) +} + +/// A primitive array's hashes, the String one only for a byte or char array of up to +/// [`STRING_HASH_MAX_BYTES`]; zero stands for not taken. +fn hash_array(ty: Ty, len: u32, body: Body) -> io::Result<[u64; 2]> { + let string = matches!(ty, Ty::Byte | Ty::Char) && body.len() <= STRING_HASH_MAX_BYTES; + let mut hash = ArrayHash::new(); + body.chunks(IO_BUFFER_SIZE, |chunk| { + if string { + hash.string(chunk); + } + hash.content(chunk); + Ok(()) + })?; + let [string_hash, content_hash] = hash.finish(len); + Ok([if string { string_hash } else { 0 }, content_hash]) +} + +/// Copy an object array's elements: an id each, zero for null. +fn copy_elements(id_size: usize, body: Body, out: &mut impl Extend) -> io::Result<()> { + body.chunks(IO_BUFFER_SIZE, |chunk| { + out.extend(chunk.chunks_exact(id_size).map(be_uint)); + Ok(()) + }) } /// The object callbacks: the same code for the sequential walk and a worker's part. @@ -79,19 +308,37 @@ macro_rules! object_callbacks { () => { fn instance(&mut self, id: u64, class_id: u64, body: Body) -> io::Result<()> { let class = self.class_index(class_id); - self.objects.push(Object { id, class, len: body.len() as u32, shallow: 0, kind: Kind::Instance }); + let (len, start) = (body.len() as u32, self.references.len()); + if self.copies() && class != NONE { + let boxed = self.boxed.get(class as usize).copied().flatten(); + let copied = (class, &self.classes[class as usize].slots[..], boxed); + copy_instance(copied, self.id_size, body, &mut self.references, &mut self.boxed_values)?; + } + self.max_id = self.max_id.max(id); + self.objects.push(raw(id, class, Kind::Instance, len)); + self.starts.push(start as u32); Ok(()) } - fn object_array(&mut self, id: u64, class_id: u64, len: u32, _body: Body) -> io::Result<()> { + fn object_array(&mut self, id: u64, class_id: u64, len: u32, body: Body) -> io::Result<()> { let class = self.class_index(class_id); - self.objects.push(Object { id, class, len, shallow: 0, kind: Kind::ObjectArray }); + let start = self.references.len(); + if self.copies() { + copy_elements(self.id_size as usize, body, &mut self.references)?; + } + self.max_id = self.max_id.max(id); + self.objects.push(raw(id, class, Kind::ObjectArray, len)); + self.starts.push(start as u32); Ok(()) } - fn primitive_array(&mut self, id: u64, ty: Ty, len: u32, _body: Body) -> io::Result<()> { + fn primitive_array(&mut self, id: u64, ty: Ty, len: u32, body: Body) -> io::Result<()> { let class = self.primitive_array_class(ty); - self.objects.push(Object { id, class, len, shallow: 0, kind: Kind::PrimitiveArray }); + let at = self.hashes.len(); + self.hashes.push(hash_array(ty, len, body)?); + self.max_id = self.max_id.max(id); + self.objects.push(raw(id, class, Kind::PrimitiveArray, len)); + self.starts.push(at as u32); Ok(()) } }; @@ -99,11 +346,11 @@ macro_rules! object_callbacks { impl Sink for Indexer { fn utf8(&mut self, id: u64, text: &[u8]) { - self.symbols.insert(id, String::from_utf8_lossy(text).into_owned()); + self.symbols.insert(id, text); } fn load_class(&mut self, serial: u32, id: u64, name_id: u64) { - let raw_name = self.symbols.get(&name_id).map_or("", String::as_str); + let raw_name = self.symbols.get(name_id).unwrap_or(""); let name = if raw_name.is_empty() { format!("") } else { dump::pretty_class_name(raw_name) }; let element_type = dump::array_element_type(raw_name, &name); @@ -132,6 +379,7 @@ impl Sink for Indexer { } fn class(&mut self, class: ClassDump) { + self.late |= self.frozen; self.class_index(class.id); self.class_dumps.push(class); } @@ -142,19 +390,35 @@ impl Sink for Indexer { /// One run of heap segments, read on a worker. A class whose load record is not seen yet is left /// to the sequential walk, which mints placeholders in file order. struct Part<'a> { + id_size: u32, class_by_id: &'a FastMap, + classes: &'a [Class], + boxed: &'a [Option<(u32, Ty)>], prim_array_classes: [u32; BASIC_TYPE_SLOTS], missed: bool, + /// Copy references: false once a class dump turned up after the layouts were fixed. + copying: bool, class_dumps: Vec, - objects: Vec, + objects: Vec, + starts: Vec, + max_id: u64, + references: Copied>, + hashes: Vec<[u64; 2]>, + boxed_values: FastMap<(u32, u64), u64>, roots: Vec, + last_class: (u64, u32), } impl Part<'_> { fn class_index(&mut self, id: u64) -> u32 { + if self.last_class.0 == id { + return self.last_class.1; + } let idx = self.class_by_id.get(&id).copied(); self.missed |= idx.is_none(); - idx.unwrap_or(NONE) + let idx = idx.unwrap_or(NONE); + self.last_class = (id, idx); + idx } fn primitive_array_class(&mut self, ty: Ty) -> u32 { @@ -162,6 +426,10 @@ impl Part<'_> { self.missed |= class == NONE; class } + + fn copies(&self) -> bool { + self.copying + } } impl Sink for Part<'_> { @@ -170,16 +438,32 @@ impl Sink for Part<'_> { } fn class(&mut self, class: ClassDump) { + self.copying = false; self.class_dumps.push(class); } object_callbacks!(); } +/// What a worker's part brought back. +struct Parsed { + class_dumps: Vec, + objects: Vec, + starts: Vec, + max_id: u64, + references: Copied>, + hashes: Vec<[u64; 2]>, + boxed_values: FastMap<(u32, u64), u64>, + copied: bool, + roots: Vec, + walked: Walked, +} + impl Indexer { /// Read the stepped-over segments, a run of adjacent ones per worker, merged in file order. False /// sends the file back to the sequential walk: segments were also walked in place, one runs past - /// the end, a part fails to parse, or a class needs a placeholder only file order can mint. + /// the end, a part fails to parse, or a class needs a placeholder only file order can mint. The + /// class dumps and roots before the first object are read first, here, to fix the layouts. pub fn read_segments(&mut self, view: &View, walked: &mut Walked, progress: &Arc) -> bool { let segments = std::mem::take(&mut walked.segments); if !walked.chunks.is_empty() || segments.last().is_some_and(|&(_, end)| end > view.len) { @@ -197,43 +481,101 @@ impl Indexer { _ => runs.push((start, end)), } } + let Some(first) = runs.first_mut() else { return true }; + let objects_start = Reader::open_at(view, first.0, self.id_size) + .and_then(|mut reader| hprof::prelude(&mut reader, first.1, self)); + let Ok(objects_start) = objects_start else { return false }; + first.0 = objects_start; + runs.retain(|&(start, end)| start < end); + self.freeze(); let mut prim_array_classes = [NONE; BASIC_TYPE_SLOTS]; for ty in [Ty::Bool, Ty::Char, Ty::Float, Ty::Double, Ty::Byte, Ty::Short, Ty::Int, Ty::Long] { let name = format!("{}[]", ty.name()); prim_array_classes[ty as usize] = class_named(&self.classes, &name).unwrap_or(NONE); } progress.begin("indexing objects", runs.iter().map(|&(start, end)| end - start).sum()); - let (id_size, class_by_id) = (self.id_size, &self.class_by_id); - let parts = parallel::items(&runs, |&(start, end)| { - let mut reader = Reader::open_at(view, start, id_size).ok()?; - reader.progress = progress.counter(start); - let mut part = Part { - class_by_id, - prim_array_classes, - missed: false, - class_dumps: Vec::new(), - objects: Vec::new(), - roots: Vec::new(), - }; - let mut part_walked = Walked::default(); - hprof::heap(&mut reader, Some(end), &mut part, &mut part_walked).ok()?; - (!part.missed).then_some((part.class_dumps, part.objects, part.roots, part_walked)) - }); - let Some(parts) = parts.into_iter().collect::>>() else { return false }; - self.objects.reserve(parts.iter().map(|(_, objects, _, _)| objects.len()).sum()); - for (class_dumps, objects, roots, part_walked) in parts { - for class_dump in class_dumps { - self.class(class_dump); - } - self.objects.extend(objects); - self.roots.extend(roots); - walked.chunks.extend(part_walked.chunks); - walked.pieces.extend(part_walked.pieces); + let (id_size, class_by_id, classes, boxed, copying, base) = + (self.id_size, &self.class_by_id, &self.classes, &self.boxed, !self.late, self.references.base); + let (mut complete, mut class_dumps) = (true, Vec::new()); + parallel::ordered( + &runs, + |&(start, end)| { + let mut reader = Reader::open_at(view, start, id_size).ok()?; + reader.progress = progress.counter(start); + let mut part = Part { + id_size, + class_by_id, + classes, + boxed, + prim_array_classes, + missed: false, + copying, + class_dumps: Vec::new(), + objects: Vec::new(), + starts: Vec::new(), + max_id: 0, + references: Copied::new(Vec::new(), base), + hashes: Vec::new(), + boxed_values: FastMap::default(), + roots: Vec::new(), + last_class: (0, NONE), + }; + let mut part_walked = Walked::default(); + hprof::heap(&mut reader, Some(end), &mut part, &mut part_walked).ok()?; + (!part.missed).then_some(Parsed { + class_dumps: part.class_dumps, + objects: part.objects, + starts: part.starts, + max_id: part.max_id, + references: part.references, + hashes: part.hashes, + boxed_values: part.boxed_values, + copied: part.copying, + roots: part.roots, + walked: part_walked, + }) + }, + |part| { + let Some(part) = part else { + complete = false; + return false; + }; + class_dumps.extend(part.class_dumps); + self.late |= !part.copied; + // Starts are the part's own: move them past what earlier parts copied. + let (reference_base, hash_base) = (self.references.len() as u64, self.hashes.len() as u64); + if !self.late { + let escapes = part.references.escapes.iter().map(|&(at, id)| (at + reference_base, id)); + self.references.escapes.extend(escapes); + self.references.ids.extend_from_slice(&part.references.ids); + } + self.hashes.extend_from_slice(&part.hashes); + for (key, count) in part.boxed_values { + *self.boxed_values.entry(key).or_default() += count; + } + self.max_id = self.max_id.max(part.max_id); + self.objects.extend_from_slice(&part.objects); + self.starts.reserve(part.starts.len()); + for (&record, &start) in part.objects.iter().zip(&part.starts) { + let base = + if raw_kind(record) == Kind::PrimitiveArray { hash_base } else { reference_base }; + self.starts.push(start.wrapping_add(base as u32)); + } + self.roots.extend(part.roots); + walked.chunks.extend(part.walked.chunks); + walked.pieces.extend(part.walked.pieces); + true + }, + ); + // Class dumps can mint placeholders, which the workers must not see change. + for class_dump in class_dumps { + self.class(class_dump); } - true + complete } - /// Resolve layouts, size every object, sort the table, resolve the roots. + /// Resolve layouts, size every object, sort the table, resolve the roots. Also hands back the + /// references the walk copied, unless a late class dump means the graph must read them itself. pub fn finish( mut self, path: &str, @@ -241,29 +583,42 @@ impl Indexer { header: Header, walked: Walked, mode: SizeMode, - ) -> Result { - let mut names = self.resolve_class_dumps(); - let value_label = intern_str(&mut names, "value"); - let name_label = intern_str(&mut names, "name"); + ) -> Result<(Dump, Option)> { + self.resolve_class_dumps(); + // Class and field names are copied out by now: only stack frames still look symbols up. + self.symbols = + self.symbols.only(self.frames.values().flat_map(|frame| [frame.method_id, frame.source_id])); + let value_label = intern_str(&mut self.names, "value"); + let name_label = intern_str(&mut self.names, "name"); // Sizes depend on the address range. - let max_id = self.objects.iter().map(|object| object.id).max().unwrap_or(0); - let sizing = Sizing::resolve(mode, self.id_size, max_id); - self.lay_out_classes(&sizing, &names); - let class_class = self.size_objects(&sizing); + let sizing = Sizing::resolve(mode, self.id_size, self.max_id); + // Fixed layouts only need their sizes for the real convention; a late class dump lays out again. + if self.frozen && !self.late { + size_classes(&mut self.classes, &sizing); + } else { + self.lay_out_classes(&sizing); + } + // Starts are four bytes: a walk that copied more ids than that reads the references again. + let keep = self.frozen && !self.late && self.references.len() < u32::MAX as usize; + let (objects, class_class, starts, array_hashes) = self.size_objects(sizing, keep); + let array_hashes = ArrayHashes::new(&objects, array_hashes); // u32 indices keep NONE free and the graph root one past the objects; labels stay below LOADER. let counts = [ - ("objects", self.objects.len(), NONE - 1), - ("classes", self.classes.len(), NONE), - ("field names", names.len(), LOADER), + ("objects", objects.len(), NONE as usize - 1), + ("classes", self.classes.len(), MAX_CLASSES - 1), + ("field names", self.names.len(), LOADER as usize), ]; for (what, count, limit) in counts { - if count > limit as usize { + if count > limit { return Err(Error::Dump(format!("too many {what} for midden ({count}, limit {limit})"))); } } - let lookup = Buckets::build(&self.objects); let string_class = class_named(&self.classes, "java.lang.String").unwrap_or(NONE); + let references = keep.then(|| References { + copied: std::mem::replace(&mut self.references, Copied::new(Column::with_capacity(0), 0)), + starts, + }); let mut dump = Dump { path: path.to_string(), @@ -275,10 +630,12 @@ impl Indexer { sizing, chunks: walked.chunks, pieces: walked.pieces, - names, + names: self.names, symbols: self.symbols, classes: self.classes, - objects: self.objects, + objects, + array_hashes, + boxed_values: keep.then_some(self.boxed_values), roots: Vec::new(), dangling_roots: 0, marked_unreachable: 0, @@ -286,53 +643,60 @@ impl Indexer { traces: self.traces, frames: self.frames, class_by_serial: self.class_by_serial, - lookup, class_class, string_class, value_label, name_label, }; resolve_roots(&mut dump, self.roots); - Ok(dump) - } - - /// Apply the class dumps to their classes, returning the interned field names. - fn resolve_class_dumps(&mut self) -> Vec { - // Names are interned by text: one `value` label whatever symbol ids the file used. - let mut names: Vec = Vec::new(); - let mut name_by_text: HashMap = HashMap::new(); - let mut name_by_id: FastMap = FastMap::default(); - let mut intern = |id: u64, symbols: &FastMap| -> u32 { - *name_by_id.entry(id).or_insert_with(|| { - let text = symbols.get(&id).cloned().unwrap_or_else(|| format!("<0x{id:x}>")); - *name_by_text.entry(text.clone()).or_insert_with(|| push(&mut names, text)) - }) - }; + Ok((dump, references)) + } + + /// Apply the class dumps read so far to their classes, interning their field names. + fn resolve_class_dumps(&mut self) { for class_dump in std::mem::take(&mut self.class_dumps) { let idx = self.class_index(class_dump.id) as usize; let superclass = if class_dump.super_id == 0 { NONE } else { self.class_index(class_dump.super_id) }; - let class = &mut self.classes[idx]; - class.superclass = superclass; - class.loader = class_dump.loader_id; - class.dumped = true; - class.fields = class_dump + let fields = class_dump .fields .iter() - .map(|&(name_id, ty)| Field { name: intern(name_id, &self.symbols), ty }) + .map(|&(name_id, ty)| Field { name: self.intern(name_id), ty }) .collect(); - class.statics = class_dump + let statics = class_dump .statics .iter() - .map(|&(name_id, ty, value)| Static { name: intern(name_id, &self.symbols), ty, value }) + .map(|&(name_id, ty, value)| Static { name: self.intern(name_id), ty, value }) .collect(); + let class = &mut self.classes[idx]; + class.superclass = superclass; + class.loader = class_dump.loader_id; + class.dumped = true; + class.fields = fields; + class.statics = statics; + } + } + + /// The label for a field name symbol. + fn intern(&mut self, id: u64) -> u32 { + if let Some(&label) = self.name_by_id.get(&id) { + return label; } - names + let text = self.symbols.get(id).map_or_else(|| format!("<0x{id:x}>"), str::to_string); + let label = if let Some(&label) = self.name_by_text.get(&text) { + label + } else { + let label = push(&mut self.names, text.clone()); + self.name_by_text.insert(text, label); + label + }; + self.name_by_id.insert(id, label); + label } - fn lay_out_classes(&mut self, sizing: &Sizing, names: &[String]) { + fn lay_out_classes(&mut self, sizing: &Sizing) { let reference = class_named(&self.classes, "java.lang.ref.Reference"); - let referent = names.iter().position(|name| name == "referent").map(|i| i as u32); + let referent = self.names.iter().position(|name| name == "referent").map(|i| i as u32); let class_count = self.classes.len(); let mut laid_out = vec![false; class_count]; for i in 0..class_count { @@ -341,40 +705,196 @@ impl Indexer { mark_reference_kinds(&mut self.classes); } - /// Add the class objects, sort the table, size every object and count instances. - /// Returns the `java.lang.Class` class. - fn size_objects(&mut self, sizing: &Sizing) -> u32 { + /// The object table: the walk's records and the class objects merged by id, one of each id kept, + /// instances counted. Also returns the `java.lang.Class` class, where each kept object's copied + /// references start when `keep`, and the primitive arrays' hashes in table order. + fn size_objects( + &mut self, + sizing: Sizing, + keep: bool, + ) -> (ObjectTable, u32, Column, Column<[u64; 2]>) { // Class objects live in the object table too: statics are edges and // sticky-class roots point at them. let class_class = class_named(&self.classes, "java.lang.Class") .unwrap_or_else(|| push(&mut self.classes, Class::placeholder("java.lang.Class".to_string(), 0))); + let mut class_objects: Vec = Vec::new(); for class in self.classes.iter().filter(|class| class.dumped) { let statics_size: u64 = class.statics.iter().map(|field| u64::from(sizing.field_size(field.ty))).sum(); - let (id, shallow) = (class.id, sizing.instance_size(statics_size)); - self.objects.push(Object { id, class: class_class, len: 0, shallow, kind: Kind::Class }); + class_objects.push(raw(class.id, class_class, Kind::Class, sizing.instance_size(statics_size))); + } + class_objects.sort_by_key(|record| record[0]); + let class_starts = vec![0u32; class_objects.len()]; + let records = std::mem::replace(&mut self.objects, Column::with_capacity(0)); + let starts = std::mem::replace(&mut self.starts, Column::with_capacity(0)); + // HotSpot writes by address in runs: merge the runs, earlier ones first on equal ids, which is + // a stable sort. The first record of each id is kept. + let mut runs: Vec = Vec::new(); + let mut start = 0; + for i in 1..=records.len() { + if i == records.len() || records[i][0] < records[i - 1][0] { + runs.push((&records[start..i], &starts[start..i])); + start = i; + } } - // One sort by id, dropping duplicates. HotSpot writes by address in runs, so the stable sort is - // nearly free. - self.objects.sort_by_key(|object| object.id); - self.objects.dedup_by_key(|object| object.id); - for object in &mut self.objects { - let class = &self.classes[object.class as usize]; - object.shallow = match object.kind { - Kind::Instance => class.shallow, - Kind::ObjectArray => sizing.array_size(u64::from(object.len), sizing.ref_size), - Kind::PrimitiveArray => sizing.array_size( - u64::from(object.len), - class.element_type.map_or(1, |ty| sizing.field_size(ty)), - ), - Kind::Class => object.shallow, - }; + runs.push((&class_objects, &class_starts)); + runs.retain(|run| !run.0.is_empty()); + let hashes = std::mem::replace(&mut self.hashes, Column::with_capacity(0)); + let merged = merge_table(&runs, &hashes, self.classes.len(), keep); + let table = ObjectTable::from_steps(&self.classes, sizing, merged.ids, merged.shapes); + for (class, count) in self.classes.iter_mut().zip(merged.instances) { + class.instances = count; } - for object in &self.objects { - self.classes[object.class as usize].instances += 1; + (table, class_class, merged.starts, merged.hashes) + } +} + +/// A run of records in id order, and where each one's references start (or its hash slot). +type Run<'a> = (&'a [Raw], &'a [u32]); + +/// The merge of sorted runs as stretches in output order, `(run, start, end)`: id order, an earlier run +/// first on equal ids, which is a stable sort. A stretch runs from the smallest head up to the next one. +fn merge_stretches(runs: &[Run]) -> Vec<(usize, usize, usize)> { + let mut at = vec![0usize; runs.len()]; + let head = |run: usize, at: &[usize]| runs[run].0.get(at[run]).map(|record| Reverse((record[0], run))); + let mut heads: BinaryHeap> = + (0..runs.len()).filter_map(|run| head(run, &at)).collect(); + let mut stretches = Vec::new(); + while let Some(Reverse((_, run))) = heads.pop() { + let rest = &runs[run].0[at[run]..]; + let len = match heads.peek() { + None => rest.len(), + Some(&Reverse(next)) => rest.partition_point(|record| (record[0], run) < next).max(1), + }; + stretches.push((run, at[run], at[run] + len)); + at[run] += len; + heads.extend(head(run, &at)); + } + stretches +} + +/// The object table's columns, merged from the walk's runs. +struct Merged { + ids: TableIds, + shapes: Shapes, + starts: Column, + hashes: Column<[u64; 2]>, + instances: Vec, +} + +/// Merge the runs into the table's columns, one of each id kept (the first), over the workers: each +/// stretch is counted first, so every worker knows where its stretches' objects go. +fn merge_table(runs: &[Run], hashes: &[[u64; 2]], class_count: usize, keep: bool) -> Merged { + let stretches = merge_stretches(runs); + let records = |&(run, start, end): &(usize, usize, usize)| &runs[run].0[start..end]; + let starts_of = |&(run, start, end): &(usize, usize, usize)| &runs[run].1[start..end]; + let last_id = |k: usize| { + k.checked_sub(1).map(|k| records(&stretches[k]).last().expect("stretches are not empty")[0]) + }; + let base = stretches.first().map_or(0, |stretch| records(stretch)[0][0]); + let fits = |id: u64| (id - base).trailing_zeros() >= 3 && u32::try_from((id - base) >> 3).is_ok(); + // Kept objects and primitive arrays per stretch, and whether its ids fit four bytes. + let counts: Vec<(usize, usize, bool)> = parallel::ranges(stretches.len(), |lo, hi| { + (lo..hi) + .map(|k| { + let (mut previous, mut kept, mut arrays, mut fit) = (last_id(k), 0, 0, true); + for record in records(&stretches[k]) { + if previous != Some(record[0]) { + kept += 1; + arrays += usize::from(raw_kind(*record) == Kind::PrimitiveArray); + fit &= fits(record[0]); + } + previous = Some(record[0]); + } + (kept, arrays, fit) + }) + .collect::>() + }) + .concat(); + let total: usize = counts.iter().map(|count| count.0).sum(); + let total_arrays: usize = counts.iter().map(|count| count.1).sum(); + let narrow = counts.iter().all(|count| count.2); + let mut steps = Column::::zeroed(if narrow { total } else { 0 }); + let mut wide = Column::::zeroed(if narrow { 0 } else { total }); + let packing = Packing::new(class_count); + let mut words = Column::::zeroed(total); + let mut starts = Column::::zeroed(if keep { total } else { 0 }); + let mut array_values = Column::<[u64; 2]>::zeroed(total_arrays); + // Stretches in groups of about equal size, one per worker, each writing its own slice of every column. + let groups = parallel::threads().max(1) * 4; + let step = total.div_ceil(groups).max(1); + let mut bounds = vec![0usize]; + let mut filled = 0; + for (k, count) in counts.iter().enumerate() { + filled += count.0; + if filled >= step * bounds.len() && k + 1 < stretches.len() { + bounds.push(k + 1); } - class_class } + bounds.push(stretches.len()); + let mut tasks = Vec::new(); + let (mut id_rest, mut wide_rest) = (&mut steps[..], &mut wide[..]); + let (mut shape_rest, mut start_rest) = (&mut words[..], &mut starts[..]); + let mut first_object = 0; + let mut value_rest = &mut array_values[..]; + for pair in bounds.windows(2) { + let (lo, hi) = (pair[0], pair[1]); + let objects: usize = counts[lo..hi].iter().map(|count| count.0).sum(); + let arrays: usize = counts[lo..hi].iter().map(|count| count.1).sum(); + let (id_part, rest) = std::mem::take(&mut id_rest).split_at_mut(if narrow { objects } else { 0 }); + id_rest = rest; + let (wide_part, rest) = std::mem::take(&mut wide_rest).split_at_mut(if narrow { 0 } else { objects }); + wide_rest = rest; + let (shape_part, rest) = std::mem::take(&mut shape_rest).split_at_mut(objects); + shape_rest = rest; + let (start_part, rest) = std::mem::take(&mut start_rest).split_at_mut(if keep { objects } else { 0 }); + start_rest = rest; + let (value_part, rest) = std::mem::take(&mut value_rest).split_at_mut(arrays); + value_rest = rest; + let (stretches, records) = (&stretches, &records); + tasks.push(move || { + let (mut instances, mut long) = (vec![0u64; class_count], Vec::new()); + let (mut at, mut array_at, mut previous) = (0, 0, last_id(lo)); + for stretch in &stretches[lo..hi] { + for (&record, &start) in records(stretch).iter().zip(starts_of(stretch)) { + if previous == Some(record[0]) { + continue; + } + previous = Some(record[0]); + let (class, kind) = (raw_class(record), raw_kind(record)); + instances[class as usize] += 1; + if kind == Kind::PrimitiveArray { + value_part[array_at] = hashes[start as usize]; + array_at += 1; + } + if narrow { + id_part[at] = ((record[0] - base) >> 3) as u32; + } else { + wide_part[at] = record[0]; + } + // An instance's size comes from its class. + let len = if kind == Kind::Instance { 0 } else { record[1] as u32 }; + let (word, too_long) = packing.pack(class, kind, len); + shape_part[at] = word; + long.extend(too_long.map(|len| ((first_object + at) as u32, len))); + if keep { + start_part[at] = start; + } + at += 1; + } + } + (instances, long) + }); + first_object += objects; + } + let parts = parallel::run_all(tasks); + let (mut instances, mut long) = (vec![0u64; class_count], Vec::new()); + for (part, part_long) in parts { + instances.iter_mut().zip(part).for_each(|(total, count)| *total += count); + long.extend(part_long); + } + let ids = if narrow { TableIds::steps(base, &steps) } else { TableIds::Wide(wide) }; + Merged { ids, shapes: Shapes::new(words, packing, long), starts, hashes: array_values, instances } } /// Resolve the roots against the object table and list each thread once. @@ -458,6 +978,33 @@ fn layout( class.slots = slots; } +/// Footprints and shallow sizes under `sizing` for layouts already fixed, in the same order and with the +/// same depth cap as [`layout`]. +fn size_classes(classes: &mut [Class], sizing: &Sizing) { + let mut sized = vec![false; classes.len()]; + for i in 0..classes.len() { + size_class(classes, i as u32, &mut sized, sizing, 0); + } +} + +fn size_class(classes: &mut [Class], idx: u32, sized: &mut [bool], sizing: &Sizing, depth: usize) { + let i = idx as usize; + if sized[i] { + return; + } + sized[i] = true; + let mut footprint: u64 = + classes[i].fields.iter().map(|field| u64::from(sizing.field_size(field.ty))).sum(); + let super_idx = classes[i].superclass; + if super_idx != NONE && depth < MAX_CLASS_DEPTH { + size_class(classes, super_idx, sized, sizing, depth + 1); + footprint += u64::from(classes[super_idx as usize].footprint); + } + let class = &mut classes[i]; + class.footprint = footprint.min(u64::from(u32::MAX)) as u32; + class.shallow = sizing.instance_size(footprint); +} + /// Tag every `Reference` subclass with its family. fn mark_reference_kinds(classes: &mut [Class]) { let bases: Vec<(u32, RefKind)> = RefKind::ALL diff --git a/src/main.rs b/src/main.rs index a13c3f2..620a91f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,6 +20,7 @@ mod report; mod shell; mod sizes; mod source; +mod store; mod strings; pub use options::Sort; @@ -73,8 +74,7 @@ fn run(args: &Args) -> Result<()> { // Baseline first, so only its snapshot outlives the load. let baseline = match &args.baseline { - Some(path) => analyse(load(path, &opts)?, view.clone(), |mut session| { - session.prepare()?; + Some(path) => analyse(load(path, &opts)?, view.clone(), |session| { let histogram = session.heap.histogram(); Ok(Some(session.heap.snapshot(&histogram, session.inputs.duplicates.as_ref()))) })?, @@ -84,7 +84,6 @@ fn run(args: &Args) -> Result<()> { analyse(load(&args.file, &opts)?, view, |mut session| { (session.json, session.color) = (args.json, use_color(args.color)); session.inputs.baseline = baseline; - session.prepare()?; progress.end(); let out = session.show(&session.view.clone())?; print!("{out}"); @@ -96,13 +95,21 @@ fn run(args: &Args) -> Result<()> { }) } -/// Build the heap of a load and hand its session to `then`. The cache write only reads the index, so it -/// runs beside the analysis. +/// Build the heap of a load, read its details and hand its session to `then`. The cache write only reads +/// the index, so it runs beside the analysis; the detail read runs beside the dominator tree. fn analyse(loaded: Loaded, view: View, then: impl FnOnce(Session) -> Result) -> Result { thread::scope(|scope| { scope.spawn(|| save(&loaded.dump, &loaded.graph, loaded.index.as_ref())); - loaded.source.progress.stage("computing dominators"); - then(Session::new(Heap::new(&loaded.dump, &loaded.graph), loaded.source, view)) + let progress = &loaded.source.progress; + progress.stage("computing dominators"); + let (heap, details) = Heap::build(&loaded.dump, &loaded.graph, &view.excludes, |heap| { + let details = read_details(heap, &loaded.source, &view); + progress.stage("computing dominators"); + details + }); + let mut session = Session::new(heap, loaded.source, view); + session.prepare(details?); + then(session) }) } @@ -196,7 +203,7 @@ fn load(path: &Path, opts: &Load) -> Result { source.seekable = hprof::View::file(&done).ok(); } } - let dump = indexer.finish(&name, &reader, header, walked, opts.sizes)?; + let (dump, references) = indexer.finish(&name, &reader, header, walked, opts.sizes)?; drop(reader); if dump.objects.is_empty() { return Err(Error::Dump(format!( @@ -204,7 +211,7 @@ fn load(path: &Path, opts: &Load) -> Result { ))); } source.id_size = dump.header.id_size; - let graph = graph::build(&dump, &source, opts.reference_policy)?; + let graph = graph::build(&dump, &source, opts.reference_policy, references)?; Ok(Loaded { dump, graph, source, index: dir.map(|dir| (dir, key)) }) } @@ -222,8 +229,7 @@ pub struct Session<'a> { } impl<'a> Session<'a> { - fn new(mut heap: Heap<'a>, source: Source, view: View) -> Session<'a> { - heap.set_excludes(&view.excludes); + fn new(heap: Heap<'a>, source: Source, view: View) -> Session<'a> { Session { heap, source, @@ -236,91 +242,14 @@ impl<'a> Session<'a> { } } - /// First detail read: thread names, array hashes, boxed values, direct buffer bodies and any search. - fn prepare(&mut self) -> Result<()> { - let (heap, view) = (&self.heap, &self.view); - let (dump, graph) = (heap.dump, heap.graph); - self.strings = strings::strings(dump, graph); - let mut request = Request::default(); - for thread in &dump.threads { - if let Some(name) = heap.field(thread.object, "name") { - strings::string_ids(dump, graph, name, &mut request.want); - } - } - let shows = |section: Section| view.sections.has(section) && !view.focused(); - let mut arrays = Vec::new(); - if view.sections.needs_hashes() && !view.focused() { - request.hash = self.strings.iter().map(|&(_, array)| dump.objects[array as usize].id).collect(); - } - if shows(Section::Arrays) { - arrays = heap.hashable_arrays(&self.strings); - request.content = arrays.iter().map(|&array| dump.objects[array as usize].id).collect(); - } - if shows(Section::Boxed) { - request.boxed = heap - .boxed_classes() - .into_iter() - .map(|(class, id, offset, ty)| (id, (class, offset, ty))) - .collect(); - } - let buffers = if shows(Section::Direct) { heap.direct_buffers() } else { Vec::new() }; - request.want.extend(heap.direct_wants(&buffers)); - self.add_search(&mut request, view.find.as_deref(), view.filter.as_deref())?; - let fetched = detail::fetch(&self.source, dump, request)?; - - let names: Vec = dump - .threads - .iter() - .map(|thread| { - heap.field(thread.object, "name") - .and_then(|name| strings::string_text(dump, graph, &fetched, name)) - .unwrap_or_default() - }) - .collect(); - let hash_of = |array: u32| fetched.hash_of(dump.objects[array as usize].id); - if shows(Section::Strings) { - self.inputs.duplicates = Some(heap.duplicates(&self.strings, hash_of)); - } - if shows(Section::Arrays) { - self.inputs.array_duplicates = Some(heap.array_duplicates(&arrays, hash_of)); - } - if shows(Section::Boxed) { - self.inputs.boxed = heap.boxed(&fetched.boxed); - } - if shows(Section::Direct) { - self.inputs.direct = Some(heap.direct(&buffers, &fetched)); - } + /// Take in the first detail read; search hits are ranked by retained size, so only now. + fn prepare(&mut self, details: Details) { + let Details { strings, fetched, names, inputs } = details; + self.strings = strings; + self.inputs = inputs; self.take_search(&fetched); self.heap.thread_names = names; self.fetched = fetched; - Ok(()) - } - - /// Add `--find` / `--where` to a request. - fn add_search(&self, request: &mut Request, find: Option<&str>, filter: Option<&str>) -> Result<()> { - let (heap, dump) = (&self.heap, self.heap.dump); - if let Some(text) = find { - let ids = self.strings.iter().map(|&(_, array)| dump.objects[array as usize].id).collect(); - request.search = Some((ids, strings::Needle::new(text))); - } - if let Some(text) = filter { - let usage = || Error::Usage(format!("`{text}`: expected CLASS.FIELD=VALUE")); - let (class_field, expect) = text.rsplit_once('=').ok_or_else(usage)?; - let (class, field) = class_field.rsplit_once('.').ok_or_else(usage)?; - let classes: dump::FastMap = heap - .classes_matching(&Pattern::parse(class)) - .into_iter() - .filter_map(|class_index| { - dump.field_offset(class_index, field) - .map(|slot| (dump.classes[class_index as usize].id, slot)) - }) - .collect(); - if classes.is_empty() { - return Err(Error::Usage(format!("no class matching `{class}` has a field `{field}`"))); - } - request.filter = Some(Filter { classes, expect: expect.trim().to_string() }); - } - Ok(()) } /// Turn a read's search hits into objects, biggest retained first. @@ -333,7 +262,7 @@ impl<'a> Session<'a> { } let mut by_array: dump::FastMap = dump::FastMap::default(); for &(string, array) in &self.strings { - by_array.entry(dump.objects[array as usize].id).or_insert(string); + by_array.entry(dump.objects.id(array as usize)).or_insert(string); } let mut found: Vec = fetched.found.iter().filter_map(|id| by_array.get(id).copied()).collect(); found.sort_by(|&a, &b| heap.retained(b).cmp(&heap.retained(a)).then(a.cmp(&b))); @@ -346,7 +275,7 @@ impl<'a> Session<'a> { /// Run a search read for the shell. pub fn search(&mut self, find: Option<&str>, filter: Option<&str>) -> Result<()> { let mut request = Request::default(); - self.add_search(&mut request, find, filter)?; + add_search(&self.heap, &self.strings, &mut request, find, filter)?; let fetched = detail::fetch(&self.source, self.heap.dump, request)?; self.take_search(&fetched); Ok(()) @@ -371,3 +300,127 @@ impl<'a> Session<'a> { }) } } + +/// What the first detail read brought back, and what the report makes of it before the dominators. +struct Details { + /// Every String with its backing array. + strings: Vec<(u32, u32)>, + fetched: Fetched, + /// One per `dump.threads`. + names: Vec, + inputs: Inputs, +} + +/// First detail read: thread names, array hashes, boxed values, direct buffer bodies and any search, then +/// the duplicate groups and the rest that need no dominators. It runs while the dominator tree is built. +fn read_details(heap: &Heap, source: &Source, view: &View) -> Result
{ + let (dump, graph) = (heap.dump, heap.graph); + let strings = strings::strings(dump, graph); + let mut request = Request::default(); + for thread in &dump.threads { + if let Some(name) = heap.field(thread.object, "name") { + strings::string_ids(dump, graph, name, &mut request.want); + } + } + let shows = |section: Section| view.sections.has(section) && !view.focused(); + let mut arrays = Vec::new(); + // Hashes and boxed tallies the index pass took are used as they are; only a String's array past its + // limit is hashed by this read. `which` is 0 for the String hash, 1 for the grouping one. + let taken = |array: u32, which: usize| { + dump.array_hashes.as_ref()?.get(array).map(|pair| pair[which]).filter(|&hash| hash != 0) + }; + if view.sections.needs_hashes() && !view.focused() { + request.hash = untaken(dump, strings.iter().map(|&(_, array)| array), |array| taken(array, 0)); + } + if shows(Section::Arrays) { + arrays = heap.hashable_arrays(&strings); + request.content = untaken(dump, arrays.iter().copied(), |array| taken(array, 1)); + } + if shows(Section::Boxed) && dump.boxed_values.is_none() { + request.boxed = heap + .boxed_classes() + .into_iter() + .map(|(class, id, offset, ty)| (id, (class, offset, ty))) + .collect(); + } + let buffers = if shows(Section::Direct) { heap.direct_buffers() } else { Vec::new() }; + request.want.extend(heap.direct_wants(&buffers)); + add_search(heap, &strings, &mut request, view.find.as_deref(), view.filter.as_deref())?; + let mut fetched = detail::fetch(source, dump, request)?; + if let Some(boxed) = dump.boxed_values.as_ref().filter(|_| shows(Section::Boxed)) { + fetched.boxed.clone_from(boxed); + } + + let names = dump + .threads + .iter() + .map(|thread| { + heap.field(thread.object, "name") + .and_then(|name| strings::string_text(dump, graph, &fetched, name)) + .unwrap_or_default() + }) + .collect(); + let hash_of = |array: u32, which: usize| { + taken(array, which).or_else(|| fetched.hash_of(dump.objects.id(array as usize))) + }; + let mut inputs = Inputs::default(); + if shows(Section::Strings) { + inputs.duplicates = Some(heap.duplicates(&strings, |array| hash_of(array, 0))); + } + if shows(Section::Arrays) { + inputs.array_duplicates = Some(heap.array_duplicates(&arrays, |array| hash_of(array, 1))); + } + if shows(Section::Boxed) { + inputs.boxed = heap.boxed(&fetched.boxed); + } + if shows(Section::Direct) { + inputs.direct = Some(heap.direct(&buffers, &fetched)); + } + Ok(Details { strings, fetched, names, inputs }) +} + +/// The ids of the arrays the index pass took no hash for, ascending, each once. +fn untaken( + dump: &dump::Dump, + arrays: impl Iterator, + taken: impl Fn(u32) -> Option, +) -> Vec { + let mut ids: Vec = + arrays.filter(|&array| taken(array).is_none()).map(|array| dump.objects.id(array as usize)).collect(); + ids.sort_unstable(); + ids.dedup(); + ids +} + +/// Add `--find` / `--where` to a request. +fn add_search( + heap: &Heap, + strings: &[(u32, u32)], + request: &mut Request, + find: Option<&str>, + filter: Option<&str>, +) -> Result<()> { + let dump = heap.dump; + if let Some(text) = find { + let ids = strings.iter().map(|&(_, array)| dump.objects.id(array as usize)).collect(); + request.search = Some((ids, strings::Needle::new(text))); + } + if let Some(text) = filter { + let usage = || Error::Usage(format!("`{text}`: expected CLASS.FIELD=VALUE")); + let (class_field, expect) = text.rsplit_once('=').ok_or_else(usage)?; + let (class, field) = class_field.rsplit_once('.').ok_or_else(usage)?; + let classes: dump::FastMap = heap + .classes_matching(&Pattern::parse(class)) + .into_iter() + .filter_map(|class_index| { + dump.field_offset(class_index, field) + .map(|slot| (dump.classes[class_index as usize].id, slot)) + }) + .collect(); + if classes.is_empty() { + return Err(Error::Usage(format!("no class matching `{class}` has a field `{field}`"))); + } + request.filter = Some(Filter { classes, expect: expect.trim().to_string() }); + } + Ok(()) +} diff --git a/src/parallel.rs b/src/parallel.rs index d693d5f..25ee23e 100644 --- a/src/parallel.rs +++ b/src/parallel.rs @@ -1,9 +1,14 @@ //! Thread fan-out over `--threads` scoped workers. Nothing is shared mutably: each worker returns its //! part and the caller merges. -use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering::Relaxed}; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use std::sync::{Condvar, Mutex}; use std::thread; +use zerocopy::FromZeros; + +use super::store::{Bits, Column}; + /// Workers when the core count is unknown, and before `--threads` is applied. pub const DEFAULT_THREADS: usize = 4; @@ -17,12 +22,6 @@ pub fn threads() -> usize { THREADS.load(Relaxed) } -/// `len` atomics at zero, on pages nothing has touched yet: the workers that -/// fill them take the page faults, not the thread that allocates. -pub fn zeroed(len: usize) -> Vec { - vec![0u32; len].into_iter().map(AtomicU32::new).collect() -} - /// Split `0..len` into one range per worker and run `f` on each, in order. pub fn ranges(len: usize, f: impl Fn(usize, usize) -> T + Sync) -> Vec { let workers = threads().min(len.max(1)); @@ -70,3 +69,113 @@ fn run<'env, T: Send + 'env>(tasks: impl Iterator T + Se handles.into_iter().map(|handle| handle.join().expect("worker panicked")).collect() }) } + +/// Run `work` over every item on the workers and hand the results to `consume` in item order, on the +/// calling thread, as each one and all before it are done. Workers stay a few items ahead, so only that +/// many results are held at once. `consume` returning false stops the rest. +pub fn ordered( + items: &[I], + work: impl Fn(&I) -> T + Sync, + mut consume: impl FnMut(T) -> bool, +) { + let workers = threads().min(items.len().max(1)); + if workers <= 1 { + for item in items { + if !consume(work(item)) { + return; + } + } + return; + } + let window = 2 * workers; + let state = Mutex::new(Ordered { + next: 0, + consumed: 0, + stop: false, + ready: (0..items.len()).map(|_| None).collect(), + }); + let (done, room) = (Condvar::new(), Condvar::new()); + thread::scope(|scope| { + for _ in 0..workers { + scope.spawn(|| { + loop { + let i = { + let mut state = state.lock().expect("worker panicked"); + while !state.stop && state.next < items.len() && state.next >= state.consumed + window + { + state = room.wait(state).expect("worker panicked"); + } + if state.stop || state.next >= items.len() { + return; + } + state.next += 1; + state.next - 1 + }; + let result = work(&items[i]); + state.lock().expect("worker panicked").ready[i] = Some(result); + done.notify_all(); + } + }); + } + for i in 0..items.len() { + let result = { + let mut state = state.lock().expect("worker panicked"); + loop { + if let Some(result) = state.ready[i].take() { + state.consumed = i + 1; + break result; + } + state = done.wait(state).expect("worker panicked"); + } + }; + room.notify_all(); + if !consume(result) { + state.lock().expect("worker panicked").stop = true; + room.notify_all(); + return; + } + } + }); +} + +/// Shared by the workers of [`ordered`]: the next item to claim, how many were consumed, the results. +struct Ordered { + next: usize, + consumed: usize, + stop: bool, + ready: Vec>, +} + +/// `len` copies of `value` in a column, filled by the workers. +pub fn filled(len: usize, value: T) -> Column { + let mut column = Column::zeroed(len); + chunks(&mut column, |_, part| part.fill(value)); + column +} + +/// `0..len` in a column. +pub fn iota(len: usize) -> Column { + let mut column = Column::zeroed(len); + chunks(&mut column, |start, part| { + for (value, i) in part.iter_mut().zip(start as u32..) { + *value = i; + } + }); + column +} + +/// A bit per index in `0..len`, set where `f` holds, filled by the workers. +pub fn bits(len: usize, f: impl Fn(usize) -> bool + Sync) -> Bits { + let mut bits = Bits::new(len); + chunks(bits.words(), |start, words| { + for (word, at) in words.iter_mut().zip((start * 64..).step_by(64)) { + *word = (at..(at + 64).min(len)).rev().fold(0, |word, index| word << 1 | u64::from(f(index))); + } + }); + bits +} + +/// Run each task on its own worker; results in task order. +pub fn run_all(tasks: Vec T + Send>) -> Vec { + run(tasks.into_iter()) +} diff --git a/src/report/focus.rs b/src/report/focus.rs index 9fdba98..03bcff0 100644 --- a/src/report/focus.rs +++ b/src/report/focus.rs @@ -104,7 +104,7 @@ impl Render<'_, '_> { pub(super) fn object(&self, out: &mut String, object: u32) { let (heap, dump) = (self.heap, self.heap.dump); - let record = &dump.objects[object as usize]; + let record = dump.objects.get(object as usize); let reach = match (heap.reachable(object), heap.weakly_reachable(object)) { (true, _) => "reachable".to_string(), (false, true) => self.dim("weakly reachable"), @@ -120,7 +120,7 @@ impl Render<'_, '_> { self.percent(heap.retained(object)) ), ); - let raw = self.fetched.raw.get(&record.id); + let raw = self.fetched.raw.get(&heap.dump.objects.id(object as usize)); match record.kind { Kind::Instance | Kind::Class => { self.dim_line(out, "fields"); diff --git a/src/report/json.rs b/src/report/json.rs index 6eb5db2..d448879 100644 --- a/src/report/json.rs +++ b/src/report/json.rs @@ -15,9 +15,9 @@ impl Render<'_, '_> { /// An object as data; strings carry their text. fn object_value(&self, object: u32) -> Value { let (heap, dump) = (self.heap, self.heap.dump); - let record = &dump.objects[object as usize]; + let record = dump.objects.get(object as usize); let mut item = json!({ - "id": format!("0x{:x}", record.id), + "id": format!("0x{:x}", dump.objects.id(object as usize)), "class": heap.kind_text(object), "shallow": heap.shallow(object), "retained": heap.retained(object), diff --git a/src/report/mod.rs b/src/report/mod.rs index 16799f9..9c98383 100644 --- a/src/report/mod.rs +++ b/src/report/mod.rs @@ -365,11 +365,11 @@ impl<'a> Plan<'a> { } } if let Some(object) = self.focus_object { - want.push(dump.objects[object as usize].id); + want.push(dump.objects.id(object as usize)); } if let Some(arrays) = &self.inputs.array_duplicates { want.extend( - arrays.groups.iter().take(self.view.top).map(|group| dump.objects[group.array as usize].id), + arrays.groups.iter().take(self.view.top).map(|group| dump.objects.id(group.array as usize)), ); } want @@ -531,17 +531,18 @@ impl Render<'_, '_> { /// An object for a line: `java.util.HashMap @0x7f3a1234`, arrays with /// their length, strings with their text, class objects by class. fn describe(&self, object: u32) -> String { - let (dump, record) = (self.heap.dump, &self.heap.dump.objects[object as usize]); + let (dump, record) = (self.heap.dump, self.heap.dump.objects.get(object as usize)); + let id = dump.objects.id(object as usize); let name = &dump.classes[record.class as usize].name; match record.kind { - Kind::Class => format!("{} @0x{:x}", self.heap.kind_text(object), record.id), + Kind::Class => format!("{} @0x{id:x}", self.heap.kind_text(object)), Kind::ObjectArray | Kind::PrimitiveArray => { let base = name.strip_suffix("[]").unwrap_or(name); - format!("{base}[{}] @0x{:x}", commas(u64::from(record.len)), record.id) + format!("{base}[{}] @0x{id:x}", commas(u64::from(record.len))) } Kind::Instance => match self.text(object) { - Some(text) => format!("{name} \"{text}\" @0x{:x}", record.id), - None => format!("{name} @0x{:x}", record.id), + Some(text) => format!("{name} \"{text}\" @0x{id:x}"), + None => format!("{name} @0x{id:x}"), }, } } diff --git a/src/report/sections.rs b/src/report/sections.rs index 8e4b872..8d63e78 100644 --- a/src/report/sections.rs +++ b/src/report/sections.rs @@ -259,7 +259,7 @@ impl Render<'_, '_> { /// The first elements of a primitive array, as text when it reads as text. pub(super) fn array_preview(&self, object: u32) -> String { let dump = self.heap.dump; - let Some(raw) = self.fetched.raw.get(&dump.objects[object as usize].id) else { return String::new() }; + let Some(raw) = self.fetched.raw.get(&dump.objects.id(object as usize)) else { return String::new() }; let readable = |text: &String| { !text.is_empty() && text.chars().all(|c| !c.is_control() && c != '�') diff --git a/src/store.rs b/src/store.rs new file mode 100644 index 0000000..b66ec55 --- /dev/null +++ b/src/store.rs @@ -0,0 +1,195 @@ +//! Big arrays as plain vectors. A large one is its own mapping from the allocator: zeroed room costs no +//! memory until written, and growing or shrinking it remaps instead of copying. + +use std::ops::{Deref, DerefMut}; +use std::sync::atomic::{AtomicU32, AtomicU64}; + +use zerocopy::{FromBytes, FromZeros, IntoBytes}; + +/// An array of plain values, zeroed when made. +pub struct Column(Vec); + +impl Column { + /// `len` zeroes. + pub fn zeroed(len: usize) -> Column { + Column(T::new_vec_zeroed(len).expect("column too large")) + } +} + +impl Column { + /// Room for `capacity` values, none yet. Untouched room costs no memory; pushing past it grows. + pub fn with_capacity(capacity: usize) -> Column { + Column(Vec::with_capacity(capacity)) + } + + pub fn push(&mut self, value: T) { + self.0.push(value); + } + + pub fn extend_from_slice(&mut self, values: &[T]) + where + T: Copy, + { + self.0.extend_from_slice(values); + } + + pub fn reserve(&mut self, additional: usize) { + self.0.reserve(additional); + } + + pub fn truncate(&mut self, len: usize) { + self.0.truncate(len); + } + + /// Hand the unused tail back. Nothing past `len` is read again. + pub fn shrink_to_fit(&mut self) { + self.0.shrink_to_fit(); + } +} + +impl Column { + /// The values as atomics, for workers filling disjoint slots. + pub fn atomics(&mut self) -> &[AtomicU32] { + <[AtomicU32]>::mut_from_bytes(self.0.as_mut_bytes()).expect("AtomicU32 has the layout of u32") + } +} + +impl Column { + /// The values as atomics, for workers adding into shared slots. + pub fn atomics(&mut self) -> &[AtomicU64] { + <[AtomicU64]>::mut_from_bytes(self.0.as_mut_bytes()).expect("AtomicU64 has the layout of u64") + } +} + +impl Deref for Column { + type Target = [T]; + + fn deref(&self) -> &[T] { + &self.0 + } +} + +impl DerefMut for Column { + fn deref_mut(&mut self) -> &mut [T] { + &mut self.0 + } +} + +impl FromIterator for Column { + fn from_iter>(values: I) -> Column { + Column(values.into_iter().collect()) + } +} + +/// One bit per index, in a column. +pub struct Bits { + words: Column, +} + +impl Bits { + pub fn new(len: usize) -> Bits { + Bits { words: Column::zeroed(len.div_ceil(64)) } + } + + pub fn get(&self, index: usize) -> bool { + self.words[index / 64] >> (index % 64) & 1 != 0 + } + + pub fn set(&mut self, index: usize) { + self.words[index / 64] |= 1 << (index % 64); + } + + /// Clear a bit, saying whether it was set. + pub fn take(&mut self, index: usize) -> bool { + let (word, bit) = (&mut self.words[index / 64], 1u64 << (index % 64)); + let was = *word & bit != 0; + *word &= !bit; + was + } + + pub fn word(&self, index: usize) -> u64 { + self.words[index] + } + + /// The words, 64 indices each, lowest bit first. + pub fn words(&mut self) -> &mut [u64] { + &mut self.words + } + + /// The words as atomics, for workers setting bits. + pub fn atomics(&mut self) -> &[AtomicU64] { + self.words.atomics() + } +} + +impl Extend for Column { + fn extend>(&mut self, values: I) { + self.0.extend(values); + } +} + +/// Values per base in a [`Blocked`] column. +pub const BLOCK: usize = 256; + +/// Ascending values in two bytes each above a base every [`BLOCK`] of them. One too far above its base +/// is all ones there and kept whole in `long`, by index. +pub struct Blocked { + bases: Column, + above: Column, + long: Vec<(u32, u64)>, +} + +impl Blocked { + /// `value(i)` for every `i` in `0..len`, ascending, laid out by the workers. + pub fn new(len: usize, value: impl Fn(usize) -> u64 + Sync) -> Blocked { + let mut bases = Column::::zeroed(len.div_ceil(BLOCK)); + let mut above = Column::::zeroed(len); + let per = bases.len().div_ceil(super::parallel::threads()).max(1); + let value = &value; + let tasks: Vec<_> = bases + .chunks_mut(per) + .zip(above.chunks_mut(per * BLOCK)) + .enumerate() + .map(|(task, (bases, above))| { + move || { + let mut long = Vec::new(); + for (block, (base, above)) in bases.iter_mut().zip(above.chunks_mut(BLOCK)).enumerate() { + let start = (task * per + block) * BLOCK; + *base = value(start); + for (i, above) in (start..).zip(above.iter_mut()) { + let value = value(i); + *above = u16::try_from(value - *base).unwrap_or(u16::MAX); + if *above == u16::MAX { + long.push((i as u32, value)); + } + } + } + long + } + }) + .collect(); + let long = super::parallel::run_all(tasks).concat(); + Blocked { bases, above, long } + } + + pub fn get(&self, i: usize) -> u64 { + match self.above[i] { + u16::MAX => self.long[self.long.partition_point(|&(at, _)| (at as usize) < i)].1, + above => self.bases[i / BLOCK] + u64::from(above), + } + } + + pub fn len(&self) -> usize { + self.above.len() + } + + /// The base of the block holding `i`. + pub fn base(&self, i: usize) -> u64 { + self.bases[i / BLOCK] + } + + /// The two-byte parts of `lo..hi`: within one block they ascend, the long ones last. + pub fn above(&self, lo: usize, hi: usize) -> &[u16] { + &self.above[lo..hi] + } +} diff --git a/src/strings.rs b/src/strings.rs index beee5d3..74938af 100644 --- a/src/strings.rs +++ b/src/strings.rs @@ -9,7 +9,7 @@ use super::parallel; pub fn string_array(dump: &Dump, graph: &Graph, string: u32) -> Option { let array = graph.field(string, dump.value_label)?; - (dump.objects[array as usize].kind == Kind::PrimitiveArray).then_some(array) + (dump.objects.kind(array as usize) == Kind::PrimitiveArray).then_some(array) } /// Every String with its backing array. @@ -19,7 +19,7 @@ pub fn strings(dump: &Dump, graph: &Graph) -> Vec<(u32, u32)> { } parallel::ranges(dump.objects.len(), |lo, hi| { (lo as u32..hi as u32) - .filter(|&object| dump.objects[object as usize].class == dump.string_class) + .filter(|&object| dump.objects.class(object as usize) == dump.string_class) .filter_map(|string| string_array(dump, graph, string).map(|array| (string, array))) .collect::>() }) @@ -28,19 +28,19 @@ pub fn strings(dump: &Dump, graph: &Graph) -> Vec<(u32, u32)> { /// The ids to fetch so String `string` can be shown. pub fn string_ids(dump: &Dump, graph: &Graph, string: u32, want: &mut Vec) { - want.push(dump.objects[string as usize].id); + want.push(dump.objects.id(string as usize)); if let Some(array) = string_array(dump, graph, string) { - want.push(dump.objects[array as usize].id); + want.push(dump.objects.id(array as usize)); } } /// Decode a String: JDK 9+ has a `byte[]` and a `coder` (0 latin-1, 1 utf-16), older JDKs a `char[]`. pub fn string_text(dump: &Dump, graph: &Graph, fetched: &Fetched, string: u32) -> Option { let backing = string_array(dump, graph, string)?; - let array = fetched.raw.get(&dump.objects[backing as usize].id)?; - let coder = fetched.raw.get(&dump.objects[string as usize].id).and_then(|raw| { + let array = fetched.raw.get(&dump.objects.id(backing as usize))?; + let coder = fetched.raw.get(&dump.objects.id(string as usize)).and_then(|raw| { let (offset, _) = dump - .field_offset(dump.objects[string as usize].class, "coder") + .field_offset(dump.objects.class(string as usize), "coder") .filter(|&(_, ty)| ty == Ty::Byte)?; raw.data.get(offset as usize).copied() });