diff --git a/.github/workflows/dwarf-perf-regression.yml b/.github/workflows/dwarf-perf-regression.yml index ab72487c..8a3775a7 100644 --- a/.github/workflows/dwarf-perf-regression.yml +++ b/.github/workflows/dwarf-perf-regression.yml @@ -167,7 +167,7 @@ jobs: --results-dir "$DWARF_PERF_RESULTS_DIR" \ --result-name "base-${target}" \ --runs 10 \ - --cargo-target-dir "$CARGO_TARGET_DIR" \ + --cargo-target-dir "$CARGO_TARGET_DIR/base" \ --parse-target "$target" done @@ -196,7 +196,7 @@ jobs: --results-dir "$DWARF_PERF_RESULTS_DIR" \ --result-name "head-${target}" \ --runs 10 \ - --cargo-target-dir "$CARGO_TARGET_DIR" \ + --cargo-target-dir "$CARGO_TARGET_DIR/head" \ --parse-target "$target" done @@ -239,7 +239,7 @@ jobs: --results-dir "$DWARF_PERF_RESULTS_DIR" \ --result-name "head-${target}" \ --runs 10 \ - --cargo-target-dir "$CARGO_TARGET_DIR" \ + --cargo-target-dir "$CARGO_TARGET_DIR/head" \ --parse-target "$target" done diff --git a/bins/dwarf-tool/src/main.rs b/bins/dwarf-tool/src/main.rs index cb638a28..7ac7f184 100644 --- a/bins/dwarf-tool/src/main.rs +++ b/bins/dwarf-tool/src/main.rs @@ -5,9 +5,9 @@ use anyhow::Result; use clap::{Parser, Subcommand}; use ghostscope_dwarf::{ - AddressQueryResult, DwarfAnalyzer, FunctionQueryResult, ModuleLoadingEvent, ModuleLoadingStats, - SectionType, SourceLanguage, ValueAdapterOutcome, ValueAdapterReport, ValueAdapterStage, - ValueCapturePlan, ValuePresentation, VariableAccessPath, + AddressQueryResult, DwarfAnalyzer, FunctionQueryResult, ModuleAddress, ModuleLoadingEvent, + ModuleLoadingStats, SectionType, SourceLanguage, ValueAdapterOutcome, ValueAdapterReport, + ValueAdapterStage, ValueCapturePlan, ValuePresentation, VariableAccessPath, }; use std::path::PathBuf; use std::sync::{Arc, Mutex}; @@ -164,6 +164,9 @@ enum Commands { /// Number of runs #[arg(long, default_value = "10")] runs: usize, + /// Number of unmeasured warmup queries + #[arg(long, default_value = "10")] + warmup_runs: usize, /// JSON output #[arg(long)] json: bool, @@ -228,6 +231,7 @@ struct SourceLineBenchmarkResult { source_location: String, loading_time_ms: u64, load_breakdown: LoadTimingBreakdown, + warmup_runs: usize, address_count: usize, total_variables: usize, first_address: Option, @@ -433,6 +437,7 @@ async fn main() -> Result<()> { Commands::BenchmarkSourceLine { location, runs, + warmup_runs, json, } => { return run_source_line_benchmark( @@ -440,6 +445,7 @@ async fn main() -> Result<()> { cli.target.as_deref(), location, *runs, + *warmup_runs, *json, ) .await; @@ -1232,6 +1238,7 @@ async fn run_source_line_benchmark( target_path: Option<&str>, source: &str, runs: usize, + warmup_runs: usize, json: bool, ) -> Result<()> { if runs == 0 { @@ -1240,9 +1247,15 @@ async fn run_source_line_benchmark( if !json { if let Some(pid) = pid { - println!("Benchmarking source-line query for PID {pid} at {source} ({runs} runs)...\n"); + println!( + "Benchmarking source-line query for PID {pid} at {source} \ + ({warmup_runs} warmup, {runs} measured runs)...\n" + ); } else if let Some(target) = target_path { - println!("Benchmarking source-line query for {target} at {source} ({runs} runs)...\n"); + println!( + "Benchmarking source-line query for {target} at {source} \ + ({warmup_runs} warmup, {runs} measured runs)...\n" + ); } } @@ -1266,6 +1279,24 @@ async fn run_source_line_benchmark( load_breakdown.index_time_ms, load_breakdown.internal_total_time_ms ); + if warmup_runs > 0 { + print!("Warming: "); + std::io::Write::flush(&mut std::io::stdout()).unwrap(); + } + } + + for _ in 0..warmup_runs { + let _ = execute_source_line_query(&analyzer, file_path, line_number)?; + if !json { + print!("."); + std::io::Write::flush(&mut std::io::stdout()).unwrap(); + } + } + + if !json { + if warmup_runs > 0 { + println!(); + } print!("Querying: "); std::io::Write::flush(&mut std::io::stdout()).unwrap(); } @@ -1277,14 +1308,8 @@ async fn run_source_line_benchmark( } let start = Instant::now(); - let addresses = analyzer.lookup_addresses_by_source_line(file_path, line_number); - let mut run_total_variables = 0usize; - - for module_address in &addresses { - let pc_context = analyzer.resolve_pc(module_address)?; - run_total_variables += analyzer.visible_variables(&pc_context)?.len(); - } - + let (addresses, run_total_variables) = + execute_source_line_query(&analyzer, file_path, line_number)?; query_times.push(start.elapsed()); if run_index == 0 { @@ -1304,6 +1329,7 @@ async fn run_source_line_benchmark( source_location: source.to_string(), loading_time_ms: loading_time.as_millis() as u64, load_breakdown, + warmup_runs, address_count, total_variables, first_address, @@ -1320,6 +1346,7 @@ async fn run_source_line_benchmark( println!(" First address: {first_address}"); } println!("\nResults:"); + println!(" Warmup runs: {}", result.warmup_runs); println!(" First run: {:.3}ms", result.benchmark.first_run_ms); println!(" Average query time: {:.3}ms", result.benchmark.average_ms); println!(" P50: {:.3}ms", result.benchmark.p50_ms); @@ -1331,6 +1358,22 @@ async fn run_source_line_benchmark( Ok(()) } +fn execute_source_line_query( + analyzer: &DwarfAnalyzer, + file_path: &str, + line_number: u32, +) -> Result<(Vec, usize)> { + let addresses = analyzer.lookup_addresses_by_source_line(file_path, line_number); + let mut total_variables = 0usize; + + for module_address in &addresses { + let pc_context = analyzer.resolve_pc(module_address)?; + total_variables += analyzer.visible_variables(&pc_context)?.len(); + } + + Ok((addresses, total_variables)) +} + fn summarize_durations(durations: &[std::time::Duration]) -> Result { if durations.is_empty() { return Err(anyhow::anyhow!("Cannot summarize an empty benchmark run")); @@ -1463,7 +1506,8 @@ fn parse_address(address_str: &str) -> Result { #[cfg(test)] mod tests { - use super::{parse_source_line, percentile_nearest_rank}; + use super::{parse_source_line, percentile_nearest_rank, Cli, Commands}; + use clap::Parser; #[test] fn parses_source_line_with_colons_in_path() { @@ -1478,6 +1522,22 @@ mod tests { assert_eq!(percentile_nearest_rank(&samples, 0.50), 3.0); assert_eq!(percentile_nearest_rank(&samples, 0.95), 5.0); } + + #[test] + fn source_line_benchmark_defaults_to_ten_warmup_queries() { + let cli = + Cli::try_parse_from(["dwarf-tool", "benchmark-source-line", "query_hotspot.c:17"]) + .unwrap(); + + let Commands::BenchmarkSourceLine { + runs, warmup_runs, .. + } = cli.command + else { + panic!("expected source-line benchmark command"); + }; + assert_eq!(runs, 10); + assert_eq!(warmup_runs, 10); + } } #[derive(Debug, serde::Serialize)] diff --git a/config-zh.toml b/config-zh.toml index 3473ea02..e33d62ce 100644 --- a/config-zh.toml +++ b/config-zh.toml @@ -144,7 +144,8 @@ max_size_bytes = 0 # 默认值：4 max_nesting_depth = 4 -# 每个嵌套序列节点最多采集多少个语义子元素，例如 Vec。如果 +# 每个嵌套序列节点最多采集多少个语义子元素，例如 Vec，以及 +# 嵌套 Hash 表中最多为多少个 bucket 采集 key/value sidecar。如果 # mem_dump_cap 无法容纳这么多子 payload，实际采集数量会更少。 # 有效范围：1 到 16。 # 默认值：4 diff --git a/config.toml b/config.toml index d7047c96..2d437a12 100644 --- a/config.toml +++ b/config.toml @@ -154,8 +154,9 @@ max_size_bytes = 0 max_nesting_depth = 4 # Maximum semantic child elements captured for each nested sequence node, such -# as Vec. The effective count can be lower when mem_dump_cap cannot fit -# this many child payloads. +# as Vec, and buckets with key/value sidecars in nested hash tables. +# The effective count can be lower when mem_dump_cap cannot fit this many child +# payloads. # Valid range: 1 to 16. # Default: 4 max_sequence_elements = 4 diff --git a/docs/configuration.md b/docs/configuration.md index 68075116..e136e159 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -559,13 +559,15 @@ enable_sysmon_for_target = true # Default independent of the depth-32 limit for native inline DWARF structure formatting. - `max_sequence_elements` limits the semantic children statically emitted for - each nested sequence node. It is a per-node width limit, not a recursion - depth. + each nested sequence node and the captured buckets that receive repeated + key/value sidecars in a nested hash table. It is a per-node width limit, not + a recursion depth. Both settings default to `4` and accept values from `1` through `16`. The -effective sequence count is also bounded by the runtime sequence length and the -argument's `ebpf.mem_dump_cap`. Increasing either setting can increase generated -eBPF program size, event size, and probe overhead. +effective collection width is also bounded by the runtime sequence length or +hash-table capacity and the argument's `ebpf.mem_dump_cap`. Increasing either +setting can increase generated eBPF program size, event size, and probe +overhead. ### Configuration Examples diff --git a/docs/limitations.md b/docs/limitations.md index b294c8ea..adf96203 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -31,11 +31,11 @@ formatting has a maximum depth of 32. **Rust** has targeted semantic support for common standard-library value families. It is selected from the concrete target DWARF rather than from a -promised Rust ABI. Nested adapters compose for projected wrappers and bounded -sequences under configured depth and per-sequence element limits plus a shared -byte budget. Hash-table and B-Tree key/value adapters do not yet compose. See -[Rust Value Presentation](scripting.md#rust-value-presentation) for the current -type list, formatting behavior, and nesting boundary. +promised Rust ABI. Nested adapters compose for projected wrappers, bounded +sequences, and hash-table entries under configured depth and collection-width +limits plus a shared byte budget. B-Tree key/value adapters do not yet compose. +See [Rust Value Presentation](scripting.md#rust-value-presentation) for the +current type list, formatting behavior, and nesting boundary. **C++** remains primarily DWARF-layout-oriented. Simple C-like layouts work best; broader standard-library and language-specific semantics are not modeled diff --git a/docs/scripting.md b/docs/scripting.md index 6c5b5dea..532e7284 100644 --- a/docs/scripting.md +++ b/docs/scripting.md @@ -263,6 +263,8 @@ Cell Cell wrapper plus captured String bytes Vec each captured element uses the String adapter Vec each captured element omits its trailing NUL Vec> each captured element uses the Vec adapter +HashMap> captured keys and values use their adapters +HashSet captured values use the String adapter Option only Some captures its String payload Result only the active Ok or Err payload is captured ``` @@ -280,18 +282,21 @@ detects repeated DWARF type identities on the active path. Traversing an ordinary Rust struct to reach an adapted field counts as one edge. Transitioning into an active enum payload also counts as one edge. Each semantic sequence reserves child captures for at most -`value_adapters.max_sequence_elements` elements. Both limits default to `4`; -the former controls recursion depth while the latter controls the width of -each sequence node. The root and every child share the argument's -`mem_dump_cap`; reducing that cap can reduce the captured element count or -fall back to the valid root presentation. Every child carries an independent -read-error or truncation status. See +`value_adapters.max_sequence_elements` elements. Nested hash tables use the +same setting to bound the number of captured buckets with key/value sidecars; +because a hash table is sparse, the bounded bucket prefix can contain fewer +occupied entries. Both limits default to `4`; the former controls recursion +depth while the latter controls each collection node's width. The root and +every child share the argument's `mem_dump_cap`; reducing that cap can reduce +the captured element or bucket count, or fall back to the valid root +presentation. Every child carries an independent read-error or truncation +status. See [Value Adapter Limits](configuration.md#value-adapter-limits). -Hash-table and B-Tree entry traversal still formats key and value bytes using -their DWARF types. For example, `HashMap>` does not recursively -run adapters for its keys or values. Known nested fields remain accessible -explicitly through the DSL. +Hash-table entry traversal recursively applies recognized adapters to keys and +values while retaining native DWARF formatting for unrecognized fields. B-Tree +entry traversal still formats key and value bytes using their DWARF types; +known nested fields remain accessible explicitly through the DSL. ## Variables diff --git a/docs/zh/configuration.md b/docs/zh/configuration.md index 8e5d9fec..7349b9e5 100644 --- a/docs/zh/configuration.md +++ b/docs/zh/configuration.md @@ -544,12 +544,13 @@ enable_sysmon_for_target = true # 默认开启 - `max_nesting_depth` 计算根值以下的语义子值边数；为了到达已适配字段而 穿过的普通 Rust struct，以及进入激活 enum payload 的转换也计算在内。 它不包含根本身，也不影响原生内联 DWARF 结构格式化的 32 层深度限制。 -- `max_sequence_elements` 限制每个嵌套序列节点静态生成的语义子元素数量。 - 它是每个节点的宽度限制，不是递归深度。 +- `max_sequence_elements` 限制每个嵌套序列节点静态生成的语义子元素数量， + 以及嵌套 Hash 表中带重复 key/value sidecar 的已采集 bucket 数量。它是 + 每个节点的宽度限制，不是递归深度。 -两项默认值均为 `4`，有效范围均为 `1..=16`。序列的实际采集数量还会受到 -运行时序列长度和参数 `ebpf.mem_dump_cap` 的限制。提高任一配置都可能增加 -生成的 eBPF 程序大小、事件大小和探针开销。 +两项默认值均为 `4`，有效范围均为 `1..=16`。集合的实际采集宽度还会受到 +运行时序列长度或 Hash 表容量，以及参数 `ebpf.mem_dump_cap` 的限制。提高 +任一配置都可能增加生成的 eBPF 程序大小、事件大小和探针开销。 ### 配置示例 diff --git a/docs/zh/limitations.md b/docs/zh/limitations.md index 5b9a1663..5551ede6 100644 --- a/docs/zh/limitations.md +++ b/docs/zh/limitations.md @@ -26,9 +26,9 @@ GhostScope 的脚本和编译器能力面只用于观测，不提供主动修改 32。 **Rust** 已针对常见标准库类型提供定向语义支持。这些适配器根据具体目标 -的 DWARF 选择，而不是假设一个稳定的 Rust ABI。投影包装器和有界序列中的 -嵌套 adapter 会在配置的递归深度、单序列元素数和共享字节预算内组合； -Hash 表和 B-Tree 的 key/value adapter 尚不会递归组合。当前类型列表、 +的 DWARF 选择，而不是假设一个稳定的 Rust ABI。投影包装器、有界序列和 +Hash 表条目中的嵌套 adapter 会在配置的递归深度、集合宽度和共享字节预算 +内组合；B-Tree 的 key/value adapter 尚不会递归组合。当前类型列表、 格式化行为和嵌套边界见 [Rust 值展示](scripting.md#rust-值展示)。 diff --git a/docs/zh/scripting.md b/docs/zh/scripting.md index 9f3cb8ce..8092421a 100644 --- a/docs/zh/scripting.md +++ b/docs/zh/scripting.md @@ -245,6 +245,8 @@ Cell 显示 Cell 包装并采集 String 字节 Vec 每个已采集元素使用 String 适配器 Vec 每个已采集元素都会排除结尾 NUL Vec> 每个已采集元素使用 Vec 适配器 +HashMap> 已采集的 key/value 使用各自的适配器 +HashSet 已采集的值使用 String 适配器 Option 只有 Some 会采集其 String payload Result 只采集当前激活的 Ok 或 Err payload ``` @@ -258,15 +260,17 @@ struct 中没有可识别的语义后代，它仍使用原有的原生 DWARF 路 `value_adapters.max_nesting_depth` 条语义子值边，并检测当前路径中重复的 DWARF 类型标识。为了到达已适配字段而穿过一个普通 Rust struct 也算一条边。 从 enum 进入其 payload 同样算一条边。每个语义序列最多为 -`value_adapters.max_sequence_elements` 个元素预留子采集槽。两项默认值均为 -`4`；前者控制递归深度，后者控制每个序列节点的宽度。根值和所有子值共享 -该参数的 `mem_dump_cap`；降低这个上限可能减少采集的元素数，或回退到仍然 +`value_adapters.max_sequence_elements` 个元素预留子采集槽。嵌套 Hash 表也使用 +该配置限制带 key/value sidecar 的 bucket 数；由于 Hash 表是稀疏的，有界 +bucket 前缀中实际占用的条目可能更少。两项默认值均为 `4`；前者控制递归 +深度，后者控制每个集合节点的宽度。根值和所有子值共享该参数的 +`mem_dump_cap`；降低这个上限可能减少采集的元素或 bucket 数，或回退到仍然 有效的根值展示。每个子值都有独立的读取错误和截断状态。配置细节见 [Value Adapter 限制](configuration.md#value-adapter-限制)。 -Hash 表和 B-Tree 的条目遍历仍按 key/value 的 DWARF 类型格式化字节。例如， -`HashMap>` 还不会为 key 或 value 递归运行适配器。已知的 -嵌套字段仍可在 DSL 中显式访问。 +Hash 表条目遍历会对 key/value 递归应用已识别的适配器，未识别字段仍使用 +原生 DWARF 格式。B-Tree 条目遍历仍按 key/value 的 DWARF 类型格式化字节； +已知的嵌套字段仍可在 DSL 中显式访问。 ## 变量 diff --git a/e2e-tests/tests/fixtures/rust_global_program/src/main.rs b/e2e-tests/tests/fixtures/rust_global_program/src/main.rs index 6359fc86..542ff386 100644 --- a/e2e-tests/tests/fixtures/rust_global_program/src/main.rs +++ b/e2e-tests/tests/fixtures/rust_global_program/src/main.rs @@ -522,6 +522,15 @@ pub mod math { + unit_set.len() } + #[inline(never)] + pub fn observe_nested_hash_collections( + map: HashMap>, + set: HashSet, + ) -> usize { + std::hint::black_box((&map, &set)); + map.values().map(Vec::len).sum::() + set.len() + } + #[inline(never)] pub fn observe_user_hash_collections( map: &crate::user_types::HashMap, @@ -957,6 +966,13 @@ fn main() { HashMap::from([((), ())]), HashSet::from([()]), ) as i64; + acc += math::observe_nested_hash_collections( + HashMap::from([ + (String::from("alpha"), vec![3, 5, 8]), + (String::from("omega"), vec![13, 21]), + ]), + HashSet::from([String::from("left"), String::from("right")]), + ) as i64; let user_map = user_types::HashMap { key: 43_i32, value: 47_u16, diff --git a/e2e-tests/tests/rust_script_execution.rs b/e2e-tests/tests/rust_script_execution.rs index 8c15dab4..c7b98b02 100644 --- a/e2e-tests/tests/rust_script_execution.rs +++ b/e2e-tests/tests/rust_script_execution.rs @@ -194,6 +194,9 @@ async fn test_rust_nested_value_plans_recurse_into_semantic_children() -> anyhow Some(ghostscope_dwarf::ValueNestedPlan::Sequence { element }) => { contains_utf8_string(element) } + Some(ghostscope_dwarf::ValueNestedPlan::HashTable { fields }) => fields + .iter() + .any(|field| contains_utf8_string(&field.value)), Some(ghostscope_dwarf::ValueNestedPlan::Variant { fields }) => fields .iter() .any(|field| contains_utf8_string(&field.value)), @@ -1491,6 +1494,71 @@ async fn test_rust_hash_collection_plans_follow_dwarf_raw_table_layout() -> anyh assert_eq!(capture_order, bucket_order); } + let nested_context = analyzer + .lookup_function_addresses("observe_nested_hash_collections") + .into_iter() + .find_map(|address| analyzer.resolve_pc(&address).ok()) + .ok_or_else(|| anyhow::anyhow!("expected observe_nested_hash_collections context"))?; + for (parameter, is_map) in [("map", true), ("set", false)] { + let parameter_plan = analyzer + .plan_variable_by_name(&nested_context, parameter)? + .ok_or_else(|| anyhow::anyhow!("expected nested {parameter} parameter plan"))?; + let parameter_type = analyzer + .resolved_type_for_plan(¶meter_plan)? + .ok_or_else(|| anyhow::anyhow!("expected nested {parameter} parameter type"))?; + let value_plan = analyzer + .value_read_plan(¶meter_type, Some(&binary_path))? + .ok_or_else(|| anyhow::anyhow!("expected nested {parameter} semantic plan"))?; + if is_map { + assert_eq!(value_plan.hash_table_fields.len(), 2); + assert_eq!(value_plan.hash_table_fields[0].field_index, 0); + assert!(value_plan.hash_table_fields[0] + .resolved_type + .summary + .type_name() + .ends_with("String")); + assert_eq!(value_plan.hash_table_fields[1].field_index, 1); + assert!(value_plan.hash_table_fields[1] + .resolved_type + .summary + .type_name() + .contains("Vec<")); + } else { + assert_eq!(value_plan.hash_table_fields.len(), 1); + assert_eq!(value_plan.hash_table_fields[0].field_index, 0); + assert!(value_plan.hash_table_fields[0] + .resolved_type + .summary + .type_name() + .ends_with("String")); + } + let Some(ghostscope_dwarf::ValueNestedPlan::HashTable { fields }) = + value_plan.nested.as_ref() + else { + anyhow::bail!("expected nested hash-table field plans for {parameter}") + }; + if is_map { + assert_eq!(fields.len(), 2); + assert_eq!(fields[0].field_index, 0); + assert_eq!( + fields[0].value.presentation, + ghostscope_dwarf::ValuePresentation::Utf8String + ); + assert_eq!(fields[1].field_index, 1); + assert!(matches!( + fields[1].value.presentation, + ghostscope_dwarf::ValuePresentation::Sequence { .. } + )); + } else { + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].field_index, 0); + assert_eq!( + fields[0].value.presentation, + ghostscope_dwarf::ValuePresentation::Utf8String + ); + } + } + for (parameter, expected_map) in [("unit_map", true), ("unit_set", false)] { let parameter_plan = analyzer .plan_variable_by_name(&context, parameter)? @@ -2497,6 +2565,56 @@ mem_dump_cap = 9 Ok(()) } +#[tokio::test] +async fn test_rust_script_print_nested_hash_collection_values() -> anyhow::Result<()> { + init(); + + let target = spawn_rust_global_program().await?; + let script = r#" +trace observe_nested_hash_collections { + print "RHASH_NESTED_MAP:{}", map; + print "RHASH_NESTED_SET:{}", set; +} +"#; + let (exit_code, stdout, stderr) = common::runner::GhostscopeRunner::new() + .with_script(script) + .with_config_content( + r#" +[ebpf] +mem_dump_cap = 1024 +"#, + ) + .attach_to(&target) + .timeout_secs(9) + .enable_sysmon_for_target(false) + .run() + .await?; + target.terminate().await?; + + assert_eq!(exit_code, 0, "stderr={stderr} stdout={stdout}"); + let map_line = stdout + .lines() + .find(|line| line.contains("RHASH_NESTED_MAP:")) + .ok_or_else(|| anyhow::anyhow!("missing nested Rust HashMap output: {stdout}"))?; + assert!(map_line.contains("HashMap(size=2)"), "{map_line}"); + assert!(map_line.contains(r#""alpha": [3, 5, 8]"#), "{map_line}"); + assert!(map_line.contains(r#""omega": [13, 21]"#), "{map_line}"); + let set_line = stdout + .lines() + .find(|line| line.contains("RHASH_NESTED_SET:")) + .ok_or_else(|| anyhow::anyhow!("missing nested Rust HashSet output: {stdout}"))?; + assert!(set_line.contains("HashSet(size=2)"), "{set_line}"); + assert!(set_line.contains(r#""left""#), "{set_line}"); + assert!(set_line.contains(r#""right""#), "{set_line}"); + assert!(!stdout.contains(" anyhow::Result<()> { init(); diff --git a/ghostscope-compiler/src/ebpf/codegen/args.rs b/ghostscope-compiler/src/ebpf/codegen/args.rs index 9f3ca276..0c6f2031 100644 --- a/ghostscope-compiler/src/ebpf/codegen/args.rs +++ b/ghostscope-compiler/src/ebpf/codegen/args.rs @@ -304,6 +304,7 @@ fn hash_table_capture_limits( cap: usize, entry_stride: u64, occupancy: ghostscope_dwarf::HashTableOccupancy, + bucket_limit: Option, ) -> Result<(usize, usize, usize)> { let stride = usize::try_from(entry_stride).map_err(|_| { CodeGenError::DwarfError(format!( @@ -319,7 +320,7 @@ fn hash_table_capture_limits( let bytes_per_bucket = stride.checked_add(occupancy_width).ok_or_else(|| { CodeGenError::DwarfError("hash-table bucket capture size overflow".to_string()) })?; - let max_buckets = cap / bytes_per_bucket; + let max_buckets = (cap / bytes_per_bucket).min(bucket_limit.unwrap_or(usize::MAX)); let max_len = max_buckets .checked_mul(bytes_per_bucket) .ok_or_else(|| CodeGenError::DwarfError("hash-table payload size overflow".to_string()))?; @@ -382,7 +383,7 @@ fn compile_nested_value_source( ) -> Result> { let Some(nested) = &plan.nested else { let Some((output_type, root, root_payload_len)) = - compile_nested_root_source(plan, budget, None)? + compile_nested_root_source(plan, budget, None, None)? else { return Ok(None); }; @@ -399,7 +400,7 @@ fn compile_nested_value_source( match nested { ghostscope_dwarf::ValueNestedPlan::ProjectedValue { value } => { let Some((output_type, root, root_payload_len)) = - compile_nested_root_source(plan, budget, None)? + compile_nested_root_source(plan, budget, None, None)? else { return Ok(None); }; @@ -435,7 +436,7 @@ fn compile_nested_value_source( } ghostscope_dwarf::ValueNestedPlan::ProjectedView { fields } => { let Some((output_type, root, root_payload_len)) = - compile_nested_root_source(plan, budget, None)? + compile_nested_root_source(plan, budget, None, None)? else { return Ok(None); }; @@ -498,7 +499,7 @@ fn compile_nested_value_source( } ghostscope_dwarf::ValueNestedPlan::Variant { fields } => { let Some((output_type, root, root_payload_len)) = - compile_nested_root_source(plan, budget, None)? + compile_nested_root_source(plan, budget, None, None)? else { return Ok(None); }; @@ -617,10 +618,115 @@ fn compile_nested_value_source( }, })) } + ghostscope_dwarf::ValueNestedPlan::HashTable { fields } => { + if fields.is_empty() || max_sequence_elements == 0 { + return Ok(None); + } + let Some((_, unconstrained_root, _)) = + compile_nested_root_source(plan, budget, None, None)? + else { + return Ok(None); + }; + let NestedValueRootSource::IndirectHashTable { max_buckets, .. } = unconstrained_root + else { + return Err(CodeGenError::DwarfError( + "nested hash-table children do not match their root capture".to_string(), + )); + }; + let max_buckets = max_buckets.min(max_sequence_elements); + for bucket_count in (1..=max_buckets).rev() { + let Some((output_type, root, root_payload_len)) = + compile_nested_root_source(plan, budget, None, Some(bucket_count))? + else { + continue; + }; + let header_bytes_per_bucket = fields + .len() + .checked_mul(ghostscope_protocol::NESTED_VALUE_CHILD_HEADER_SIZE) + .ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table field header size overflow".to_string(), + ) + })?; + let child_budget = budget + .saturating_sub(root_payload_len) + .checked_div(bucket_count) + .unwrap_or(0) + .saturating_sub(header_bytes_per_bucket) + .checked_div(fields.len()) + .unwrap_or(0); + let mut field_slot_offset = 0usize; + let mut compiled_fields = Vec::with_capacity(fields.len()); + for field in fields { + let Some(child) = compile_nested_value_source( + &field.value, + child_budget, + max_sequence_elements, + )? + else { + break; + }; + let entry_offset = + nested_hash_table_field_offset(&plan.presentation, field.field_index)?; + compiled_fields.push(NestedHashTableFieldSource { + field_index: field.field_index, + entry_offset, + slot_offset: field_slot_offset, + child: Box::new(child), + }); + field_slot_offset = field_slot_offset + .checked_add(ghostscope_protocol::NESTED_VALUE_CHILD_HEADER_SIZE) + .and_then(|offset| { + offset.checked_add( + compiled_fields + .last() + .expect("compiled hash-table field was just pushed") + .child + .total_len, + ) + }) + .ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table field payload size overflow".to_string(), + ) + })?; + } + if compiled_fields.len() != fields.len() { + continue; + } + let total_len = bucket_count + .checked_mul(field_slot_offset) + .and_then(|children_len| root_payload_len.checked_add(children_len)) + .ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table payload size overflow".to_string(), + ) + })?; + if total_len > budget { + continue; + } + let metadata = nested_hash_table_metadata(&plan.capture)?; + return Ok(Some(NestedValueSource { + output_type, + presentation: plan.presentation.clone(), + root_payload_len, + total_len, + root, + children: NestedValueChildrenSource::HashTable { + first_slot_offset: root_payload_len, + bucket_slot_stride: field_slot_offset, + bucket_count, + fields: compiled_fields, + metadata, + }, + })); + } + Ok(None) + } ghostscope_dwarf::ValueNestedPlan::Sequence { element } => { for slot_count in (1..=max_sequence_elements).rev() { let Some((output_type, root, root_payload_len)) = - compile_nested_root_source(plan, budget, Some(slot_count))? + compile_nested_root_source(plan, budget, Some(slot_count), None)? else { continue; }; @@ -682,6 +788,7 @@ fn compile_nested_root_source( plan: &ghostscope_dwarf::ValueReadPlan, budget: usize, sequence_elements: Option, + hash_buckets: Option, ) -> Result> { let output = match &plan.capture { ghostscope_dwarf::ValueCapturePlan::ProjectedValue { value } => { @@ -868,6 +975,7 @@ fn compile_nested_root_source( budget - ghostscope_protocol::HASH_TABLE_HEADER_SIZE, *entry_stride, *occupancy, + hash_buckets, )?; ( plan.root_type.summary.clone(), @@ -1069,6 +1177,84 @@ fn nested_sequence_metadata( } } +fn nested_hash_table_field_offset( + presentation: &ghostscope_dwarf::ValuePresentation, + field_index: usize, +) -> Result { + let ghostscope_dwarf::ValuePresentation::HashTable { entry, .. } = presentation else { + return Err(CodeGenError::DwarfError( + "nested hash-table fields do not match their root presentation".to_string(), + )); + }; + match (entry, field_index) { + (ghostscope_dwarf::HashTableEntryPresentation::Map { key, .. }, 0) => Ok(key.offset), + (ghostscope_dwarf::HashTableEntryPresentation::Map { value, .. }, 1) + | (ghostscope_dwarf::HashTableEntryPresentation::Set { value }, 0) => Ok(value.offset), + _ => Err(CodeGenError::DwarfError( + "nested hash-table field index is out of bounds".to_string(), + )), + } +} + +fn nested_hash_table_metadata( + capture: &ghostscope_dwarf::ValueCapturePlan, +) -> Result { + let ghostscope_dwarf::ValueCapturePlan::IndirectHashTable { + control, + entry_stride, + occupancy, + buckets, + bucket_order, + .. + } = capture + else { + return Err(CodeGenError::DwarfError( + "nested hash-table metadata does not match its root capture".to_string(), + )); + }; + let (control_offset, control_access_size) = + metadata_member(control, "nested hash-table child control")?; + let buckets = match (buckets, bucket_order) { + ( + ghostscope_dwarf::HashTableBucketSource::Forward { data }, + ghostscope_dwarf::HashTableBucketOrder::Forward, + ) => { + let (data_offset, data_access_size) = + metadata_member(data, "nested hash-table child data")?; + HashTableBucketSource::Forward { + data_offset, + data_access_size, + } + } + ( + ghostscope_dwarf::HashTableBucketSource::ReverseFromControl, + ghostscope_dwarf::HashTableBucketOrder::Reverse, + ) => HashTableBucketSource::ReverseFromControl, + ( + ghostscope_dwarf::HashTableBucketSource::LegacyAfterControl { + entry_alignment, + pointer_tag_mask, + }, + ghostscope_dwarf::HashTableBucketOrder::Forward, + ) => HashTableBucketSource::LegacyAfterControl { + entry_alignment: *entry_alignment, + pointer_tag_mask: *pointer_tag_mask, + }, + _ => { + return Err(CodeGenError::DwarfError( + "nested hash-table bucket source and order do not match".to_string(), + )); + } + }; + Ok(NestedHashTableMetadataSource { + control_offset, + control_access_size, + entry_stride: *entry_stride, + occupancy: *occupancy, + buckets, + }) +} + fn nested_value_presentation( value: &NestedValueSource, ) -> Result { @@ -1111,6 +1297,49 @@ fn nested_value_presentation( .collect::>>()?, } } + NestedValueChildrenSource::HashTable { + first_slot_offset, + bucket_slot_stride, + bucket_count, + fields, + .. + } => ghostscope_protocol::NestedValueChildrenPresentation::HashTable { + first_slot_offset: u64::try_from(*first_slot_offset).map_err(|_| { + CodeGenError::DwarfError( + "nested hash-table offset does not fit the protocol".to_string(), + ) + })?, + bucket_slot_stride: u64::try_from(*bucket_slot_stride).map_err(|_| { + CodeGenError::DwarfError( + "nested hash-table stride does not fit the protocol".to_string(), + ) + })?, + bucket_count: u64::try_from(*bucket_count).map_err(|_| { + CodeGenError::DwarfError( + "nested hash-table bucket count does not fit the protocol".to_string(), + ) + })?, + fields: fields + .iter() + .map(|field| { + Ok(ghostscope_protocol::NestedValueHashTableFieldPresentation { + field_index: u64::try_from(field.field_index).map_err(|_| { + CodeGenError::DwarfError( + "nested hash-table field index does not fit the protocol" + .to_string(), + ) + })?, + slot_offset: u64::try_from(field.slot_offset).map_err(|_| { + CodeGenError::DwarfError( + "nested hash-table field offset does not fit the protocol" + .to_string(), + ) + })?, + value: Box::new(nested_child_presentation(&field.child)?), + }) + }) + .collect::>>()?, + }, NestedValueChildrenSource::Variant { fields } => { ghostscope_protocol::NestedValueChildrenPresentation::Variant { fields: fields @@ -1300,6 +1529,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { presentation, capture, sequence_element: _, + hash_table_fields: _, nested: _, } = plan; let mut output_type = dwarf_type; @@ -1510,7 +1740,7 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { )); } let (max_buckets, max_len, data_len) = - hash_table_capture_limits(cap, entry_stride, occupancy)?; + hash_table_capture_limits(cap, entry_stride, occupancy, None)?; ( data_len, ComplexArgSource::IndirectHashTable { @@ -2866,9 +3096,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { mod semantic_value_tests { use super::*; use ghostscope_dwarf::{ - DiscriminantRange, DiscriminantValue, MemoryAccessSize, ProjectedValueRead, - ProjectedValueStep, ProjectedViewField, ResolvedType, StructMember, TypeIdentity, TypeInfo, - TypeProjection, TypeProjectionLayout, ValueCapturePlan, ValueNestedPlan, + DiscriminantRange, DiscriminantValue, HashTableBucketOrder, HashTableBucketSource, + HashTableEntryPresentation, HashTableFieldPresentation, HashTableOccupancy, + MemoryAccessSize, ProjectedValueRead, ProjectedValueStep, ProjectedViewField, ResolvedType, + StructMember, TypeIdentity, TypeInfo, TypeProjection, TypeProjectionLayout, + ValueCapturePlan, ValueNestedHashTableFieldPlan, ValueNestedPlan, ValueNestedVariantCondition, ValueNestedVariantFieldPlan, ValuePresentation, ValueReadPlan, }; @@ -3205,6 +3437,7 @@ mod semantic_value_tests { excluded_tail_bytes: 0, }, sequence_element: None, + hash_table_fields: Vec::new(), nested: None, } } @@ -3223,6 +3456,7 @@ mod semantic_value_tests { element_stride: 24, }, sequence_element: Some(string.root_type.clone()), + hash_table_fields: Vec::new(), nested: Some(ValueNestedPlan::Sequence { element: Box::new(string), }), @@ -3267,10 +3501,58 @@ mod semantic_value_tests { fields: Vec::new(), }, sequence_element: None, + hash_table_fields: Vec::new(), nested: Some(ValueNestedPlan::Variant { fields }), } } + fn nested_string_hash_map_plan() -> ValueReadPlan { + let key = string_value_plan(); + let value = string_value_plan(); + let entry_type = key.root_type.summary.clone(); + ValueReadPlan { + root_type: resolved_struct("HashMap", 48), + presentation: ValuePresentation::HashTable { + entry_stride: 48, + bucket_order: HashTableBucketOrder::Reverse, + occupancy: HashTableOccupancy::ControlByteHighBitClear, + entry: HashTableEntryPresentation::Map { + key: HashTableFieldPresentation { + offset: 0, + field_type: Box::new(entry_type.clone()), + }, + value: HashTableFieldPresentation { + offset: 24, + field_type: Box::new(entry_type), + }, + }, + }, + capture: ValueCapturePlan::IndirectHashTable { + control: projection(8), + length: projection(8), + bucket_mask: projection(8), + entry_stride: 48, + occupancy: HashTableOccupancy::ControlByteHighBitClear, + buckets: HashTableBucketSource::ReverseFromControl, + bucket_order: HashTableBucketOrder::Reverse, + }, + sequence_element: None, + hash_table_fields: Vec::new(), + nested: Some(ValueNestedPlan::HashTable { + fields: vec![ + ValueNestedHashTableFieldPlan { + field_index: 0, + value: Box::new(key), + }, + ValueNestedHashTableFieldPlan { + field_index: 1, + value: Box::new(value), + }, + ], + }), + } + } + #[test] fn nested_variant_capture_reuses_slots_across_mutually_exclusive_branches() { let source = compile_nested_value_source(&many_variant_strings_plan(16), 256, 4) @@ -3333,4 +3615,43 @@ mod semantic_value_tests { }; assert_eq!(slot_count, 2); } + + #[test] + fn nested_hash_table_capture_reuses_field_plans_for_bounded_buckets() { + let plan = nested_string_hash_map_plan(); + + let source = compile_nested_value_source(&plan, 512, 4) + .unwrap() + .expect("512 bytes should fit four hash-table bucket sidecars"); + let NestedValueChildrenSource::HashTable { + bucket_count, + bucket_slot_stride, + fields, + .. + } = &source.children + else { + panic!("expected nested hash-table source"); + }; + assert_eq!(*bucket_count, 4); + assert_eq!(*bucket_slot_stride, 70); + assert_eq!(fields.len(), 2); + assert_eq!(source.total_len, 508); + + let tight = compile_nested_value_source(&plan, 256, 4) + .unwrap() + .expect("256 bytes should reduce the nested hash-table bucket count"); + let NestedValueChildrenSource::HashTable { bucket_count, .. } = tight.children else { + panic!("expected nested hash-table source"); + }; + assert_eq!(bucket_count, 2); + assert_eq!(tight.total_len, 254); + + let configured = compile_nested_value_source(&plan, 512, 2) + .unwrap() + .expect("configured nested element limit should retain a valid hash-table plan"); + let NestedValueChildrenSource::HashTable { bucket_count, .. } = configured.children else { + panic!("expected nested hash-table source"); + }; + assert_eq!(bucket_count, 2); + } } diff --git a/ghostscope-compiler/src/ebpf/codegen/format.rs b/ghostscope-compiler/src/ebpf/codegen/format.rs index 600306f7..77f1be6a 100644 --- a/ghostscope-compiler/src/ebpf/codegen/format.rs +++ b/ghostscope-compiler/src/ebpf/codegen/format.rs @@ -5206,7 +5206,11 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { "nested_root_ok", ) .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; - let can_capture = if matches!(value.children, NestedValueChildrenSource::Sequence { .. }) { + let can_capture = if matches!( + value.children, + NestedValueChildrenSource::Sequence { .. } + | NestedValueChildrenSource::HashTable { .. } + ) { let truncated = self .builder .build_int_compare( @@ -5293,6 +5297,22 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { metadata, finish_block, )?, + NestedValueChildrenSource::HashTable { + first_slot_offset, + bucket_slot_stride, + bucket_count, + fields, + metadata, + } => self.emit_nested_hash_table_children( + payload_ptr, + descriptor, + *first_slot_offset, + *bucket_slot_stride, + *bucket_count, + fields, + metadata, + finish_block, + )?, } if self .builder @@ -5897,6 +5917,416 @@ impl<'ctx, 'dw> EbpfContext<'ctx, 'dw> { Ok(()) } + #[allow(clippy::too_many_arguments)] + fn emit_nested_hash_table_children( + &mut self, + parent_payload: PointerValue<'ctx>, + descriptor: &RuntimeAddress<'ctx>, + first_slot_offset: usize, + bucket_slot_stride: usize, + bucket_count: usize, + fields: &[NestedHashTableFieldSource], + metadata: &NestedHashTableMetadataSource, + finish_block: inkwell::basic_block::BasicBlock<'ctx>, + ) -> Result<()> { + let function = self.current_function("compile nested hash-table children")?; + let i8_type = self.context.i8_type(); + let i32_type = self.context.i32_type(); + let i64_type = self.context.i64_type(); + let occupancy_width = metadata + .occupancy + .byte_width() + .and_then(|width| usize::try_from(width).ok()) + .ok_or_else(|| { + CodeGenError::DwarfError("invalid nested hash-table occupancy width".to_string()) + })?; + + for bucket_index in 0..bucket_count { + let bucket_slot_offset = bucket_index + .checked_mul(bucket_slot_stride) + .and_then(|offset| first_slot_offset.checked_add(offset)) + .ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table bucket slot offset overflow".to_string(), + ) + })?; + for field in fields { + let slot_offset = bucket_slot_offset + .checked_add(field.slot_offset) + .ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table field slot offset overflow".to_string(), + ) + })?; + self.initialize_nested_child_header( + parent_payload, + slot_offset, + VariableStatus::AccessError, + )?; + } + } + + let (pointer_offset, pointer_access_size) = match metadata.buckets { + HashTableBucketSource::Forward { + data_offset, + data_access_size, + } => (data_offset, data_access_size), + HashTableBucketSource::ReverseFromControl + | HashTableBucketSource::LegacyAfterControl { .. } => { + (metadata.control_offset, metadata.control_access_size) + } + }; + let pointer_member = self + .builder + .build_int_add( + descriptor.value, + i64_type.const_int(pointer_offset, false), + "nested_hash_table_pointer_member", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let pointer_read = self.generate_memory_read_with_diagnostics( + descriptor.with_value(pointer_member), + pointer_access_size, + None, + "nested_hash_table_pointer", + )?; + let metadata_ok_block = self + .context + .append_basic_block(function, "nested_hash_table_metadata_ok"); + self.builder + .build_conditional_branch(pointer_read.combined_fail, finish_block, metadata_ok_block) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + self.builder.position_at_end(metadata_ok_block); + + // SAFETY: the hash-table root capture reserves the complete fixed + // header. + let captured_buckets_ptr_i8 = unsafe { + self.builder + .build_gep( + i8_type, + parent_payload, + &[i32_type.const_int( + ghostscope_protocol::HASH_TABLE_CAPTURED_BUCKETS_OFFSET as u64, + false, + )], + "nested_hash_table_captured_buckets_i8", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + }; + let captured_buckets_ptr = self + .builder + .build_pointer_cast( + captured_buckets_ptr_i8, + self.context.ptr_type(AddressSpace::default()), + "nested_hash_table_captured_buckets", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let captured_buckets = self + .builder + .build_load(i64_type, captured_buckets_ptr, "nested_hash_table_captured") + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + .into_int_value(); + + let raw_pointer = pointer_read.value.into_int_value(); + let bucket_base = match metadata.buckets { + HashTableBucketSource::Forward { .. } => raw_pointer, + HashTableBucketSource::ReverseFromControl => raw_pointer, + HashTableBucketSource::LegacyAfterControl { + entry_alignment, + pointer_tag_mask, + } => { + let control_address = self + .builder + .build_and( + raw_pointer, + i64_type.const_int(!pointer_tag_mask, false), + "nested_hash_table_legacy_control", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + // SAFETY: the hash-table root capture reserves the complete + // fixed header. + let capacity_ptr_i8 = unsafe { + self.builder + .build_gep( + i8_type, + parent_payload, + &[i32_type.const_int( + ghostscope_protocol::HASH_TABLE_CAPACITY_OFFSET as u64, + false, + )], + "nested_hash_table_capacity_i8", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + }; + let capacity_ptr = self + .builder + .build_pointer_cast( + capacity_ptr_i8, + self.context.ptr_type(AddressSpace::default()), + "nested_hash_table_capacity", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let capacity = self + .builder + .build_load(i64_type, capacity_ptr, "nested_hash_table_capacity_value") + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + .into_int_value(); + let hash_words_len = self + .builder + .build_int_mul( + capacity, + i64_type.const_int(occupancy_width as u64, false), + "nested_hash_table_legacy_hash_words_len", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let padded_len = self + .builder + .build_int_add( + hash_words_len, + i64_type.const_int(entry_alignment - 1, false), + "nested_hash_table_legacy_padded_len", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let aligned_len = self + .builder + .build_and( + padded_len, + i64_type.const_int(!(entry_alignment - 1), false), + "nested_hash_table_legacy_aligned_len", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + self.builder + .build_int_add( + control_address, + aligned_len, + "nested_hash_table_legacy_bucket_base", + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + } + }; + let entry_stride = i64_type.const_int(metadata.entry_stride, false); + + for bucket_index in 0..bucket_count { + let index_value = i64_type.const_int(bucket_index as u64, false); + let captured = self + .builder + .build_int_compare( + inkwell::IntPredicate::ULT, + index_value, + captured_buckets, + &format!("nested_hash_table_{bucket_index}_captured"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let occupancy_offset = ghostscope_protocol::HASH_TABLE_HEADER_SIZE + .checked_add(bucket_index.checked_mul(occupancy_width).ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table occupancy offset overflow".to_string(), + ) + })?) + .ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table occupancy offset overflow".to_string(), + ) + })?; + let mut nonzero = None; + for byte_index in 0..occupancy_width { + let byte_offset = occupancy_offset.checked_add(byte_index).ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table occupancy byte offset overflow".to_string(), + ) + })?; + // SAFETY: planning bounds every occupancy byte to the reserved + // hash-table root payload. + let byte_ptr = unsafe { + self.builder + .build_gep( + i8_type, + parent_payload, + &[i32_type.const_int(byte_offset as u64, false)], + &format!("nested_hash_table_{bucket_index}_{byte_index}_occupancy_ptr"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + }; + let byte = self + .builder + .build_load( + i8_type, + byte_ptr, + &format!("nested_hash_table_{bucket_index}_{byte_index}_occupancy"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + .into_int_value(); + let byte_nonzero = self + .builder + .build_int_compare( + inkwell::IntPredicate::NE, + byte, + i8_type.const_zero(), + &format!("nested_hash_table_{bucket_index}_{byte_index}_occupancy_nonzero"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + nonzero = Some(match nonzero { + Some(previous) => self + .builder + .build_or( + previous, + byte_nonzero, + &format!("nested_hash_table_{bucket_index}_occupancy_nonzero"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?, + None => byte_nonzero, + }); + } + let occupied = match metadata.occupancy { + ghostscope_dwarf::HashTableOccupancy::ControlByteHighBitClear => { + // SAFETY: occupancy_offset selects the first reserved byte + // for this bucket. + let byte_ptr = unsafe { + self.builder + .build_gep( + i8_type, + parent_payload, + &[i32_type.const_int(occupancy_offset as u64, false)], + &format!("nested_hash_table_{bucket_index}_control_ptr"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + }; + let control = self + .builder + .build_load( + i8_type, + byte_ptr, + &format!("nested_hash_table_{bucket_index}_control"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + .into_int_value(); + let high_bit = self + .builder + .build_and( + control, + i8_type.const_int(0x80, false), + &format!("nested_hash_table_{bucket_index}_control_high_bit"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + self.builder + .build_int_compare( + inkwell::IntPredicate::EQ, + high_bit, + i8_type.const_zero(), + &format!("nested_hash_table_{bucket_index}_occupied"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))? + } + ghostscope_dwarf::HashTableOccupancy::NonZeroWord { .. } => { + nonzero.ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table occupancy has no bytes".to_string(), + ) + })? + } + }; + let active = self + .builder + .build_and( + captured, + occupied, + &format!("nested_hash_table_{bucket_index}_active"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let active_block = self.context.append_basic_block( + function, + &format!("nested_hash_table_{bucket_index}_capture"), + ); + let continue_block = self.context.append_basic_block( + function, + &format!("nested_hash_table_{bucket_index}_continue"), + ); + self.builder + .build_conditional_branch(active, active_block, continue_block) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + self.builder.position_at_end(active_block); + + let entry_offset = self + .builder + .build_int_mul( + index_value, + entry_stride, + &format!("nested_hash_table_{bucket_index}_entry_offset"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let entry_address = match metadata.buckets { + HashTableBucketSource::ReverseFromControl => self + .builder + .build_int_sub( + bucket_base, + self.builder + .build_int_add( + entry_offset, + entry_stride, + &format!("nested_hash_table_{bucket_index}_reverse_entry_end"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?, + &format!("nested_hash_table_{bucket_index}_entry_address"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?, + HashTableBucketSource::Forward { .. } + | HashTableBucketSource::LegacyAfterControl { .. } => self + .builder + .build_int_add( + bucket_base, + entry_offset, + &format!("nested_hash_table_{bucket_index}_entry_address"), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?, + }; + let bucket_slot_offset = bucket_index + .checked_mul(bucket_slot_stride) + .and_then(|offset| first_slot_offset.checked_add(offset)) + .ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table bucket slot offset overflow".to_string(), + ) + })?; + for field in fields { + let field_address = self + .builder + .build_int_add( + entry_address, + i64_type.const_int(field.entry_offset, false), + &format!( + "nested_hash_table_{bucket_index}_field_{}_address", + field.field_index + ), + ) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + let slot_offset = bucket_slot_offset + .checked_add(field.slot_offset) + .ok_or_else(|| { + CodeGenError::DwarfError( + "nested hash-table field slot offset overflow".to_string(), + ) + })?; + self.emit_nested_child_slot( + parent_payload, + slot_offset, + &RuntimeAddress::available(field_address, self.context), + &field.child, + )?; + } + if self + .builder + .get_insert_block() + .is_some_and(|block| block.get_terminator().is_none()) + { + self.builder + .build_unconditional_branch(continue_block) + .map_err(|error| CodeGenError::LLVMError(error.to_string()))?; + } + self.builder.position_at_end(continue_block); + } + Ok(()) + } + /// Generate eBPF code for PrintComplexFormat instruction with runtime reads for variables pub(super) fn generate_print_complex_format_instruction( &mut self, @@ -6283,6 +6713,103 @@ mod complex_format_layout_tests { ebpf.module.print_to_string().to_string() } + fn nested_hash_table_emitter_ir( + buckets: HashTableBucketSource, + occupancy: ghostscope_dwarf::HashTableOccupancy, + ) -> String { + let context = Context::create(); + let options = crate::CompileOptions::default(); + let mut ebpf = EbpfContext::new(&context, "nested_hash_table", Some(0), &options) + .expect("create eBPF context"); + let function_type = context.i64_type().fn_type(&[], false); + let function = ebpf + .module + .add_function("emit_nested_hash_table", function_type, None); + let entry = context.append_basic_block(function, "entry"); + let finish = context.append_basic_block(function, "finish"); + ebpf.builder.position_at_end(entry); + + let bucket_count = 2usize; + let entry_stride = 8u64; + let occupancy_width = usize::try_from( + occupancy + .byte_width() + .expect("test occupancy has a known width"), + ) + .expect("test occupancy width fits usize"); + let root_payload_len = ghostscope_protocol::HASH_TABLE_HEADER_SIZE + + bucket_count * (occupancy_width + entry_stride as usize); + let child = NestedValueSource { + output_type: ghostscope_dwarf::TypeInfo::BaseType { + name: "u8".to_string(), + size: 1, + encoding: ghostscope_dwarf::constants::DW_ATE_unsigned.0 as u16, + }, + presentation: ghostscope_dwarf::ValuePresentation::Dwarf, + root_payload_len: 1, + total_len: 1, + root: NestedValueRootSource::ProjectedValue { offset: 0, len: 1 }, + children: NestedValueChildrenSource::None, + }; + let bucket_slot_stride = + ghostscope_protocol::NESTED_VALUE_CHILD_HEADER_SIZE + child.total_len; + let total_len = root_payload_len + bucket_count * bucket_slot_stride; + let parent_payload = ebpf + .builder + .build_alloca( + context.i8_type().array_type(total_len as u32), + "parent_payload", + ) + .expect("allocate nested hash-table payload"); + let descriptor = + RuntimeAddress::available(context.i64_type().const_int(0x1000, false), &context); + let control_offset = match buckets { + HashTableBucketSource::LegacyAfterControl { .. } => 16, + HashTableBucketSource::Forward { .. } | HashTableBucketSource::ReverseFromControl => 0, + }; + let fields = [NestedHashTableFieldSource { + field_index: 0, + entry_offset: 0, + slot_offset: 0, + child: Box::new(child), + }]; + let metadata = NestedHashTableMetadataSource { + control_offset, + control_access_size: ghostscope_dwarf::MemoryAccessSize::U64, + entry_stride, + occupancy, + buckets, + }; + + ebpf.emit_nested_hash_table_children( + parent_payload, + &descriptor, + root_payload_len, + bucket_slot_stride, + bucket_count, + &fields, + &metadata, + finish, + ) + .expect("emit nested hash-table capture"); + if ebpf + .builder + .get_insert_block() + .is_some_and(|block| block.get_terminator().is_none()) + { + ebpf.builder + .build_unconditional_branch(finish) + .expect("finish nested hash-table capture"); + } + ebpf.builder.position_at_end(finish); + ebpf.builder + .build_return(Some(&context.i64_type().const_zero())) + .expect("return from test function"); + ebpf.module.verify().expect("verify generated LLVM IR"); + + ebpf.module.print_to_string().to_string() + } + fn btree_emitter_ir(capture: BTreeCaptureConfig, max_len: usize) -> String { let context = Context::create(); let options = crate::CompileOptions::default(); @@ -6605,6 +7132,45 @@ mod complex_format_layout_tests { assert!(!legacy_ir.contains("hash_table_data_metadata")); } + #[test] + fn nested_hash_table_emitter_uses_each_dwarf_storage_layout() { + let reverse_ir = nested_hash_table_emitter_ir( + HashTableBucketSource::ReverseFromControl, + ghostscope_dwarf::HashTableOccupancy::ControlByteHighBitClear, + ); + assert!( + reverse_ir.contains("%nested_hash_table_0_entry_address = sub i64"), + "{reverse_ir}" + ); + assert!( + reverse_ir.contains("nested_hash_table_0_field_0_address"), + "{reverse_ir}" + ); + + let forward_ir = nested_hash_table_emitter_ir( + HashTableBucketSource::Forward { + data_offset: 8, + data_access_size: ghostscope_dwarf::MemoryAccessSize::U64, + }, + ghostscope_dwarf::HashTableOccupancy::ControlByteHighBitClear, + ); + assert!(forward_ir.contains("%nested_hash_table_0_entry_address = add i64")); + assert!(!forward_ir.contains("nested_hash_table_legacy_bucket_base")); + assert!(forward_ir.contains("nested_hash_table_0_field_0_address")); + + let legacy_ir = nested_hash_table_emitter_ir( + HashTableBucketSource::LegacyAfterControl { + entry_alignment: 4, + pointer_tag_mask: 1, + }, + ghostscope_dwarf::HashTableOccupancy::NonZeroWord { word_size: 8 }, + ); + assert!(legacy_ir.contains("nested_hash_table_legacy_control")); + assert!(legacy_ir.contains("nested_hash_table_legacy_aligned_len")); + assert!(legacy_ir.contains("nested_hash_table_legacy_bucket_base")); + assert!(legacy_ir.contains("%nested_hash_table_0_entry_address = add i64")); + } + #[test] fn btree_emitter_uses_dwarf_node_layout_without_runtime_loops() { let record_size = ghostscope_protocol::BTREE_NODE_HEADER_SIZE + 2 * (4 + 2); diff --git a/ghostscope-compiler/src/ebpf/codegen/mod.rs b/ghostscope-compiler/src/ebpf/codegen/mod.rs index 1bb1b57b..eaacd4dc 100644 --- a/ghostscope-compiler/src/ebpf/codegen/mod.rs +++ b/ghostscope-compiler/src/ebpf/codegen/mod.rs @@ -197,6 +197,13 @@ enum NestedValueChildrenSource { element: Box, metadata: NestedSequenceMetadataSource, }, + HashTable { + first_slot_offset: usize, + bucket_slot_stride: usize, + bucket_count: usize, + fields: Vec, + metadata: NestedHashTableMetadataSource, + }, Variant { fields: Vec, }, @@ -210,6 +217,14 @@ struct NestedValueFieldSource { child: Box, } +#[derive(Debug, Clone)] +struct NestedHashTableFieldSource { + field_index: usize, + entry_offset: u64, + slot_offset: usize, + child: Box, +} + #[derive(Debug, Clone)] struct NestedValueVariantFieldSource { part_index: usize, @@ -240,6 +255,15 @@ struct NestedSequenceMetadataSource { ring: Option, } +#[derive(Debug, Clone, Copy)] +struct NestedHashTableMetadataSource { + control_offset: u64, + control_access_size: ghostscope_dwarf::MemoryAccessSize, + entry_stride: u64, + occupancy: ghostscope_dwarf::HashTableOccupancy, + buckets: HashTableBucketSource, +} + #[derive(Debug, Clone, Copy)] struct NestedRingMetadataSource { start_offset: u64, diff --git a/ghostscope-dwarf/src/analyzer/type_context.rs b/ghostscope-dwarf/src/analyzer/type_context.rs index d440b56d..53b149e2 100644 --- a/ghostscope-dwarf/src/analyzer/type_context.rs +++ b/ghostscope-dwarf/src/analyzer/type_context.rs @@ -4,8 +4,9 @@ use crate::{ CompilationUnitMetadata, CuId, MemberLayout, ModuleId, PcContext, ProjectedValueRead, ProjectedValueStep, ResolvedType, Result, SemanticType, TypeId, TypeIdentity, TypeInfo, TypeLayoutError, TypeOrigin, TypeProjection, TypeProjectionLayout, ValueAdapterOutcome, - ValueAdapterReport, ValueAdapterStage, ValueCapturePlan, ValueNestedFieldPlan, ValueNestedPlan, - ValueReadPlan, ValueReadPlanOptions, VariableAccessSegment, VariableReadPlan, + ValueAdapterReport, ValueAdapterStage, ValueCapturePlan, ValueNestedFieldPlan, + ValueNestedHashTableFieldPlan, ValueNestedPlan, ValueReadPlan, ValueReadPlanOptions, + VariableAccessSegment, VariableReadPlan, }; use std::path::Path; @@ -663,9 +664,29 @@ impl DwarfAnalyzer { .map(|element| ValueNestedPlan::Sequence { element: Box::new(element), }), - ValueCapturePlan::IndirectBytes { .. } - | ValueCapturePlan::IndirectHashTable { .. } - | ValueCapturePlan::IndirectBTree { .. } => None, + ValueCapturePlan::IndirectHashTable { .. } => { + let nested_fields = plan + .hash_table_fields + .iter() + .filter_map(|field| { + self.try_nested_value_read_plan( + &field.resolved_type, + type_module_path, + depth + 1, + max_nesting_depth, + ancestors, + ) + .map(|value| ValueNestedHashTableFieldPlan { + field_index: field.field_index, + value: Box::new(value), + }) + }) + .collect::>(); + (!nested_fields.is_empty()).then_some(ValueNestedPlan::HashTable { + fields: nested_fields, + }) + } + ValueCapturePlan::IndirectBytes { .. } | ValueCapturePlan::IndirectBTree { .. } => None, }; let nested = match language_nested { crate::language::NestedValuePlanResolution::Handled(nested) => nested, @@ -1051,6 +1072,20 @@ impl crate::language::ValueAdapterContext for DwarfAnalyzer { DwarfAnalyzer::tuple_member_layout(self, type_id, aggregate_type, index) } + fn project_tuple_member( + &self, + current: &ResolvedType, + index: u32, + type_module_path: Option<&Path>, + ) -> Result { + DwarfAnalyzer::project_resolved_type( + self, + current, + &VariableAccessSegment::TupleIndex(index), + type_module_path, + ) + } + fn resolve_aggregate_type_in_module( &self, anchor: TypeId, diff --git a/ghostscope-dwarf/src/language/adapter.rs b/ghostscope-dwarf/src/language/adapter.rs index 68525e32..d946aa5a 100644 --- a/ghostscope-dwarf/src/language/adapter.rs +++ b/ghostscope-dwarf/src/language/adapter.rs @@ -245,6 +245,13 @@ pub(crate) trait ValueAdapterContext { index: u32, ) -> Result; + fn project_tuple_member( + &self, + current: &ResolvedType, + index: u32, + type_module_path: Option<&Path>, + ) -> Result; + fn resolve_aggregate_type_in_module( &self, anchor: TypeId, diff --git a/ghostscope-dwarf/src/language/mod.rs b/ghostscope-dwarf/src/language/mod.rs index 657dceed..db2e5bb5 100644 --- a/ghostscope-dwarf/src/language/mod.rs +++ b/ghostscope-dwarf/src/language/mod.rs @@ -208,6 +208,15 @@ mod tests { panic!("non-Rust composition must not query DWARF") } + fn project_tuple_member( + &self, + _current: &ResolvedType, + _index: u32, + _type_module_path: Option<&Path>, + ) -> crate::Result { + panic!("non-Rust composition must not query DWARF") + } + fn resolve_aggregate_type_in_module( &self, _anchor: TypeId, diff --git a/ghostscope-dwarf/src/language/rust/plan.rs b/ghostscope-dwarf/src/language/rust/plan.rs index 2318b8de..a293d8ec 100644 --- a/ghostscope-dwarf/src/language/rust/plan.rs +++ b/ghostscope-dwarf/src/language/rust/plan.rs @@ -3,7 +3,7 @@ use crate::{ BTreeFieldPresentation, HashTableBucketOrder, HashTableBucketSource, HashTableEntryPresentation, HashTableFieldPresentation, HashTableOccupancy, ResolvedType, Result, StructMember, TypeInfo, TypeProjection, TypeProjectionLayout, ValueCapturePlan, - ValuePresentation, ValueReadPlan, VariableAccessSegment, + ValueHashTableField, ValuePresentation, ValueReadPlan, VariableAccessSegment, }; use std::path::Path; @@ -97,26 +97,72 @@ pub(crate) fn hash_table_value_read_plan( }; let entry_stride = entry_type.summary.size(); let field = |index| { - context + let field = context .tuple_member_layout(entry_type_id, &entry_type.summary, index) + .ok()?; + let field_end = field.offset.checked_add(field.member_type.size())?; + if field_end > entry_stride { + return None; + } + let resolved_type = context + .project_tuple_member(&entry_type, index, type_module_path) .ok() - .map(|field| HashTableFieldPresentation { + .and_then(|projection| { + let TypeProjectionLayout::Member { offset } = projection.layout else { + return None; + }; + (offset == field.offset + && strip_type_aliases(&projection.resolved_type.summary) + == strip_type_aliases(&field.member_type)) + .then_some(projection.resolved_type) + }); + Some(( + HashTableFieldPresentation { offset: field.offset, field_type: Box::new(field.member_type), - }) + }, + resolved_type, + )) }; - let entry = match layout.kind { + let (entry, hash_table_fields) = match layout.kind { HashTableKind::Map => { let (Some(key), Some(value)) = (field(0), field(1)) else { return Ok(None); }; - HashTableEntryPresentation::Map { key, value } + let mut fields = Vec::new(); + if let Some(resolved_type) = key.1 { + fields.push(ValueHashTableField { + field_index: 0, + resolved_type, + }); + } + if let Some(resolved_type) = value.1 { + fields.push(ValueHashTableField { + field_index: 1, + resolved_type, + }); + } + ( + HashTableEntryPresentation::Map { + key: key.0, + value: value.0, + }, + fields, + ) } HashTableKind::Set => { let Some(value) = field(0) else { return Ok(None); }; - HashTableEntryPresentation::Set { value } + let fields = value + .1 + .map(|resolved_type| ValueHashTableField { + field_index: 0, + resolved_type, + }) + .into_iter() + .collect(); + (HashTableEntryPresentation::Set { value: value.0 }, fields) } }; @@ -158,7 +204,7 @@ pub(crate) fn hash_table_value_read_plan( let bucket_mask = context.project_member_path(current, &layout.bucket_mask_path, type_module_path)?; - Ok(Some(ValueReadPlan::new( + let mut plan = ValueReadPlan::new( current.clone(), ValuePresentation::HashTable { entry_stride, @@ -175,7 +221,9 @@ pub(crate) fn hash_table_value_read_plan( buckets, bucket_order: layout.bucket_order, }, - ))) + ); + plan.hash_table_fields = hash_table_fields; + Ok(Some(plan)) } pub(crate) fn btree_value_read_plan<'a>( diff --git a/ghostscope-dwarf/src/lib.rs b/ghostscope-dwarf/src/lib.rs index 8fdb985c..fd9a84d3 100644 --- a/ghostscope-dwarf/src/lib.rs +++ b/ghostscope-dwarf/src/lib.rs @@ -52,7 +52,8 @@ pub use semantics::{ RingSequenceLength, RuntimeComputedExpr, RuntimeComputedKind, RustcVersion, SemanticType, SourceLanguage, SyntheticTypeKind, TypeIdentity, TypeLayoutError, TypeOrigin, TypeProjection, TypeProjectionLayout, UnwindDiagnostic, UnwindDiagnosticKind, ValueAdapterOutcome, - ValueAdapterReport, ValueAdapterStage, ValueCapturePlan, ValueNestedFieldPlan, ValueNestedPlan, + ValueAdapterReport, ValueAdapterStage, ValueCapturePlan, ValueHashTableField, + ValueNestedFieldPlan, ValueNestedHashTableFieldPlan, ValueNestedPlan, ValueNestedVariantCondition, ValueNestedVariantFieldPlan, ValueReadPlan, ValueReadPlanOptions, VariableAccessPath, VariableAccessSegment, VariableLoweringKind, VariableLoweringPlan, VariableMaterialization, VariableMaterializationPlan, VariablePlan, VariableQueryDiagnostic, diff --git a/ghostscope-dwarf/src/semantics/mod.rs b/ghostscope-dwarf/src/semantics/mod.rs index cea36343..31e03ef3 100644 --- a/ghostscope-dwarf/src/semantics/mod.rs +++ b/ghostscope-dwarf/src/semantics/mod.rs @@ -41,8 +41,8 @@ pub use value::{ BTreeArrayCapture, BTreeEdgesCapture, HashTableBucketSource, ProjectedValueRead, ProjectedValueStep, ProjectedViewField, ProjectedViewFieldCapture, RingSequenceLength, ValueAdapterOutcome, ValueAdapterReport, ValueAdapterStage, ValueCapturePlan, - ValueNestedFieldPlan, ValueNestedPlan, ValueNestedVariantCondition, - ValueNestedVariantFieldPlan, ValueReadPlan, ValueReadPlanOptions, + ValueHashTableField, ValueNestedFieldPlan, ValueNestedHashTableFieldPlan, ValueNestedPlan, + ValueNestedVariantCondition, ValueNestedVariantFieldPlan, ValueReadPlan, ValueReadPlanOptions, DEFAULT_VALUE_ADAPTER_NESTING_DEPTH, MAX_VALUE_ADAPTER_NESTING_DEPTH, }; pub(crate) use variable_plan::PlanError; diff --git a/ghostscope-dwarf/src/semantics/value.rs b/ghostscope-dwarf/src/semantics/value.rs index 8d4978bb..7f9e7b63 100644 --- a/ghostscope-dwarf/src/semantics/value.rs +++ b/ghostscope-dwarf/src/semantics/value.rs @@ -36,6 +36,10 @@ pub struct ValueReadPlan { /// erase the physical allocation pointer to `u8`, so this cannot always be /// recovered by dereferencing the data projection. pub sequence_element: Option, + /// Exact field identities for one hash-table entry. The physical entry + /// presentation retains only normalized DWARF types, which is insufficient + /// for recursively selecting source-language adapters. + pub hash_table_fields: Vec, /// Recursively selected adapters for values embedded in this root capture. /// /// The root capture remains valid on its own. Consumers that do not @@ -55,11 +59,19 @@ impl ValueReadPlan { presentation, capture, sequence_element: None, + hash_table_fields: Vec::new(), nested: None, } } } +/// Exact source-level identity for one hash-table entry field. +#[derive(Debug, Clone, PartialEq)] +pub struct ValueHashTableField { + pub field_index: usize, + pub resolved_type: crate::ResolvedType, +} + /// Adapter plans selected for semantic values embedded in a root capture. #[derive(Debug, Clone, PartialEq)] pub enum ValueNestedPlan { @@ -69,6 +81,11 @@ pub enum ValueNestedPlan { ProjectedView { fields: Vec }, /// Every captured sequence element uses the same semantic adapter. Sequence { element: Box }, + /// Selected fields in every occupied hash-table bucket use the same + /// semantic adapters. + HashTable { + fields: Vec, + }, /// Selected payload fields in active DWARF variant branches have semantic /// adapters. Variant { @@ -83,6 +100,13 @@ pub struct ValueNestedFieldPlan { pub value: Box, } +/// One source-level hash-table entry field with a recursively selected adapter. +#[derive(Debug, Clone, PartialEq)] +pub struct ValueNestedHashTableFieldPlan { + pub field_index: usize, + pub value: Box, +} + /// One enum payload field with a recursively selected semantic adapter. #[derive(Debug, Clone, PartialEq)] pub struct ValueNestedVariantFieldPlan { diff --git a/ghostscope-protocol/src/format_printer.rs b/ghostscope-protocol/src/format_printer.rs index ad0df14c..bd3fb3c4 100644 --- a/ghostscope-protocol/src/format_printer.rs +++ b/ghostscope-protocol/src/format_printer.rs @@ -13,7 +13,8 @@ use crate::type_info::TypeInfo; use crate::{ BTreeEntryPresentation, BTreeFieldPresentation, HashTableBucketOrder, HashTableEntryPresentation, HashTableFieldPresentation, HashTableOccupancy, - NestedValueChildrenPresentation, NestedValueFieldPresentation, NestedValuePresentation, + NestedValueChildrenPresentation, NestedValueFieldPresentation, + NestedValueHashTableFieldPresentation, NestedValuePresentation, NestedValueVariantFieldPresentation, ValuePresentation, BTREE_CAPTURED_ITEM_COUNT_OFFSET, BTREE_HEADER_SIZE, BTREE_NODE_HEADER_SIZE, BTREE_NODE_HEIGHT_OFFSET, BTREE_NODE_LENGTH_OFFSET, BTREE_NODE_SLOT_COUNT_OFFSET, HASH_TABLE_BUCKET_DATA_OFFSET, HASH_TABLE_CAPACITY_OFFSET, @@ -35,6 +36,14 @@ struct NestedVariantContext<'a> { fields: &'a [NestedValueVariantFieldPresentation], } +struct NestedHashTableContext<'a> { + full_data: &'a [u8], + first_slot_offset: usize, + bucket_slot_stride: usize, + bucket_count: usize, + fields: &'a [NestedValueHashTableFieldPresentation], +} + // Removed legacy simple variable wrapper; use complex paths only. /// A parsed complex variable from PrintComplexVariable instruction data @@ -867,6 +876,7 @@ impl FormatPrinter { *bucket_order, *occupancy, entry, + None, ), ValuePresentation::BTree { node_capacity, @@ -909,6 +919,91 @@ impl FormatPrinter { NestedValueChildrenPresentation::ProjectedView { fields } => { Self::format_nested_projected_view(root_data, data, type_info, root, fields) } + NestedValueChildrenPresentation::HashTable { + first_slot_offset, + bucket_slot_stride, + bucket_count, + fields, + } => { + let ValuePresentation::HashTable { + entry_stride, + bucket_order, + occupancy, + entry, + } = root + else { + return "".to_string(); + }; + let (Ok(first_slot_offset), Ok(bucket_slot_stride), Ok(bucket_count)) = ( + usize::try_from(*first_slot_offset), + usize::try_from(*bucket_slot_stride), + usize::try_from(*bucket_count), + ) else { + return "".to_string(); + }; + if first_slot_offset < root_payload_len + || bucket_slot_stride == 0 + || bucket_count == 0 + || fields.is_empty() + { + return "".to_string(); + } + let field_count = match entry { + HashTableEntryPresentation::Map { .. } => 2, + HashTableEntryPresentation::Set { .. } => 1, + }; + let mut seen_fields = vec![false; field_count]; + for field in fields { + let Ok(field_index) = usize::try_from(field.field_index) else { + return "".to_string(); + }; + let Some(seen) = seen_fields.get_mut(field_index) else { + return "".to_string(); + }; + if *seen { + return "".to_string(); + } + *seen = true; + let Some(field_end) = usize::try_from(field.slot_offset) + .ok() + .and_then(|offset| offset.checked_add(NESTED_VALUE_CHILD_HEADER_SIZE)) + .and_then(|end| { + usize::try_from(field.value.payload_len) + .ok() + .and_then(|payload_len| end.checked_add(payload_len)) + }) + else { + return "".to_string(); + }; + if field_end > bucket_slot_stride { + return "".to_string(); + } + } + let Some(sidecar_end) = bucket_count + .checked_mul(bucket_slot_stride) + .and_then(|len| first_slot_offset.checked_add(len)) + else { + return "".to_string(); + }; + if sidecar_end > data.len() { + return "".to_string(); + } + let nested = NestedHashTableContext { + full_data: data, + first_slot_offset, + bucket_slot_stride, + bucket_count, + fields, + }; + Self::format_hash_table_payload( + root_data, + *entry_stride, + *bucket_order, + *occupancy, + entry, + Some(&nested), + ) + } NestedValueChildrenPresentation::Variant { fields } => { let ( ValuePresentation::Dwarf, @@ -1472,7 +1567,12 @@ impl FormatPrinter { ) -> Result, String> { let mut truncated = false; loop { - let ValuePresentation::Nested { children, .. } = presentation else { + let ValuePresentation::Nested { + root, + root_payload_len, + children, + } = presentation + else { return Ok(RawPresentationPayload { data, presentation, @@ -1482,12 +1582,13 @@ impl FormatPrinter { }; let NestedValueChildrenPresentation::ProjectedValue { child } = children.as_ref() else { - return Ok(RawPresentationPayload { - data, - presentation, - truncated, - zero_length: false, - }); + let root_len = usize::try_from(*root_payload_len) + .map_err(|_| "".to_string())?; + data = data + .get(..root_len) + .ok_or_else(|| "".to_string())?; + presentation = root; + continue; }; let slot_offset = usize::try_from(child.slot_offset) @@ -1773,7 +1874,28 @@ impl FormatPrinter { entry_data: &[u8], entry_stride: u64, field: &HashTableFieldPresentation, + field_index: usize, + control_index: usize, + nested: Option<&NestedHashTableContext<'_>>, ) -> Option { + if let Some(nested) = nested { + if let Some(nested_field) = nested + .fields + .iter() + .find(|candidate| candidate.field_index == field_index as u64) + { + let field_slot_offset = usize::try_from(nested_field.slot_offset).ok()?; + let slot_offset = control_index + .checked_mul(nested.bucket_slot_stride) + .and_then(|offset| nested.first_slot_offset.checked_add(offset)) + .and_then(|offset| offset.checked_add(field_slot_offset))?; + return Some(Self::format_nested_value_at( + nested.full_data, + slot_offset, + &nested_field.value, + )); + } + } let field_end = field.offset.checked_add(field.field_type.size())?; if field_end > entry_stride { return None; @@ -1794,10 +1916,14 @@ impl FormatPrinter { bucket_order: HashTableBucketOrder, occupancy: HashTableOccupancy, entry: &HashTableEntryPresentation, + nested: Option<&NestedHashTableContext<'_>>, ) -> String { let Some(payload) = Self::parse_hash_table_payload(data, entry_stride, occupancy) else { return "".to_string(); }; + if nested.is_some_and(|nested| payload.captured_buckets > nested.bucket_count) { + return "".to_string(); + } let type_name = match entry { HashTableEntryPresentation::Map { .. } => "HashMap", HashTableEntryPresentation::Set { .. } => "HashSet", @@ -1823,13 +1949,24 @@ impl FormatPrinter { } match entry { HashTableEntryPresentation::Map { key, value } => { - let Some(key) = Self::format_hash_table_field(entry_data, entry_stride, key) - else { + let Some(key) = Self::format_hash_table_field( + entry_data, + entry_stride, + key, + 0, + control_index, + nested, + ) else { return "".to_string(); }; - let Some(value) = - Self::format_hash_table_field(entry_data, entry_stride, value) - else { + let Some(value) = Self::format_hash_table_field( + entry_data, + entry_stride, + value, + 1, + control_index, + nested, + ) else { return "".to_string(); }; result.push_str(&key); @@ -1837,9 +1974,14 @@ impl FormatPrinter { result.push_str(&value); } HashTableEntryPresentation::Set { value } => { - let Some(value) = - Self::format_hash_table_field(entry_data, entry_stride, value) - else { + let Some(value) = Self::format_hash_table_field( + entry_data, + entry_stride, + value, + 0, + control_index, + nested, + ) else { return "".to_string(); }; result.push_str(&value); @@ -3768,6 +3910,116 @@ mod tests { ); } + #[test] + fn nested_hash_table_presentation_formats_semantic_entry_fields() { + let controls = [0xff, 0x12]; + let buckets = vec![0u8; 2 * 48]; + let root_data = hash_table_payload(1, 2, &controls, &buckets); + let root_payload_len = root_data.len(); + + let key_payload = indirect_bytes_payload(10, b"alpha"); + let mut values = Vec::new(); + for value in [3_i32, 5, 8] { + values.extend_from_slice(&value.to_le_bytes()); + } + let value_payload = indirect_sequence_payload(3, 3, &values); + let key_slot_len = NESTED_VALUE_CHILD_HEADER_SIZE + key_payload.len(); + let value_slot_offset = key_slot_len; + let bucket_slot_stride = + key_slot_len + NESTED_VALUE_CHILD_HEADER_SIZE + value_payload.len(); + + let mut data = root_data; + data.extend_from_slice(&nested_child_slot( + VariableStatus::AccessError, + &[], + key_payload.len(), + )); + data.extend_from_slice(&nested_child_slot( + VariableStatus::AccessError, + &[], + value_payload.len(), + )); + data.extend_from_slice(&nested_child_slot( + VariableStatus::Truncated, + &key_payload, + key_payload.len(), + )); + data.extend_from_slice(&nested_child_slot( + VariableStatus::Ok, + &value_payload, + value_payload.len(), + )); + + let string_type = TypeInfo::StructType { + name: "String".to_string(), + size: 24, + members: Vec::new(), + }; + let vector_type = TypeInfo::StructType { + name: "Vec".to_string(), + size: 24, + members: Vec::new(), + }; + let root = ValuePresentation::HashTable { + entry_stride: 48, + bucket_order: HashTableBucketOrder::Forward, + occupancy: HashTableOccupancy::ControlByteHighBitClear, + entry: HashTableEntryPresentation::Map { + key: HashTableFieldPresentation { + offset: 0, + field_type: Box::new(string_type.clone()), + }, + value: HashTableFieldPresentation { + offset: 24, + field_type: Box::new(vector_type.clone()), + }, + }, + }; + let presentation = ValuePresentation::Nested { + root: Box::new(root), + root_payload_len: root_payload_len as u64, + children: Box::new(NestedValueChildrenPresentation::HashTable { + first_slot_offset: root_payload_len as u64, + bucket_slot_stride: bucket_slot_stride as u64, + bucket_count: 2, + fields: vec![ + NestedValueHashTableFieldPresentation { + field_index: 0, + slot_offset: 0, + value: Box::new(NestedValuePresentation { + payload_len: key_payload.len() as u64, + type_info: Box::new(string_type), + presentation: Box::new(ValuePresentation::Utf8String), + }), + }, + NestedValueHashTableFieldPresentation { + field_index: 1, + slot_offset: value_slot_offset as u64, + value: Box::new(NestedValuePresentation { + payload_len: value_payload.len() as u64, + type_info: Box::new(vector_type), + presentation: Box::new(ValuePresentation::Sequence { + element_type: Box::new(signed_i32_type()), + element_stride: 4, + }), + }), + }, + ], + }), + }; + + assert_eq!( + FormatPrinter::format_data_with_presentation( + &data, + &TypeInfo::UnknownType { + name: "HashMap".to_string(), + }, + &presentation, + ), + "HashMap(size=1) {\"alpha\" : [3, 5, 8]}" + ); + } + #[test] fn test_legacy_hash_table_presentation_uses_nonzero_hash_words() { let mut occupancy = Vec::new(); @@ -3916,7 +4168,7 @@ mod tests { TypeInfo::UnknownType { name: "HashSet".to_string(), }, - presentation, + presentation.clone(), ) .unwrap(); let name_index = trace_context.add_variable_name("set".to_string()).unwrap(); @@ -3931,7 +4183,60 @@ mod tests { assert_eq!( FormatPrinter::format_complex_print_data( format_index, - &[variable.clone(), variable], + &[variable.clone(), variable.clone()], + &trace_context, + ), + "41 00 42 00|A" + ); + + let root_payload_len = variable.data.len(); + let child_payload_len = 2; + let bucket_slot_stride = NESTED_VALUE_CHILD_HEADER_SIZE + child_payload_len; + let mut nested_data = variable.data.clone(); + for _ in 0..controls.len() { + nested_data.extend_from_slice(&nested_child_slot( + VariableStatus::AccessError, + &[], + child_payload_len, + )); + } + let nested_presentation = ValuePresentation::Nested { + root: Box::new(presentation), + root_payload_len: root_payload_len as u64, + children: Box::new(NestedValueChildrenPresentation::HashTable { + first_slot_offset: root_payload_len as u64, + bucket_slot_stride: bucket_slot_stride as u64, + bucket_count: controls.len() as u64, + fields: vec![NestedValueHashTableFieldPresentation { + field_index: 0, + slot_offset: 0, + value: Box::new(NestedValuePresentation { + payload_len: child_payload_len as u64, + type_info: Box::new(unsigned_u16_type()), + presentation: Box::new(ValuePresentation::Dwarf), + }), + }], + }), + }; + let nested_type_index = trace_context + .add_type_with_presentation( + TypeInfo::UnknownType { + name: "HashSet".to_string(), + }, + nested_presentation, + ) + .unwrap(); + let nested_variable = ParsedComplexVariable { + var_name_index: name_index, + type_index: nested_type_index, + access_path: String::new(), + status: VariableStatus::Ok as u8, + data: nested_data, + }; + assert_eq!( + FormatPrinter::format_complex_print_data( + format_index, + &[nested_variable.clone(), nested_variable], &trace_context, ), "41 00 42 00|A" diff --git a/ghostscope-protocol/src/lib.rs b/ghostscope-protocol/src/lib.rs index 97eecb22..fb4b8feb 100644 --- a/ghostscope-protocol/src/lib.rs +++ b/ghostscope-protocol/src/lib.rs @@ -18,11 +18,11 @@ pub use value_presentation::{ BTreeEntryPresentation, BTreeFieldPresentation, HashTableBucketOrder, HashTableEntryPresentation, HashTableFieldPresentation, HashTableOccupancy, NestedValueChildPresentation, NestedValueChildrenPresentation, NestedValueFieldPresentation, - NestedValuePresentation, NestedValueVariantFieldPresentation, ValuePresentation, - BTREE_CAPTURED_ITEM_COUNT_OFFSET, BTREE_HEADER_SIZE, BTREE_NODE_HEADER_SIZE, - BTREE_NODE_HEIGHT_OFFSET, BTREE_NODE_LENGTH_OFFSET, BTREE_NODE_SLOT_COUNT_OFFSET, - HASH_TABLE_BUCKET_DATA_OFFSET, HASH_TABLE_CAPACITY_OFFSET, HASH_TABLE_CAPTURED_BUCKETS_OFFSET, - HASH_TABLE_HEADER_SIZE, INDIRECT_BYTES_LENGTH_PREFIX_SIZE, + NestedValueHashTableFieldPresentation, NestedValuePresentation, + NestedValueVariantFieldPresentation, ValuePresentation, BTREE_CAPTURED_ITEM_COUNT_OFFSET, + BTREE_HEADER_SIZE, BTREE_NODE_HEADER_SIZE, BTREE_NODE_HEIGHT_OFFSET, BTREE_NODE_LENGTH_OFFSET, + BTREE_NODE_SLOT_COUNT_OFFSET, HASH_TABLE_BUCKET_DATA_OFFSET, HASH_TABLE_CAPACITY_OFFSET, + HASH_TABLE_CAPTURED_BUCKETS_OFFSET, HASH_TABLE_HEADER_SIZE, INDIRECT_BYTES_LENGTH_PREFIX_SIZE, INDIRECT_SEQUENCE_CAPTURED_COUNT_OFFSET, INDIRECT_SEQUENCE_HEADER_SIZE, NESTED_VALUE_CHILD_HEADER_SIZE, NESTED_VALUE_CHILD_STATUS_OFFSET, }; diff --git a/ghostscope-protocol/src/value_presentation.rs b/ghostscope-protocol/src/value_presentation.rs index d45e5ded..e065b7a2 100644 --- a/ghostscope-protocol/src/value_presentation.rs +++ b/ghostscope-protocol/src/value_presentation.rs @@ -149,6 +149,15 @@ pub struct NestedValueFieldPresentation { pub child: NestedValueChildPresentation, } +/// One hash-table entry field repeated in every statically reserved bucket +/// sidecar. `slot_offset` is relative to that bucket's sidecar base. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct NestedValueHashTableFieldPresentation { + pub field_index: u64, + pub slot_offset: u64, + pub value: Box, +} + /// One DWARF variant payload field whose raw bytes have a semantic child /// sidecar. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -175,6 +184,12 @@ pub enum NestedValueChildrenPresentation { slot_count: u64, element: Box, }, + HashTable { + first_slot_offset: u64, + bucket_slot_stride: u64, + bucket_count: u64, + fields: Vec, + }, Variant { fields: Vec, }, diff --git a/scripts/dwarf-perf/corpus/README.md b/scripts/dwarf-perf/corpus/README.md index e97b1fcb..880ef00a 100644 --- a/scripts/dwarf-perf/corpus/README.md +++ b/scripts/dwarf-perf/corpus/README.md @@ -108,11 +108,12 @@ The script prints three separate sections: DWARF fast-parse and index-build cost. It reports `average_ms`, `p50_ms`, `p95_ms`, `min_ms`, and `max_ms`. -The source-line benchmark keeps one analyzer instance warm, repeats the same -query multiple times, and records `first_run_ms`, `average_ms`, `p50_ms`, -`p95_ms`, `min_ms`, and `max_ms`. Its latency covers the full source-line -query path: source-line lookup, matched address resolution, and variable -collection for those addresses. +The source-line benchmark keeps one analyzer instance warm, performs 10 +unmeasured warmup queries by default, and then records `first_run_ms`, +`average_ms`, `p50_ms`, `p95_ms`, `min_ms`, and `max_ms` from the measured +queries. Use `--query-warmup-runs` to override the baseline warmup count. Its +latency covers the full source-line query path: source-line lookup, matched +address resolution, and variable collection for those addresses. ## Tuning diff --git a/scripts/dwarf-perf/run_baseline.sh b/scripts/dwarf-perf/run_baseline.sh index 01b7acaf..ca2010ed 100755 --- a/scripts/dwarf-perf/run_baseline.sh +++ b/scripts/dwarf-perf/run_baseline.sh @@ -6,6 +6,7 @@ REPO_ROOT=$(cd -- "$SCRIPT_DIR/../.." && pwd) BUILD_CORPUS=1 RUNS=10 +QUERY_WARMUP_RUNS=10 CORPUS_DIR="$REPO_ROOT/scripts/dwarf-perf/corpus/out" RESULTS_DIR="$REPO_ROOT/perf-results" RESULT_NAME="" @@ -21,7 +22,8 @@ Options: --corpus-dir PATH corpus output directory (default: scripts/dwarf-perf/corpus/out) --results-dir PATH result directory (default: perf-results) --result-name NAME result file stem (default: timestamp-based) - --runs N benchmark runs for parse and query baselines (default: 10) + --runs N measured runs for parse and query baselines (default: 10) + --query-warmup-runs N unmeasured source-line query warmups (default: 10) --parse-target NAME run only one parse artifact from the manifest --cargo-target-dir P cargo target dir for dwarf-tool (default: .target_tmp/dwarf-perf) -h, --help show this help @@ -50,6 +52,10 @@ while [[ $# -gt 0 ]]; do RUNS=$2 shift 2 ;; + --query-warmup-runs) + QUERY_WARMUP_RUNS=$2 + shift 2 + ;; --parse-target) PARSE_TARGET_NAME=$2 shift 2 @@ -159,6 +165,7 @@ print_query_summary() { echo " source: ${QUERY_SOURCE_ABS}:${QUERY_LINE}" echo " binary: $QUERY_BINARY" echo " loading time: ${QUERY_LOADING_MS}ms" + echo " warmup runs: $QUERY_REPORTED_WARMUP_RUNS" echo " runs: $RUNS" echo " first run: ${QUERY_FIRST_RUN_MS}ms" echo " average: ${QUERY_AVG_MS}ms" @@ -186,10 +193,12 @@ run_query_benchmark() { -t "$QUERY_BINARY" \ benchmark-source-line "${QUERY_SOURCE_ABS}:${QUERY_LINE}" \ --runs "$RUNS" \ + --warmup-runs "$QUERY_WARMUP_RUNS" \ --json) query_json=$(printf '%s\n' "$query_output" | sed -n '/^{/,$p') QUERY_LOADING_MS=$(require_json_value "query loading time" "$(printf '%s\n' "$query_json" | jq '.loading_time_ms')" "$query_json") + QUERY_REPORTED_WARMUP_RUNS=$(require_json_value "query warmup runs" "$(printf '%s\n' "$query_json" | jq '.warmup_runs')" "$query_json") QUERY_FIRST_RUN_MS=$(require_json_value "query first run" "$(printf '%s\n' "$query_json" | jq '.benchmark.first_run_ms')" "$query_json") QUERY_AVG_MS=$(require_json_value "query average" "$(printf '%s\n' "$query_json" | jq '.benchmark.average_ms')" "$query_json") QUERY_P50_MS=$(require_json_value "query p50" "$(printf '%s\n' "$query_json" | jq '.benchmark.p50_ms')" "$query_json") @@ -283,6 +292,7 @@ write_baseline_json() { --argjson internal_total_min_ms "$INTERNAL_TOTAL_MIN_MS" \ --argjson internal_total_max_ms "$INTERNAL_TOTAL_MAX_MS" \ --argjson query_loading_ms "$QUERY_LOADING_MS" \ + --argjson query_warmup_runs "$QUERY_REPORTED_WARMUP_RUNS" \ --argjson query_first_run_ms "$QUERY_FIRST_RUN_MS" \ --argjson query_avg_ms "$QUERY_AVG_MS" \ --argjson query_p50_ms "$QUERY_P50_MS" \ @@ -344,6 +354,7 @@ write_baseline_json() { line: ($query_line | tonumber) }, loading_time_ms: $query_loading_ms, + warmup_runs: $query_warmup_runs, runs: $runs, metrics_ms: { first_run: $query_first_run_ms,