diff --git a/crates/logical-optimizer/src/accuracy/allocation.rs b/crates/logical-optimizer/src/accuracy/allocation.rs deleted file mode 100644 index 77849d9c..00000000 --- a/crates/logical-optimizer/src/accuracy/allocation.rs +++ /dev/null @@ -1,156 +0,0 @@ -//! Allocate end-to-end error and failure budgets across approximate layers. -use super::*; - -/// The shape of a composition an allocator splits a budget across. -#[derive(Debug, Clone, PartialEq)] -pub struct CompositionShape { - /// The metric the composed guarantee will carry — decides whether the - /// budget composes additively (`Σ ε_i ≤ ε`) or multiplicatively - /// (`Π(1+ε_i) ≤ 1+ε`). - pub metric: ErrorMetric, - /// How many approximate layers share the budget (≥ 1). - pub approximate_layer_count: usize, -} - -/// One way of splitting an end-to-end target across a composition's -/// approximate layers. `layers[0]` is the outermost layer's local target; -/// the remainder are the inner layers', outermost first. -#[derive(Debug, Clone, PartialEq)] -pub struct AccuracyAllocation { - pub allocator: &'static str, - pub layers: Vec, -} - -impl AccuracyAllocation { - /// The end-to-end budget left for everything below `layers[0]` — what - /// the inner sub-DAG must satisfy as a whole (it re-splits internally). - /// `None` for a single-layer allocation. - pub fn inner_target(&self, shape: &CompositionShape) -> Option { - let inner = &self.layers[1..]; - if inner.is_empty() { - return None; - } - let (eps, delta): (Vec, Vec>) = inner - .iter() - .map(|t| match t { - AccuracyTarget::Exact => (0.0, Some(0.0)), - AccuracyTarget::Epsilon(e) => (*e, None), - AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, Some(*delta)), - }) - .unzip(); - let epsilon = match shape.metric { - ErrorMetric::RelativeValue => eps.iter().map(|e| 1.0 + e).product::() - 1.0, - _ => eps.iter().sum(), - }; - Some(match delta.iter().copied().sum::>() { - Some(delta) => AccuracyTarget::EpsilonDelta { epsilon, delta }, - None => AccuracyTarget::Epsilon(epsilon), - }) - } -} - -/// Enumerates the finite set of budget splits the search tries for one -/// composition. Exposed as its own hook because equal splitting is rarely -/// cost-optimal; a deployment can return several candidate splits and let -/// cost ranking pick among the legal ones. -pub trait AccuracyBudgetAllocator { - fn allocations( - &self, - target: &AccuracyTarget, - composition: &CompositionShape, - ) -> Vec; -} - -/// The initial deterministic allocator: every approximate layer gets an -/// equal share — `ε_i = ε / n`, `δ_i = δ / n` for an additively composed -/// metric, and `ε_i = (1 + ε)^{1/n} − 1` for a multiplicatively composed -/// one — so the composed bound meets the target exactly with no slack. -/// `AccuracyTarget::Exact` yields no allocation: no approximate layer can -/// meet it. -#[derive(Debug, Default, Clone, Copy)] -pub struct EqualSplitAllocator; - -impl AccuracyBudgetAllocator for EqualSplitAllocator { - fn allocations( - &self, - target: &AccuracyTarget, - composition: &CompositionShape, - ) -> Vec { - let n = composition.approximate_layer_count.max(1); - let (epsilon, delta) = match target { - AccuracyTarget::Exact => return Vec::new(), - AccuracyTarget::Epsilon(e) => (*e, None), - AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, Some(*delta)), - }; - if !(epsilon.is_finite() && epsilon > 0.0) { - return Vec::new(); - } - let local_epsilon = match composition.metric { - ErrorMetric::RelativeValue => (1.0 + epsilon).powf(1.0 / n as f64) - 1.0, - _ => epsilon / n as f64, - }; - let layer = match delta { - Some(delta) => AccuracyTarget::EpsilonDelta { - epsilon: local_epsilon, - delta: delta / n as f64, - }, - None => AccuracyTarget::Epsilon(local_epsilon), - }; - vec![AccuracyAllocation { - allocator: "EqualSplitAllocator", - layers: vec![layer; n], - }] - } -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn equal_split_respects_the_root_epsilon_and_delta() { - let target = AccuracyTarget::EpsilonDelta { - epsilon: 0.1, - delta: 0.02, - }; - let shape = CompositionShape { - metric: ErrorMetric::AbsoluteValue, - approximate_layer_count: 2, - }; - let allocations = EqualSplitAllocator.allocations(&target, &shape); - assert_eq!(allocations.len(), 1); - let layers = &allocations[0].layers; - assert_eq!(layers.len(), 2); - let (eps, deltas): (Vec, Vec) = layers - .iter() - .map(|t| match t { - AccuracyTarget::EpsilonDelta { epsilon, delta } => (*epsilon, *delta), - other => panic!("unexpected {other:?}"), - }) - .unzip(); - assert!((eps.iter().sum::() - 0.1).abs() < 1e-12); - assert!((deltas.iter().sum::() - 0.02).abs() < 1e-12); - assert_eq!( - allocations[0].inner_target(&shape), - Some(AccuracyTarget::EpsilonDelta { - epsilon: 0.05, - delta: 0.01 - }) - ); - - // Multiplicative composition: (1+ε_i)^2 = 1+ε, not 2ε_i = ε. - let rel_shape = CompositionShape { - metric: ErrorMetric::RelativeValue, - approximate_layer_count: 2, - }; - let allocations = - EqualSplitAllocator.allocations(&AccuracyTarget::Epsilon(0.21), &rel_shape); - let AccuracyTarget::Epsilon(e) = allocations[0].layers[0] else { - panic!() - }; - assert!((e - 0.1).abs() < 1e-12); - - assert!(EqualSplitAllocator - .allocations(&AccuracyTarget::Exact, &shape) - .is_empty()); - } -} diff --git a/crates/logical-optimizer/src/accuracy/composition.rs b/crates/logical-optimizer/src/accuracy/composition.rs deleted file mode 100644 index 17efd657..00000000 --- a/crates/logical-optimizer/src/accuracy/composition.rs +++ /dev/null @@ -1,1122 +0,0 @@ -//! Registered propagation rules for computations over uncertain inputs. -use super::*; - -impl DefaultAccuracyModel { - fn additive( - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: &ResultGuarantee, - rule: &str, - ) -> ResultGuarantee { - let mut terms: Vec = inputs.iter().map(|g| g.bound.clone()).collect(); - terms.push(local.bound.clone()); - let mut deltas: Vec = inputs - .iter() - .map(|g| g.failure_probability.clone()) - .collect(); - deltas.push(local.failure_probability.clone()); - ResultGuarantee { - metric: local.metric, - bound: BoundExpr::Sum { terms }, - failure_probability: ProbabilityExpr::UnionBound { terms: deltas }, - provenance: composed_provenance(op, inputs, local, rule), - } - } - - /// `(1 + ε_total) = Π (1 + ε_i)` ⇒ for two factors - /// `ε_in + ε_out + ε_in·ε_out`; written out as the sum of all - /// cross-products so the expression DAG is exact for any input count. - fn multiplicative( - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: &ResultGuarantee, - ) -> ResultGuarantee { - let factors: Vec<&BoundExpr> = inputs - .iter() - .map(|g| &g.bound) - .chain(std::iter::once(&local.bound)) - .collect(); - // Every non-empty subset's product: Π(1+ε_i) − 1 = Σ_{S≠∅} Π_{i∈S} ε_i. - let mut terms = Vec::new(); - for mask in 1..(1u32 << factors.len()) { - let subset: Vec = factors - .iter() - .enumerate() - .filter(|(i, _)| mask & (1 << i) != 0) - .map(|(_, b)| (*b).clone()) - .collect(); - terms.push(if subset.len() == 1 { - subset.into_iter().next().expect("one element") - } else { - BoundExpr::Product { factors: subset } - }); - } - let mut deltas: Vec = inputs - .iter() - .map(|g| g.failure_probability.clone()) - .collect(); - deltas.push(local.failure_probability.clone()); - ResultGuarantee { - metric: ErrorMetric::RelativeValue, - bound: BoundExpr::Sum { terms }, - failure_probability: ProbabilityExpr::UnionBound { terms: deltas }, - provenance: composed_provenance(op, inputs, local, "relative_cross_term_union_bound"), - } - } - - fn lipschitz( - op: &CompositionOperator, - constant: f64, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - ) -> ResultGuarantee { - let input = &inputs[0]; - let scaled = BoundExpr::Scaled { - factor: constant, - inner: Box::new(input.bound.clone()), - }; - let (bound, delta) = match local { - Some(local) => ( - BoundExpr::Sum { - terms: vec![scaled, local.bound.clone()], - }, - ProbabilityExpr::UnionBound { - terms: vec![ - input.failure_probability.clone(), - local.failure_probability.clone(), - ], - }, - ), - None => (scaled, input.failure_probability.clone()), - }; - let exact_local = ResultGuarantee::exact("deterministic Lipschitz transformation"); - ResultGuarantee { - metric: ErrorMetric::AbsoluteValue, - bound, - failure_probability: delta, - provenance: composed_provenance( - op, - inputs, - local.unwrap_or(&exact_local), - "lipschitz_union_bound", - ), - } - } - - /// Exact `sum` over approximate inputs: `B ≤ Σ B_i`, `δ ≤ Σ δ_i`. The - /// planner composes one *per-value* child guarantee over an unknown - /// number of input rows, so both the bound and the union bound scale by - /// `stats.input_row_count` — an [`BoundExpr::Unknown`] leaf when it is - /// not supplied. Each input's normalized bound is first converted to - /// absolute units via the statistic its metric is normalized by (also - /// unknown unless supplied); a `Rank` input has no such conversion. - fn exact_sum( - op: &CompositionOperator, - inputs: &[ResultGuarantee], - stats: &PropagationStats, - ) -> Result { - let mut terms = Vec::with_capacity(inputs.len()); - let mut deltas = Vec::with_capacity(inputs.len()); - let mut provenance = Vec::new(); - for (i, input) in inputs.iter().enumerate() { - let absolute = - absolute_bound(input).ok_or_else(|| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.metric).collect(), - local_metric: None, - reason: format!( - "input {i} carries a {:?} guarantee, which has no registered \ - conversion to an absolute value error", - input.metric - ), - })?; - if let BoundExpr::Product { factors } = &absolute { - for f in factors { - if let BoundExpr::Unknown { statistic } = f { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: statistic.clone(), - }); - } - } - } - terms.push(absolute); - deltas.push(input.failure_probability.clone()); - } - let count = row_count(stats, &mut provenance); - let exact_local = ResultGuarantee::exact("ExactAggregate(Sum)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_sum_union_bound", - )); - Ok(ResultGuarantee { - metric: ErrorMetric::AbsoluteValue, - bound: BoundExpr::Product { - factors: vec![count.clone(), BoundExpr::Sum { terms }], - }, - failure_probability: ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { terms: deltas }), - }, - provenance, - }) - } - - /// Exact arithmetic mean over values with absolute-error guarantees. - /// Averaging cannot amplify the largest absolute input error. The event - /// that every row respects its bound is still protected conservatively - /// by a union bound over the input row count. - fn exact_average( - op: &CompositionOperator, - inputs: &[ResultGuarantee], - stats: &PropagationStats, - ) -> Result { - if inputs - .iter() - .any(|input| input.metric != ErrorMetric::AbsoluteValue) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.metric).collect(), - local_metric: None, - reason: "exact average requires AbsoluteValue input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = ResultGuarantee::exact("ExactAggregate(Average)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_average_union_bound", - )); - Ok(ResultGuarantee { - metric: ErrorMetric::AbsoluteValue, - bound: BoundExpr::Max { - terms: inputs.iter().map(|g| g.bound.clone()).collect(), - }, - failure_probability: ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.failure_probability.clone()) - .collect(), - }), - }, - provenance, - }) - } - - /// Exact `max`/`min` over approximate inputs of one shared metric: the - /// returned value's error is at most the largest input bound (order - /// statistics are monotone under a uniform perturbation), with - /// probability by the union bound over every input row. This bounds the - /// returned *value*; it does not identify the true winning key. - fn exact_extremum( - op: &CompositionOperator, - inputs: &[ResultGuarantee], - stats: &PropagationStats, - ) -> Result { - let metric = inputs[0].metric; - if inputs - .iter() - .any(|g| g.metric != ErrorMetric::AbsoluteValue) - { - return Err(AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.metric).collect(), - local_metric: None, - reason: "exact max/min requires AbsoluteValue input guarantees".into(), - }); - } - let mut provenance = Vec::new(); - let count = row_count(stats, &mut provenance); - let exact_local = ResultGuarantee::exact("ExactAggregate(Max)"); - provenance.extend(composed_provenance( - op, - inputs, - &exact_local, - "exact_extremum_union_bound", - )); - Ok(ResultGuarantee { - metric, - bound: BoundExpr::Max { - terms: inputs.iter().map(|g| g.bound.clone()).collect(), - }, - failure_probability: ProbabilityExpr::Scaled { - count, - inner: Box::new(ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.failure_probability.clone()) - .collect(), - }), - }, - provenance, - }) - } - - /// Exact division of two relative-value estimates. If the numerator is - /// within `a` and the denominator within `b`, their ratio is within - /// `(a + b) / (1 - b)`. DDSketch supplies those deterministic bounds. - fn exact_division( - op: &CompositionOperator, - inputs: &[ResultGuarantee], - stats: &PropagationStats, - ) -> Result { - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: inputs.iter().map(|g| g.metric).collect(), - local_metric: None, - reason, - }; - if inputs.len() != 2 - || inputs - .iter() - .any(|input| input.metric != ErrorMetric::RelativeValue && !input.is_exact()) - { - return Err(unsupported( - "division needs two relative-value or exact guarantees".into(), - )); - } - let Some(numerator) = inputs[0].bound.evaluate() else { - return Err(unsupported( - "numerator relative bound is unavailable".into(), - )); - }; - let Some(denominator) = inputs[1].bound.evaluate() else { - return Err(unsupported( - "denominator relative bound is unavailable".into(), - )); - }; - if !(numerator.is_finite() - && denominator.is_finite() - && numerator >= 0.0 - && (0.0..1.0).contains(&denominator)) - { - return Err(unsupported( - "division needs finite non-negative bounds and a denominator bound below one" - .into(), - )); - } - let Some(domains) = &stats.division_operand_domains else { - return Err(unsupported( - "division needs finite operand domains and a nonzero denominator proof".into(), - )); - }; - for domain in domains { - if !domain.lower.is_finite() - || !domain.upper.is_finite() - || domain.lower > domain.upper - || domain.contract.trim().is_empty() - { - return Err(unsupported("invalid division operand domain".into())); - } - } - if domains[1].lower <= 0.0 && domains[1].upper >= 0.0 { - return Err(unsupported("denominator domain includes zero".into())); - } - // Keep the true and perturbed quotients finite and out of the - // subnormal range, where Float64 division loses relative accuracy. - // A nonzero numerator interval touching zero cannot prove this. - let num = &domains[0]; - if num.lower <= 0.0 && num.upper >= 0.0 && (num.lower != 0.0 || num.upper != 0.0) { - return Err(unsupported( - "numerator domain cannot exclude underflow near zero".into(), - )); - } - for n in [num.lower, num.upper] { - for d in [domains[1].lower, domains[1].upper] { - for nf in [1.0 - numerator, 1.0 + numerator] { - for df in [1.0 - denominator, 1.0 + denominator] { - let quotient = (n * nf) / (d * df); - if !(n * nf).is_finite() - || !(d * df).is_finite() - || !quotient.is_finite() - || (n != 0.0 && quotient.abs() < f64::MIN_POSITIVE) - { - return Err(unsupported( - "division may overflow or underflow Float64".into(), - )); - } - } - } - } - } - Ok(ResultGuarantee { - metric: ErrorMetric::RelativeValue, - bound: BoundExpr::Constant { - value: (numerator + denominator) / (1.0 - denominator), - }, - failure_probability: ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|input| input.failure_probability.clone()) - .collect(), - }, - provenance: inputs - .iter() - .enumerate() - .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(guarantee.clone()), - }) - .chain(domains.iter().enumerate().map(|(input_index, domain)| { - GuaranteeSource::InputValueDomain { - input_index, - lower: domain.lower, - upper: domain.upper, - max_samples: domain.max_samples, - contract: domain.contract.clone(), - } - })) - .chain(std::iter::once(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "relative_division".into(), - })) - .collect(), - }) - } -} - -/// `stats.input_row_count` as a bound factor, or an `Unknown` leaf (recorded -/// in `provenance`) when absent. -fn row_count(stats: &PropagationStats, provenance: &mut Vec) -> BoundExpr { - match stats.input_row_count { - Some(n) => BoundExpr::Constant { value: n as f64 }, - None => { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: "input_row_count".into(), - }); - BoundExpr::Unknown { - statistic: "input_row_count".into(), - } - } - } -} - -/// `input`'s bound converted to absolute value units, multiplying a -/// normalized metric by the (unknown) statistic it is normalized by. `None` -/// for a metric with no such conversion (`Rank`, `TopKMembership`). -fn absolute_bound(input: &ResultGuarantee) -> Option { - let normalizer = match input.metric { - ErrorMetric::AbsoluteValue => return Some(input.bound.clone()), - ErrorMetric::RelativeValue => "true_value_magnitude", - ErrorMetric::Cardinality => "true_cardinality", - ErrorMetric::Frequency => "stream_l1_norm", - ErrorMetric::L2Frequency => "stream_l2_norm", - // `Rank` has no distribution-free conversion to a value error; a - // metric this crate does not know has no registered conversion. - ErrorMetric::Rank | ErrorMetric::TopKMembership | _ => return None, - }; - if input.bound.is_zero() { - return Some(BoundExpr::Zero); - } - Some(BoundExpr::Product { - factors: vec![ - input.bound.clone(), - BoundExpr::Unknown { - statistic: normalizer.into(), - }, - ], - }) -} - -fn composed_provenance( - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: &ResultGuarantee, - rule: &str, -) -> Vec { - let mut provenance: Vec = inputs - .iter() - .enumerate() - .map(|(input_index, g)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g.clone()), - }) - .collect(); - provenance.extend(local.provenance.iter().cloned()); - provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: rule.into(), - }); - provenance -} - -pub(super) fn exact_operation_rule(operation: &ExactOperation) -> Option { - let ExactOperation::Aggregate { measures, .. } = operation; - match measures.as_slice() { - [intent] => { - crate::pass1::function_rules::function_rules(intent).map(|rules| rules.accuracy) - } - // The remaining functions are exact over exact samples, but have - // no definition-backed rule over approximate values yet. - _ => None, - } -} - -pub(super) fn propagate( - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - stats: &PropagationStats, -) -> Result { - // Exact input: only the local guarantee remains (or the value is exact). - if inputs.iter().all(ResultGuarantee::is_exact) - && !matches!(op, CompositionOperator::TopKSelection) - { - return Ok(match local { - Some(local) => { - let mut out = local.clone(); - out.provenance - .extend(inputs.iter().enumerate().map(|(input_index, g)| { - GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(g.clone()), - } - })); - out.provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "exact_input".into(), - }); - out - } - None => { - let mut out = ResultGuarantee::exact(format!("{op:?} over exact inputs")); - out.provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "exact_input".into(), - }); - out - } - }); - } - - let input_metrics: Vec = inputs.iter().map(|g| g.metric).collect(); - let unsupported = |reason: String| AccuracyError::UnsupportedComposition { - operator: op.clone(), - input_metrics: input_metrics.clone(), - local_metric: local.map(|g| g.metric), - reason, - }; - // An exact input is compatible with every metric; only approximate - // inputs constrain the rule. - let approximate: Vec<&ResultGuarantee> = inputs.iter().filter(|g| !g.is_exact()).collect(); - let same_metric = |metric: ErrorMetric| approximate.iter().all(|g| g.metric == metric); - - match op { - CompositionOperator::CheckedRelativeDivision => { - if inputs.len() != 2 || local.is_some() || !same_metric(ErrorMetric::RelativeValue) { - return Err(unsupported( - "checked division requires two exact/relative-value operands".into(), - )); - } - let a = inputs[0] - .bound - .evaluate() - .ok_or_else(|| unsupported("unknown numerator bound".into()))?; - let b = inputs[1] - .bound - .evaluate() - .ok_or_else(|| unsupported("unknown denominator bound".into()))?; - if !(0.0..1.0).contains(&b) || a < 0.0 || !a.is_finite() { - return Err(unsupported("invalid relative division bounds".into())); - } - Ok(ResultGuarantee { - metric: ErrorMetric::RelativeValue, - bound: BoundExpr::Constant { - value: (a + b) / (1.0 - b) + 4.0 * f64::EPSILON, - }, - failure_probability: ProbabilityExpr::UnionBound { - terms: inputs - .iter() - .map(|g| g.failure_probability.clone()) - .collect(), - }, - provenance: composed_provenance( - op, - inputs, - &ResultGuarantee::exact("checked floating-point division"), - "checked_relative_division_union_bound", - ), - }) - } - CompositionOperator::ApproximateAggregate => { - let local = local.ok_or_else(|| { - unsupported("approximate operator has no local guarantee to compose".into()) - })?; - if !same_metric(local.metric) { - return Err(unsupported(format!( - "no registered cross-metric rule from {input_metrics:?} to {:?}", - local.metric - ))); - } - match local.metric { - ErrorMetric::AbsoluteValue => Ok(DefaultAccuracyModel::additive( - op, - inputs, - local, - "additive_union_bound", - )), - ErrorMetric::RelativeValue => { - if stats.values_non_negative == Some(false) { - return Err(unsupported( - "relative-error composition cannot use a known signed input".into(), - )); - } - if stats.values_non_negative.is_none() { - let mut provenance = composed_provenance( - op, - inputs, - local, - "relative_value_sign_unverified", - ); - provenance.extend(stats.evidence_provenance.clone()); - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: "values_non_negative".into(), - }); - return Ok(ResultGuarantee { - metric: ErrorMetric::RelativeValue, - bound: BoundExpr::Unknown { - statistic: "values_non_negative".into(), - }, - failure_probability: ProbabilityExpr::Unknown { - statistic: "values_non_negative".into(), - }, - provenance, - }); - } - Ok(DefaultAccuracyModel::multiplicative(op, inputs, local)) - } - ErrorMetric::Rank - | ErrorMetric::Cardinality - | ErrorMetric::Frequency - | ErrorMetric::L2Frequency - | ErrorMetric::TopKMembership - | _ => Err(unsupported(format!( - "no registered same-metric composition rule for {:?} over {:?}", - local.metric, local.metric - ))), - } - } - CompositionOperator::Lipschitz { constant } => { - if !(constant.is_finite() && *constant >= 0.0) { - return Err(unsupported(format!( - "Lipschitz constant {constant} is not a finite non-negative number" - ))); - } - if inputs.len() != 1 || !same_metric(ErrorMetric::AbsoluteValue) { - return Err(unsupported( - "Lipschitz rule is registered for exactly one AbsoluteValue input".into(), - )); - } - if local.is_some_and(|g| g.metric != ErrorMetric::AbsoluteValue) { - return Err(unsupported( - "Lipschitz rule needs an AbsoluteValue local guarantee".into(), - )); - } - Ok(DefaultAccuracyModel::lipschitz( - op, *constant, inputs, local, - )) - } - CompositionOperator::ExactSum => DefaultAccuracyModel::exact_sum(op, inputs, stats), - CompositionOperator::ExactAverage => DefaultAccuracyModel::exact_average(op, inputs, stats), - CompositionOperator::ExactExtremum => { - DefaultAccuracyModel::exact_extremum(op, inputs, stats) - } - CompositionOperator::ExactDivision => { - DefaultAccuracyModel::exact_division(op, inputs, stats) - } - CompositionOperator::CounterRate - | CompositionOperator::InstantCounterRate - | CompositionOperator::CounterIncrease => Err(unsupported( - "counter reset detection and boundary extrapolation have no distribution-free \ - accuracy bound over approximate samples; exact samples remain exact" - .into(), - )), - CompositionOperator::TopKSelection => { - let selected = stats.topk_selected_lower_bound; - let excluded = stats.topk_excluded_upper_bound; - let delta = stats.topk_interval_failure_probability; - if selected.is_some_and(|value| !value.is_finite()) - || excluded.is_some_and(|value| !value.is_finite()) - || delta.is_some_and(|value| !value.is_finite() || !(0.0..=1.0).contains(&value)) - || selected - .zip(excluded) - .is_some_and(|(lower, upper)| lower <= upper) - { - return Err(unsupported( - "top-k confidence intervals overlap or contain invalid evidence".into(), - )); - } - let mut provenance = inputs - .iter() - .enumerate() - .map(|(input_index, guarantee)| GuaranteeSource::ChildGuarantee { - input_index, - guarantee: Box::new(guarantee.clone()), - }) - .collect::>(); - provenance.extend(stats.evidence_provenance.clone()); - if let Some(local) = local { - provenance.extend(local.provenance.clone()); - } - for (name, missing) in [ - ("topk_selected_lower_bound", selected.is_none()), - ("topk_excluded_upper_bound", excluded.is_none()), - ("topk_interval_failure_probability", delta.is_none()), - ] { - if missing { - provenance.push(GuaranteeSource::UnavailableStatistic { - statistic: name.into(), - }); - } - } - provenance.push(GuaranteeSource::CompositionStep { - operator: op.clone(), - rule: "topk_membership_margin_certificate".into(), - }); - let certified = selected.is_some() && excluded.is_some() && delta.is_some(); - Ok(ResultGuarantee { - metric: ErrorMetric::TopKMembership, - bound: if certified { - BoundExpr::Zero - } else { - BoundExpr::Unknown { - statistic: "topk_membership_margin".into(), - } - }, - failure_probability: delta.map_or_else( - || ProbabilityExpr::Unknown { - statistic: "topk_interval_failure_probability".into(), - }, - |value| ProbabilityExpr::Constant { value }, - ), - provenance, - }) - } - // An operator this crate does not know has no registered rule. - _ => Err(unsupported("no registered rule for this operator".into())), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::AggIntent; - fn abs(bound: f64, delta: f64) -> ResultGuarantee { - ResultGuarantee { - metric: ErrorMetric::AbsoluteValue, - bound: BoundExpr::Constant { value: bound }, - failure_probability: ProbabilityExpr::Constant { value: delta }, - provenance: vec![], - } - } - fn rel(bound: f64) -> ResultGuarantee { - ResultGuarantee { - metric: ErrorMetric::RelativeValue, - bound: BoundExpr::Constant { value: bound }, - failure_probability: ProbabilityExpr::Zero, - provenance: vec![], - } - } - fn domain(lower: f64, upper: f64) -> QuantileInputDomain { - QuantileInputDomain { - lower, - upper, - max_samples: 1000, - contract: "enforced test population".into(), - } - } - fn with_metric(metric: ErrorMetric, bound: f64) -> ResultGuarantee { - ResultGuarantee { - metric, - ..abs(bound, 0.0) - } - } - #[test] - fn checked_division_propagates_value_bounds_and_rejects_rank_bounds() { - let op = CompositionOperator::CheckedRelativeDivision; - let inputs = [rel(0.01), rel(0.01)]; - let g = DefaultAccuracyModel - .propagate(&op, &inputs, None, &Default::default()) - .unwrap(); - assert!((g.bound.evaluate().unwrap() - 0.02 / 0.99).abs() < 1e-14); - let mut rank = inputs[0].clone(); - rank.metric = ErrorMetric::Rank; - assert!(DefaultAccuracyModel - .propagate(&op, &[rank.clone(), rank], None, &Default::default()) - .is_err()); - assert!(DefaultAccuracyModel - .propagate(&op, &[rel(0.01), rel(1.0)], None, &Default::default()) - .is_err()); - } - #[test] - fn relative_division_requires_operand_domains() { - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(0.01)], - None, - &PropagationStats::default() - ) - .is_err()); - } - #[test] - fn relative_division_preserves_asymmetric_bound_and_domain_provenance() { - for denominator in [domain(1., 10.), domain(-10., -1.)] { - let stats = PropagationStats { - division_operand_domains: Some([domain(-20., -2.), denominator]), - ..Default::default() - }; - let got = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.02), rel(0.03)], - None, - &stats, - ) - .unwrap(); - assert!((got.bound.evaluate().unwrap() - 0.05 / 0.97).abs() < 1e-14); - assert_eq!(got.failure_probability.evaluate(), Some(0.)); - assert_eq!( - got.provenance - .iter() - .filter(|p| matches!(p, GuaranteeSource::InputValueDomain { .. })) - .count(), - 2 - ); - } - } - #[test] - fn relative_division_rejects_zero_special_and_extreme_domains() { - for domains in [ - [domain(1., 2.), domain(0., 0.)], - [domain(1., 2.), domain(-1., 1.)], - [domain(f64::NAN, 2.), domain(1., 2.)], - [domain(1., 2.), domain(1., f64::INFINITY)], - [domain(1e250, 1e250), domain(1e-250, 1e-250)], - [domain(1e-250, 1e-250), domain(1e250, 1e250)], - ] { - let stats = PropagationStats { - division_operand_domains: Some(domains), - ..Default::default() - }; - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(0.01)], - None, - &stats - ) - .is_err()); - } - let stats = PropagationStats { - division_operand_domains: Some([domain(1., 2.), domain(1., 2.)]), - ..Default::default() - }; - for bound in [1., f64::NAN, f64::INFINITY, -0.1] { - assert!(DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactDivision, - &[rel(0.01), rel(bound)], - None, - &stats - ) - .is_err()); - } - } - #[test] - fn exact_child_contributes_zero_error() { - let local = abs(0.05, 0.01); - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[ResultGuarantee::exact("sum")], - Some(&local), - &PropagationStats::default(), - ) - .unwrap(); - assert_eq!(out.bound.evaluate(), Some(0.05)); - assert_eq!(out.failure_probability.evaluate(), Some(0.01)); - assert_eq!(out.metric, ErrorMetric::AbsoluteValue); - } - #[test] - fn additive_bounds_and_delta_union_bound_compose() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[abs(0.02, 0.01)], - Some(&abs(0.03, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); - assert!((out.bound.evaluate().unwrap() - 0.05).abs() < 1e-12); - // Union bound, not 1 − (1−0.01)(1−0.02) = 0.0298. - assert!((out.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); - assert!(out.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::CompositionStep { rule, .. } if rule == "additive_union_bound" - ))); - } - #[test] - fn relative_error_includes_the_cross_term() { - let stats = PropagationStats { - values_non_negative: Some(true), - ..Default::default() - }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &stats, - ) - .unwrap(); - // 0.1 + 0.2 + 0.1·0.2 = 0.32, not 0.3. - assert!((out.bound.evaluate().unwrap() - 0.32).abs() < 1e-12); - assert_eq!(out.metric, ErrorMetric::RelativeValue); - } - #[test] - fn relative_error_without_sign_knowledge_remains_symbolic() { - let unknown = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats::default(), - ) - .unwrap(); - assert!(unknown.has_unknown()); - let signed = DefaultAccuracyModel.propagate( - &CompositionOperator::ApproximateAggregate, - &[rel(0.1)], - Some(&rel(0.2)), - &PropagationStats { - values_non_negative: Some(false), - ..Default::default() - }, - ); - assert!(signed.is_err()); - } - #[test] - fn incompatible_metrics_are_rejected_not_treated_as_exact() { - // HLL cardinality error under a CMS frequency guarantee. - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(ErrorMetric::Cardinality, 0.01)], - Some(&with_metric(ErrorMetric::Frequency, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!( - err, - AccuracyError::UnsupportedComposition { - input_metrics, - local_metric: Some(ErrorMetric::Frequency), - .. - } if input_metrics == vec![ErrorMetric::Cardinality] - )); - // Quantile rank error under value-additive logic. - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(ErrorMetric::Rank, 0.01)], - Some(&abs(0.01, 0.0)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); - } - #[test] - fn same_metric_rank_over_rank_has_no_registered_rule() { - let err = DefaultAccuracyModel - .propagate( - &CompositionOperator::ApproximateAggregate, - &[with_metric(ErrorMetric::Rank, 0.01)], - Some(&with_metric(ErrorMetric::Rank, 0.01)), - &PropagationStats::default(), - ) - .unwrap_err(); - assert!(matches!(err, AccuracyError::UnsupportedComposition { .. })); - } - #[test] - fn lipschitz_scales_the_input_bound() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::Lipschitz { constant: 3.0 }, - &[abs(0.1, 0.01)], - Some(&abs(0.05, 0.02)), - &PropagationStats::default(), - ) - .unwrap(); - assert!((out.bound.evaluate().unwrap() - 0.35).abs() < 1e-12); - assert!((out.failure_probability.evaluate().unwrap() - 0.03).abs() < 1e-12); - } - #[test] - fn exact_sum_over_approximate_sums_bounds_and_keeps_unknown_row_count_unknown() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); - assert_eq!(out.metric, ErrorMetric::AbsoluteValue); - assert_eq!( - out.bound.evaluate(), - None, - "unknown row count stays unknown" - ); - assert!(out.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::UnavailableStatistic { statistic } if statistic == "input_row_count" - ))); - assert!(!DefaultAccuracyModel.satisfies(&out, &AccuracyTarget::Epsilon(1.0))); - - let known = PropagationStats { - input_row_count: Some(4), - ..Default::default() - }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactSum, - &[abs(0.1, 0.01)], - None, - &known, - ) - .unwrap(); - assert!((out.bound.evaluate().unwrap() - 0.4).abs() < 1e-12); - assert!((out.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); - } - #[test] - fn exact_extremum_takes_the_max_bound() { - let known = PropagationStats { - input_row_count: Some(2), - ..Default::default() - }; - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactExtremum, - &[abs(0.1, 0.01), abs(0.3, 0.01)], - None, - &known, - ) - .unwrap(); - assert!((out.bound.evaluate().unwrap() - 0.3).abs() < 1e-12); - assert!((out.failure_probability.evaluate().unwrap() - 0.04).abs() < 1e-12); - } - #[test] - fn exact_average_has_its_own_absolute_error_rule() { - let out = DefaultAccuracyModel - .propagate( - &CompositionOperator::ExactAverage, - &[abs(0.25, 0.01)], - None, - &PropagationStats { - input_row_count: Some(4), - ..PropagationStats::default() - }, - ) - .unwrap(); - assert_eq!(out.metric, ErrorMetric::AbsoluteValue); - assert_eq!(out.bound.evaluate(), Some(0.25)); - assert_eq!(out.failure_probability.evaluate(), Some(0.04)); - } - #[test] - fn counter_functions_have_distinct_definition_rules() { - let operation = |intent| ExactOperation::Aggregate { - reduction: asap_types::ir::operator::Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - }; - assert_eq!( - DefaultAccuracyModel.exact_operation_rule(&operation(AggIntent::Rate)), - Some(CompositionOperator::CounterRate) - ); - assert_eq!( - DefaultAccuracyModel.exact_operation_rule(&operation(AggIntent::IRate)), - Some(CompositionOperator::InstantCounterRate) - ); - assert_eq!( - DefaultAccuracyModel.exact_operation_rule(&operation(AggIntent::Increase)), - Some(CompositionOperator::CounterIncrease) - ); - } - #[test] - fn topk_selection_requires_a_separated_margin_certificate() { - let unknown = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats::default(), - ) - .unwrap(); - assert!(unknown.has_unknown()); - - let certified = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); - assert_eq!(certified.metric, ErrorMetric::TopKMembership); - assert_eq!(certified.bound.evaluate(), Some(0.0)); - assert_eq!(certified.failure_probability.evaluate(), Some(0.005)); - - let overlapping = DefaultAccuracyModel.propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(100.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ); - assert!(overlapping.is_err()); - - let partial = DefaultAccuracyModel - .propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - }, - ) - .unwrap(); - assert!(partial.has_unknown()); - assert_eq!(partial.failure_probability.evaluate(), Some(0.005)); - - let invalid_partial = DefaultAccuracyModel.propagate( - &CompositionOperator::TopKSelection, - &[abs(0.1, 0.01)], - None, - &PropagationStats { - topk_selected_lower_bound: Some(f64::NAN), - ..Default::default() - }, - ); - assert!(invalid_partial.is_err()); - } -} diff --git a/crates/logical-optimizer/src/accuracy/estimators/mod.rs b/crates/logical-optimizer/src/accuracy/estimators/mod.rs index 3ffeba42..1f81938b 100644 --- a/crates/logical-optimizer/src/accuracy/estimators/mod.rs +++ b/crates/logical-optimizer/src/accuracy/estimators/mod.rs @@ -172,92 +172,6 @@ pub(crate) fn saturating_ceil(x: f64, lo: u32, hi: u32) -> u32 { } (x.ceil() as u32).clamp(lo, hi) } -pub(crate) struct EstimatorAccuracy<'a> { - base: &'a dyn AccuracyModel, - contract: Option, - epsilon: f64, - delta: f64, -} - -impl<'a> EstimatorAccuracy<'a> { - pub(crate) fn new( - base: &'a dyn AccuracyModel, - contract: Option, - target: Option<&AccuracyTarget>, - ) -> Self { - let (epsilon, delta) = target - .map(crate::pass1::realization::accuracy_budget) - .unwrap_or((0.0, 0.0)); - Self { - base, - contract, - epsilon, - delta, - } - } - - fn hll(&self) -> Option { - let EstimatorContract::ClassicHll { - max_distinct_per_evaluation, - } = self.contract?; - hll::ClassicHllConfidence::new(max_distinct_per_evaluation, self.epsilon) - } - - pub(crate) fn size_params(&self, algorithm: &SketchAlgorithm) -> Option { - if *algorithm != SketchAlgorithm::Hll || self.contract.is_none() { - return None; - } - // Retain the strongest supported parameter for diagnostics if sizing - // is infeasible. The normal guarantee check rejects it below. - Some(SketchParams::Hll { - precision: self - .hll() - .and_then(|model| model.precision(self.delta)) - .unwrap_or(18), - }) - } -} - -impl AccuracyModel for EstimatorAccuracy<'_> { - fn exact_operation_rule(&self, operation: &ExactOperation) -> Option { - self.base.exact_operation_rule(operation) - } - fn local_guarantee( - &self, - family: &FieldDataType, - query: &SketchStatistic, - ) -> Option { - if let (Some(_), FieldDataType::Sketch(kind, grouping), SketchStatistic::Cardinality) = - (self.contract, family, query) - { - if let (SketchAlgorithm::Hll, SketchParams::Hll { precision }) = - (kind.algorithm(), kind.params()) - { - if *grouping != GroupingStrategy::PerSubpopulationInstance { - return None; - } - return self.hll()?.guarantee(*precision); - } - } - self.base.local_guarantee(family, query) - } - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - stats: &PropagationStats, - ) -> Result { - self.base.propagate(op, inputs, local, stats) - } - fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { - self.base.satisfies(guarantee, target) - } - fn answers(&self, statistic: &SketchStatistic, guarantee: &ResultGuarantee) -> bool { - self.base.answers(statistic, guarantee) - } -} - /// Compose the inner per-subpopulation guarantee with Hydra's outer shared /// grid. The paper's collision term depends on deployment/data statistics; /// keeping those leaves symbolic makes the formula explicit while ensuring diff --git a/crates/logical-optimizer/src/accuracy/evidence.rs b/crates/logical-optimizer/src/accuracy/evidence.rs index 7ba65067..84365ac7 100644 --- a/crates/logical-optimizer/src/accuracy/evidence.rs +++ b/crates/logical-optimizer/src/accuracy/evidence.rs @@ -24,29 +24,6 @@ pub struct QuantileInputDomain { pub contract: String, } -impl QuantileInputDomain { - pub(crate) fn supports_ddsketch(&self, alpha: f64) -> bool { - if !alpha.is_finite() - || alpha <= 0.0 - || alpha >= 1.0 - || !self.lower.is_finite() - || !self.upper.is_finite() - || self.lower > self.upper - || self.max_samples == 0 - || self.max_samples > (1u64 << 53) - || self.contract.trim().is_empty() - { - return false; - } - let (min, max) = asap_sketchlib::sketches::ddsketch::ddsketch_indexable_bounds(alpha); - // Same-sign interpolation preserves relative error. Zero alone is - // exact; an interval touching zero also admits tiny zero-mapped values. - (self.lower >= min && self.upper <= max) - || (self.upper <= -min && self.lower >= -max) - || (self.lower == 0.0 && self.upper == 0.0) - } -} - /// Statistics a propagation rule may consult. Every field is optional and /// defaults to "unknown": a rule that needs a missing statistic emits a /// [`BoundExpr::Unknown`] leaf (or rejects) rather than guessing. diff --git a/crates/logical-optimizer/src/accuracy/mod.rs b/crates/logical-optimizer/src/accuracy/mod.rs index b609bd03..e5b635aa 100644 --- a/crates/logical-optimizer/src/accuracy/mod.rs +++ b/crates/logical-optimizer/src/accuracy/mod.rs @@ -1,47 +1,29 @@ //! Planner accuracy interfaces and model dispatch. //! -//! Estimator models derive local guarantees, composition propagates them, -//! allocation proposes local budgets, and evidence supplies scoped contracts. -//! Unknown evidence may retain a candidate but does not authorize selection. -//! See `docs/design_docs/concepts/accuracy-models.md` for the design. +//! Estimator models derive local guarantees, and evidence supplies scoped +//! contracts. Unknown evidence may retain a candidate but does not authorize +//! selection. See `docs/design_docs/concepts/accuracy-models.md` for the +//! design. -pub mod allocation; -pub mod composition; pub mod estimators; pub mod evidence; -pub use allocation::{ - AccuracyAllocation, AccuracyBudgetAllocator, CompositionShape, EqualSplitAllocator, -}; -pub(crate) use estimators::EstimatorAccuracy; pub use evidence::{ AccuracyEvidenceProvider, EstimatorContract, NoAccuracyEvidence, PropagationStats, QuantileInputDomain, WorkloadAccuracyEvidence, }; use asap_types::ir::properties::{ - AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, - ResultGuarantee, + BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, ResultGuarantee, }; use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams, SketchStatistic}; use asap_types::ir::OperatorNode; use asap_types::types::AccuracyTarget; -use crate::pass1::exact_composition::ExactOperation; - -/// The deployment-extensible accuracy algebra. `asap-logical-optimizer` ships -/// [`DefaultAccuracyModel`]; a deployment with a proof for a composition the -/// default rejects (a registered cross-metric conversion, say) implements -/// this trait and passes it to -/// [`crate::pass1::replacement::ASAPStrategies::new_with_planning_inputs`]. +/// The deployment-extensible accuracy model. `asap-logical-optimizer` ships +/// [`DefaultAccuracyModel`]; a deployment with its own error model for a +/// family implements this trait and passes it to Stage 3. pub trait AccuracyModel { - /// The definition-registered rule for applying `operation` to an - /// approximate input. `None` means the function is exact only over exact - /// inputs; callers must fail closed for approximate input. - fn exact_operation_rule(&self, _operation: &ExactOperation) -> Option { - None - } - /// The guarantee of reading `query` out of a summary of family `family` /// built over an **exact** input — derived from the family's committed /// parameters by inverting the same sizing formulas @@ -54,17 +36,6 @@ pub trait AccuracyModel { query: &SketchStatistic, ) -> Option; - /// Compose `inputs`' guarantees (in the parent's child order) with the - /// parent's own `local` guarantee under `op`. `Err` is the fail-closed - /// answer: no registered rule, or a missing input guarantee. - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - stats: &PropagationStats, - ) -> Result; - /// Compare the dimensions requested by `target`. Unknown required /// dimensions fail; selection separately excludes missing accuracy evidence. fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool; @@ -98,7 +69,7 @@ pub trait AccuracyModel { } } -/// The built-in estimator and composition models, with conservative target checks. +/// The built-in estimator models, with conservative target checks. #[derive(Debug, Default, Clone, Copy)] pub struct DefaultAccuracyModel; @@ -119,9 +90,6 @@ impl DefaultAccuracyModel { } impl AccuracyModel for DefaultAccuracyModel { - fn exact_operation_rule(&self, operation: &ExactOperation) -> Option { - composition::exact_operation_rule(operation) - } fn local_guarantee( &self, family: &FieldDataType, @@ -129,15 +97,6 @@ impl AccuracyModel for DefaultAccuracyModel { ) -> Option { estimators::local_guarantee(family, query) } - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - stats: &PropagationStats, - ) -> Result { - composition::propagate(op, inputs, local, stats) - } fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { let within = |value: Option, limit: f64| { value.is_some_and(|v| v <= limit * (1.0 + SATISFACTION_TOLERANCE) + f64::EPSILON) @@ -153,8 +112,6 @@ impl AccuracyModel for DefaultAccuracyModel { } } -pub(crate) use estimators::topk_capacity; - #[cfg(test)] mod tests { use super::*; diff --git a/crates/logical-optimizer/src/lib.rs b/crates/logical-optimizer/src/lib.rs index 6ca519c8..1cec80c4 100644 --- a/crates/logical-optimizer/src/lib.rs +++ b/crates/logical-optimizer/src/lib.rs @@ -1,20 +1,17 @@ //! `asap-logical-optimizer` — #509 Stage 1: logical candidate generation. //! //! It takes the pre-ASAP [`OperatorNode`](asap_types::ir::OperatorNode) DAGs a -//! front end produces and proposes the logical alternatives for each target -//! sub-DAG: which summary (if any) realizes each approximate intent, and which -//! semantic rewrites, roll-ups, groupings and exact compositions apply. It -//! never prices a plan: only Stage 3 uses the cost model (#572, decision -//! Q36(a)). Cargo enforces the stage order: this crate depends only on -//! `asap-types`, never on a front end, a later stage or the executor. -//! -//! - [`pass1`] — local alternatives per target sub-DAG. +//! front end produces and lists the logical alternatives for each target +//! sub-DAG: which summary (if any) realizes each aggregate intent. It never +//! prices a plan: only Stage 3 uses the cost model (#572, decision Q36(a)). +//! Cargo enforces the stage order: this crate depends only on `asap-types`, +//! never on a front end, a later stage or the executor. +//! +//! - [`pass1`] — local alternatives per target sub-DAG +//! ([`pass1::logical_candidates`], the stage pipeline's Stage 1 entry point). //! - [`pass2`] — ASAP-aware sharing across targets. //! - [`accuracy`] — the analytical accuracy model: per-family error bounds and -//! sizing ([`accuracy::estimators`]), propagation through compositions -//! ([`accuracy::composition`]) and error-budget allocation -//! ([`accuracy::allocation`]). Candidates no analytical rule can prove -//! invalid are kept. +//! sizing ([`accuracy::estimators`]). //! //! **Common sub-expression elimination (CSE) of identical sub-DAGs is not //! implemented here.** It runs over the pre-ASAP IR itself @@ -23,31 +20,6 @@ //! pipeline keeps a variant with and without it. Pass 2 also recognizes //! sharing that is invisible at that level, such as `Quantile(x, 0.99)` and //! `Quantile(x, 0.95)` reading one built sketch. -//! -//! ## Candidate search -//! -//! Search returns [`CandidateLogicalASAPDAGs`](pass1::replacement::CandidateLogicalASAPDAGs), -//! a compact logical choice space with one [`TargetSubDAGCandidates`] per -//! target sub-DAG. [`ReplacementStrategy`] implementations propose local -//! alternatives; search applies the applicable semantic and accuracy checks. -//! Candidate presence does not certify physical deployability or an unknown -//! accuracy guarantee. [`GlobalSelection`] assembles a DAG from given per-target -//! choices; choosing them is a later stage's job. -//! -//! | Term | Meaning | Entry point | -//! |---|---|---| -//! | Realization | Enumerate the physical forms for one aggregate intent | `pass1::replacement::realizations_for_intent` | -//! | Replacement | Construct each candidate summary sub-DAG | [`ASAPStrategies`] | -//! | Search | Enumerate alternatives across a workload | [`search_workload`] | -//! | Local candidates | The #509 stage pipeline's Stage 1 entry point | [`pass1::logical_candidates`] | -//! -//! [`Matcher`] asks whether an already available `Realization` satisfies a -//! required one. It has no shipped implementation: which realizations are -//! available is a deployment's concern. -//! -//! [`explanation`](pass1::explanation) reports, for each discovered target -//! with a non-trivial candidate list, why a replacement exists, reusing the -//! candidate's own rationale. pub mod accuracy; pub mod pass1; @@ -56,25 +28,7 @@ pub mod pass2; mod test_support; pub use accuracy::{ - AccuracyAllocation, AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, - CompositionShape, DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, + AccuracyEvidenceProvider, AccuracyModel, DefaultAccuracyModel, NoAccuracyEvidence, PropagationStats, WorkloadAccuracyEvidence, }; -pub use pass1::exact_composition::{ - ExactComposition, ExactCompositionStrategy, OperationPlacement, -}; -pub use pass1::explanation::{ - explain_replacements, explain_replacements_with, ExplanationKind, ReplacementExplanation, -}; -pub use pass1::grouping::HydraGroupingStrategy; pub use pass1::realization::{has_subpopulations, summary_candidates, Realization}; -pub use pass1::replacement::{ - default_strategies, is_logical_rewrite, search_workload, search_workload_with, - search_workload_with_targets, ASAPStrategies, CandidateLogicalASAPDAGs, GlobalSelection, - Matcher, Proposals, RealizationError, RejectedCandidate, Replacement, ReplacementProvenance, - ReplacementStrategy, ReplacementSubDAG, SharedSubDAGStrategy, TargetSubDAG, - TargetSubDAGCandidates, TargetSubDAGSelection, MAX_SEARCH_ITERATIONS, -}; -pub use pass1::rewrite::{AvgToSumOverCountStrategy, SemanticEquivalentRewriteStrategy}; -pub use pass2::reconciliation::AccuracyReconciliationStrategy; -pub use pass2::topk_reuse::TopKLimitReuseStrategy; diff --git a/crates/logical-optimizer/src/pass1/exact_composition.rs b/crates/logical-optimizer/src/pass1/exact_composition.rs deleted file mode 100644 index db5a0654..00000000 --- a/crates/logical-optimizer/src/pass1/exact_composition.rs +++ /dev/null @@ -1,679 +0,0 @@ -//! [`ExactCompositionStrategy`] composes an exact function with a summary -//! plan across an explicit maintenance/read-time boundary (issue #171). -//! -//! `construct_summary_agg` already nests accumulator realizations, such as -//! KLL over exact `Sum` state or a quantile over `Rate` state. This strategy -//! covers the more general cases where an exact function must consume a -//! summary evaluation, or where a maintained summary consumes the values of an -//! exact function that has no accumulator realization. -//! -//! Both cases use an ordinary `NonASAPOp::Aggregate` node over the child -//! plan. The node carries no timing: it runs when its consumer runs, so the -//! same operator serves both placements and adding a function does not -//! require adding a new physical node type. [`OperationPlacement`] is the -//! search-time placement choice. -//! -//! ## Reference, don't select -//! -//! A composed candidate needs a child plan to compose *with* — the inner -//! quantile's own summary evaluation, say. This strategy deliberately does -//! **not** pick that child itself (the way `construct_summary_agg`'s -//! `realize_child` takes the head of the child's own ranking): a -//! [`Replacement::ExactComposition`] carries only the child *target* -//! (`ExactComposition::child_target`, the same `Rc` whose -//! `TargetSubDAGCandidates` in `CandidateLogicalASAPDAGs` already holds every candidate for it). It is -//! `candidate_selection::global_selection` -//! that commits the compatible parent/child pair — so the child's own -//! cost-model ranking, workload-wide effective consumer count, and shared -//! `Rc` identity (one inner summary serving two outer folds) all stay -//! correct, and a child that is also shared by an unrelated consumer is -//! maintained exactly once. `GlobalSelection::assemble_selected_dag` then links the -//! committed pair into one validated post-ASAP DAG. -//! -//! ## Proposal conditions -//! -//! A candidate is proposed only when all of these hold: -//! -//! - the target is a single-measure, `HAVING`-free exact aggregate; -//! - read-time operation: the child is a bindable aggregate that has at least one -//! evaluation-producing summary implementation (a sketch/sample/wavelet/ -//! model — the shapes a maintained accumulator can't sit above), and the -//! target's grouping keys resolve in the child's output schema; -//! transform: the target is a per-entity exact function with no -//! accumulator form (its only implementation is `PassThrough`); -//! - the exact operator consumes only `Plain` values in its data_state — checked -//! again, structurally, when the pair is composed. -//! -//! Runtime support is not checked here: selection admits a composition only -//! with explicit positive support evidence from its cost model. -//! -//! `avg` gets a read-time operation candidate *and* keeps -//! [`crate::pass1::rewrite::AvgToSumOverCountStrategy`]'s rewrite in the same -//! group; the cost model picks between them, nothing here hard-codes one. -//! -//! ## What this strategy never does -//! -//! - Propose an `ExactRead` for a position beneath a maintained -//! summary — data_state validation at composition rejects it as a typed -//! `RealizationError` regardless. -//! - Decide whether a composition is *worth it*: that is -//! `global_selection`'s job, using the issue's cost-units-per-second -//! formulas (see `cost_model::read_operation_plan_cost_rate` and -//! siblings). Missing statistics keep the conservative kept sub-DAG. - -use asap_types::ir::operator::non_asap::any_measure_filtered; -use std::rc::Rc; - -use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::properties::timing::{planned_data_state, validate_maintained}; -use asap_types::ir::properties::{ - AccuracyError, ExecutionDataState, ExecutionDataStateError, ResultGuarantee, -}; -use asap_types::ir::schema::aggregate_schema::aggregate_output_schema; -use asap_types::ir::schema::Schema; -use asap_types::ir::{NonASAPOp, Operator, OperatorNode, Predicate}; -use asap_types::physical::execution_data_state::lift_plain; -use asap_types::physical::ExactOperationSchemaError; -use asap_types::types::AccuracyTarget; - -use crate::pass1::realization::Realization; -use crate::pass1::replacement::{ - bindable_intent, describe_intent, realizations_for_intent, RealizationError, Replacement, - ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, -}; -use crate::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; - -#[cfg(test)] -use asap_types::ir::properties::ExecutionTiming; - -/// Which side of the maintenance/read boundary an [`ExactComposition`]'s -/// exact function executes on. -/// The exact function an [`ExactComposition`] applies: the parameters of -/// the `NonASAPOp::Aggregate` node the composition builds over its child. -#[derive(Debug, Clone, PartialEq)] -pub enum ExactOperation { - Aggregate { - reduction: Reduction, - measures: Vec, - output_names: Vec, - filters: Vec>, - having: Option, - }, -} - -impl ExactOperation { - /// Output schema of this operation over a child whose edge carries - /// `input` — the same canonical derivation the pre-ASAP `Aggregate` - /// node uses. `Err` when the child carries non-plain state the operator - /// cannot read. - pub fn output_schema(&self, input: &Schema) -> Result { - if !input.is_all_plain() { - return Err(ExactOperationSchemaError::NonPlainInput); - } - let plain = lift_plain(input); - let ExactOperation::Aggregate { - reduction, - measures, - output_names, - .. - } = self; - let out = aggregate_output_schema(&plain, reduction, measures, output_names)?; - Ok(lift_plain(&out)) - } - - fn into_op(self, child: Rc) -> NonASAPOp { - let ExactOperation::Aggregate { - reduction, - measures, - output_names, - filters, - having, - } = self; - NonASAPOp::Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum OperationPlacement { - /// After the child's summary evaluation. - Read, - /// On the maintenance path, feeding - /// maintained state above. - Maintenance, -} - -impl OperationPlacement { - /// The availability the composed operator consumes and produces. - pub fn data_state(self) -> ExecutionDataState { - match self { - Self::Read => ExecutionDataState::QUERY_ROWS, - Self::Maintenance => ExecutionDataState::INGESTION_ROWS, - } - } - - pub fn provenance(self) -> ReplacementProvenance { - match self { - Self::Read => ReplacementProvenance::ValueOperationAtQueryTime, - Self::Maintenance => ReplacementProvenance::ValueOperationAtIngestionTime, - } - } -} - -/// The payload of a [`Replacement::ExactComposition`] candidate: an exact -/// operator, the placement it runs at, and a *reference* to the child target -/// it composes over — never an already-selected child plan (see the module -/// docs' "Reference, don't select"). -#[derive(Debug, Clone)] -pub struct ExactComposition { - pub placement: OperationPlacement, - pub op: ExactOperation, - /// The pre-ASAP child the operator consumes; its `TargetSubDAGCandidates` holds the - /// candidates `global_selection` may commit this composition with. - pub child_target: Rc, - /// The composed node's output schema — the target's own pre-ASAP - /// output schema, lifted with every column `Plain` (an exact operator - /// only ever produces plain values). - pub schema: Schema, -} - -impl ExactComposition { - /// The data state `child` produces when this operation (its consumer) - /// runs at the placement's timing. - fn child_data_state(&self, child: &Rc) -> ExecutionDataState { - planned_data_state(child, self.placement.data_state().timing) - } - - /// Can `child` legally be this composition's input? Phase legality - /// (the child's produced data_state — a kept pre-ASAP sub-DAG takes the - /// phase this edge assigns) plus the plain-operand rule, checked - /// through the same schema derivation [`Self::compose`] uses. - pub fn accepts_child(&self, child: &Rc) -> bool { - self.child_data_state(child) == self.placement.data_state() - && self.op.output_schema(&child.schema).is_ok() - } - - /// Build the composed, data_state-validated node over `child`. Every edge of - /// the result (including everything beneath `child`) is checked by - /// `asap_types::ir::properties::timing::validate_maintained`; an illegal - /// placement is a typed [`RealizationError::ExecutionDataState`], never deferred to a - /// runtime. - pub fn compose(&self, child: Rc) -> Result, RealizationError> { - self.compose_with_accuracy(child, &DefaultAccuracyModel) - } - - /// Compose using the caller's accuracy algebra. Exact operators do not - /// erase an approximate child's error: supported folds propagate it; - /// unsupported folds fail closed with a typed accuracy error. - pub fn compose_with_accuracy( - &self, - child: Rc, - accuracy_model: &dyn AccuracyModel, - ) -> Result, RealizationError> { - let produced = self.child_data_state(&child); - if produced != self.placement.data_state() { - let edge = match self.placement { - OperationPlacement::Maintenance => "exact operation child (maintenance time)", - OperationPlacement::Read => "exact operation child (read time)", - }; - return Err(RealizationError::ExecutionDataState( - ExecutionDataStateError::IllegalChildDataState { - edge, - child: produced, - }, - )); - } - let schema = self.op.output_schema(&child.schema)?; - let guarantee = match &child.guarantee { - None => None, - Some(input) if input.is_exact() => Some(ResultGuarantee::exact(format!( - "exact function {:?} over exact input", - self.op - ))), - Some(input) => match accuracy_model.exact_operation_rule(&self.op) { - Some(operator) => match accuracy_model.propagate( - &operator, - std::slice::from_ref(input), - None, - &PropagationStats::default(), - ) { - Ok(guarantee) => Some(guarantee), - // The plan remains executable without a declared accuracy - // target, but an unknown guarantee cannot satisfy a later - // target check. Never replace this with an exact/default - // bound. - Err(AccuracyError::UnsupportedComposition { .. }) => None, - Err(error) => return Err(RealizationError::Accuracy(error)), - }, - // No definition-registered rule: preserve "unknown". This is - // the fail-closed value used by accuracy-target filtering. - None => None, - }, - }; - let node = Rc::new( - OperatorNode::with_schema(Operator::NonASAP(self.op.clone().into_op(child)), schema) - .with_guarantee(guarantee), - ); - validate_maintained(&node, self.placement.data_state().timing)?; - Ok(node) - } - - /// Structural identity for `TargetSubDAGCandidates` dedup: same placement, same - /// operator, same child `Rc`. - pub fn same_as(&self, other: &Self) -> bool { - self.placement == other.placement - && self.op == other.op - && Rc::ptr_eq(&self.child_target, &other.child_target) - } -} - -/// Which exact reducers may run as a query-time fold over evaluation rows. -/// `Count` only at `Exact` accuracy (an approximate count is a sketch -/// target, not an exact fold). -fn is_query_time_reducer(intent: &AggIntent) -> bool { - matches!( - intent, - AggIntent::Sum { .. } - | AggIntent::Min { .. } - | AggIntent::Max { .. } - | AggIntent::Avg { .. } - | AggIntent::StdDev { .. } - | AggIntent::Variance { .. } - | AggIntent::Count { - accuracy: AccuracyTarget::Exact - } - ) -} - -/// Does `implementation` need a `SummaryEstimate` evaluation to yield a value -/// — i.e. is it a shape a maintained accumulator can't legally sit above? -fn needs_evaluation(implementation: &Realization) -> bool { - matches!( - implementation, - Realization::Sketch(_) - | Realization::Sample { .. } - | Realization::Wavelet { .. } - | Realization::StatModel { .. } - ) -} - -/// The `(op, child)` of a read-time operation-shaped target, or `None`. -fn query_time_shape(root: &OperatorNode) -> Option<(ExactOperation, Rc, AggIntent)> { - let Some(NonASAPOp::Aggregate { - reduction, - measures, - output_names, - filters, - having: None, - child, - }) = root.non_asap() - else { - return None; - }; - if any_measure_filtered(filters) { - return None; - } - let Reduction::Reduce(by) = reduction else { - return None; - }; - if by.is_without() { - return None; - } - let [intent] = measures.as_slice() else { - return None; - }; - if !is_query_time_reducer(intent) { - return None; - } - let child_intent = bindable_intent(child)?; - if !realizations_for_intent(child_intent) - .iter() - .any(needs_evaluation) - { - return None; - } - // Grouping keys must resolve in the child's output schema — the same - // derivation the composed node's own schema will use. - Some(( - ExactOperation::Aggregate { - reduction: reduction.clone(), - measures: measures.clone(), - output_names: output_names.clone(), - filters: filters.clone(), - having: None, - }, - Rc::clone(child), - intent.clone(), - )) -} - -/// The `(op, child)` of a function-shaped target — a per-entity exact -/// transform with no accumulator form — or `None`. -fn ingestion_time_shape( - root: &OperatorNode, -) -> Option<(ExactOperation, Rc, AggIntent)> { - let Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures, - output_names, - filters, - having: None, - child, - }) = root.non_asap() - else { - return None; - }; - if any_measure_filtered(filters) { - return None; - } - let [intent] = measures.as_slice() else { - return None; - }; - if !intent.is_per_series() { - return None; - } - // Exact accumulators (`Rate`/`Increase`) are already directly nestable - // as `SummaryAgg(ExactAggregate)`; only a pass-through function needs - // an explicit update-path node. - if realizations_for_intent(intent) - .iter() - .any(|i| *i != Realization::PassThrough) - { - return None; - } - Some(( - ExactOperation::Aggregate { - reduction: Reduction::PerEntity, - measures: measures.clone(), - output_names: output_names.clone(), - filters: filters.clone(), - having: None, - }, - Rc::clone(child), - intent.clone(), - )) -} - -/// Proposes [`Replacement::ExactComposition`] candidates — see the module -/// docs. -pub struct ExactCompositionStrategy; - -impl ExactCompositionStrategy { - fn candidates(&self, target: &TargetSubDAG<'_>) -> Vec { - let schema = lift_plain(&target.root.schema); - let mut out = Vec::new(); - - if let Some((op, child, intent)) = query_time_shape(target.root) { - let child_desc = - describe_intent(bindable_intent(&child).expect("checked by query_time_shape")); - out.push(ReplacementSubDAG { - strategy: "ExactCompositionStrategy", - replacement: Replacement::ExactComposition(ExactComposition { - placement: OperationPlacement::Read, - op, - child_target: child, - schema: schema.clone(), - }), - provenance: ReplacementProvenance::ValueOperationAtQueryTime, - rationale: format!( - "{} is an exact fold whose input is the evaluation of {} — a maintained \ - accumulator cannot consume query-time values, so instead of keeping \ - the whole tree pre-ASAP this applies the fold as an \ - ExactRead over whichever summary evaluation global_selection \ - commits for the child target (asap_logical_optimizer::pass1::exact_composition)", - describe_intent(&intent), - child_desc - ), - }); - } - - if let Some((op, child, intent)) = ingestion_time_shape(target.root) { - out.push(ReplacementSubDAG { - strategy: "ExactCompositionStrategy", - replacement: Replacement::ExactComposition(ExactComposition { - placement: OperationPlacement::Maintenance, - op, - child_target: child, - schema, - }), - provenance: ReplacementProvenance::ValueOperationAtIngestionTime, - rationale: format!( - "{} is an exact per-entity function with no accumulator form; as an \ - explicit ExactMaintenance on the update path its output can feed a \ - maintained summary above it instead of being handed over as an opaque \ - raw kept sub_dag (asap_logical_optimizer::pass1::exact_composition)", - describe_intent(&intent) - ), - }); - } - out - } -} - -impl ReplacementStrategy for ExactCompositionStrategy { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - !self.candidates(target).is_empty() - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - self.candidates(target) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pass1::replacement::retain_exact; - use crate::test_support::{agg, agg_per_entity as per_entity, metric_scan, timed}; - use asap_types::ir::operator::agg_intent::default_quantile; - use asap_types::ir::properties::ExecutionDataStateError; - use asap_types::ir::schema::FieldDataType; - use asap_types::ir::ASAPOp; - - /// `max by (zone) (quantile by (zone, host) (m))`. - fn max_over_quantile() -> Rc { - let inner = agg( - vec![2, 3], - default_quantile(0.99), - metric_scan(&["zone", "host"]), - ); - agg(vec![0], AggIntent::Max { col: None }, inner) - } - - #[test] - fn proposes_query_time_operation_for_max_over_quantile() { - let root = max_over_quantile(); - let target = TargetSubDAG::new(&root); - let strategy = ExactCompositionStrategy; - assert!(strategy.matches(&target)); - let candidates = strategy.replacements(&target); - assert_eq!(candidates.len(), 1); - let Replacement::ExactComposition(comp) = &candidates[0].replacement else { - panic!( - "expected a composition, got {:?}", - candidates[0].replacement - ); - }; - assert_eq!(comp.placement, OperationPlacement::Read); - assert_eq!( - candidates[0].provenance, - ReplacementProvenance::ValueOperationAtQueryTime - ); - let Some(NonASAPOp::Aggregate { child, .. }) = root.non_asap() else { - unreachable!() - }; - assert!( - Rc::ptr_eq(&comp.child_target, child), - "the candidate references the child target's own Rc — nothing selected" - ); - let names: Vec<_> = comp.schema.fields.iter().map(|f| f.name.as_str()).collect(); - assert_eq!(names, vec!["zone", "max"]); - } - - #[test] - fn proposes_query_time_operation_for_avg_over_quantile_alongside_the_rewrite() { - let inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); - let root = agg(vec![0], AggIntent::Avg { col: None }, inner); - let target = TargetSubDAG::new(&root); - assert_eq!(ExactCompositionStrategy.replacements(&target).len(), 1); - // `avg` competes with AvgToSumOverCountStrategy in the same group. - assert!(crate::pass1::rewrite::AvgToSumOverCountStrategy.matches(&target)); - } - - #[test] - fn proposes_ingestion_time_operation_for_a_per_entity_pass_through_over_raw_input() { - let root = per_entity(AggIntent::Deriv, metric_scan(&["zone"])); - let target = TargetSubDAG::new(&root); - let candidates = ExactCompositionStrategy.replacements(&target); - assert_eq!(candidates.len(), 1); - assert_eq!( - candidates[0].provenance, - ReplacementProvenance::ValueOperationAtIngestionTime - ); - } - - #[test] - fn does_not_propose_for_shapes_already_covered_by_accumulators() { - // sum by (zone) over an exact Sum child: the child has no evaluation, - // so SummaryAgg(Sum) over SummaryAgg(Sum) is already legal. - let inner = agg( - vec![2, 3], - AggIntent::Sum { col: None }, - metric_scan(&["zone", "host"]), - ); - let root = agg(vec![0], AggIntent::Sum { col: None }, inner); - assert!(!ExactCompositionStrategy.matches(&TargetSubDAG::new(&root))); - // rate is an exact accumulator — directly nestable, no separate value operation. - let rate = per_entity(AggIntent::Rate, metric_scan(&[])); - assert!(!ExactCompositionStrategy.matches(&TargetSubDAG::new(&rate))); - // A sketch-capable outer intent is not an exact fold. - let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["zone"])); - let root = agg(vec![0], default_quantile(0.99), inner); - assert!(!ExactCompositionStrategy.matches(&TargetSubDAG::new(&root))); - } - - #[test] - fn compose_rejects_a_maintained_state_child_for_a_query_time_operation() { - let root = max_over_quantile(); - let target = TargetSubDAG::new(&root); - let candidates = ExactCompositionStrategy.replacements(&target); - let Replacement::ExactComposition(comp) = &candidates[0].replacement else { - unreachable!() - }; - // A bare SummaryAgg (state, no evaluation) is not a legal read-time operation - // input — the operator would be consuming sketch state. - let state_child = crate::pass1::replacement::realize_child(&comp.child_target).unwrap(); - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &state_child.operator - else { - panic!("expected the child to realize to a evaluation"); - }; - assert!(!comp.accepts_child(summary_input)); - assert!(matches!( - comp.compose(Rc::clone(summary_input)), - Err(RealizationError::ExecutionDataState( - ExecutionDataStateError::IllegalChildDataState { .. } - )) - )); - // The evaluation itself is accepted and composes to a plain schema. - assert!(comp.accepts_child(&state_child)); - let composed = comp.compose(state_child).unwrap(); - assert!( - composed.guarantee.is_none(), - "rank error has no registered conversion through max" - ); - assert!(matches!( - composed.operator, - Operator::NonASAP(NonASAPOp::Aggregate { .. }) - )); - // Timing is no longer stored by composition: under the default - // materialization assignment the composed read-time operation runs at - // query time. - assert_eq!(timed(&composed).timing, Some(ExecutionTiming::QueryTime)); - assert!(composed - .schema - .fields - .iter() - .all(|f| matches!(f.dtype, FieldDataType::Plain(_)))); - } - - #[test] - fn compose_rejects_a_evaluation_child_for_a_ingestion_time_operation() { - let inner = agg(vec![2], default_quantile(0.99), metric_scan(&["zone"])); - let root = per_entity(AggIntent::Deriv, inner); - let candidates = ExactCompositionStrategy.replacements(&TargetSubDAG::new(&root)); - let Replacement::ExactComposition(comp) = &candidates[0].replacement else { - unreachable!() - }; - let evaluation = crate::pass1::replacement::realize_child(&comp.child_target).unwrap(); - assert!(!comp.accepts_child(&evaluation)); - assert!(matches!( - comp.compose(evaluation), - Err(RealizationError::ExecutionDataState( - ExecutionDataStateError::IllegalChildDataState { .. } - )) - )); - // Raw update input is fine. - let raw = retain_exact(&comp.child_target).unwrap(); - assert!(comp.accepts_child(&raw)); - // Timing is no longer stored by composition: the composition's - // placement is maintenance time, the composed exact operation is a - // plain Aggregate over the raw rows, and it is legal (and planned to - // run) at ingestion time. - assert_eq!(comp.placement, OperationPlacement::Maintenance); - let composed = comp.compose(raw).unwrap(); - assert!(matches!( - composed.operator, - Operator::NonASAP(NonASAPOp::Aggregate { .. }) - )); - validate_maintained(&composed, ExecutionTiming::IngestionTime).unwrap(); - assert_eq!( - planned_data_state(&composed, ExecutionTiming::IngestionTime).timing, - ExecutionTiming::IngestionTime - ); - } - - fn max_op(by: Vec) -> ExactOperation { - ExactOperation::Aggregate { - reduction: Reduction::by(by), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - filters: vec![], - having: None, - } - } - - #[test] - fn exact_operator_schema_matches_pre_asap_aggregate_derivation() { - let child = lift_plain(&metric_scan(&["zone"]).schema); - let out = max_op(vec![2]).output_schema(&child).unwrap(); - let names: Vec<_> = out.fields.iter().map(|f| f.name.as_str()).collect(); - assert_eq!(names, vec!["zone", "max"]); - assert!(out.is_all_plain()); - } - - #[test] - fn exact_operator_rejects_non_plain_input() { - let state = Schema::lifted( - vec![asap_types::ir::schema::Field::new( - "state", - FieldDataType::ExactAggregate( - asap_types::ir::schema::ExactKind::Sum, - asap_types::ir::schema::ExactParams::Sum, - ), - false, - )], - None, - ); - assert!(matches!( - max_op(vec![]).output_schema(&state), - Err(ExactOperationSchemaError::NonPlainInput) - )); - } -} diff --git a/crates/logical-optimizer/src/pass1/explanation.rs b/crates/logical-optimizer/src/pass1/explanation.rs deleted file mode 100644 index 8392aa22..00000000 --- a/crates/logical-optimizer/src/pass1/explanation.rs +++ /dev/null @@ -1,786 +0,0 @@ -//! This crate's **explanation of a replacement**: for a `TargetSubDAG` that -//! [`crate::pass1::replacement::search_workload`] found something to say about, why -//! does that candidate exist? (issue #33: "Add logic to detect which -//! optimizations are applicable to a query workload"; this module: issue -//! #257.) -//! -//! This module does not answer "is optimization X applicable here, yes or -//! no" — that framing implies a classifier deciding admissibility from -//! scratch. What it actually does is narrower and more mechanical: reuse a -//! matching candidate's own [`crate::pass1::replacement::ReplacementSubDAG::rationale`] -//! to explain, in the candidate's own words, why a [`Replacement`] exists at -//! a given target. No new prose is invented here; see "The reframing" below -//! for exactly what's being reused and why. -//! -//! ## The reframing: an explanation *is* "this `TargetSubDAG`'s candidate -//! list is non-trivial" -//! -//! Earlier (PR #247, superseded by this module — see "What this replaces" -//! below), "is optimization X applicable here?" was a yes/no fact each rule -//! re-derived by walking the DAG itself. That made sense before there was -//! any other structure to consult. But [`crate::pass1::replacement::search_workload`] -//! (issue #252) now *already* computes, for every -//! `TargetSubDAG` in the workload, every -//! semantically valid [`crate::pass1::replacement::ReplacementSubDAG`] a registered -//! [`ReplacementStrategy`] can propose — a [`CandidateLogicalASAPDAGs`] of [`TargetSubDAGCandidates`]s. A -//! rule re-deriving the same yes/no fact from scratch would be answering a -//! question the search already answered, via a second, independently -//! maintained traversal that has to keep agreeing with the first one. -//! -//! Once that candidate space exists, "which optimizations are applicable" -//! collapses into a single question this module asks of *that* data instead: -//! **for a given `TargetSubDAG`, does its candidate list contain anything -//! other than the trivial, no-op realization?** A `TargetSubDAG` whose only -//! candidate is "the one thing `ASAPStrategies` would have committed -//! to anyway, with no alternative" has no optimization to report — that -//! candidate isn't an *opportunity*, it's just the target's existing shape -//! reflected back. A `TargetSubDAG` with more than one candidate (several -//! sketch families to choose between), or one candidate that is itself a -//! genuine alternative to the status quo (share this already-shared sub-DAG -//! instead of recomputing it at every consumer), *is* an applicability -//! finding — [`explain_replacements`] and -//! [`explain_replacements_with`] just translate [`CandidateLogicalASAPDAGs`]'s -//! [`TargetSubDAGCandidates`]s into that shape: -//! -//! - [`ExplanationKind::SketchApproximation`] — the `TargetSubDAG`'s -//! candidate list contains at least one summary-realization [`Replacement::SubDAG`] that -//! actually realizes a sketch family (`FieldDataType::Sketch`), i.e. -//! [`ASAPStrategies`] found something to offer beyond whatever -//! exact/pass-through candidate [`crate::pass1::replacement`]'s own -//! `realizations_for_intent` would have committed to on its own. -//! - [`ExplanationKind::CommonSubexpressionReuse`] — the `TargetSubDAG` -//! has two or more consumers *and* its candidate list contains the -//! [`SharedSubDAGStrategy`] "build once and share" candidate (the one -//! whose `Rc` is the group's own `target`) — i.e. sharing this sub-DAG -//! instead of recomputing it independently is a real, reported choice, not -//! just an accident of how the workload happened to be built. -//! -//! Each finding's `reason` is literally the matching candidate's own -//! [`crate::pass1::replacement::ReplacementSubDAG::rationale`] (joined, if more than one candidate -//! qualifies) — this module invents no new prose to explain *why* a -//! candidate is valid; that explanation already exists on the candidate a -//! [`ReplacementStrategy`] produced, and repeating it here (rather than -//! re-describing the same fact in different words) keeps exactly one place -//! that has to be right about "why is this a valid alternative". -//! -//! ## What this replaces, and what carries over unmodified -//! -//! [`ReplacementExplanation`] and [`ExplanationKind`] keep PR #247's original -//! shape and contract — a struct/enum pair meant for a downstream -//! DAG-visualization consumer, `#[non_exhaustive]` discipline (only an -//! optimization backed by a real, registered [`ReplacementStrategy`] gets a -//! variant; see the catalog table below for everything still deliberately -//! unrepresented). So do the two top-level entry points, -//! [`explain_replacements`] and [`explain_replacements_with`] -//! — same "workload roots in, findings out" contract, mirroring -//! [`crate::pass1::replacement::search_workload`]/[`crate::pass1::replacement::search_workload_with`]'s -//! own signature shape. Only the *data source* changed: this module now -//! calls those two functions and translates the result, rather than running -//! its own rules and their supporting traversal over the DAG a second time. -//! All of that old traversal is deleted, not kept alongside the new -//! implementation — see "Two guarantees the old traversal made, re-verified" -//! below for the two properties it's important that deletion didn't quietly -//! lose. -//! -//! ## Why a second, applicability-specific rule trait doesn't exist here -//! -//! PR #247 gave this module its own extension-point trait, `ApplicabilityRule` -//! (`fn optimization(&self) -> ExplanationKind` + `fn evaluate(&self, roots) -//! -> Vec`), the same shape `cost_model::CostModel` -//! and [`crate::pass1::replacement::Matcher`] use elsewhere in this crate. Once -//! findings are a *view* over [`CandidateLogicalASAPDAGs`] rather than an independent -//! computation, that trait would be a second extension point answering a -//! question [`ReplacementStrategy`] (issue #251) already answers: "does this -//! `TargetSubDAG` have an alternative worth reporting, and why". A caller who -//! wants a new optimization represented as a finding needs a new -//! `impl ReplacementStrategy` wired into -//! [`crate::pass1::replacement::search_workload_with`]'s strategy set *regardless* -//! (that's the only way its candidates end up in the [`CandidateLogicalASAPDAGs`] this -//! module reads) — adding an `ApplicabilityRule` too would mean maintaining -//! two extension points for the same new capability, one of which (the rule) -//! would just be re-describing candidates the other (the strategy) already -//! produced. So this module ships no extension-point trait of its own: -//! [`ReplacementStrategy`] already *is* that extension point, one layer -//! down, and [`explain_replacements_with`]'s own `strategies` -//! parameter is where a caller plugs in a custom one — the identical spot -//! [`crate::pass1::replacement::search_workload_with`] itself exposes. -//! -//! ## Two guarantees the old traversal made, re-verified against the new one -//! -//! 1. **A finding is reported at the maximal `TargetSubDAG`, never once more -//! per subsumed descendant.** [`crate::pass1::replacement`]'s own -//! `discover_targets` (used by [`crate::pass1::replacement::search_workload_with`], -//! and so by this module) walks every workload root's whole DAG but only -//! *recurses into a node's children the first time that node's `Rc` is -//! seen*; every subsequent occurrence still counts towards -//! `consumer_count`, but never triggers a second descent. A node nested -//! under an already-discovered shared ancestor therefore only becomes its -//! own `TargetSubDAG` if something *outside* that ancestor also -//! references it — identical to PR #247's own discovery pass, which -//! reported "the highest point sharing starts," not a finding at every -//! subsumed level below it. Same guarantee, same mechanism, just living in -//! [`crate::pass1::replacement`] now instead of here. -//! 2. **A node reachable via more than one path is one finding, not one per -//! path.** [`TargetSubDAGCandidates`]s are keyed by `Rc` pointer identity in -//! [`CandidateLogicalASAPDAGs`]'s internal map — there is exactly one group per distinct -//! `Rc`, full stop, so a shared `Aggregate` reached via two different -//! `BinaryOp` branches (or two different workload roots) is exactly one -//! group, hence at most one [`ExplanationKind::SketchApproximation`] -//! finding, no matter how many paths reach it. -//! [`tests::a_shared_sketchable_aggregate_is_reported_only_once`] pins -//! this directly. -//! -//! ## One thing [`CandidateLogicalASAPDAGs`] doesn't carry that this module still needs: -//! human-readable `location` text -//! -//! [`TargetSubDAGCandidates`]/[`CandidateLogicalASAPDAGs`] deliberately track only `Rc` -//! pointer identity — the currency the search itself needs — not -//! caller-facing prose. [`ReplacementExplanation::location`] is prose (a -//! breadcrumb like `root "dash_a" > lhs`), so this module keeps one small, -//! self-contained walk of its own, [`collect_locations`], whose *only* job -//! is turning "this `Rc`" into "the human-readable place(s) it occurs" for a -//! finding already decided by [`CandidateLogicalASAPDAGs`]. This is not a reincarnation of -//! the deleted rule traversal: it makes no applicability decision (it runs -//! the same regardless of what any strategy found), and duplicating this -//! small, self-contained shape rather than threading location strings -//! through [`crate::pass1::replacement`]'s own `discover_targets` matches the same -//! call that module's own docs already make for its (test-only) -//! `count_consumers` counterpart — see [`crate::pass1::replacement`]'s "Where -//! `TargetSubDAG` discovery comes from" section. -//! -//! ## Catalog primitives deliberately left as future work -//! -//! The internal catalog -//! (`ProjectASAP/internal-docs/catalog_of_optimizations.md`) lists several -//! primitives with **no [`ReplacementStrategy`] implementation anywhere in -//! this codebase today**. Faking a variant for one of them would report a -//! finding this codebase cannot back with a real candidate, so none of the -//! below get an [`ExplanationKind`] variant yet — each gets one once a real -//! strategy exists and is wired into [`crate::pass1::replacement::default_strategies`]: -//! -//! | Catalog entry | Status | Where a future `ExplanationKind` would come from | -//! |---|---|---| -//! | Semantic-equivalent rewriting (e.g. `avg` → `sum`/`count`) | `AvgToSumOverCountStrategy` exists and is wired into `default_strategies()` (issue #253) — but still no `ExplanationKind` of its own below, since this table is about *direct* findings for a catalog entry, and this strategy's whole point is indirect: its `Replacement::SubDAG` rewrite candidate exposes `sum`/`count` as independently bindable discovered targets, which can then earn `CommonSubexpressionReuse` findings when the workload actually reuses them | A dedicated variant would need `findings_from_candidate_logical_asap_dags` to recognize a `LogicalRewrite`-provenance candidate as a finding in its own right, not just rely on what it exposes downstream | -//! | Roll-ups (fine-to-coarse group-by reuse) | `RollupStrategy`, derived from workload siblings after CSE/target discovery (issue #254) | Any `Replacement::SubDAG` rewrite candidate that rolls a coarse aggregate up from a compatible finer aggregate | -//! | Wavelets/OMP | Params type exists (`WaveletKind`/`WaveletParams`), not reachable (no core `AggIntent` dispatch picks it) | A `ReplacementStrategy` for it, once some intent shape actually maps to `Realization::Wavelet` | -//! | Sampling | Same story as Wavelets: `SamplingKind`/`SamplingParams` exist, unreachable from core dispatch | Same hook as Wavelets, for `Realization::Sample` | -//! | Deep generative compression | No representation at all — no `Realization`/`FieldDataType` variant | Needs a new summary family added to `asap_types::ir::schema::state_type` first | -//! | Approximation frameworks for windows | No representation — `TimeRange`/`PromqlSubquery` windows are always evaluated exactly | Would key off those node types once an approximate-window operator exists | -//! | Function decomposition | No representation anywhere | No hook point identified yet | -//! | Continuous distributed monitoring | No representation — `RepeatingEntry`/`RepetitionInterval` in `asap_types::workload` describe *that* a query repeats, not any monitoring-specific decomposition | Would likely key off `RepeatingEntry` once such logic exists | -//! | Incremental computation across time | No representation — nothing carries state across repeated evaluations of a `RepeatingEntry` today | Would key off `RepeatingEntry` + `TimeShift`/`TimeRange` once incremental state-carry exists | -//! | Delta encoding | No representation — `AggIntent::Delta`/`IDelta` are PromQL *value*-difference semantics, not a wire/storage delta-encoding optimization | Would plug into a future deployment-side wire/storage encoding decision (post-ASAP), not this crate's IR-level dispatch | -//! -//! [`ReplacementStrategy`]: crate::pass1::replacement::ReplacementStrategy -//! [`ReplacementSubDAG`]: crate::pass1::replacement::ReplacementSubDAG -//! [`Replacement`]: crate::pass1::replacement::Replacement -//! [`Replacement::SubDAG`]: crate::pass1::replacement::Replacement::SubDAG -//! [`ASAPStrategies`]: crate::pass1::replacement::ASAPStrategies -//! [`SharedSubDAGStrategy`]: crate::pass1::replacement::SharedSubDAGStrategy -//! [`CandidateLogicalASAPDAGs`]: crate::pass1::replacement::CandidateLogicalASAPDAGs -//! [`TargetSubDAGCandidates`]: crate::pass1::replacement::TargetSubDAGCandidates - -use std::collections::HashMap; -use std::fmt::Display; -use std::rc::Rc; - -use asap_types::ir::cse::{structural_hash, HashCache}; -use asap_types::ir::schema::FieldDataType; -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; - -use crate::pass1::replacement::{ - self, CandidateLogicalASAPDAGs, Replacement, ReplacementStrategy, TargetSubDAGCandidates, -}; - -/// Which kind of replacement a [`ReplacementExplanation`] is about. -/// -/// `#[non_exhaustive]`: only optimizations with a real [`ReplacementStrategy`] -/// behind them get a variant (see the module docs' "Catalog primitives -/// deliberately left as future work" table for everything else in the -/// catalog). -/// -/// [`ReplacementStrategy`]: crate::pass1::replacement::ReplacementStrategy -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[non_exhaustive] -pub enum ExplanationKind { - /// A `TargetSubDAG`'s candidate list contains at least one - /// [`Replacement::SubDAG`] that realizes a sketch family — - /// [`crate::pass1::replacement::ASAPStrategies`] found a genuine sketch - /// alternative for this `Aggregate`, beyond whatever exact/pass-through - /// candidate `crate::pass1::replacement`'s own `realizations_for_intent` would - /// have committed to on its own. - SketchApproximation, - /// A `TargetSubDAG` has two or more consumers *and* its candidate list - /// contains [`crate::pass1::replacement::SharedSubDAGStrategy`]'s "build once - /// and share" candidate — the catalog's cross-statistic / cross-metrics / - /// cross-subpopulation reuse entries, all the same underlying structural - /// fact. - CommonSubexpressionReuse, - /// A `TargetSubDAG`'s candidate list contains at least one - /// [`Replacement::ExactComposition`] — - /// [`crate::pass1::exact_composition::ExactCompositionStrategy`] found an exact - /// operator that can be composed with a summary plan across an explicit - /// update/evaluation boundary instead of keeping the whole tree as it is - /// (issue #171). - ExactComposition, -} - -/// Why a [`Replacement`] of `kind` exists at `location` (a human-readable -/// breadcrumb into the workload — e.g. `root "dashboard_p99"` or -/// `root "ratio" > lhs`): `reason` (human-readable, meant for a report/log, -/// not machine parsing — literally the matching candidate's own -/// [`crate::pass1::replacement::ReplacementSubDAG::rationale`]). -/// -/// `node_hash` is [`structural_hash`](asap_types::ir::cse::structural_hash) -/// of the `TargetSubDAG`'s own `target` sub-DAG. A downstream consumer that -/// hashed the same node can match this explanation by first comparing hashes -/// and then confirming structural equality with -/// [`ReplacementExplanation::target`]. -#[derive(Debug, Clone, PartialEq)] -pub struct ReplacementExplanation { - pub kind: ExplanationKind, - pub location: String, - pub reason: String, - pub node_hash: u64, - /// The exact target expression the explanation describes. Reporting - /// integrations use this together with `node_hash`: the hash narrows the - /// search, and structural equality makes the final match collision-safe. - pub target: Rc, -} - -/// Explain every replacement [`crate::pass1::replacement::search_workload`] finds -/// across a workload's pre-ASAP query roots, using -/// [`crate::pass1::replacement::default_strategies`]. -/// -/// `roots` — like [`crate::pass1::replacement::search_workload`]'s own `Id` type -/// parameter — is caller-chosen: a `QueryWorkload` entry's own key, an index, -/// a query name. It only needs [`Display`], since a finding's `location` is -/// prose, not a structured key back to the caller. -/// -/// Internally runs [`crate::pass1::replacement::search_workload`] to build the -/// candidate-plan space, then reads findings off it — see the module docs' -/// "The reframing" section for what that translation actually checks. -pub fn explain_replacements( - roots: Vec<(Id, Rc)>, -) -> Vec { - explain_replacements_with(roots, &replacement::default_strategies()) -} - -/// Like [`explain_replacements`], but searches with `strategies` -/// instead of [`crate::pass1::replacement::default_strategies`] — the extension -/// point for a deployment-specific [`ReplacementStrategy`]. -/// -/// [`ReplacementStrategy`]: crate::pass1::replacement::ReplacementStrategy -pub fn explain_replacements_with<'s, Id: Display>( - roots: Vec<(Id, Rc)>, - strategies: &[Box], -) -> Vec { - let ided: Vec<(String, Rc)> = roots - .into_iter() - .map(|(id, expr)| (id.to_string(), expr)) - .collect(); - let space = replacement::search_workload_with(ided, strategies); - findings_from_candidate_logical_asap_dags(&space) -} - -/// Translate every discovered [`TargetSubDAGCandidates`] in `space` into zero, one, or two -/// [`ReplacementExplanation`]s (a `TargetSubDAG` can be both sketch-approximable -/// *and* shared — the two optimizations are independent axes, not mutually -/// exclusive). -/// -/// `space`'s own `Id` is always `String` here: [`explain_replacements_with`] -/// already converted the caller's `Id: Display` into a `String` (via -/// `to_string()`) before calling [`crate::pass1::replacement::search_workload_with`], -/// so this function (and [`collect_locations`], which formats `id` with -/// [`std::fmt::Debug`] for the breadcrumb text) doesn't need its own generic -/// `Id` bound. -fn findings_from_candidate_logical_asap_dags( - space: &CandidateLogicalASAPDAGs, -) -> Vec { - let locations = collect_locations(&space.roots); - // One cache for the whole pass — this is a bottom-up pass over every - // discovered group, so amortizing the cache across groups (rather than - // resetting it per group) is real, not just a micro-optimization. - let mut hash_cache = HashCache::new(); - let mut findings = Vec::new(); - for group in space.target_subdag_candidates() { - let location = locations - .get(&Rc::as_ptr(&group.target)) - .map(|locs| locs.join(", ")) - .unwrap_or_default(); - let node_hash = structural_hash(&group.target, &mut hash_cache); - - if let Some(reason) = sketch_finding_reason(group) { - findings.push(ReplacementExplanation { - kind: ExplanationKind::SketchApproximation, - location: location.clone(), - reason, - node_hash, - target: Rc::clone(&group.target), - }); - } - if let Some(reason) = shared_subexpr_finding_reason(group) { - findings.push(ReplacementExplanation { - kind: ExplanationKind::CommonSubexpressionReuse, - location: location.clone(), - reason, - node_hash, - target: Rc::clone(&group.target), - }); - } - if let Some(reason) = composition_finding_reason(group) { - findings.push(ReplacementExplanation { - kind: ExplanationKind::ExactComposition, - location, - reason, - node_hash, - target: Rc::clone(&group.target), - }); - } - } - findings -} - -/// Does `group`'s candidate list contain an exact composition (issue -/// #171)? If so, the finding's `reason` is every such candidate's own -/// `rationale`, joined. -fn composition_finding_reason(group: &TargetSubDAGCandidates) -> Option { - let reasons: Vec<&str> = group - .candidates - .iter() - .filter(|c| matches!(c.replacement, Replacement::ExactComposition(_))) - .map(|c| c.rationale.as_str()) - .collect(); - if reasons.is_empty() { - None - } else { - Some(reasons.join("; ")) - } -} - -/// Does `group`'s candidate list contain a genuine sketch-family realization? -/// If so, the finding's `reason` is every such candidate's own `rationale`, -/// joined — this module does not invent new prose to restate why a candidate -/// is valid. -fn sketch_finding_reason(group: &TargetSubDAGCandidates) -> Option { - let reasons: Vec<&str> = group - .candidates - .iter() - .filter( - |c| matches!(&c.replacement, Replacement::SubDAG(node) if is_sketch_realization(node)), - ) - .map(|c| c.rationale.as_str()) - .collect(); - if reasons.is_empty() { - None - } else { - Some(reasons.join("; ")) - } -} - -/// Does `group` have two or more consumers *and* a "build once and share" -/// candidate (the [`Replacement::SubDAG`] whose `Rc` is the group's own -/// `target`) in its candidate list? If so, the finding's `reason` is that -/// candidate's own `rationale`. -fn shared_subexpr_finding_reason(group: &TargetSubDAGCandidates) -> Option { - if group.consumer_count < 2 { - return None; - } - group - .candidates - .iter() - .find( - |c| matches!(&c.replacement, Replacement::SubDAG(rc) if Rc::ptr_eq(rc, &group.target)), - ) - .map(|c| c.rationale.clone()) -} - -/// Does `node` (unwrapping any `SummaryEstimate` layer, the same shape -/// [`crate::pass1::replacement`]'s own private `sketch_kind_of` unwraps) ultimately -/// realize a [`FieldDataType::Sketch`] family? This module only needs the -/// yes/no fact (a candidate's own `rationale` already names the specific -/// `SketchKind`/`SketchAlgorithm` for a finding's `reason` text), so unlike -/// `replacement.rs`'s counterpart this returns `bool`, not the kind itself. -fn is_sketch_realization(node: &OperatorNode) -> bool { - if node - .guarantee - .as_ref() - .is_some_and(|guarantee| guarantee.is_exact()) - { - return false; - } - match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - is_sketch_realization(summary_input) - } - Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) => { - matches!(family, FieldDataType::Sketch(..)) - } - _ => false, - } -} - -// ── location breadcrumbs ───────────────────────────────────────────────── - -/// Build `location` text for every distinct `TargetSubDAG` reachable from -/// `roots` — see the module docs' "One thing `CandidateLogicalASAPDAGs` doesn't carry" -/// section for why this module needs its own small walk for this. Returns -/// every breadcrumb path that reaches a given `Rc`, not just the first: a -/// shared node referenced from two workload roots (or two branches of one -/// root) needs both breadcrumbs in its finding's `location`, not just one. -fn collect_locations( - roots: &[(String, Rc)], -) -> HashMap<*const OperatorNode, Vec> { - let mut locations: HashMap<*const OperatorNode, Vec> = HashMap::new(); - for (id, root) in roots { - visit(root, format!("root {id:?}"), &mut locations); - } - locations -} - -/// Record `label` as one of `node`'s breadcrumbs, then propagate that path -/// through its children. A shared ancestor is intentionally traversed once -/// per incoming path so every descendant receives every valid breadcrumb. -fn visit( - node: &Rc, - label: String, - locations: &mut HashMap<*const OperatorNode, Vec>, -) { - let ptr = Rc::as_ptr(node); - locations.entry(ptr).or_default().push(label.clone()); - visit_children(node, &label, locations); -} - -/// `node`'s own **relational-skeleton** operator children — the same scope -/// `crate::pass1::replacement`'s own target-discovery `walk_children` (and -/// `asap_types::ir::cse::share_common_sub_dags`) use. Exhaustive over every -/// `NonASAPOp` variant: a new variant fails to compile here until this match -/// is extended too. An ASAP node never occurs in a workload root. -fn visit_children( - node: &OperatorNode, - label: &str, - locations: &mut HashMap<*const OperatorNode, Vec>, -) { - use asap_types::ir::{NonASAPOp::*, ScalarExpr}; - let Operator::NonASAP(op) = &node.operator else { - return; - }; - match op { - Scan { .. } | Values { .. } => {} - PromqlVectorFromScalar(ScalarExpr::PromqlScalarFromVector(c)) => { - visit(c, format!("{label} > child"), locations) - } - PromqlVectorFromScalar(_) => {} - PromqlRelabel { child, .. } - | PromqlInfoEnrich { child, .. } - | PromqlSeriesSample { child, .. } - | Filter { child, .. } - | Project { child, .. } - | Aggregate { child, .. } - | Dedup { child, .. } - | PromqlSubquery { child, .. } - | TimeRange { child, .. } - | TimeShift { child, .. } - | SQLWindowFunc { child, .. } - | Sort { child, .. } - | Limit { child, .. } => visit(child, format!("{label} > child"), locations), - Concat { children, .. } => { - for (i, c) in children.iter().enumerate() { - visit(c, format!("{label} > concat[{i}]"), locations); - } - } - Join { left, right, .. } | SetOp { left, right, .. } => { - visit(left, format!("{label} > left"), locations); - visit(right, format!("{label} > right"), locations); - } - BinaryOp { lhs, rhs, .. } => { - visit(lhs, format!("{label} > lhs"), locations); - visit(rhs, format!("{label} > rhs"), locations); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::agg_intent::{default_quantile, AggIntent}; - use asap_types::ir::operator::operator_properties::{BinaryOpKind, Reduction, Source}; - use asap_types::ir::schema::{DataType, Field, Schema}; - use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, Predicate, ScalarExpr}; - - use asap_types::types::AccuracyTarget; - - fn metric_scan(labels: &[&str]) -> Rc { - let mut columns = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ]; - columns.extend( - labels - .iter() - .map(|n| Field::plain(*n, DataType::Utf8, true)), - ); - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index(columns, 0, vec![]), - })) - .unwrap() - } - - fn agg(by: Vec, intent: AggIntent, child: Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child, - })) - .unwrap() - } - - fn binary( - kind: BinaryOpKind, - lhs: Rc, - rhs: Rc, - ) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - kind, - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - return_bool: false, - lhs, - rhs, - })) - .unwrap() - } - - // ── SketchApproximation ────────────────────────────────────────────── - - #[test] - fn approximate_quantile_is_a_sketch_applicability_finding() { - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let findings = explain_replacements(vec![("dashboard_p99", q)]); - let sketch: Vec<_> = findings - .iter() - .filter(|f| f.kind == ExplanationKind::SketchApproximation) - .collect(); - assert_eq!( - sketch.len(), - 1, - "expected one sketch finding, got {findings:?}" - ); - assert!(sketch[0].location.contains("dashboard_p99")); - assert!(sketch[0].reason.to_lowercase().contains("kll")); - } - - #[test] - fn exact_quantile_is_not_a_sketch_applicability_finding() { - let q = agg( - vec![2], - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }, - metric_scan(&["job"]), - ); - let findings = explain_replacements(vec![("exact_p99", q)]); - assert!( - findings - .iter() - .all(|f| f.kind != ExplanationKind::SketchApproximation), - "an Exact accuracy target must not report sketch-applicability, got {findings:?}" - ); - } - - #[test] - fn nested_aggregate_still_finds_the_inner_sketchable_node() { - // avg(quantile(0.9, sum by (job) (m))) shaped test isn't representable - // (avg is PassThrough, not a wrapper we recurse through structurally - // the way an Aggregate's own child is) — instead nest a sketchable - // quantile under an exact sum, the same nesting replacement.rs's own - // nested_aggregates_bind_per_node test uses. - let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let outer = agg(vec![], default_quantile(0.9), inner); - let findings = explain_replacements(vec![("q", outer)]); - let sketch_count = findings - .iter() - .filter(|f| f.kind == ExplanationKind::SketchApproximation) - .count(); - assert_eq!( - sketch_count, 1, - "expected the outer quantile only, got {findings:?}" - ); - } - - #[test] - fn pass_through_intent_reports_no_sketch_finding() { - let q = agg(vec![2], AggIntent::Avg { col: None }, metric_scan(&["job"])); - let findings = explain_replacements(vec![("avg_latency", q)]); - assert!(findings - .iter() - .all(|f| f.kind != ExplanationKind::SketchApproximation)); - } - - /// A sketch-applicable `Aggregate` reachable via two paths that CSE - /// collapses onto one `Rc` — the same `median(x) == median(x)` shape - /// `ir::cse`'s own `single_query_shares_its_own_repeated_sub-DAG` - /// test uses — must be reported once, not once per path: it is exactly - /// one [`crate::pass1::replacement::TargetSubDAGCandidates`], keyed by `Rc` pointer identity, - /// not one per path that reaches it. - #[test] - fn a_shared_sketchable_aggregate_is_reported_only_once() { - let quantile = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let root = binary( - BinaryOpKind::Compare(asap_types::ir::scalar::CompareOpKind::Eq), - Rc::clone(&quantile), - quantile, - ); - let findings = explain_replacements(vec![("ratio", root)]); - let sketch: Vec<_> = findings - .iter() - .filter(|f| f.kind == ExplanationKind::SketchApproximation) - .collect(); - assert_eq!( - sketch.len(), - 1, - "a single shared Aggregate must produce one finding, not one \ - per path that reaches it: got {findings:?}" - ); - } - - // ── CommonSubexpressionReuse ───────────────────────────────────────── - - #[test] - fn two_roots_with_the_same_grouped_aggregate_share_a_reuse_finding() { - // Grouped (`by (job)`), so the shared `Aggregate`'s output schema - // carries a provable unique key — share_common_sub_dags's legality - // gate — and identical across both roots, so it is shareable. - let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let findings = explain_replacements(vec![("dash_a", a), ("dash_b", b)]); - let reuse: Vec<_> = findings - .iter() - .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) - .collect(); - assert_eq!( - reuse.len(), - 1, - "expected one reuse finding, got {findings:?}" - ); - assert!(reuse[0].location.contains("dash_a")); - assert!(reuse[0].location.contains("dash_b")); - } - - #[test] - fn descendant_of_a_shared_root_keeps_every_root_breadcrumb() { - let inner = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let outer = agg(vec![0], AggIntent::Sum { col: None }, inner); - let findings = explain_replacements(vec![("dash_a", Rc::clone(&outer)), ("dash_b", outer)]); - let inner_sketch = findings - .iter() - .find(|f| { - f.kind == ExplanationKind::SketchApproximation && f.location.contains("child") - }) - .expect("expected the nested sketch explanation"); - assert!(inner_sketch.location.contains("dash_a")); - assert!(inner_sketch.location.contains("dash_b")); - } - - #[test] - fn distinct_queries_report_no_reuse_finding() { - let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let b = agg( - vec![2], - AggIntent::Sum { col: None }, - metric_scan(&["route"]), - ); - let findings = explain_replacements(vec![("dash_a", a), ("dash_b", b)]); - assert!( - findings - .iter() - .all(|f| f.kind != ExplanationKind::CommonSubexpressionReuse), - "structurally different queries must not report reuse, got {findings:?}" - ); - } - - #[test] - fn ungrouped_identical_aggregates_are_not_shareable_so_no_finding() { - // Empty `by`: no provable unique key — share_common_sub_dags never - // hoists these, so consumer_count stays 1 for each and this module - // must not report a finding either. - let a = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let b = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let findings = explain_replacements(vec![("a", a), ("b", b)]); - assert!(findings - .iter() - .all(|f| f.kind != ExplanationKind::CommonSubexpressionReuse)); - } - - #[test] - fn single_query_repeated_subexpression_is_a_reuse_finding() { - // The same shared branch appearing twice within one query (an `a/a` - // shape) — single-query CSE. - let branch = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let q = binary( - BinaryOpKind::Arithmetic(asap_types::ir::scalar::ArithmeticOpKind::Div), - Rc::clone(&branch), - branch, - ); - let findings = explain_replacements(vec![("ratio", q)]); - let reuse: Vec<_> = findings - .iter() - .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) - .collect(); - assert_eq!( - reuse.len(), - 1, - "expected one reuse finding, got {findings:?}" - ); - assert!(reuse[0].location.contains("lhs")); - assert!(reuse[0].location.contains("rhs")); - } - - /// A shared node nested three levels under two *different*, unshared - /// `Filter` parents (mirrors `crate::pass1::replacement::tests:: - /// nested_shared_sub-DAG_below_an_unshared_parent_is_still_discovered`) - /// must still be exactly one finding — the maximal-`TargetSubDAG` - /// guarantee the module docs describe, now provided by - /// `crate::pass1::replacement`'s own target discovery rather than this module's - /// (deleted) traversal. - #[test] - fn a_deeply_shared_sub_dag_under_different_parents_is_reported_once() { - use asap_types::ir::scalar::ScalarValue; - - let shared = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let root_a = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Int64(1))), - child: Rc::clone(&shared), - })) - .unwrap(); - let root_b = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Int64(2))), - child: shared, - })) - .unwrap(); - let findings = explain_replacements(vec![("a", root_a), ("b", root_b)]); - let reuse: Vec<_> = findings - .iter() - .filter(|f| f.kind == ExplanationKind::CommonSubexpressionReuse) - .collect(); - assert_eq!( - reuse.len(), - 1, - "a node shared under two different parents must be one finding, got {findings:?}" - ); - assert!(reuse[0].location.contains('a')); - assert!(reuse[0].location.contains('b')); - } -} diff --git a/crates/logical-optimizer/src/pass1/function_rules.rs b/crates/logical-optimizer/src/pass1/function_rules.rs deleted file mode 100644 index 3c83f2e2..00000000 --- a/crates/logical-optimizer/src/pass1/function_rules.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! Function facts shared by value-operation propagation and accumulator realization. -//! Runtime support remains a deployment decision in `CostModel`. -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::CompositionOperator; -use asap_types::ir::schema::{ExactKind, ExactParams}; - -pub(crate) struct FunctionRules { - pub accuracy: CompositionOperator, - pub accumulator: Option<(ExactKind, ExactParams)>, -} - -/// Unregistered functions have no approximate-input propagation rule. -pub(crate) fn function_rules(intent: &AggIntent) -> Option { - let (accuracy, accumulator) = match intent { - AggIntent::Sum { .. } => ( - CompositionOperator::ExactSum, - Some((ExactKind::Sum, ExactParams::Sum)), - ), - AggIntent::Min { .. } => ( - CompositionOperator::ExactExtremum, - Some((ExactKind::Min, ExactParams::Min)), - ), - AggIntent::Max { .. } => ( - CompositionOperator::ExactExtremum, - Some((ExactKind::Max, ExactParams::Max)), - ), - AggIntent::Avg { .. } => (CompositionOperator::ExactAverage, None), - AggIntent::Rate => ( - CompositionOperator::CounterRate, - Some((ExactKind::Rate, ExactParams::Rate)), - ), - AggIntent::IRate => ( - CompositionOperator::InstantCounterRate, - Some((ExactKind::IRate, ExactParams::IRate)), - ), - AggIntent::Increase => ( - CompositionOperator::CounterIncrease, - Some((ExactKind::Increase, ExactParams::Increase)), - ), - _ => return None, - }; - Some(FunctionRules { - accuracy, - accumulator, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - // The maintained extrema state must encode the direction independently of query text. - #[test] - fn minimum_and_maximum_have_distinct_accumulator_contracts() { - assert_ne!( - function_rules(&AggIntent::Min { col: None }) - .unwrap() - .accumulator, - function_rules(&AggIntent::Max { col: None }) - .unwrap() - .accumulator - ); - } -} diff --git a/crates/logical-optimizer/src/pass1/grouping.rs b/crates/logical-optimizer/src/pass1/grouping.rs deleted file mode 100644 index 5ec864cd..00000000 --- a/crates/logical-optimizer/src/pass1/grouping.rs +++ /dev/null @@ -1,671 +0,0 @@ -//! `GroupingStrategy` (issue #256, part of #33): the axis deciding whether a -//! grouped aggregate's summary state is built as one independent instance -//! per `by` subpopulation (today's only, implicit behavior) or as one -//! shared Hydra-family structure serving all of them — orthogonal to -//! *which* summary family/kind answers the intent, the same way -//! [`asap_types::ir::schema::GroupingStrategy`]'s own doc explains. -//! -//! ## Placement: planning metadata and edge-state type -//! -//! `ASAPOp::SummaryAgg` carries the grouping choice next to the -//! `Reduction` whose `by` keys determine legality. The same choice is also -//! committed to `FieldDataType::Sketch` on the aggregate's output edge. -//! That duplication is intentional: the node field makes the choice easy to -//! inspect during planning, while the edge type ensures an independent KLL/ -//! CMS state and a Hydra-backed state cannot be accepted as compatible inputs -//! to a downstream `SummaryMerge`. [`with_grouping`] updates both atomically. -//! -//! ## Legality vs. cost (same split [`crate::pass1::replacement::realizations_for_intent`] -//! already draws) -//! -//! This module only answers "is `SharedMultiSubpopulation` valid here at -//! all", never "is it worth it": -//! -//! - **Non-empty `by`** ([`has_subpopulations`]): an aggregate with no -//! subpopulation concept (a global reduction, or a per-entity reduction -//! with no grouping concept at all) has nothing for a -//! shared-multi-subpopulation structure to multiplex across. -//! - **The family has a Hydra variant** -//! ([`asap_types::ir::schema::hydra_kind_for`]): `Cms` and `CountSketch` -//! have structural Hydra mappings. `HydraKll` remains an explicit -//! experimental IR value, but the paper excludes quantiles and search -//! therefore never emits it. The shared-grid term is represented -//! symbolically; missing statistics leave an uncertified candidate visible. -//! -//! Whether Hydra is *worth it* for a given estimated subpopulation -//! cardinality is a cost-model question, deliberately out of scope here — -//! candidates with missing error evidence remain visible for downstream review. -//! -//! ## No `ForceSketchKind`-style steering — bind one already-known candidate directly -//! -//! An earlier draft of this module (written against the very first draft of -//! #251) reused a `CostModel`-wrapping adapter that "steered" a -//! whole-recursive-bind decision procedure toward a specific `SketchKind`, -//! the same pattern [`crate::pass1::replacement::ASAPStrategies`]'s own module -//! docs explain was deliberately deleted from this crate as an anti-pattern: -//! forcing a choice via a whole-DAG `CostModel` adapter had a real bug where -//! the forced choice could leak into a target's own nested aggregates. This -//! module never needs that: [`crate::pass1::replacement::realizations_for_intent`] -//! already returns every ranked candidate `Realization` directly, so -//! [`build_candidate`](HydraGroupingStrategy::build_candidate) just finds the -//! one whose `Realization::Sketch(kind)` has `kind.algorithm()` matching -//! the Hydra-eligible `sketch_kind` it's building a candidate for, and -//! passes that exact, -//! already-decided `Realization` to -//! [`crate::pass1::replacement::construct_summary`] — the same first-class, -//! one-candidate-at-a-time primitive [`crate::pass1::replacement::ASAPStrategies`] -//! itself calls once per candidate. No adapter, no steering, no risk of a -//! forced choice leaking into nested aggregates. -//! -//! ## Cross-axis legality with roll-up (issue #254) -//! -//! Roll-up and Hydra currently operate on disjoint candidates. Roll-up -//! rewrites exact `Sum`/`Min`/`Max`/`Count` aggregates in the pre-ASAP DAG; -//! Hydra is offered only for approximate quantile/count intents (KLL, CMS, -//! Count-Sketch) and produces a terminal post-ASAP summary candidate. -//! Consequently neither strategy can -//! presently offer the other's candidate as a source. If roll-up support is -//! extended to mergeable sketches, that extension must consult -//! `rollup::is_legal_rollup_source` and add explicit Hydra merge semantics; -//! grouping alone must not imply that a sketch can be rolled up. - -use std::rc::Rc; - -use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::properties::{AccuracyError, CompositionOperator}; -use asap_types::ir::schema::{ - default_hydra_params, hydra_kind_for, FieldDataType, GroupingStrategy, HydraKind, - SketchAlgorithm, SketchParams, -}; -use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; - -use crate::accuracy::estimators::hydra_guarantee; -use crate::accuracy::{ - AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, PropagationStats, -}; -use crate::pass1::realization::{ - accuracy_target, has_subpopulations, summary_candidates, Realization, -}; -use crate::pass1::replacement::{ - bindable_intent, construct_summary_with, describe_intent, realizations_for_intent, - CandidatePlanningInputs, Proposals, RejectedCandidate, Replacement, ReplacementStrategy, - ReplacementSubDAG, TargetSubDAG, -}; - -/// Wraps the `GroupingStrategy` axis (issue #256) as a -/// [`ReplacementStrategy`]: for a target `ASAPStrategies` -/// already has an opinion on, offers an additional -/// `GroupingStrategy::SharedMultiSubpopulation` candidate wherever the -/// legality conditions in the module docs above hold — alongside, not -/// instead of, the per-subpopulation candidates `ASAPStrategies` -/// itself enumerates. The workload search composes both strategies over the -/// same target, so it sees every summary-family alternative *and* the Hydra -/// alternative; the built-in workload search registers both strategies, and -/// this strategy's own `replacements()` reports only the -/// latter, matching every other strategy in this crate's "one strategy, one -/// concern" shape. -pub struct HydraGroupingStrategy<'a> { - planning_inputs: CandidatePlanningInputs<'a>, -} - -impl Default for HydraGroupingStrategy<'static> { - /// The built-in accuracy models, the same default - /// [`crate::pass1::replacement::ASAPStrategies`] uses. - fn default() -> Self { - Self { - planning_inputs: CandidatePlanningInputs::with_default_accuracy(), - } - } -} - -impl<'a> HydraGroupingStrategy<'a> { - pub fn new_with_planning_inputs_and_evidence( - accuracy_model: &'a dyn AccuracyModel, - allocator: &'a dyn AccuracyBudgetAllocator, - evidence: &'a dyn AccuracyEvidenceProvider, - ) -> Self { - Self { - planning_inputs: CandidatePlanningInputs { - accuracy: accuracy_model, - allocator, - evidence, - }, - } - } - - /// Every legal `SharedMultiSubpopulation` candidate for `target` — empty - /// when `target` isn't a bindable aggregate, has no subpopulation - /// concept, or its intent's candidate summary families have no Hydra - /// variant modeled. - fn hydra_proposals(&self, target: &TargetSubDAG<'_>) -> Proposals { - let mut proposals = Proposals::default(); - let Some(NonASAPOp::Aggregate { reduction, .. }) = target.root.non_asap() else { - return proposals; - }; - if !has_subpopulations(reduction) { - return proposals; - } - let Some(intent) = bindable_intent(target.root) else { - return proposals; - }; - for (sketch_kind, hydra_kind) in summary_candidates(intent) - .iter() - .filter_map(|kind| hydra_kind_for(kind).map(|hydra_kind| (kind.clone(), hydra_kind))) - { - if let Some(candidate) = self.build_candidate( - target.root, - intent, - sketch_kind, - hydra_kind, - &mut proposals.rejected, - ) { - proposals.candidates.push(candidate); - } - } - proposals - } - - /// Find the already-ranked candidate [`Realization::Sketch`] matching - /// `sketch_kind` among [`realizations_for_intent`]'s exhaustive list for - /// `intent`, bind `root` to that exact, already-decided candidate via - /// [`crate::pass1::replacement::construct_summary_with`] (no steering/forcing — see - /// the module docs' "No `ForceSketchKind`-style steering"), then swap the - /// resulting `SummaryAgg`'s `grouping` field from the default - /// `PerSubpopulationInstance` to - /// `SharedMultiSubpopulation { kind: hydra_kind, .. }` — reusing the - /// entire bind decision procedure (schema derivation, column resolution, - /// evaluation construction) unchanged, patching only the one field this - /// axis owns. - fn build_candidate( - &self, - root: &Rc, - intent: &AggIntent, - sketch_kind: SketchAlgorithm, - hydra_kind: HydraKind, - rejected: &mut Vec, - ) -> Option { - let realization = realizations_for_intent(intent) - .into_iter() - .find(|candidate| { - matches!(candidate, Realization::Sketch(kind) if *kind.algorithm() == sketch_kind) - })?; - let node = - construct_summary_with(root, intent, realization, self.planning_inputs, None, None) - .ok()?; - let per_subpopulation_params = per_subpopulation_sketch_params(&node)?; - let params = default_hydra_params(hydra_kind.clone(), &per_subpopulation_params)?; - let grouping = GroupingStrategy::SharedMultiSubpopulation { - kind: hydra_kind.clone(), - params, - }; - - let (family, query) = match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) => match &summary_input.operator { - Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) => (family, Some(query)), - _ => return None, - }, - _ => return None, - }; - let stats = self.planning_inputs.evidence.propagation_stats( - &CompositionOperator::ApproximateAggregate, - family, - query, - ); - if stats - .hydra_shared_grid_collision_bound - .is_some_and(|bound| !bound.is_finite() || bound < 0.0) - || stats - .hydra_shared_grid_failure_probability - .is_some_and(|probability| { - !probability.is_finite() || !(0.0..=1.0).contains(&probability) - }) - { - rejected.push(RejectedCandidate { - strategy: "HydraGroupingStrategy", - description: format!("{hydra_kind:?} over {sketch_kind:?}"), - error: AccuracyError::UnsupportedComposition { - operator: CompositionOperator::ApproximateAggregate, - input_metrics: node - .guarantee - .as_ref() - .map_or_else(Vec::new, |g| vec![g.metric]), - local_metric: None, - reason: "invalid Hydra shared-grid collision or failure-probability evidence" - .into(), - }, - }); - return None; - } - let patched = with_grouping(node, grouping, &stats); - if let (Some(target), Some(guarantee)) = (accuracy_target(intent), &patched.guarantee) { - if !self - .planning_inputs - .accuracy - .satisfies(&guarantee.optimistic_floor(), target) - { - rejected.push(RejectedCandidate { - strategy: "HydraGroupingStrategy", - description: format!("{hydra_kind:?} over {sketch_kind:?}"), - error: AccuracyError::TargetNotSatisfied { - metric: guarantee.metric, - bound: guarantee.bound.evaluate(), - failure_probability: guarantee.failure_probability.evaluate(), - target: target.clone(), - }, - }); - return None; - } - } - Some(ReplacementSubDAG { - strategy: "HydraGroupingStrategy", - replacement: Replacement::SubDAG(patched), - provenance: crate::pass1::replacement::ReplacementProvenance::SummaryRealization, - rationale: format!( - "{} realizes as a shared {hydra_kind:?} structure over {sketch_kind:?} \ - serving every subpopulation of this grouped aggregate, instead of one \ - {sketch_kind:?} instance per distinct `by` key — legal because this \ - aggregate has a non-empty subpopulation concept and {sketch_kind:?} has a \ - modeled Hydra variant (asap_types::ir::schema::hydra_kind_for); whether it's \ - *worth* the shared/independent trade-off for the actual subpopulation \ - cardinality is a CostModel's call, not this strategy's", - describe_intent(intent) - ), - }) - } -} - -impl ReplacementStrategy for HydraGroupingStrategy<'_> { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - let Some(NonASAPOp::Aggregate { reduction, .. }) = target.root.non_asap() else { - return false; - }; - if !has_subpopulations(reduction) { - return false; - } - let Some(intent) = bindable_intent(target.root) else { - return false; - }; - realizations_for_intent(intent) - .into_iter() - .any(|realization| { - matches!(realization, - Realization::Sketch(kind) if hydra_kind_for(kind.algorithm()).is_some()) - }) - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - self.hydra_proposals(target).candidates - } - - fn propose(&self, target: &TargetSubDAG<'_>) -> Proposals { - self.hydra_proposals(target) - } -} - -/// The [`SketchParams`] a bound sketch candidate's `SummaryAgg` committed -/// to, if its family is `Sketch(_)` at all — `None` for any other bound -/// shape (an exact accumulator, a pass-through, or a family with no -/// `SketchParams`), never expected here in practice since `hydra_candidates` -/// only calls this for a `sketch_kind` it already confirmed has a -/// `HydraKind` via `hydra_kind_for`, but degrading to "no candidate" rather -/// than panicking keeps this as conservative as the rest of this module. -/// -/// Deliberately returns the *whole* [`SketchParams`], not one scalar field -/// pulled out of it (an earlier version of this function assumed -/// `SketchParams::Kll { k }` specifically and returned a bare `k: u32`). -/// This axis is a "sketch of sketches" framework: the inner sketch a Hydra -/// structure wraps isn't always KLL, and each [`HydraKind`] variant needs -/// its own inner sketch's own knobs — `HydraCms`/`HydraCountSketch` need -/// (`width`, `depth`), not a `k`. [`default_hydra_params`] is what actually -/// destructures the right variant for `kind`; this function's only job is -/// to find whatever `SketchParams` the bind decision already committed to -/// and hand the whole thing over unchanged. -fn per_subpopulation_sketch_params(node: &OperatorNode) -> Option { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - per_subpopulation_sketch_params(summary_input) - } - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.params().clone()), - _ => None, - } -} - -/// Rebuild `node`, replacing its `SummaryAgg`'s `grouping` field with -/// `grouping` — patching the one field this axis owns onto an -/// already-correctly-bound node rather than re-deriving the rest of it. -/// Recurses through a `SummaryEstimate` evaluation wrapper (the shape every -/// sketch candidate this module builds actually has) to reach the -/// `SummaryAgg` underneath. -fn with_grouping( - node: Rc, - grouping: GroupingStrategy, - stats: &PropagationStats, -) -> Rc { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) => std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: with_grouping(Rc::clone(summary_input), grouping, stats), - query: query.clone(), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.as_ref().map(|g| hydra_guarantee(g, stats))), - ), - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - .. - }) => { - let grouped_family = match family { - FieldDataType::Sketch(kind, _) => { - FieldDataType::Sketch(kind.clone(), grouping.clone()) - } - _ => family.clone(), - }; - let mut grouped_schema = node.schema.clone(); - for field in &mut grouped_schema.fields { - if let FieldDataType::Sketch(kind, _) = &field.dtype { - field.dtype = FieldDataType::Sketch(kind.clone(), grouping.clone()); - } - } - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(child), - family: grouped_family, - input: input.clone(), - reduction: reduction.clone(), - grouping, - filter: None, - }), - grouped_schema, - ) - .with_guarantee(None), - ) - } - // Never reached by this module's own callers (they only ever pass a - // node `construct_summary_with` just bound for a `Sketch` - // candidate, which is always `SummaryAgg` or - // `SummaryEstimate(SummaryAgg)`) — returning the node unchanged - // rather than panicking keeps this as conservative as the rest of - // the module if that ever stops holding. - _ => node, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::accuracy::{DefaultAccuracyModel, EqualSplitAllocator}; - use crate::test_support::{agg, agg_per_entity, metric_scan}; - use asap_types::ir::operator::agg_intent::{default_cardinality, default_quantile}; - use asap_types::ir::operator::operator_properties::Reduction; - use asap_types::types::AccuracyTarget; - - // ── HydraGroupingStrategy ───────────────────────────────────────────── - - #[test] - fn matches_a_grouped_count_with_an_unprovable_accuracy_target() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }; - let q = agg(vec![2], intent, metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - assert!(HydraGroupingStrategy::default().matches(&target)); - } - - #[test] - fn does_not_match_an_empty_by_aggregate() { - // Global reduction — no subpopulation concept, no Hydra alternative. - let q = agg(vec![], default_quantile(0.99), metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let strategy = HydraGroupingStrategy::default(); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn does_not_match_a_per_entity_aggregate() { - let q = agg_per_entity(default_quantile(0.99), metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let strategy = HydraGroupingStrategy::default(); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn does_not_match_a_non_aggregate_node() { - let scan = metric_scan(&["job"]); - let target = TargetSubDAG::new(&scan); - assert!(!HydraGroupingStrategy::default().matches(&target)); - } - - #[test] - fn quantile_has_no_hydra_candidate_without_a_modeled_error_bound() { - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let replacements = HydraGroupingStrategy::default().replacements(&target); - assert!(replacements.is_empty(), "{replacements:?}"); - } - - #[test] - fn count_with_an_accuracy_target_keeps_uncertified_hydra_candidates() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }; - let q = agg(vec![2], intent, metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let replacements = HydraGroupingStrategy::default().replacements(&target); - assert_eq!(replacements.len(), 2, "{replacements:?}"); - assert!(replacements.iter().all(|candidate| matches!( - &candidate.replacement, - Replacement::SubDAG(node) - if node.guarantee.as_ref().is_some_and(|guarantee| - guarantee.bound.evaluate().is_none() - && guarantee.failure_probability.evaluate().is_none()) - ))); - } - - struct ZeroSharedGridEvidence; - - impl AccuracyEvidenceProvider for ZeroSharedGridEvidence { - fn propagation_stats( - &self, - _op: &CompositionOperator, - _family: &FieldDataType, - _query: Option<&asap_types::ir::schema::SketchStatistic>, - ) -> PropagationStats { - PropagationStats { - hydra_shared_grid_collision_bound: Some(0.0), - hydra_shared_grid_failure_probability: Some(0.0), - ..Default::default() - } - } - } - - #[test] - fn hydra_shared_grid_evidence_is_consumed_by_candidate_construction() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }; - let q = agg(vec![2], intent, metric_scan(&["job"])); - let strategy = HydraGroupingStrategy::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &ZeroSharedGridEvidence, - ); - let replacements = strategy.replacements(&TargetSubDAG::new(&q)); - assert_eq!(replacements.len(), 2, "{replacements:?}"); - assert!(replacements.iter().all(|candidate| matches!( - &candidate.replacement, - Replacement::SubDAG(node) - if node.guarantee.as_ref().is_some_and(|g| - g.bound.evaluate().is_some() - && g.failure_probability.evaluate().is_some()) - ))); - } - - #[test] - fn invalid_partial_hydra_evidence_is_rejected_with_a_reason() { - struct InvalidEvidence; - impl AccuracyEvidenceProvider for InvalidEvidence { - fn propagation_stats( - &self, - _op: &CompositionOperator, - _family: &FieldDataType, - _query: Option<&asap_types::ir::schema::SketchStatistic>, - ) -> PropagationStats { - PropagationStats { - hydra_shared_grid_failure_probability: Some(1.5), - ..Default::default() - } - } - } - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }; - let q = agg(vec![2], intent, metric_scan(&["job"])); - let strategy = HydraGroupingStrategy::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &InvalidEvidence, - ); - let proposals = strategy.propose(&TargetSubDAG::new(&q)); - assert!(proposals.candidates.is_empty()); - assert_eq!(proposals.rejected.len(), 2); - assert!(proposals - .rejected - .iter() - .all(|r| matches!(r.error, AccuracyError::UnsupportedComposition { .. }))); - - let space = crate::pass1::replacement::search_workload_with( - vec![("q", Rc::clone(&q))], - &[Box::new(strategy)], - ); - let group = space.candidates_for_target(&space.roots[0].1).unwrap(); - assert!(group.candidates.is_empty()); - assert_eq!(group.rejected.len(), 2); - } - - #[test] - fn known_hydra_bound_over_target_is_rejected_despite_unknown_probability() { - struct ExcessiveCollision; - impl AccuracyEvidenceProvider for ExcessiveCollision { - fn propagation_stats( - &self, - _op: &CompositionOperator, - _family: &FieldDataType, - _query: Option<&asap_types::ir::schema::SketchStatistic>, - ) -> PropagationStats { - PropagationStats { - hydra_shared_grid_collision_bound: Some(0.1), - ..Default::default() - } - } - } - let q = agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }, - metric_scan(&["job"]), - ); - let strategy = HydraGroupingStrategy::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &ExcessiveCollision, - ); - let proposals = strategy.propose(&TargetSubDAG::new(&q)); - assert!(proposals.candidates.is_empty()); - assert_eq!(proposals.rejected.len(), 2); - assert!(proposals - .rejected - .iter() - .all(|r| matches!(r.error, AccuracyError::TargetNotSatisfied { .. }))); - } - - #[test] - fn cardinality_has_no_hydra_candidate_yet() { - // summary_candidates(Cardinality) = [Hll, Theta, Kmv] — none have a - // modeled Hydra variant, so no candidate at all (not an error, just - // an empty result, same conservatism as every other strategy here). - let q = agg(vec![2], default_cardinality(), metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let strategy = HydraGroupingStrategy::default(); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn exact_accuracy_target_has_no_hydra_candidate() { - // AccuracyTarget::Exact never binds a sketch at all — nothing for - // this axis to offer a shared-structure alternative to. - let intent = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }; - let q = agg(vec![2], intent, metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let strategy = HydraGroupingStrategy::default(); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn exact_mergeable_intent_has_no_hydra_candidate() { - // Sum's exact accumulator has no candidate summary families at all - // (summary_candidates only covers approximate-capable intents). - let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let strategy = HydraGroupingStrategy::default(); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn does_not_match_a_multi_intent_or_having_aggregate() { - let strategy = HydraGroupingStrategy::default(); - - let multi = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: metric_scan(&["job"]), - })) - .unwrap(); - let target = TargetSubDAG::new(&multi); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } -} diff --git a/crates/logical-optimizer/src/pass1/logical_candidates.rs b/crates/logical-optimizer/src/pass1/logical_candidates.rs index d773fca8..0c5a7c4f 100644 --- a/crates/logical-optimizer/src/pass1/logical_candidates.rs +++ b/crates/logical-optimizer/src/pass1/logical_candidates.rs @@ -3,7 +3,7 @@ //! Alternatives are nominal realization descriptors attached to their original //! target, not ranked plans or accuracy certificates. Workload composition and //! physical planning consume this inventory later; empirical models belong to -//! selection. The legacy search API remains until planner cutover. +//! selection. use std::collections::{BTreeMap, HashMap, HashSet}; use std::rc::Rc; @@ -22,8 +22,7 @@ use thiserror::Error; use crate::pass1::realization::{ accuracy_budget, accuracy_target, column_ref, default_size_params, has_subpopulations, - realize_keyed_additive_summary_input, summarised_input, summary_candidates, - PhysicalSummaryInputRuleResult, Realization, + realize_keyed_additive_summary_input, summarised_input, summary_candidates, Realization, }; use crate::pass2::window_composition::{tumbling_state, WindowForm}; @@ -181,7 +180,7 @@ pub fn enumerate_local_logical_candidates( /// or count is realized as one heap sketch over the inner aggregate's input, /// keyed by the ranked item and weighted by the summed value, instead of a /// sketch over the inner aggregate's exact result. The decision is the -/// legacy keyed-additive rule's. Offered only when the inner target has no +/// keyed-additive rule's ([`realize_keyed_additive_summary_input`]). Offered only when the inner target has no /// other consumer, so absorbing it removes its work. fn add_whole_expression_alternatives( targets: &mut [LocalLogicalTarget], @@ -312,7 +311,7 @@ fn count_item<'a>( } /// The input and update of a whole-expression top-k over `target`'s inner -/// aggregate, by the legacy keyed-additive rule, or `None` when it does not +/// aggregate, by the keyed-additive rule, or `None` when it does not /// apply. Rows that carry the full series identity rank it as a column, as /// [`summary_update`] does. fn whole_expression_input(target: &OperatorNode) -> Option<(Rc, SummaryUpdate)> { @@ -346,11 +345,7 @@ fn whole_expression_input(target: &OperatorNode) -> Option<(Rc, Su ), GroupingStrategy::default(), ); - let PhysicalSummaryInputRuleResult::Realized(realized) = - realize_keyed_additive_summary_input(intent, &family, reduction, child) - else { - return None; - }; + let realized = realize_keyed_additive_summary_input(intent, &family, reduction, child)?; let mut input = realized.input; let per_series = matches!( child.non_asap(), @@ -759,7 +754,7 @@ fn counter_samples(node: &OperatorNode, metric_types: &BTreeMap>, @@ -309,24 +306,6 @@ impl MaintainedPopulationStrategy { )) } } -impl ReplacementStrategy for MaintainedPopulationStrategy { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - recognize(target.root).is_some() - } - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - self.candidate(target.root) - .map(|node| ReplacementSubDAG { - strategy: "MaintainedPopulationStrategy", - replacement: Replacement::SubDAG(node), - provenance: ReplacementProvenance::SummaryRealization, - rationale: - "share an exact maintained population across compatible aggregate evaluations" - .into(), - }) - .into_iter() - .collect() - } -} #[cfg(test)] mod tests { @@ -388,18 +367,6 @@ mod tests { .map(lower) .into(); let strategy = MaintainedPopulationStrategy::new(&roots); - let space = crate::search_workload_with( - roots - .iter() - .enumerate() - .map(|(i, r)| (i, Rc::clone(r))) - .collect(), - &[Box::new(MaintainedPopulationStrategy::new(&roots))], - ); - assert!(space - .target_subdag_candidates() - .flat_map(|g| &g.candidates) - .any(|c| c.strategy == "MaintainedPopulationStrategy")); let plans = share_common_sub_dags( roots .iter() diff --git a/crates/logical-optimizer/src/pass1/mod.rs b/crates/logical-optimizer/src/pass1/mod.rs index 95bd58f6..657a0171 100644 --- a/crates/logical-optimizer/src/pass1/mod.rs +++ b/crates/logical-optimizer/src/pass1/mod.rs @@ -1,14 +1,7 @@ -//! Pass 1: local logical alternatives for each target sub-DAG. Strategies -//! propose rewrites and summary realizations; a candidate is pruned only when -//! it is provably invalid. +//! Pass 1: local logical alternatives for each target sub-DAG +//! ([`logical_candidates`]), from the realizations of each aggregate intent +//! ([`realization`]). A candidate is pruned only when it is provably invalid. -pub mod exact_composition; -pub mod explanation; -pub(crate) mod function_rules; -pub mod grouping; pub mod logical_candidates; pub mod maintained_population; pub mod realization; -pub mod replacement; -pub mod rewrite; -pub mod rollup; diff --git a/crates/logical-optimizer/src/pass1/realization.rs b/crates/logical-optimizer/src/pass1/realization.rs index cafc290b..c64dcd2c 100644 --- a/crates/logical-optimizer/src/pass1/realization.rs +++ b/crates/logical-optimizer/src/pass1/realization.rs @@ -156,21 +156,14 @@ pub fn default_size_params( crate::accuracy::estimators::size_params(kind, intent, eps, delta) } -/// The physical input consumed by one summary realization. Most summaries -/// consume the logical aggregate's immediate child and summarize its declared -/// input value. Composite realizations can instead consume a larger -/// logical sub-DAG and bind a different key or value. +/// The physical input consumed by one summary realization: a larger logical +/// sub-DAG than the aggregate's immediate child, bound with its own key or +/// value. pub(crate) struct PhysicalSummaryInput { pub(crate) child: Rc, pub(crate) input: SummaryUpdate, } -pub(crate) enum PhysicalSummaryInputRuleResult { - NotApplicable, - Realized(PhysicalSummaryInput), - Unsupported(&'static str), -} - /// Realize the composite heavy-hitter realization for /// `TopK(Count GROUP BY key)`. The heap sketch consumes the raw keyed stream; /// it does not consume an independently materialized Count result. @@ -179,19 +172,19 @@ pub(crate) fn realize_keyed_additive_summary_input( family: &FieldDataType, output_reduction: &Reduction, child: &Rc, -) -> PhysicalSummaryInputRuleResult { +) -> Option { if !matches!(intent, AggIntent::TopK { .. }) { - return PhysicalSummaryInputRuleResult::NotApplicable; + return None; } let FieldDataType::Sketch(kind, _) = family else { - return PhysicalSummaryInputRuleResult::NotApplicable; + return None; }; let heap_algorithm = kind.algorithm(); if !matches!( heap_algorithm, SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap ) { - return PhysicalSummaryInputRuleResult::NotApplicable; + return None; } let Some(NonASAPOp::Aggregate { reduction, @@ -201,7 +194,7 @@ pub(crate) fn realize_keyed_additive_summary_input( .. }) = child.non_asap() else { - return PhysicalSummaryInputRuleResult::NotApplicable; + return None; }; let counter_input = matches!(measures.as_slice(), [AggIntent::Sum { .. }]) && matches!(raw_child.non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) @@ -216,13 +209,12 @@ pub(crate) fn realize_keyed_additive_summary_input( Some(index) => match schema_column_ref(raw_child, *index) { Some(column) => column, None => { - return PhysicalSummaryInputRuleResult::Unsupported( - "sum-ranked Top-K value column is outside the raw input schema", - ) + // Sum-ranked Top-K value column is outside the raw input schema. + return None; } }, }), - _ => return PhysicalSummaryInputRuleResult::NotApplicable, + _ => return None, }; let weight_domain = match measures.as_slice() { [AggIntent::Count { .. }] => WeightDomain::NonNegative { @@ -236,9 +228,8 @@ pub(crate) fn realize_keyed_additive_summary_input( if matches!(heap_algorithm, SketchAlgorithm::CmsWithHeap) && !matches!(weight_domain, WeightDomain::NonNegative { .. }) { - return PhysicalSummaryInputRuleResult::Unsupported( - "value-weighted CMS requires non-negative update evidence; use CountSketch for arbitrary values", - ); + // Value-weighted CMS requires non-negative update evidence; use CountSketch for arbitrary values. + return None; } let subpopulation_columns = match output_reduction { Reduction::PerEntity => vec![], @@ -257,9 +248,8 @@ pub(crate) fn realize_keyed_additive_summary_input( .map(|index| schema_column_ref(raw_child, *index)) .collect::>>() else { - return PhysicalSummaryInputRuleResult::Unsupported( - "ranked item column is outside the raw input schema", - ); + // Ranked item column is outside the raw input schema. + return None; }; let item_columns: Vec<_> = columns .into_iter() @@ -267,9 +257,8 @@ pub(crate) fn realize_keyed_additive_summary_input( .collect(); match item_columns.as_slice() { [] => { - return PhysicalSummaryInputRuleResult::Unsupported( - "subpopulation columns consume the complete ranked item identity", - ) + // Subpopulation columns consume the complete ranked item identity. + return None; } [column] => SummaryInputExpr::Column(column.clone()), _ => SummaryInputExpr::Tuple( @@ -281,12 +270,11 @@ pub(crate) fn realize_keyed_additive_summary_input( } } Reduction::Reduce(_) => { - return PhysicalSummaryInputRuleResult::Unsupported( - "an empty or without grouping does not identify ranked items", - ) + // An empty or without grouping does not identify ranked items. + return None; } }; - PhysicalSummaryInputRuleResult::Realized(PhysicalSummaryInput { + Some(PhysicalSummaryInput { child: Rc::clone(raw_child), input: SummaryUpdate { item: Some(item), diff --git a/crates/logical-optimizer/src/pass1/replacement.rs b/crates/logical-optimizer/src/pass1/replacement.rs deleted file mode 100644 index a672c551..00000000 --- a/crates/logical-optimizer/src/pass1/replacement.rs +++ /dev/null @@ -1,8125 +0,0 @@ -//! `TargetSubDAG` / `ReplacementSubDAG` / `ReplacementStrategy` — the -//! candidate-replacement vocabulary `docs/design_docs/asap_aware_mapping.md` stubs out -//! under "Key concepts (not yet implemented)", implemented for real (issue -//! #251, part of #33). -//! -//! ## One step, not two: `ASAPStrategies::replacements()` decides *and* builds -//! -//! For a bindable `Aggregate`, `ASAPStrategies::replacements()` is the -//! single place this crate both decides what an `AggIntent` may become and -//! turns each of those candidates into a real, executable -//! [`ReplacementSubDAG`]: -//! -//! 1. **Decide**: [`realizations_for_intent`] enumerates every valid -//! [`Realization`] for the target's intent — exhaustive, in a static -//! order (candidate sketch family/kind, already sized to the target's own -//! accuracy target: `Realization::Sketch`'s `params` are the output of -//! inverting that accuracy target through the analytical estimators, not a -//! placeholder filled in later). -//! 2. **Build**: for each candidate in that list, [`construct_summary`] -//! mechanically turns the already-decided `(kind, params)` into a real -//! [`OperatorNode`] — derives the child schema, resolves the summarized -//! column, builds the evaluation query, recurses into the child (via -//! [`realize_child`], so a nested aggregate gets its own -//! independent enumeration, never the outer target's forced choice), and -//! assembles the `SummaryAgg`/`SummaryEstimate` node. -//! -//! There is no separate decision step and construction step living in -//! different modules bridged by a named "given a `Realization`, bind it" -//! function — step 2 is *not* a second decision (nothing about which -//! candidate to prefer happens there), it is mechanical construction that -//! has to run regardless of how `(kind, params)` were chosen, so it lives -//! directly inside the one method that needs it. -//! -//! - [`TargetSubDAG`] — a reference to a pre-ASAP [`OperatorNode`] that is a -//! candidate for replacement, plus how many places in the workload already -//! reference it (its `consumer_count`) — the one piece of cross-node -//! context [`SharedSubDAGStrategy`] needs that a bare node reference alone -//! doesn't carry. -//! - [`ReplacementSubDAG`] — one candidate replacement for a `TargetSubDAG`: -//! either a fully bound summary sub-DAG or a pre-ASAP logical rewrite -//! (still logical, structurally different from the target but semantically -//! equivalent) — see [`Replacement`] — plus a human-readable `rationale`. -//! - [`ReplacementStrategy`] — `matches` + `replacements`, the same -//! extension-point shape `CostModel` and [`Matcher`] already use in this -//! crate: a new replacement source is a new `impl ReplacementStrategy`, not -//! a restructuring of this trait or of any existing strategy. `replacements` -//! is **exhaustive, not ranked, not filtered** — reporting "every valid -//! candidate" is core's job; picking the best one is left to the caller. -//! [`crate::pass1::explanation`] (issue #257) is this trait's own downstream -//! consumer, not a second extension point: it explains why a replacement -//! exists as a pure view over the candidates strategies registered here -//! already produced, rather than re-deriving that explanation with a rule -//! of its own. -//! -//! A caller may inspect local replacements, but taking the first candidate -//! does not establish a compatible workload plan or physical deployability. -//! For Planner-owned logical selection, call `candidate_selection::global_selection` -//! once and [`GlobalSelection::assemble_selected_dag`] for each wanted query -//! root. Physical binding, deployment, and execution remain downstream. -//! -//! Internally, [`realize_child`] and [`realize_one`] may take a preferred local -//! realization while constructing or costing a candidate. That local operation -//! is not the public workload-selection workflow and does not create runtime state. -//! -//! This means an ordinary single-target bind sizes and fully constructs -//! *every* sketch candidate at every sketch-capable node (not just the one a -//! caller keeps) — a deliberate tradeoff, made so there is exactly one place -//! in this crate that decides what an `AggIntent` may become, at the cost of -//! extra work per bind proportional to each node's own candidate count. -//! -//! ## The two strategies, and why these two -//! -//! - [`ASAPStrategies`] wraps [`realizations_for_intent`]'s exhaustive, -//! ranked list directly: for the same bindable-`Aggregate` shape this crate -//! binds (single intent, no `HAVING`), every entry becomes its own bound -//! candidate. -//! - [`SharedSubDAGStrategy`] wraps -//! `asap_types::ir::cse::share_common_sub_dags`'s sharing decision. -//! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. -//! `share_common_sub_dags` already collapsed two or more workload -//! locations onto the same `Rc` — [`discover_targets`] below -//! does the identical workload-wide discovery for [`search_workload_with`]; -//! this module's own tests reuse the same dedup logic to build realistic -//! fixtures), it reports the two-way candidate CSE's own detection pass -//! deliberately declines to pick between on its own: build once and share -//! the already-interned sub-DAG, or build it independently at each -//! consumer. `cost_model::CostModel::cse_share_decision` is where -//! that choice actually gets made *today* (a fixed comparison, not a -//! search) — this strategy exposes the same two-way choice as an explicit, -//! inspectable pair of candidates instead of a cost model's already-decided -//! boolean. -//! -//! ## Non-goals (tracked separately, not attempted here) -//! -//! - **[`realizations_for_intent`]'s own outward-facing behavior is -//! unchanged.** Same inputs still produce the same exhaustive, ranked -//! list — only its home moved (from a separate `implementation` module -//! into this one) and its own visibility dropped to module-private, since -//! [`ASAPStrategies`] is now its only caller. -//! -//! ## Workload-wide search — merged in from the former `search.rs` (issue #252, part of #33) -//! -//! This section used to carry two more "non-goals" bullets here — "no -//! search/selection-across-a-whole-plan logic" and "no workload-wide -//! `TargetSubDAG` discovery pass" — describing work deliberately left for a -//! future Cascades/Volcano-style search engine (PR #263, -//! `feat/cascades-search-252`, over the [`ReplacementStrategy`] extension -//! point above). That engine is [`CandidateLogicalASAPDAGs`]/[`TargetSubDAGCandidates`]/ -//! [`search_workload`]/[`search_workload_with`] below, merged into this -//! module rather than kept as a separate `search` module — the same "one -//! module, one step" reasoning the top of this file already uses for -//! decide-and-build: searching *across* a whole workload's worth of -//! [`TargetSubDAG`]s is a natural continuation of deciding and building -//! replacements *for* one, not a different concern that deserves its own -//! file. What follows (through "Cost-based final selection" below) is that -//! engine's own design documentation, preserved from `search.rs`. -//! -//! ### The pseudocode, and the two things it deliberately leaves open -//! -//! ```text -//! candidate_plans = { input_workload_plan } -//! loop: -//! new_plans = {} -//! for plan in candidate_plans: -//! for site in plan.bindable_sites(): -//! for strategy in registered_strategies: -//! if strategy.matches(site): -//! for replacement in strategy.replacements(site): -//! new_plans += substitute(plan, site, replacement) -//! new_plans -= candidate_plans -//! candidate_plans += new_plans -//! until new_plans is empty -//! return candidate_plans.sorted_by(cost_model) -//! ``` -//! -//! Read literally, this enumerates whole *plans* — full copies of the -//! workload's DAG, one per combination of per-target choices. A workload -//! with `N` independently-choosable targets would produce up to `2^N` flat -//! plans, each one duplicating every untouched sibling sub-DAG. This module -//! does not do that: -//! -//! 1. **Per-target candidates, not flat plans.** [`TargetSubDAGCandidates`] -//! stores the alternatives for one distinct [`TargetSubDAG`] (identified by -//! its own `Rc` pointer identity — the same currency -//! [`asap_types::ir::cse::share_common_sub_dags`] already -//! established across the workload) holding every -//! [`ReplacementSubDAG`] alternative discovered for it. [`CandidateLogicalASAPDAGs`] is -//! a collection of these groups, keyed by `TargetSubDAG` — a candidate -//! "plan" is never materialized as a distinct top-level `Rc` -//! at all; two logically-different overall choices at two different -//! targets are just two different entries in two different groups, -//! sharing every other node in the workload by construction (they *are* -//! the same `Rc`s — nothing was copied to make a second "plan"). -//! 2. **Dedup by structural hash + `PartialEq`, reusing `pre_asap::cse`'s own -//! discipline.** [`asap_types::ir::cse::structural_hash`] (made -//! `pub` for exactly this reuse) is only ever a candidate-narrowing -//! filter; [`TargetSubDAGCandidates::add_candidate`]'s actual duplicate check is -//! `OperatorNode`'s derived `PartialEq` — the same "hash is a filter, -//! `PartialEq` is the decision, no exceptions" rule `cse.rs`'s own -//! "Correctness" section states and this module inherits rather than -//! reinvents. See [`is_duplicate_rewrite`] for the one deliberate -//! wrinkle this reuse needs (a `Rc`-identity case pure value equality -//! would get wrong). -//! -//! ### Where `TargetSubDAG` discovery comes from -//! -//! [`discover_targets`] is the workload-wide `TargetSubDAG` discovery pass -//! this section used to flag as explicitly *not* implemented ("no -//! workload-wide `TargetSubDAG` discovery pass is shipped either... wiring -//! it up automatically belongs to the same future search engine, not this -//! issue") — this is that future engine, so it's this module's job now, and -//! it's what the quoted pseudocode's `for site in plan.bindable_sites()` -//! line above stands for: every `TargetSubDAG` this pass discovers is one -//! iteration of that loop. It walks every workload root's whole DAG (the -//! same **relational-skeleton** operator-child scope -//! `asap_types::ir::cse::share_common_sub_dags` itself uses — see -//! that module's "Algorithm" section), discovering one `TargetSubDAG` per -//! distinct `Rc` and a *real* `consumer_count`: how many operator-child -//! positions anywhere in the workload reference that exact `Rc`, not just -//! how many of the workload's own top-level roots happen to be it — a -//! `SharedSubDAGStrategy` candidate three levels under an unshared -//! `Filter` is exactly as real a target as a shared whole root, so this -//! module's discovery can't stop at the top level. -//! -//! `discover_targets` duplicates (rather than reuses) this module's own -//! `#[cfg(test)]`-only `count_consumers` traversal (in the test module -//! below), which mirrors this exact shape for this module's own test -//! fixtures — that copy is intentionally test-only, so it isn't reachable -//! from this module's production code without either moving it into -//! shared, non-test-gated code or duplicating the (small, self-contained) -//! traversal here. Duplicating was judged simpler than restructuring a test -//! helper into shared production code for one caller. -//! -//! ### Termination -//! -//! Every discovered target is asked *once* per registered strategy, never -//! re-asked — [`search_workload_with`]'s loop processes each round's -//! frontier of not-yet-visited targets exactly one time each, so there is -//! no scenario where the same `(target, strategy)` pair is queried twice -//! (the `new_plans -= candidate_plans` dedup step the module-level -//! pseudocode describes is therefore never asked to recognize "the same -//! candidate, proposed again" as a special case — see -//! [`TargetSubDAGCandidates::add_candidate`]'s own doc on why that distinction matters -//! for [`Replacement::Summary`] specifically, where no real equality check -//! exists to make it safely). -//! -//! What *can* grow the frontier is a candidate's own reachable structure: -//! after a target is processed, every [`Replacement::Rewrite`] candidate's -//! **children** (never the candidate's own top-level node — that value is -//! an alternative *for* the target just processed, not a new target of its -//! own; see [`discover_new_descendant_targets`]) are scanned for pointers -//! not already known, and any found become next round's frontier. Both shipped -//! strategies are idempotent in exactly this sense: [`ASAPStrategies`] -//! produces terminal bound-summary [`Replacement::SubDAG`] candidates (no -//! logical-rewrite children to scan at all), and [`SharedSubDAGStrategy`]'s -//! two logical-rewrite [`Replacement::SubDAG`] candidates both reuse the target's own -//! already-known child `Rc`s verbatim (`Rc::clone`/a shallow top-level -//! `.clone()` — see that strategy's own doc). So for both, the frontier is -//! always empty after round one: real workloads converge in exactly one -//! round, regardless of size. -//! -//! That said, a future strategy whose `Replacement::Rewrite` candidates -//! invent brand-new descendant structure every time they're computed (e.g. -//! internal state that fabricates a fresh child node on every call) could -//! in principle keep the frontier non-empty forever. Since this crate has -//! no principled bound on strategy-generated descendant sites, -//! [`search_workload_with`] enforces a generous, documented round cap -//! ([`MAX_SEARCH_ITERATIONS`]) instead: exceeding it panics with a clear -//! message naming the actual cause, rather than hanging silently — a test -//! ([`tests::a_pathologically_growing_strategy_trips_the_iteration_cap`]) -//! pins that this guard actually fires, by using exactly that shape of -//! pathological strategy. -//! -//! ### Cost-based final selection — reusing `CostModel`, not a second interface -//! -//! `candidate_selection::cost_sorted` is the `sorted_by(cost_model)` step, and it -//! reuses this crate's existing `CostModel` trait rather than inventing a -//! second cost interface (`docs/design_docs/cse-cost-model-decision.md`, -//! issue #237, explicitly reasoned about *why* a narrow, direct cost -//! comparison was enough for the CSE share/recompute decision alone, and -//! flagged that a real search engine — this module — is where that stops -//! being the whole story; it isn't a contradiction of #237, it's the scope -//! change #237 itself named). Concretely, per [`TargetSubDAGCandidates`]: -//! -//! - A group whose candidates are the [`SharedSubDAGStrategy`] -//! share-vs-recompute pair is ranked by calling -//! `CostModel::cse_share_decision` via this module's own -//! [`cse_preference`] — rather than re-deriving a competing comparison. -//! - A group whose candidates are [`ASAPStrategies`]'s sketch-family -//! candidates is ranked via `CostModel::rank_candidates` (the same hook -//! `realizations_for_intent` itself consults), applied to the -//! candidates' own [`SketchAlgorithm`]s. -//! - Any other shape (a single candidate, or a mix this module doesn't have -//! a defined comparison for) keeps discovery order — there is nothing to -//! rank, or no `CostModel` hook this module knows how to apply; it never -//! invents a comparison `CostModel` doesn't already define. -//! -//! ## Whole-plan (cross-group) selection — issue #271 -//! -//! `candidate_selection::cost_sorted` above ranks every group's candidates -//! independently: it never lets one group's choice influence how another -//! group is costed. That's the right behavior when groups genuinely don't -//! interact — which both shipped strategies' one-round convergence (see -//! "Termination" above) makes the common case — but it's the wrong answer -//! whenever they do. Concretely: `CostModel::cse_share_decision` costs a -//! [`SharedSubDAGStrategy`] group by comparing a `consumer_count`-scaled -//! recompute cost against a fixed maintenance cost — but a **nested** -//! `SharedSubDAGStrategy` group's *true* recompute burden isn't its own -//! raw [`TargetSubDAGCandidates::consumer_count`] (how many operator-child positions -//! directly reference it) whenever an ancestor on the path to it is -//! *itself* being recomputed independently rather than shared: recomputing -//! that ancestor independently at each of *its own* uses recomputes -//! everything underneath it that many times too, even though nothing -//! underneath gained a single new direct reference. `cost_sorted`'s -//! per-group ranking has no way to see this — it only ever looks at one -//! group's own `candidates`, in isolation. -//! -//! `candidate_selection::global_selection` is that missing step: a single -//! **top-down dynamic-programming pass** over the discovered sites, -//! processed in the topological order [`topological_order`] computes over a -//! small [`ReferenceDAG`] built for exactly this purpose (parent before -//! every child, so a site's `effective_consumer_count` is always computed -//! from *already-decided* ancestors). For every site it computes the -//! **effective consumer count** — how many times that site actually runs -//! once every ancestor's own selected candidate is accounted for — and, for -//! every [`SharedSubDAGStrategy`]-shaped group, re-decides -//! `CostModel::cse_share_decision` against *that* corrected count instead -//! of the group's raw structural one. When that group also contains a -//! non-CSE alternative such as a semantic rewrite, the chosen CSE candidate -//! and the cheapest non-CSE candidate additionally compete through -//! `CostModel::estimate_cost`; the CSE pair is no longer allowed to hide an -//! otherwise valid logical alternative. See [`multiplier`]'s doc for the -//! exact recurrence: a group that chooses `Share` collapses its own -//! multiplicity to exactly `1` for everything beneath it (one shared -//! execution backs every use of it); a group that chooses -//! `RecomputeIndependently` — or has no Share/Recompute decision of its own -//! at all, i.e. isn't itself a `SharedSubDAGStrategy` shape — passes its -//! *own* effective count straight through to whatever it references, -//! transitively composing contributions from every ancestor on the path, -//! not just the immediate parent. -//! -//! This is genuine dynamic programming in the classical sense: overlapping -//! subproblems (a site reachable through more than one parent path is -//! solved once, memoized in `effective_uses`, and reused for every path -//! into it) combined via a real recurrence — not just the MEMO-group -//! sharing [`CandidateLogicalASAPDAGs`] itself already does for *storing* candidates. That -//! distinction is exactly what issue #271 raised: this module already looks -//! like a Cascades/Volcano MEMO, but `candidate_selection::cost_sorted` alone never -//! actually performed this composition step; `global_selection` is that -//! step, added alongside `cost_sorted` rather than replacing it (both stay -//! available — see [`RankedTargetSubDAGCandidates`] vs. [`TargetSubDAGSelection`]'s own docs for when -//! to reach for which). -//! -//! Two things this deliberately does **not** attempt, both left as -//! documented follow-up rather than silently overclaimed: -//! -//! - `CostModel::rank_candidates` — the hook -//! [`ASAPStrategies`] groups rank by — takes no `consumer_count` -//! parameter at all today, so a `ASAPStrategies` group's selection -//! here still falls back to [`rank_group`]'s ordinary (consumer-count- -//! blind) local ranking, even though its own -//! [`TargetSubDAGSelection::effective_consumer_count`] is computed and exposed -//! correctly regardless. Wiring sketch sizing/ranking to actually consume -//! it needs a `CostModel` interface change — out of scope here per this -//! issue's own "reuse `CostModel`, don't invent a new interface" ask; a -//! correct `effective_consumer_count` is the input such a future hook -//! would need, and this module now computes it for every group, sketch -//! groups included. -//! - This is not an exhaustive search over combinations of choices for a -//! provably-global optimum in every case. `CostModel::cse_share_decision` -//! is still a *local*, pairwise comparison at each `SharedSubDAGStrategy` -//! site (recompute-total vs. one fixed maintenance cost) — this module -//! just now feeds it a *correct* input instead of an *incorrect* one. Two -//! sibling `SharedSubDAGStrategy` groups that could trade off against -//! each other under some shared resource budget (memory, say) still -//! aren't jointly optimized here — this crate has no -//! cardinality/statistics estimation to bound a combinatorial search like -//! that with (the same constraint #237/#263 already navigated), so real -//! multi-group joint optimization beyond this per-site recurrence is left -//! for whenever that changes. - -use crate::accuracy::estimators::{ - cms::{cms_depth, cms_width}, - saturating_ceil, size_params, -}; -use asap_types::ir::operator::non_asap::any_measure_filtered; -use asap_types::ir::scalar::resolve_column_ref; -use std::cell::RefCell; -use std::collections::{HashMap, HashSet, VecDeque}; - -use asap_types::ir::cse::{share_common_sub_dags, structural_hash, HashCache}; -use asap_types::ir::operator::agg_intent::{agg_is_mergeable, AggIntent}; -use asap_types::ir::operator::operator_properties::{BinaryOpKind, JoinKind, Reduction}; -use asap_types::ir::properties::timing::validate_maintained; -use asap_types::ir::properties::{ - AccuracyError, CompositionOperator, GuaranteeSource, ResultGuarantee, -}; -use asap_types::ir::properties::{ExecutionDataStateError, ExecutionTiming}; -use asap_types::ir::scalar::{ArithmeticOpKind, ColumnRef}; -use asap_types::ir::schema::{ - ColumnId, EntityIdentity, ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, - NonNegativeWeightProof, Schema, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic as PostAsapSketchStatistic, SummaryInputExpr, SummaryUpdate, WeightDomain, -}; -use asap_types::ir::SchemaDerivationError; -use asap_types::ir::{ - ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, Predicate, ProjectItem, ScalarExpr, - SortKey, -}; -use asap_types::physical::ExactOperationSchemaError; -use asap_types::types::AccuracyTarget; -use std::rc::{Rc, Weak}; -use thiserror::Error; - -use crate::accuracy::{ - AccuracyBudgetAllocator, AccuracyEvidenceProvider, AccuracyModel, CompositionShape, - DefaultAccuracyModel, EqualSplitAllocator, NoAccuracyEvidence, -}; -use crate::pass1::exact_composition::{ - ExactComposition, ExactCompositionStrategy, OperationPlacement, -}; -use crate::pass1::grouping::HydraGroupingStrategy; -use crate::pass1::realization::{ - accuracy_budget, accuracy_target, column_ref, default_size_params, - realize_keyed_additive_summary_input, schema_column_ref, summarised_column, summarised_input, - summary_candidates, PhysicalSummaryInput, PhysicalSummaryInputRuleResult, Realization, - DEFAULT_DELTA, -}; -use crate::pass1::rollup::RollupStrategy; -use crate::pass2::reconciliation::AccuracyReconciliationStrategy; -use crate::pass2::topk_reuse::TopKLimitReuseStrategy; - -/// Errors from the pre-ASAP → post-ASAP replacement/construction path -/// ([`realize_child`] and [`retain_exact`]). Moved here from the former -/// `bind.rs` (issue #251): this is what a [`ReplacementStrategy`] -/// implementor's own construction path can realistically fail with — -/// schema derivation over a pre-ASAP [`OperatorNode`] sub-DAG — not -/// something specific to workload-wide orchestration. -#[derive(Debug, Error)] -pub enum RealizationError { - /// Schema derivation failed while lifting an edge to `Schema`. - #[error("schema derivation failed during pre-ASAP → post-ASAP binding: {0}")] - Schema(#[from] SchemaDerivationError), - /// The candidate is accuracy-illegal (issue #172): its composed - /// guarantee has no sound propagation rule, or misses the applicable - /// `AccuracyTarget`. Fail-closed — the candidate is never constructed - /// with the child "treated as exact". [`ASAPStrategies::propose`] - /// records it as a [`RejectedCandidate`] instead of a candidate. - #[error("accuracy-illegal candidate: {0}")] - Accuracy(#[from] AccuracyError), - /// The selected summary family has a physical realization rule for this - /// logical shape, but the rule cannot represent the complete input. The - /// candidate must not fall back to ordinary one-node binding because that - /// would change its semantics. - #[error("unsupported physical summary realization: {0}")] - PhysicalRealization(&'static str), - /// A constructed plan violates the update/evaluation phase contract - /// (issue #171) — e.g. a summary evaluation placed beneath a maintained - /// `SummaryAgg`. Detected at construction, never at runtime. - #[error("execution-data_state violation in post-ASAP plan: {0}")] - ExecutionDataState(#[from] ExecutionDataStateError), - /// An `ExactOperator`'s output schema could not be derived over its - /// child — the child carries summary state the operator can't read. - #[error("exact operator schema derivation failed: {0}")] - ExactOperationSchema(#[from] ExactOperationSchemaError), -} - -/// A pre-ASAP sub-DAG a [`ReplacementStrategy`] knows how to replace. -/// -/// `root` is a reference into the workload's own [`OperatorNode`] DAG (an -/// `Rc`, the same currency [`search_workload`] and -/// `asap_types::ir::cse::share_common_sub_dags` already thread through -/// this crate's public API — not a bare `&OperatorNode` — so a strategy that -/// needs the node's own `Rc` identity, not just its shape, has it available -/// without the caller re-deriving it). -/// -/// `consumer_count` is how many locations across the workload reference this -/// exact `Rc` — 1 for an ordinary single-use node and 2+ for a shared sub-DAG. -/// [`search_workload_with`] computes the workload-wide value during target -/// discovery. [`TargetSubDAG::new`] defaults it to `1` for callers invoking a -/// strategy against one node in isolation. A strategy that only cares about -/// `root`'s shape (for example, [`ASAPStrategies`]) can ignore the -/// count; [`SharedSubDAGStrategy`] consults it directly. -/// -/// `strictest_sibling_accuracy` is the strictest accuracy among workload -/// siblings that read the same summary input as `root`, when stricter than -/// `root`'s own. [`search_workload_with`] sets it; [`ASAPStrategies`] -/// also sizes a candidate to it. -#[derive(Debug, Clone, Copy)] -pub struct TargetSubDAG<'a> { - pub root: &'a Rc, - pub consumer_count: usize, - pub strictest_sibling_accuracy: Option<&'a AccuracyTarget>, -} - -impl<'a> TargetSubDAG<'a> { - /// A target assumed to have exactly one consumer — the common case for a - /// caller that isn't already tracking cross-workload sharing. - pub fn new(root: &'a Rc) -> Self { - Self { - root, - consumer_count: 1, - strictest_sibling_accuracy: None, - } - } - - /// A target with an explicit `consumer_count`, used by workload discovery - /// and by callers that already know how many locations reference `root`. - pub fn with_consumer_count(root: &'a Rc, consumer_count: usize) -> Self { - Self { - root, - consumer_count, - strictest_sibling_accuracy: None, - } - } -} - -/// What a [`ReplacementSubDAG`] actually substitutes a [`TargetSubDAG`] with. -/// -/// Generalizes [`realizations_for_intent`]'s two possible *kinds* of answer -/// — a post-ASAP binding decision, or a still-pre-ASAP structural alternative -/// — into "one candidate among several", each with its own -/// [`ReplacementSubDAG`]. -#[derive(Debug, Clone)] -#[allow(clippy::large_enum_variant)] // Keep the public strategy API value-based. -pub enum Replacement { - /// A sub-DAG that replaces the target: either a bound summary decision - /// (a DAG containing ASAP operators, for one particular candidate - /// realization of the target) or a pre-ASAP rewrite (a logical sub-DAG - /// with no ASAP operator, structurally different from the target's own - /// `root` — e.g. sharing vs. not sharing a sub-DAG — but semantically - /// equivalent to it). [`is_logical_rewrite`] tells the two apart. - SubDAG(Rc), - /// An exact operator composed over another target's *own* selected - /// decision across an explicit update/evaluation boundary (issue #171): - /// `ValueOperationAtQueryTime` over a child's summary evaluation, or - /// `ValueOperationAtIngestionTime` feeding a maintained summary above. Carries only a - /// reference to the child target — `candidate_selection::global_selection` - /// commits the compatible parent/child pair and - /// [`GlobalSelection::assemble_selected_dag`] links it into one validated - /// `OperatorNode` DAG. See [`crate::pass1::exact_composition`]. - ExactComposition(ExactComposition), -} - -/// Whether a [`Replacement::SubDAG`] is a pure logical rewrite: a sub-DAG -/// with no ASAP operator and no guarantee established yet (the shape every -/// front end emits and every rewrite strategy builds). A bound summary -/// decision contains an ASAP operator, or is a kept pre-ASAP sub-DAG that -/// already carries its exact guarantee. -pub fn is_logical_rewrite(node: &OperatorNode) -> bool { - node.guarantee.is_none() && !node.contains_asap() -} - -/// One candidate replacement for a [`TargetSubDAG`], plus a human-readable -/// `rationale` explaining why it's a valid candidate (meant for a -/// report/log/debugging a search engine's choices, not machine parsing — -/// [`crate::pass1::explanation::ReplacementExplanation::reason`] literally reuses -/// this same string rather than inventing new prose of its own. -#[derive(Debug, Clone)] -pub struct ReplacementSubDAG { - pub replacement: Replacement, - /// Name of the [`ReplacementStrategy`] that proposed this candidate. - /// Search fills this from `ReplacementStrategy::name`; consumers must not - /// infer it from the replacement's shape or provenance. - pub strategy: &'static str, - /// Machine-readable origin/role of this alternative. Selection uses this - /// instead of inferring strategy semantics from replacement shape or - /// pointer identity when several strategies contribute to one memo group. - pub provenance: ReplacementProvenance, - pub rationale: String, -} - -impl ReplacementSubDAG { - /// Whether this summary still needs accuracy/domain evidence before it can - /// be treated as certified. A missing guarantee on any summary candidate - /// (a sub-DAG whose root is an ASAP operator) is unknown; a kept - /// pre-ASAP sub-DAG carries an explicit exact guarantee. - pub fn has_missing_accuracy_evidence(&self) -> bool { - matches!( - &self.replacement, - Replacement::SubDAG(node) if !is_logical_rewrite(node) && has_missing_accuracy_evidence(node) - ) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ReplacementProvenance { - SummaryRealization, - CseShare, - CseRecompute, - LogicalRewrite, - /// [`crate::pass2::reconciliation::AccuracyReconciliationStrategy`]'s - /// "read a strictly-tighter sibling instead of building an independent, - /// looser copy" candidate (issue #273). Kept distinct from - /// `LogicalRewrite` — even though both are structurally-different, - /// semantically-equivalent rewrites — because - /// `cost_model::DefaultCostModel::estimate_cost` needs to price - /// it differently: `LogicalRewrite` candidates (`RollupStrategy`, - /// `TopKLimitReuseStrategy`) still rebuild `target` itself from a - /// different source, so pricing them like an independent rebuild is - /// correct; this candidate never rebuilds `target` at all; it reads a - /// sibling that (per this strategy's own safety argument) is built - /// regardless, so pricing it like a full independent rebuild would be - /// the wrong shape of cost, not just the wrong number. - AccuracyReconciliation, - /// [`Replacement::ExactComposition`] with - /// [`OperationPlacement::Read`] (issue #171). - ValueOperationAtQueryTime, - /// [`Replacement::ExactComposition`] with - /// [`OperationPlacement::Maintenance`] (issue #171). - ValueOperationAtIngestionTime, - /// A finalized whole-query result over rows carrying the PromQL series - /// identity, which the logical root does not expose (see - /// [`ReplacementStrategy::propose_for_root`]). Default selection never - /// commits it, because its evaluation must be validated and priced by - /// deployment; otherwise it would silently replace the logical plan. - RootPhysicalRealization, -} - -/// A candidate a strategy considered for a target but refused to propose on -/// accuracy-legality grounds (issue #172) — kept alongside the group's -/// legal candidates in [`TargetSubDAGCandidates::rejected`] so a rejection is as -/// inspectable (and exportable) as a selection. Never ranked: a -/// `CostModel` only ever sees [`TargetSubDAGCandidates::candidates`]. -#[derive(Debug, Clone)] -pub struct RejectedCandidate { - /// Name of the [`ReplacementStrategy`] that considered it. - pub strategy: &'static str, - /// What the candidate would have been (the same prose a - /// [`ReplacementSubDAG::rationale`] would have carried). - pub description: String, - /// The typed reason it is illegal. - pub error: AccuracyError, -} - -/// Everything one [`ReplacementStrategy`] has to say about one target: the -/// legal candidates it proposes plus the accuracy-illegal ones it refused — -/// the output of [`ReplacementStrategy::propose`]. -#[derive(Debug, Clone, Default)] -pub struct Proposals { - pub candidates: Vec, - pub rejected: Vec, - domain_error: Option, -} - -/// A replacement strategy: given a [`TargetSubDAG`], does this strategy have -/// an opinion on it at all (`matches`), and if so, every semantically valid -/// replacement (`replacements`)? -/// -/// The extension point this module exists for — the same shape -/// `CostModel` and [`Matcher`] already use elsewhere in this crate: a new -/// replacement source is a new `impl ReplacementStrategy`, no restructuring -/// of this trait or any existing strategy required. -/// -/// `replacements` is only meaningful when `matches` would return `true` for -/// the same target; both [`ASAPStrategies`] and [`SharedSubDAGStrategy`] -/// return an empty `Vec` rather than panicking when called on a target they -/// don't match, so a caller that skips the `matches` check first still gets a -/// safe (merely uninformative) answer instead of a crash. -pub trait ReplacementStrategy { - /// Stable, human-readable strategy name carried into every proposed - /// candidate and ultimately into planner diagnostics/visualizations. - fn name(&self) -> &'static str { - let short = std::any::type_name::() - .rsplit("::") - .next() - .expect("a Rust type name always has a final segment"); - short.split_once('<').map_or(short, |(base, _)| base) - } - - /// Does this strategy have any replacement to offer for `target`? - fn matches(&self, target: &TargetSubDAG<'_>) -> bool; - - /// Every valid replacement for `target` — not ranked, not filtered. - /// Reporting "every valid candidate" is this method's whole job; picking - /// the best one is a `CostModel`'s job, out of scope here. - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec; - - /// [`replacements`](Self::replacements) plus the accuracy-illegal - /// candidates this strategy refused to propose (issue #172). Default: - /// every candidate from `replacements`, no rejections — a strategy that - /// never performs an accuracy check need not override this. - /// [`search_workload_with`] calls this (not `replacements`) so the - /// rejections land in [`TargetSubDAGCandidates::rejected`]. - fn propose(&self, target: &TargetSubDAG<'_>) -> Proposals { - Proposals { - candidates: self.replacements(target), - rejected: Vec::new(), - domain_error: None, - } - } - - /// Whole-query logical alternatives for a workload root under its - /// end-to-end `target`. These may need input rows the root does not expose - /// (for example, the PromQL series identity), so - /// [`search_workload_with_targets`] asks only workload roots, once each. - /// They decide what to compute, never placement. Default: none. - fn propose_for_root(&self, _root: &Rc, _target: &AccuracyTarget) -> Proposals { - Proposals::default() - } -} - -/// Does an already-**available** [`Realization`] — e.g. a summary -/// instance a downstream deployment already materialized somewhere, found -/// via whatever inventory/index that deployment keeps — satisfy a -/// **required** [`Realization`] (one of the candidates -/// [`realizations_for_intent`] produced for some [`AggIntent`])? -/// -/// This is the query-optimization-literature "materialized view matching" -/// / "answering queries using views" question, narrowed to this crate's -/// summary vocabulary: not "can I build this from scratch" (that's what -/// [`realizations_for_intent`] answers) but "does something that already -/// exists answer this". -/// -/// `asap-plan` deliberately ships no implementation of this trait and no -/// default method body — unlike [`realizations_for_intent`], which decision -/// an available `Realization` satisfies a required one is not a fact this -/// crate can settle on its own. Two real, reasonable answers already -/// diverge outside this crate: -/// -/// - A **pure sketch-algebra** answer would say a `Sketch{kind: Kll, ..}` -/// requirement is satisfied by an available `DDSketch` (both quantile -/// sketches), and that a heap-bearing top-k sketch also answers a bare -/// frequency point-query (the heap is additional info on the same -/// underlying matrix) — but not the reverse. -/// - A **deployment with its own storage-layout rules** may need more: -/// e.g. whether a multi-population accumulator can serve a -/// single-population query via re-aggregation is a fact about that -/// deployment's storage layout, not about any summary family's kind at -/// all — a family's own kind doesn't encode grouping (grouping lives on -/// the post-ASAP node's `by` instead), so there is nothing in this -/// crate's own vocabulary to subsume. -/// -/// Implementations are expected to consult `required`/`available`'s -/// `kind` (and whatever grouping/placement context the deployment tracks -/// alongside `Realization`, which this trait's signature doesn't carry -/// because this crate has no inventory concept to carry it in). -pub trait Matcher { - fn is_satisfied_by(&self, required: &Realization, available: &Realization) -> bool; -} - -/// Every valid [`Realization`] for `intent`, exhaustive, in -/// [`summary_candidates`]' static order — the *only* place this crate -/// decides what an `AggIntent` may become. Nothing in this crate computes -/// "the one" `Realization` independently of this list: -/// [`ASAPStrategies`] keeps every entry as a candidate, and a caller -/// that wants a single executable answer takes the head of *that* strategy's -/// output itself. -/// -/// Exhaustive over the [`AggIntent`] vocabulary — adding a variant without an -/// explicit realization is a compile error, and the coverage-matrix test pins -/// each variant's category. -/// -/// `pub(crate)`: [`ASAPStrategies::replacements`] is this module's -/// own caller; `grouping::HydraGroupingStrategy` (issue #256) is the one -/// caller outside it, needing the exact same already-ranked candidate list -/// to find the `Realization::Sketch` matching the Hydra-eligible kind it -/// is building a candidate for. -pub(crate) fn realizations_for_intent(intent: &AggIntent) -> Vec { - match intent { - // ── Approximate-capable intents — the AccuracyTarget decides ──────── - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::FrequencyL2 { accuracy, .. } - | AggIntent::FrequencyEntropy { accuracy, .. } - | AggIntent::Count { accuracy } - | AggIntent::TopK { accuracy, .. } => match accuracy { - AccuracyTarget::Exact if matches!(intent, AggIntent::Count { .. }) => vec![ - exact_realization(intent), - Realization::Sketch(SketchKind::new( - SketchAlgorithm::UnivMon, - default_size_params(SketchAlgorithm::UnivMon, intent, 0.0, DEFAULT_DELTA), - )), - ], - AccuracyTarget::Exact => vec![exact_realization(intent)], - _ if matches!(intent, AggIntent::Count { .. }) => { - let mut candidates = sketch_realizations(intent, accuracy); - candidates.push(exact_realization(intent)); - candidates - } - _ => sketch_realizations(intent, accuracy), - }, - - // ── Exact mergeable accumulators ───────────────────────────────────── - AggIntent::Sum { .. } - | AggIntent::Min { .. } - | AggIntent::Max { .. } - | AggIntent::Rate - | AggIntent::IRate - | AggIntent::Increase => { - let (kind, params) = crate::pass1::function_rules::function_rules(intent) - .and_then(|rules| rules.accumulator) - .expect("exact accumulator intents have registered realizations"); - vec![exact_accumulator(intent, kind, params)] - } - - // ── Exact, non-mergeable reducers — richer partial state than a - // single value (see `agg_is_mergeable`), so no accumulator form. - AggIntent::Avg { .. } - | AggIntent::StdDev { .. } - | AggIntent::Variance { .. } - | AggIntent::PearsonCorr { .. } => { - vec![Realization::PassThrough] - } - - // ── Classic-bucket histogram_quantile (#79): exact `le`-bucket - // interpolation over pre-aggregated counts — NOT re-sketchable. - // (The native/raw form lowers to the generic `Quantile` above.) - AggIntent::HistogramQuantile { .. } => vec![Realization::PassThrough], - - // ── Per-series transforms and reductions with no sketch realization: - // counter-derivatives (#44), math (#45), time/calendar (#46), - // presence (#47), native-histogram accessors (#43), and the - // `*OverTime` reducers (#51). All exact by construction. - AggIntent::Changes - | AggIntent::Delta - | AggIntent::IDelta - | AggIntent::Deriv - | AggIntent::Resets - | AggIntent::PredictLinear { .. } - | AggIntent::DoubleExpSmoothing { .. } - | AggIntent::HistogramCount - | AggIntent::HistogramSum - | AggIntent::HistogramAvg - | AggIntent::HistogramStdDev - | AggIntent::HistogramStdVar - | AggIntent::HistogramFraction { .. } - | AggIntent::Math(_) - | AggIntent::Absent - | AggIntent::AbsentOverTime - | AggIntent::PresentOverTime - | AggIntent::TimeFn(_) - | AggIntent::LastOverTime - | AggIntent::FirstOverTime - | AggIntent::MadOverTime - | AggIntent::TsOfMinOverTime - | AggIntent::TsOfMaxOverTime - | AggIntent::TsOfFirstOverTime - | AggIntent::TsOfLastOverTime => vec![Realization::PassThrough], - - // ── Group / count_values (#49): exact per `agg_is_exact`, but their - // output is structural (constant-1 / a synthesized label column), - // not a value a summary accumulator carries. - AggIntent::Group | AggIntent::CountValues { .. } => vec![Realization::PassThrough], - - // ── Extension (deployment-model-specific, issue #131) — core has no - // realization opinion for a shape it doesn't know, so it stays - // logical. - AggIntent::Extension { .. } => vec![Realization::PassThrough], - } -} - -/// Exact realization of an approximate-capable intent whose target is -/// `AccuracyTarget::Exact`. `Count` has a mergeable exact accumulator; exact -/// quantile / top-k / cardinality have no single-value summary form (they -/// need the full multiset / heap / set) and pass through. -fn exact_realization(intent: &AggIntent) -> Realization { - match intent { - AggIntent::Count { .. } => exact_accumulator(intent, ExactKind::Count, ExactParams::Count), - _ => Realization::PassThrough, - } -} - -fn exact_accumulator(intent: &AggIntent, kind: ExactKind, params: ExactParams) -> Realization { - // An exact accumulator is only sound when partial states merge - // (`agg(A ∪ B) = combine(agg(A), agg(B))`). - debug_assert!( - agg_is_mergeable(intent), - "accumulator for non-mergeable {intent:?}" - ); - Realization::ExactAggregate { kind, params } -} - -/// Every candidate sketch [`Realization`] for an approximate-capable -/// intent, sized analytically to `accuracy`, in [`summary_candidates`]' -/// order — [`realizations_for_intent`]'s Sketch branch. -fn sketch_realizations(intent: &AggIntent, accuracy: &AccuracyTarget) -> Vec { - let (eps, delta) = accuracy_budget(accuracy); - summary_candidates(intent) - .iter() - .cloned() - .filter_map(|algorithm| { - let params = size_params(algorithm.clone(), intent, eps, delta); - sketch_state_bytes(¶ms) - .is_none_or(|bytes| bytes <= DEFAULT_MAX_SKETCH_STATE_BYTES) - .then(|| Realization::Sketch(SketchKind::new(algorithm, params))) - }) - .collect() -} - -/// Fail-safe ceiling used when a deployment has not supplied a tighter -/// resource model. It applies to one physical keyed state; grouped instance -/// multiplicity must be charged separately by deployment-aware costing. -pub const DEFAULT_MAX_SKETCH_STATE_BYTES: u64 = 512 * 1024 * 1024; - -/// Conservative dense-counter allocation for CMS-family states. Returning -/// `None` leaves non-CMS families to their family-specific resource models. -pub fn sketch_state_bytes(params: &SketchParams) -> Option { - if let SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } = params - { - return u64::from(*sketch_rows) - .checked_mul(u64::from(*sketch_cols))? - .checked_mul(8)? - .checked_add(u64::from(*heap_size).checked_mul(64)?)? - .checked_mul(u64::from(*layers)); - } - let (width, depth, heap_size) = match params { - SketchParams::Cms { width, depth } | SketchParams::CountSketch { width, depth } => { - (*width, *depth, 0) - } - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } - | SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - } => (*width, *depth, *heap_size), - _ => return None, - }; - // Eight-byte counters plus a conservative 64 bytes for each heap entry. - u64::from(width) - .checked_mul(u64::from(depth))? - .checked_mul(8)? - .checked_add(u64::from(heap_size).checked_mul(64)?) -} - -/// A deployment's explicit bet about how "typical" (non-adversarial) its -/// workload's collision pattern is expected to be, consumed only by -/// [`posterior_aware_size_params`]. -/// -/// This is **not** derived from Chen et al.'s posterior-error-estimation -/// technique (issue #239, `asap_types::post_asap::query_time::error_estimation`) -/// — that technique computes a tighter bound *at query time* from a -/// sketch's real counter values, and this repo has no sketch runtime yet -/// for a real counter array to size against (see that module's docs, and -/// `asap_types::post_asap::query_time`'s module doc for why it's a -/// deliberately separate folder from this crate's own *plan-time* code). -/// This struct is this crate's own *plan-time* analogue of the same -/// underlying intuition — an expected-case (skewed / non-adversarial) -/// workload needs a smaller sketch than the adversarial worst case — -/// expressed as an explicit, caller-supplied assumption rather than -/// anything observed or proven. Issue #250 tracks actually connecting the -/// two: feeding query-time-observed posterior error back into a future -/// replan's `width_relaxation` instead of a bare caller guess. -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct ExpectedCaseSizing { - /// Fraction, in `(0, 1]`, of the traditional worst-case width - /// ([`cms_width`]) the caller is betting is enough. `1.0` (or any - /// value outside `(0, 1)`) reproduces the worst-case width exactly — - /// no risk taken. A smaller value shrinks the sketch proportionally, - /// at the cost documented on [`posterior_aware_size_params`]. - pub width_relaxation: f64, -} - -/// Opt-in alternative to [`default_size_params`] for the CMS-family kinds -/// (`Cms` / `CmsWithHeap` / `CountSketch` / `CountSketchWithHeap`): sizes -/// width to `assumption.width_relaxation` of the worst-case [`cms_width`], -/// trading the unconditional worst-case `(ε,δ)` guarantee for a smaller -/// sketch under an explicit, caller-stated non-adversarial-workload bet — -/// see [`ExpectedCaseSizing`]. -/// -/// **The tradeoff, spelled out:** [`default_size_params`]'s width guarantees -/// `Pr[error > ε·|F|₁] < δ` for *any* input, including an adversarial one -/// built to maximize collisions (§3.3 of the posterior-error-estimation -/// paper this issue is about — see -/// `asap_types::post_asap::query_time::error_estimation`'s module docs). -/// Shrinking -/// width below that only keeps the same `(ε,δ)` guarantee if the real -/// workload's collision load stays within `width_relaxation` of the -/// worst-case assumption — this function does not check that, cannot check -/// it (no data exists at plan time), and does not change the formal -/// guarantee's statement; it only changes how much hardware is spent -/// chasing it. Callers accept that gap explicitly by choosing -/// `width_relaxation < 1.0`. -/// -/// Depth ([`cms_depth`]) is left unchanged from [`default_size_params`]: -/// depth trades away confidence *exponentially* (`Pr[all r rows bad] = -/// p^r` — each extra row multiplies the failure probability down), a -/// differently-shaped and materially riskier tradeoff than width's linear -/// relaxation. Issue #239 asks for *a* tighter-sizing option under a -/// stated assumption, not a full redesign of the depth/width tradeoff -/// space, so depth relaxation is left as explicit future scope. -/// -/// For every `SketchAlgorithm` outside the CMS family, this is identical to -/// [`default_size_params`] — `width_relaxation` only ever touches the -/// [`cms_width`]-sized formulas this issue is about. -/// -/// [`default_size_params`]'s own behavior is completely unchanged by this -/// function's existence — this is a separate, additive entry point, never -/// called from [`default_size_params`] or [`realizations_for_intent`]. -pub fn posterior_aware_size_params( - kind: SketchAlgorithm, - intent: &AggIntent, - eps: f64, - delta: f64, - assumption: ExpectedCaseSizing, -) -> SketchParams { - let relaxed_width = |eps: f64| -> u32 { - let base = cms_width(eps); - let f = assumption.width_relaxation; - if !(f.is_finite() && f > 0.0 && f < 1.0) { - return base; // out-of-range bet: no relaxation, fall back to worst case - } - saturating_ceil(base as f64 * f, 2, base) - }; - match kind { - SketchAlgorithm::Cms => SketchParams::Cms { - width: relaxed_width(eps), - depth: cms_depth(delta), - }, - SketchAlgorithm::CmsWithHeap => { - let k = match intent { - AggIntent::TopK { k, .. } => *k, - _ => unreachable!("CmsWithHeap is only a TopK candidate"), - }; - SketchParams::CmsWithHeap { - width: relaxed_width(eps), - depth: cms_depth(delta), - heap_size: crate::accuracy::topk_capacity(k, eps), - } - } - SketchAlgorithm::CountSketch => { - // CMS's expected-L1 collision relaxation is not a CountSketch - // L2 theorem; retain the formal CountSketch sizing unchanged. - default_size_params(kind, intent, eps, delta) - } - SketchAlgorithm::CountSketchWithHeap => { - // As above, do not apply CMS's L1 relaxation to CountSketch. - default_size_params(kind, intent, eps, delta) - } - // Every other kind is untouched by this issue's CMS-specific - // relaxation — defer to the existing formula verbatim. Spelled out - // exhaustively, matching `default_size_params`'s own match, rather - // than a wildcard arm: a future `SketchAlgorithm` variant then fails to - // compile *here* too, instead of silently inheriting worst-case - // sizing with no signal that this function never considered it. - SketchAlgorithm::Kll => default_size_params(kind, intent, eps, delta), - SketchAlgorithm::Hll => default_size_params(kind, intent, eps, delta), - SketchAlgorithm::DDSketch => default_size_params(kind, intent, eps, delta), - SketchAlgorithm::Theta => default_size_params(kind, intent, eps, delta), - SketchAlgorithm::Kmv | SketchAlgorithm::UnivMon => { - default_size_params(kind, intent, eps, delta) - } - } -} - -// ── ASAPStrategies ───────────────────────────────────────────────── - -static DEFAULT_ACCURACY_MODEL: DefaultAccuracyModel = DefaultAccuracyModel; -static DEFAULT_ALLOCATOR: EqualSplitAllocator = EqualSplitAllocator; -static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; - -/// The accuracy, allocation, and evidence inputs consulted during -/// candidate construction, bundled so the construction path threads one -/// argument. `accuracy` and `allocator` decide legality (issue #172) — see -/// [`crate::accuracy`]'s module docs. -#[derive(Clone, Copy)] -pub(crate) struct CandidatePlanningInputs<'a> { - pub accuracy: &'a dyn AccuracyModel, - pub allocator: &'a dyn AccuracyBudgetAllocator, - pub evidence: &'a dyn AccuracyEvidenceProvider, -} - -impl CandidatePlanningInputs<'static> { - /// The built-in [`DefaultAccuracyModel`]/[`EqualSplitAllocator`], with no - /// planning-time evidence. - pub(crate) fn with_default_accuracy() -> Self { - Self { - accuracy: &DEFAULT_ACCURACY_MODEL, - allocator: &DEFAULT_ALLOCATOR, - evidence: &NO_ACCURACY_EVIDENCE, - } - } -} - -/// Proposes the supported ASAP realizations for a bindable aggregate, including -/// exact accumulators, approximate sketches, and supported maintained populations. -/// Each valid realization becomes its own [`ReplacementSubDAG`]. -/// -/// [`realizations_for_intent`] enumerates summary families. This is not -/// limited to sketch algorithms. Nothing here ranks candidates by cost; that -/// is plan selection's job. -/// -/// The one thing that drops a candidate is accuracy legality (issue -/// #172), decided by the [`AccuracyModel`]: a -/// sketch over an approximate child is proposed only if its composed -/// guarantee has a sound propagation rule and satisfies the node's own -/// `AccuracyTarget`; otherwise it is reported through -/// [`ReplacementStrategy::propose`] as a [`RejectedCandidate`]. See -/// [`crate::accuracy`]'s module docs for the rules and the precedence -/// between root and per-node targets. -pub struct ASAPStrategies<'a> { - planning_inputs: CandidatePlanningInputs<'a>, -} - -impl Default for ASAPStrategies<'static> { - /// The built-in [`DefaultAccuracyModel`]/[`EqualSplitAllocator`]. - fn default() -> Self { - Self { - planning_inputs: CandidatePlanningInputs::with_default_accuracy(), - } - } -} - -impl<'a> ASAPStrategies<'a> { - /// A strategy with every model plugged in explicitly: `accuracy_model` - /// for guarantee derivation/propagation/satisfaction, `allocator` for - /// end-to-end budget splits. - pub fn new_with_planning_inputs( - accuracy_model: &'a dyn AccuracyModel, - allocator: &'a dyn AccuracyBudgetAllocator, - ) -> Self { - Self { - planning_inputs: CandidatePlanningInputs { - accuracy: accuracy_model, - allocator, - evidence: &NO_ACCURACY_EVIDENCE, - }, - } - } - - /// Like [`Self::new_with_planning_inputs`], with typed planning-time evidence for - /// rules such as TopK membership and Hydra shared-grid composition. - pub fn new_with_planning_inputs_and_evidence( - accuracy_model: &'a dyn AccuracyModel, - allocator: &'a dyn AccuracyBudgetAllocator, - evidence: &'a dyn AccuracyEvidenceProvider, - ) -> Self { - Self { - planning_inputs: CandidatePlanningInputs { - accuracy: accuracy_model, - allocator, - evidence, - }, - } - } - - /// Preserve the canonical Sort/Limit representation while exploring heap - /// realizations of an instant-vector ranking under the caller's target. - /// The input must carry the complete dynamic series identity. This never - /// treats a range of historical samples as the instant vector. - pub fn current_series_topk_candidates( - &self, - root: &Rc, - accuracy: &AccuracyTarget, - ) -> Proposals { - let Some(NonASAPOp::Limit { - n: Some(n), - offset: 0, - child, - .. - }) = root.non_asap() - else { - return Proposals::default(); - }; - let Some(NonASAPOp::Sort { - keys, - partition_by, - child, - }) = child.non_asap() - else { - return Proposals::default(); - }; - let [key] = keys.as_slice() else { - return Proposals::default(); - }; - let ScalarExpr::Column(value) = key.expr else { - return Proposals::default(); - }; - let schema = &child.schema; - if key.ascending - || key.nulls_first - || partition_by.is_without() - || !schema.has_promql_series_identity() - || !schema - .fields - .get(value) - .is_some_and(|column| column.name == "value") - || !is_current_series_source(child) - { - return Proposals::default(); - } - let Ok(ranked) = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(partition_by.clone()), - measures: vec![AggIntent::TopK { - k: *n, - accuracy: accuracy.clone(), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(child), - })) - else { - return Proposals::default(); - }; - self.propose_with(&ranked, None, None) - } - - /// Fixed-window maintenance can finalize each series' counter state and - /// build a fresh heap or grouped Sum for that evaluation window. Deployment must provide - /// a complete, synchronized population and bind the matching window; this - /// candidate never incrementally adds one window's rates to another. - /// - pub fn fixed_window_rate_candidates(&self, root: &Rc) -> Proposals { - fn place(node: &Rc) -> Option> { - retime_rate_finalize(node, ExecutionTiming::IngestionTime, true) - } - // Legal only if the candidate stays executable with its states maintained. - let timed = |node: &Rc| { - asap_types::ir::apply_materialization_timings( - node, - &asap_types::ir::MaterializationAssignment::all_ingestion_time(), - &mut asap_types::ir::TimingMemo::new(), - ) - .ok() - .and_then(|timed| { - asap_types::ir::physical_export::compile_physical_asap_dag(&timed).ok() - }) - }; - let mut proposals = self.propose_with(root, None, None); - proposals.candidates.retain_mut(|candidate| { - let Replacement::SubDAG(node) = &candidate.replacement else { - return false; - }; - let Some(dag) = timed(node) else { - return false; - }; - if !dag.nodes.iter().any(|node| match &node.payload { - asap_types::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - asap_types::ir::ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }, - ) => matches!( - kind.algorithm(), - SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap - ), - asap_types::ir::physical_export::PhysicalASAPOperatorPayload::ASAP( - asap_types::ir::ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, _), - .. - }, - ) => true, - _ => false, - }) { - return false; - } - let Some(placed) = place(node) else { - return false; - }; - if timed(&placed).is_none() { - return false; - } - let Ok(placed) = finalize_query_candidate(placed, root) else { - return false; - }; - candidate.replacement = Replacement::SubDAG(placed); - candidate - .rationale - .push_str("; fixed-window precompute over complete per-series counter states"); - true - }); - proposals - } - - /// Retain grouped Sum after a per-series Rate evaluation as a query-time - /// candidate alongside its complete-window maintenance placement. - /// - pub fn query_time_rate_aggregation_candidates(&self, root: &Rc) -> Proposals { - let mut proposals = self.fixed_window_rate_candidates(root); - proposals.candidates.retain_mut(|candidate| { - let Replacement::SubDAG(node) = &candidate.replacement else { return false }; - if !matches!(&node.operator, Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) - if matches!(&child.operator, Operator::ASAP(ASAPOp::SummaryAgg { family: FieldDataType::ExactAggregate(ExactKind::Sum, _), .. }))) { return false; } - let Some(query_time) = retime_rate_finalize(node, ExecutionTiming::QueryTime, false) else { return false }; - candidate.replacement = Replacement::SubDAG(query_time); - candidate.rationale = "query-time grouped Sum over complete per-series Rate evaluations".into(); - true - }); - proposals - } - - pub(crate) fn from_planning_inputs(planning_inputs: CandidatePlanningInputs<'a>) -> Self { - Self { planning_inputs } - } - - /// The whole enumeration for one target, with `intent_override` - /// substituting the target's own intent (only ever its `AccuracyTarget` - /// differs — see [`realize_child_with`]). `strictest_sibling` adds each - /// sketch resized to that stricter sibling accuracy (#509 summary - /// capability): alone it only costs more, but post-ASAP CSE shares it - /// with the sibling that needs it. - fn propose_with( - &self, - root: &Rc, - intent_override: Option<&AggIntent>, - strictest_sibling: Option<&AccuracyTarget>, - ) -> Proposals { - let mut proposals = Proposals::default(); - // A selected logical rewrite otherwise stays a kept pre-ASAP sub-DAG - // during DAG assembly. Also expose its concrete summary realization - // for selection. - if intent_override.is_none() { - if let Some(rewritten) = crate::pass1::rewrite::composed_aggregate_rewrite(root) { - if let Ok(node) = realize_child_with(&rewritten, self.planning_inputs, None) { - if node.contains_asap() { - proposals.candidates.push(ReplacementSubDAG { - replacement: Replacement::SubDAG(node), - strategy: "ASAPStrategies", - provenance: ReplacementProvenance::SummaryRealization, - rationale: "realize a schema-preserving composition of temporal and grouped accumulators".into(), - }); - } - } - } - } - if let Ok(Some(node)) = exact_topk_over_temporal_values(root, self.planning_inputs) { - proposals.candidates.push(ReplacementSubDAG { - replacement: Replacement::SubDAG(node), - strategy: "ASAPStrategies", - provenance: ReplacementProvenance::SummaryRealization, - rationale: "select exact Top-K from independently maintained temporal values" - .into(), - }); - } - if intent_override.is_none() { - if let Ok(Some(node)) = realize_temporal_average(root, self.planning_inputs, None) { - proposals.candidates.push(ReplacementSubDAG { - replacement: Replacement::SubDAG(node), - strategy: "ASAPStrategies", - provenance: ReplacementProvenance::SummaryRealization, - rationale: "read temporal average from sum/count only within the finite arithmetic domain; otherwise execute the original average".into(), - }); - } - } - if intent_override.is_none() && is_supported_exact_binary(root) { - if let Ok(Some(node)) = realize_binary(root, self.planning_inputs, None) { - let rationale = if node.guarantee.is_none() { - "DDSketch quantile ratio without a certified end-to-end accuracy guarantee" - } else { - "preserve exact PromQL arithmetic over independently realized summary operands" - }; - proposals.candidates.push(ReplacementSubDAG { - replacement: Replacement::SubDAG(node), - strategy: "ASAPStrategies", - provenance: ReplacementProvenance::SummaryRealization, - rationale: rationale.into(), - }); - } - return proposals; - } - let Some(declared) = bindable_intent(root) else { - return proposals; - }; - let intent = intent_override.unwrap_or(declared); - let planning_inputs = self.planning_inputs; - - // Is the child approximate? Probed once, up front: a candidate over - // an approximate child needs the end-to-end budget split across both - // layers, which changes which candidates exist at all. - let child_layers = aggregate_child(root) - .and_then(|child| realize_child_with(child, planning_inputs, None).ok()) - .and_then(|child| { - child - .guarantee - .as_ref() - .filter(|g| !g.is_exact()) - .map(ResultGuarantee::approximate_layer_count) - }); - - // `realizations_for_intent` is already exhaustive and ranked — no - // separate dispatch needed here. Only `Sketch` has more than one - // candidate in practice (every other variant's own dispatch produces - // exactly one `Realization`), but this loop doesn't need to know - // that; it just constructs whatever the list contains. - for realization in realizations_for_intent(intent) { - let rationale = describe_realization(intent, &realization); - // The as-declared composition: every layer sized to its own - // declared `AccuracyTarget`. Legal iff the composed guarantee - // satisfies this node's target — a front end copying one target - // onto every node does not make that so. - proposals.record( - rationale.clone(), - construct_summary_with( - root, - intent, - realization.clone(), - planning_inputs, - None, - None, - ), - ); - - // Sized for the strictest sibling reading the same summary input, - // when that changes the parameters. - if let (Some(stricter), Realization::Sketch(kind)) = (strictest_sibling, &realization) { - let (eps, delta) = accuracy_budget(stricter); - let algorithm = kind.algorithm().clone(); - let params = size_params(algorithm.clone(), intent, eps, delta); - if params != *kind.params() { - proposals.record( - format!( - "{rationale}; sized for the strictest sibling consumer {stricter:?}" - ), - construct_summary_with( - root, - &override_accuracy(intent, stricter), - Realization::Sketch(SketchKind::new(algorithm, params)), - planning_inputs, - None, - None, - ), - ); - } - } - - // Budget-split alternatives (issue #172, PR 2): re-size this - // layer and the approximate child under each allocation of this - // node's target across every approximate layer. - let (Some(child_layers), Realization::Sketch(kind), Some(target)) = - (child_layers, &realization, accuracy_target(intent)) - else { - continue; - }; - let family = FieldDataType::Sketch(kind.clone(), GroupingStrategy::default()); - let Some(child) = aggregate_child(root) else { - continue; - }; - let Some(NonASAPOp::Aggregate { reduction, .. }) = root.non_asap() else { - continue; - }; - let Ok(input) = realize_physical_summary_input(intent, &family, reduction, child) - else { - continue; - }; - let evaluation_query = evaluation(intent, &input.input); - let Some(local) = planning_inputs - .accuracy - .local_guarantee(&family, &evaluation_query) - else { - continue; - }; - let shape = CompositionShape { - metric: local.metric, - approximate_layer_count: 1 + child_layers, - }; - let allocations = planning_inputs.allocator.allocations(target, &shape); - if allocations.is_empty() { - proposals.rejected.push(RejectedCandidate { - strategy: "ASAPStrategies", - description: rationale.clone(), - error: AccuracyError::NoLegalAllocation { - target: target.clone(), - layer_count: shape.approximate_layer_count, - }, - }); - continue; - } - let declared_child_target = aggregate_child(root) - .and_then(|child| bindable_intent(child)) - .and_then(accuracy_target); - for allocation in allocations { - let outer_target = &allocation.layers[0]; - let inner_target = allocation.inner_target(&shape); - let (eps, delta) = accuracy_budget(outer_target); - let resized = Realization::Sketch(SketchKind::new( - kind.algorithm().clone(), - size_params(kind.algorithm().clone(), intent, eps, delta), - )); - // Identical to the as-declared composition already recorded - // above — nothing new to propose. - if resized == realization && inner_target.as_ref() == declared_child_target { - continue; - } - let note = GuaranteeSource::BudgetAllocation { - allocator: allocation.allocator.to_string(), - layer: 0, - layer_count: shape.approximate_layer_count, - local_target: outer_target.clone(), - end_to_end_target: target.clone(), - }; - proposals.record( - format!( - "{rationale}; sized under {} budget split of {target:?} across \ - {} approximate layers (this layer {outer_target:?}, child sub-DAG \ - {inner_target:?})", - allocation.allocator, shape.approximate_layer_count - ), - construct_summary_with( - root, - intent, - resized, - planning_inputs, - inner_target.as_ref(), - Some(note), - ), - ); - } - } - if proposals.candidates.is_empty() { - if let Some(error) = &proposals.domain_error { - if let Ok(node) = retain_exact(root) { - proposals.candidates.push(ReplacementSubDAG { - strategy: "ASAPStrategies", - replacement: Replacement::SubDAG(node), - provenance: ReplacementProvenance::SummaryRealization, - rationale: format!( - "{} stays pre-ASAP because summary construction crosses an illegal \ - execution-data_state boundary ({error})", - describe_intent(intent) - ), - }); - } - } - } - proposals - } -} - -impl Proposals { - /// File one construction attempt: a legal node becomes a candidate, an - /// [`RealizationError::Accuracy`] becomes a [`RejectedCandidate`], and a - /// schema-derivation failure is skipped exactly as it always was. - fn record(&mut self, rationale: String, built: Result, RealizationError>) { - match built { - Ok(node) => self.candidates.push(ReplacementSubDAG { - strategy: "ASAPStrategies", - replacement: Replacement::SubDAG(node), - provenance: ReplacementProvenance::SummaryRealization, - rationale, - }), - Err(RealizationError::Accuracy(error)) => self.rejected.push(RejectedCandidate { - strategy: "ASAPStrategies", - description: rationale, - error, - }), - Err(RealizationError::ExecutionDataState(error)) => { - self.domain_error.get_or_insert(error); - } - Err( - RealizationError::Schema(_) - | RealizationError::ExactOperationSchema(_) - | RealizationError::PhysicalRealization(_), - ) => {} - } - } -} - -/// The `child` of a [`bindable_intent`]-shaped `Aggregate`. -fn aggregate_child(node: &OperatorNode) -> Option<&Rc> { - match node.non_asap() { - Some(NonASAPOp::Aggregate { child, .. }) => Some(child), - _ => None, - } -} - -impl ReplacementStrategy for ASAPStrategies<'_> { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - bindable_intent(target.root).is_some() || is_supported_exact_binary(target.root) - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - self.propose(target).candidates - } - - fn propose(&self, target: &TargetSubDAG<'_>) -> Proposals { - self.propose_with(target.root, None, target.strictest_sibling_accuracy) - } - - /// Heap realizations of an instant-vector ranking (current-series TopK). - /// They rank rows that carry the complete PromQL series identity, which - /// the logical root does not expose, so each is a finalized query result - /// for the identity-carrying root. Placement variants (for example, - /// fixed-window or query-time Rate aggregation) are not listed here: the - /// materialization assigns timing and the physical compiler reads it. - fn propose_for_root(&self, root: &Rc, target: &AccuracyTarget) -> Proposals { - let Ok(typed) = asap_types::ir::schema_support::with_promql_series_identity(root) else { - return Proposals::default(); - }; - - let mut proposals = self.current_series_topk_candidates(&typed, target); - for mut candidate in std::mem::take(&mut proposals.candidates) { - let Replacement::SubDAG(node) = candidate.replacement else { - continue; - }; - let Ok(node) = finalize_query_candidate(node, &typed) else { - continue; - }; - let duplicate = proposals.candidates.iter().any(|existing| { - matches!(&existing.replacement, Replacement::SubDAG(other) if *other == node) - }); - if !duplicate { - candidate.replacement = Replacement::SubDAG(node); - candidate.provenance = ReplacementProvenance::RootPhysicalRealization; - proposals.candidates.push(candidate); - } - } - proposals - } -} - -/// A human-readable rationale for one candidate `Realization`, for -/// [`ReplacementSubDAG::rationale`] text. -fn describe_realization(intent: &AggIntent, realization: &Realization) -> String { - match realization { - Realization::Sketch(kind) => format!( - "{} realizes as a {:?} sketch — one of summary_candidates' \ - candidates for this intent (asap_logical_optimizer::pass1::replacement::realizations_for_intent)", - describe_intent(intent), - kind.algorithm() - ), - Realization::ExactAggregate { kind, .. } => format!( - "{} realizes as an exact {kind:?} accumulator — the only realization \ - realizations_for_intent produces for this intent (no approximate \ - candidate applies)", - describe_intent(intent) - ), - Realization::PassThrough => format!( - "{} has no summary realization and stays a logical pass-through — the \ - only realization realizations_for_intent produces for this intent", - describe_intent(intent) - ), - Realization::Sample { kind, .. } => format!( - "{} realizes as a {kind:?} sample — the only realization produced for \ - this intent", - describe_intent(intent) - ), - Realization::Wavelet { kind, .. } => format!( - "{} realizes as a {kind:?} wavelet transform — the only realization \ - produced for this intent", - describe_intent(intent) - ), - Realization::StatModel { kind, .. } => format!( - "{} realizes as a {kind:?} statistical model — the only realization \ - produced for this intent", - describe_intent(intent) - ), - } -} - -/// A short human-readable label for an `AggIntent`, for -/// [`ReplacementSubDAG::rationale`] text. Not exhaustive by design (unlike -/// this crate's other `AggIntent` matches, e.g. [`realizations_for_intent`]'s) -/// — this is prose for a rationale string, not a decision, so an unlisted -/// variant just falls back to its `Debug` tag rather than forcing every -/// future intent to be named here too. [`crate::pass1::explanation`] needs no -/// counterpart of its own: it reads a candidate's `rationale` — built from -/// this text — straight off [`ReplacementSubDAG`], rather than re-describing -/// the same intent a second time. -/// -/// `pub(crate)`: `grouping::HydraGroupingStrategy` (issue #256) reuses this -/// for its own rationale strings, for the same reason. -pub(crate) fn describe_intent(intent: &AggIntent) -> String { - match intent { - AggIntent::Quantile { q, .. } => format!("quantile(q={q})"), - AggIntent::Cardinality { cols, .. } if cols.len() > 1 => { - format!("cardinality (distinct count over {} columns)", cols.len()) - } - AggIntent::Cardinality { .. } => "cardinality (distinct count)".to_string(), - AggIntent::TopK { k, .. } => format!("top-{k} heavy-hitters"), - AggIntent::Count { .. } => "count".to_string(), - other => format!("{other:?}"), - } -} - -// ── realize_child / retain_exact: rank-and-take-first, and its fallback ── - -/// Rank-and-take-first selector for a single [`OperatorNode`]: enumerate -/// every candidate via [`ASAPStrategies::replacements`], keep the -/// `cost_model`-preferred (first) one, and fall back to [`retain_exact`] -/// when there's no candidate at all — **not** a general single-answer API -/// for a whole workload. Use `candidate_selection::global_selection` and DAG assembly -/// for coordinated logical selection; physical deployment remains downstream. -/// `root` must already be the caller's own -/// `Rc`, never fabricated per call, so this never allocates beyond what the -/// caller already held. -/// -/// Public because Stage 3 cost models need one representative bound node for -/// a target: `DefaultCostModel::estimate_cost` and the legacy CSE ranking in -/// `plan_selection::candidate_selection`. Every other caller goes through -/// [`ASAPStrategies::replacements`] directly and decides for itself. -pub fn realize_child(root: &Rc) -> Result, RealizationError> { - realize_child_with(root, CandidatePlanningInputs::with_default_accuracy(), None) -} - -/// [`realize_child`] with every model explicit, plus an optional -/// `end_to_end_target` for `root`'s own value (issue #172): when an -/// [`AccuracyBudgetAllocator`] hands an approximate child a share of its -/// parent's budget, the child is re-enumerated with that share substituted -/// for its declared `AccuracyTarget` — sizing its sketch (and, recursively, -/// re-splitting for its own approximate children) under the allocated -/// budget. A child whose declared target is `Exact` keeps it: an allocation -/// never approximates something the caller declared exact. -fn exact_topk_over_temporal_values( - root: &Rc, - planning_inputs: CandidatePlanningInputs<'_>, -) -> Result>, RealizationError> { - let Some(NonASAPOp::Aggregate { - reduction, - measures, - output_names: _, - filters, - having: None, - child, - }) = root.non_asap() - else { - return Ok(None); - }; - if any_measure_filtered(filters) { - return Ok(None); - } - let [AggIntent::TopK { k, .. }] = measures.as_slice() else { - return Ok(None); - }; - let Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - child: input, - .. - }) = child.non_asap() - else { - return Ok(None); - }; - if !matches!(input.non_asap(), Some(NonASAPOp::TimeRange { .. })) { - return Ok(None); - } - let values = realize_child_with(child, planning_inputs, Some(&AccuracyTarget::Exact))?; - if !values.contains_asap() - || !values - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) - { - return Ok(None); - } - let values = finalize_query_candidate(values, child)?; - let partition_by = reduction - .group_keys() - .ok_or(RealizationError::PhysicalRealization( - "temporal ranking requires explicit grouping", - ))? - .clone(); - let score = ranking_score_index(child, &values.schema)?; - let guarantee = values.guarantee.clone(); - let schema = values.schema.clone(); - let sorted = Rc::new( - OperatorNode::with_schema( - Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(score), - ascending: false, - nulls_first: false, - }], - partition_by: partition_by.clone(), - child: values, - }), - schema.clone(), - ) - .with_guarantee(guarantee.clone()), - ); - let node = Rc::new( - OperatorNode::with_schema( - Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by, - child: sorted, - }), - schema, - ) - .with_guarantee(guarantee), - ); - validate_maintained(&node, ExecutionTiming::QueryTime)?; - Ok(Some(node)) -} - -fn realize_temporal_average( - root: &Rc, - planning_inputs: CandidatePlanningInputs<'_>, - target: Option<&AccuracyTarget>, -) -> Result>, RealizationError> { - let Some(components) = crate::pass1::rewrite::temporal_average_components(root) else { - return Ok(None); - }; - let mut node = realize_child_with(&components, planning_inputs, target)?; - let Operator::NonASAP(NonASAPOp::BinaryOp { operator, .. }) = - &mut Rc::make_mut(&mut node).operator - else { - return Ok(None); - }; - operator.checked_finite_division = true; - validate_maintained(&node, ExecutionTiming::QueryTime)?; - Ok(Some(node)) -} - -pub(crate) fn realize_child_with( - root: &Rc, - planning_inputs: CandidatePlanningInputs<'_>, - end_to_end_target: Option<&AccuracyTarget>, -) -> Result, RealizationError> { - if let Some(node) = realize_temporal_average(root, planning_inputs, end_to_end_target)? { - return Ok(node); - } - if let Some(composed) = realize_binary(root, planning_inputs, end_to_end_target)? { - return Ok(composed); - } - let overridden = end_to_end_target.and_then(|target| { - let declared = bindable_intent(root)?; - match accuracy_target(declared) { - Some(AccuracyTarget::Exact) | None => None, - Some(_) => Some(override_accuracy(declared, target)), - } - }); - match ASAPStrategies::from_planning_inputs(planning_inputs) - .propose_with(root, overridden.as_ref(), None) - .candidates - .into_iter() - .next() - { - Some(ReplacementSubDAG { - replacement: Replacement::SubDAG(node), - .. - }) => Ok(node), - Some(ReplacementSubDAG { - replacement: Replacement::ExactComposition(_), - .. - }) => { - unreachable!("ASAPStrategies never returns a composition candidate") - } - // No candidate at all: `root` isn't `bindable_intent` shape (or its - // intent has no realization `realizations_for_intent` can't - // produce — never happens, that match is exhaustive), or every - // candidate was accuracy-illegal — either way the same conservative - // fallback `ASAPStrategies::matches` uses: keep the - // pre-ASAP sub-DAG, executed exactly. - None => retain_exact(root), - } -} - -/// Preserve an exact arithmetic root while allowing each vector operand to -/// select its own summary realization. If either vector arm cannot be -/// accelerated, return `None` so the caller keeps the whole query exact; -/// mixed raw/summary snapshots are never constructed. -fn realize_binary( - root: &Rc, - planning_inputs: CandidatePlanningInputs<'_>, - end_to_end_target: Option<&AccuracyTarget>, -) -> Result>, RealizationError> { - let Some(NonASAPOp::BinaryOp { - operator, - return_bool, - lhs, - rhs, - }) = root.non_asap() - else { - return Ok(None); - }; - let (op, vector_match) = (&operator.kind, &operator.vector_match); - if !matches!(op, BinaryOpKind::Arithmetic(_)) || vector_match.is_some() { - return Ok(None); - } - - let mut lhs_node = realize_binary_operand(lhs, planning_inputs, None)?; - let mut rhs_node = realize_binary_operand(rhs, planning_inputs, None)?; - - let direct_ddsketch_ratio = matches!(op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)) - && shared_quantile_target(lhs, rhs).is_some(); - let ratio_target = end_to_end_target - .cloned() - .or_else(|| shared_quantile_target(lhs, rhs)); - - let mut ratio_domains = None; - if direct_ddsketch_ratio { - if let Some(target) = ratio_target - .as_ref() - .and_then(ddsketch_ratio_operand_target) - { - let (alpha, _) = accuracy_budget(&target); - let lhs_domain = planning_inputs.evidence.quantile_input_domain(lhs); - let rhs_domain = planning_inputs.evidence.quantile_input_domain(rhs); - if [&lhs_domain, &rhs_domain] - .into_iter() - .flatten() - .any(|domain| !domain.supports_ddsketch(alpha)) - { - return Ok(None); - } - let domains = lhs_domain.zip(rhs_domain).map(|(lhs, rhs)| [lhs, rhs]); - let has_mean = [lhs, rhs] - .iter() - .any(|expr| matches!(bindable_intent(expr), Some(AggIntent::Avg { .. }))); - if has_mean - && domains.as_ref().is_none_or(|domains| { - domains.iter().any(|domain| { - !(domain.lower.abs().max(domain.upper.abs()) * domain.max_samples as f64) - .is_finite() - }) - }) - { - return Ok(None); - } - lhs_node = realize_ddsketch_quantile_operand(lhs, planning_inputs, &target)?; - rhs_node = realize_ddsketch_quantile_operand(rhs, planning_inputs, &target)?; - if let Some(domains) = domains.as_ref() { - for (domain, node) in domains.iter().zip([&lhs_node, &rhs_node]) { - if !ddsketch_quantile_alpha(node) - .or_else(|| { - node.guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) - .then_some(alpha) - }) - .is_some_and(|alpha| domain.supports_ddsketch(alpha)) - { - return Ok(None); - } - } - } - ratio_domains = domains; - } - } else if let Some(target) = end_to_end_target { - let operand_guarantees = [lhs_node.guarantee.as_ref(), rhs_node.guarantee.as_ref()]; - if operand_guarantees.iter().any(Option::is_none) { - return Ok(None); - } - let approximate = operand_guarantees - .into_iter() - .enumerate() - .filter_map(|(index, guarantee)| { - guarantee - .filter(|guarantee| !guarantee.is_exact()) - .map(|guarantee| (index, guarantee.metric)) - }) - .collect::>(); - if !approximate.is_empty() { - let metric = approximate[0].1; - if approximate - .iter() - .any(|(_, candidate)| *candidate != metric) - { - return Ok(None); - } - let shape = CompositionShape { - metric, - approximate_layer_count: approximate.len(), - }; - let Some(allocation) = planning_inputs - .allocator - .allocations(target, &shape) - .into_iter() - .next() - else { - return Ok(None); - }; - for ((operand_index, _), local_target) in - approximate.into_iter().zip(allocation.layers.iter()) - { - if operand_index == 0 { - lhs_node = realize_binary_operand(lhs, planning_inputs, Some(local_target))?; - } else { - rhs_node = realize_binary_operand(rhs, planning_inputs, Some(local_target))?; - } - } - } - } - // Only direct quantile ratios may consume approximate division operands - // without domain proof; their root guarantee remains unknown. - if ratio_domains.is_none() - && !direct_ddsketch_ratio - && matches!(op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)) - && [&lhs_node, &rhs_node].iter().any(|node| { - !node - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) - }) - { - return Ok(None); - } - - let lhs_accelerated = lhs_node.contains_asap(); - let rhs_accelerated = rhs_node.contains_asap(); - if !lhs_accelerated || !rhs_accelerated { - return Ok(None); - } - - lhs_node = finalize_query_candidate(lhs_node, lhs)?; - rhs_node = finalize_query_candidate(rhs_node, rhs)?; - - let has_ratio_domains = ratio_domains.is_some(); - let guarantee = if matches!(op, BinaryOpKind::Arithmetic(ArithmeticOpKind::Div)) - && direct_ddsketch_ratio - && has_ratio_domains - && [&lhs_node, &rhs_node].iter().all(|node| { - ddsketch_quantile_alpha(node).is_some() - || node - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) - }) { - [lhs_node.guarantee.clone(), rhs_node.guarantee.clone()] - .into_iter() - .collect::>>() - .and_then(|inputs| { - planning_inputs - .accuracy - .propagate( - &CompositionOperator::ExactDivision, - &inputs, - None, - &crate::accuracy::PropagationStats { - division_operand_domains: ratio_domains, - ..Default::default() - }, - ) - .ok() - }) - } else { - [lhs_node.guarantee.as_ref(), rhs_node.guarantee.as_ref()] - .into_iter() - .all(|guarantee| guarantee.is_some_and(ResultGuarantee::is_exact)) - .then(|| ResultGuarantee::exact("BinaryOp over exact operands")) - }; - - if direct_ddsketch_ratio && has_ratio_domains && guarantee.is_none() { - return Ok(None); - } - - Ok(Some(Rc::new( - OperatorNode::with_schema( - Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: op.clone(), - vector_match: vector_match.clone(), - }, - return_bool: *return_bool, - lhs: lhs_node, - rhs: rhs_node, - }), - root.schema.clone(), - ) - // Exact arithmetic does not erase approximation error. Until the - // accuracy algebra has an operator-specific rule (and any value-range - // evidence needed by multiplication/division), unknown stays unknown. - .with_guarantee(guarantee), - ))) -} - -/// Rebuild the summary chain above a per-series `Rate` accumulator with its -/// `FinalizeExactAccumulator` placed at `timing`. `strict` additionally -/// requires the fixed-window shape (a `PerEntity` Rate over a `TimeRange`); -/// `None` when no such boundary exists (strict only). -fn retime_rate_finalize( - node: &Rc, - timing: ExecutionTiming, - strict: bool, -) -> Option> { - let is_rate_boundary = |child: &OperatorNode| match &child.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Rate, _), - reduction, - child: source, - .. - }) => { - !strict - || (matches!(reduction, Reduction::PerEntity) - && matches!(source.non_asap(), Some(NonASAPOp::TimeRange { .. }))) - } - _ => false, - }; - let rebuilt = |operator: Operator, timing: Option| { - let mut copy = node.as_ref().clone(); - copy.operator = operator; - copy.timing = timing; - Rc::new(copy) - }; - match &node.operator { - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) if is_rate_boundary(child) => { - Some(rebuilt(node.operator.clone(), Some(timing))) - } - Operator::ASAP( - ASAPOp::FinalizeExactAccumulator { child } - | ASAPOp::SummaryAgg { child, .. } - | ASAPOp::SummaryEstimate { - summary_input: child, - .. - }, - ) => { - let placed = match retime_rate_finalize(child, timing, strict) { - Some(placed) => placed, - None if strict => return None, - None => return Some(Rc::clone(node)), - }; - let operator = node.operator.map_children(|_| Rc::clone(&placed)); - Some(rebuilt(operator, node.timing)) - } - _ if strict => None, - _ => Some(Rc::clone(node)), - } -} - -/// Put an explicit read boundary between maintained exact state and a -/// query-time value consumer. Approximate summaries must already carry a -/// `SummaryEstimate`, so they deliberately do not pass this predicate. -pub fn finalize_query_candidate( - node: Rc, - logical_output: &OperatorNode, -) -> Result, RealizationError> { - finalize_exact_accumulator(node, logical_output, ExecutionTiming::QueryTime) -} - -/// The read boundary's placement is fixed here, where the candidate's -/// semantics decide it (a fresh query-time summary over this evaluation's -/// finalized values vs. finalized values feeding maintenance); the materialization -/// timing pass honors it. -fn finalize_exact_accumulator( - node: Rc, - logical_output: &OperatorNode, - placement: ExecutionTiming, -) -> Result, RealizationError> { - let is_exact_state = matches!( - node.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(..), - .. - }) - ); - if !is_exact_state { - return Ok(node); - } - // The child edge carries accumulator state, while this explicit read - // boundary produces the logical operator's ordinary values. Preserve the - // canonical pre-ASAP output types instead of leaking ExactAggregate into - // query-time operators that follow this node. - let schema = logical_output.schema.clone(); - let guarantee = node.guarantee.clone(); - Ok(Rc::new( - OperatorNode::with_schema( - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: node }), - schema, - ) - .with_guarantee(guarantee) - .with_timing(Some(placement)), - )) -} - -fn is_supported_exact_binary(root: &OperatorNode) -> bool { - matches!( - root.non_asap(), - Some(NonASAPOp::BinaryOp { - operator: BinaryOperator { - kind: BinaryOpKind::Arithmetic(_), - vector_match: None, - .. - }, - .. - }) - ) -} - -/// Quantile operands inherit one workload target. A temporal mean is exact -/// on its checked finite domain and needs no approximation budget. -fn shared_quantile_target(lhs: &OperatorNode, rhs: &OperatorNode) -> Option { - let quantile_target = |expr: &OperatorNode| match bindable_intent(expr) { - Some(AggIntent::Quantile { accuracy, q, .. }) - if q.is_finite() && (0.0..=1.0).contains(q) => - { - Some(accuracy.clone()) - } - _ => None, - }; - match (quantile_target(lhs), quantile_target(rhs)) { - (Some(lhs), Some(rhs)) => (lhs == rhs).then_some(lhs), - (Some(target), None) if matches!(bindable_intent(rhs), Some(AggIntent::Avg { .. })) => { - Some(target) - } - (None, Some(target)) if matches!(bindable_intent(lhs), Some(AggIntent::Avg { .. })) => { - Some(target) - } - _ => None, - } -} - -/// For `a / b`, two DDSketches with the same relative bound `alpha` produce -/// at most `2 * alpha / (1 - alpha)` relative error. Inverting that bound -/// gives `alpha = epsilon / (2 + epsilon)`. -fn ddsketch_ratio_operand_target(target: &AccuracyTarget) -> Option { - let tighten = - |epsilon: f64| (epsilon.is_finite() && epsilon > 0.0).then_some(epsilon / (2.0 + epsilon)); - match target { - AccuracyTarget::Exact => None, - AccuracyTarget::Epsilon(epsilon) => tighten(*epsilon).map(AccuracyTarget::Epsilon), - AccuracyTarget::EpsilonDelta { epsilon, delta } => { - tighten(*epsilon).map(|epsilon| AccuracyTarget::EpsilonDelta { - epsilon, - delta: *delta, - }) - } - } -} - -fn ddsketch_quantile_alpha(node: &OperatorNode) -> Option { - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query: PostAsapSketchStatistic::Quantile { .. }, - }) = &node.operator - else { - return None; - }; - let Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &summary_input.operator - else { - return None; - }; - match (kind.algorithm(), kind.params()) { - (SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }) => Some(*alpha), - _ => None, - } -} - -fn has_missing_accuracy_evidence(node: &OperatorNode) -> bool { - node.guarantee - .as_ref() - .is_none_or(ResultGuarantee::has_unknown) -} - -/// A direct ratio has an operator-specific DDSketch proof, so it must select -/// DDSketch rather than the cost model's generally preferred KLL candidate. -fn realize_ddsketch_quantile_operand( - operand: &Rc, - planning_inputs: CandidatePlanningInputs<'_>, - target: &AccuracyTarget, -) -> Result, RealizationError> { - let intent = bindable_intent(operand).and_then(|intent| match intent { - AggIntent::Quantile { .. } => Some(override_accuracy(intent, target)), - _ => None, - }); - let Some(intent) = intent else { - return realize_binary_operand(operand, planning_inputs, Some(target)); - }; - let (epsilon, delta) = accuracy_budget(target); - let realization = Realization::Sketch(SketchKind::new( - SketchAlgorithm::DDSketch, - size_params(SketchAlgorithm::DDSketch, &intent, epsilon, delta), - )); - construct_summary_with(operand, &intent, realization, planning_inputs, None, None) -} - -fn realize_binary_operand( - operand: &Rc, - planning_inputs: CandidatePlanningInputs<'_>, - end_to_end_target: Option<&AccuracyTarget>, -) -> Result, RealizationError> { - realize_child_with(operand, planning_inputs, end_to_end_target) -} - -/// `intent` with its `AccuracyTarget` replaced by `target` — a no-op for an -/// intent that carries none (see [`accuracy_target`]). -fn override_accuracy(intent: &AggIntent, target: &AccuracyTarget) -> AggIntent { - let mut out = intent.clone(); - match &mut out { - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::Count { accuracy } - | AggIntent::TopK { accuracy, .. } => *accuracy = target.clone(), - _ => {} - } - out -} - -/// Keep an unrewritten pre-ASAP sub-DAG as it is. There is no wrapper node: -/// the sub-DAG itself is the plan, carrying an exact guarantee. The same -/// `Rc` is returned when the node already has a guarantee; otherwise a copy -/// with `guarantee = exact("RetainedExact")` — only for a sub-DAG with no -/// ASAP operator (a sub-DAG containing one keeps whatever its construction -/// established). `pub` so a caller can fall back to this explicitly — e.g. -/// when `ASAPStrategies::replacements()` returns no candidate for a -/// target, or a deployment wants to force a node its own runtime can't -/// actually implement — through the same fallback this crate's own dispatch -/// uses. -pub fn retain_exact(expr: &Rc) -> Result, RealizationError> { - retain_exact_rc(Rc::clone(expr)) -} - -fn retain_exact_rc(expr: Rc) -> Result, RealizationError> { - if expr.guarantee.is_some() || expr.contains_asap() { - return Ok(expr); - } - // Keeping the same sub-DAG twice (e.g. one `Scan` read by an exact - // aggregate and by a sketch, or by two candidates) must yield one node: - // sharing is pointer identity. Memoize the kept copy per input node while - // both are alive; weak references keep the memo from extending lifetimes - // or matching a reused address. - type KeptMemo = HashMap<*const OperatorNode, (Weak, Weak)>; - thread_local! { - static KEPT: RefCell = RefCell::new(HashMap::new()); - } - let key = Rc::as_ptr(&expr); - if let Some(kept) = KEPT.with(|memo| { - memo.borrow().get(&key).and_then(|(input, kept)| { - input - .upgrade() - .filter(|input| Rc::ptr_eq(input, &expr)) - .and_then(|_| kept.upgrade()) - }) - }) { - return Ok(kept); - } - let kept = Rc::new( - expr.as_ref() - .clone() - // A kept pre-ASAP sub-DAG is executed exactly by the runtime - // (`Realization::PassThrough`'s contract) — zero error. - .with_guarantee(Some(ResultGuarantee::exact("RetainedExact"))), - ); - KEPT.with(|memo| { - let mut memo = memo.borrow_mut(); - if memo.len() > 4096 { - memo.retain(|_, (input, kept)| input.strong_count() > 0 && kept.strong_count() > 0); - } - memo.insert(key, (Rc::downgrade(&expr), Rc::downgrade(&kept))); - }); - Ok(kept) -} - -// ── Construction: turn one already-decided Realization into an OperatorNode ─ - -/// The bindable shape [`ASAPStrategies`] targets: a single intent, no -/// `HAVING`. A multi-intent node (SQL `SELECT SUM(a), AVG(b)`), or one with a -/// `HAVING` predicate (the filter would need the estimate first), stays -/// logical. Unsupported logical parents are conservatively kept as pre-ASAP -/// sub-DAGs ([`retain_exact`]). Relational operators are retained during -/// final DAG assembly so their independently planned children remain -/// visible. -pub fn bindable_intent(node: &OperatorNode) -> Option<&AggIntent> { - if let Some(NonASAPOp::Aggregate { - measures, - filters, - having, - .. - }) = node.non_asap() - { - if let ([intent], None) = (measures.as_slice(), having) { - if !any_measure_filtered(filters) { - return Some(intent); - } - } - } - None -} - -/// `expr` must still be the [`bindable_intent`] shape for `realization` to -/// have any effect; anything else falls back to [`retain_exact`]. -/// Only `expr`'s own top-level decision is forced — recursion into `expr`'s -/// child goes back through [`realize_child`] (fresh candidate -/// enumeration, not a forced pick), so choosing one candidate for a target -/// never leaks into that target's own nested aggregates. -/// -/// `pub(crate)`: `grouping::HydraGroupingStrategy` (issue #256) is the one -/// caller outside this module — the same first-class, -/// one-candidate-at-a-time primitive [`ASAPStrategies`] itself -/// calls once per candidate, reused rather than duplicated so a Hydra -/// candidate gets exactly the same schema derivation/column -/// resolution/evaluation construction as every other candidate, patching only -/// the `grouping` field this axis owns. -/// Construct a summary with every model explicit (issue #172). `intent` -/// is `expr`'s own [`bindable_intent`], or a copy of it with an allocated -/// `AccuracyTarget` substituted (see [`realize_child_with`]). -/// `child_target`, when set, is the end-to-end budget the child sub-DAG is -/// re-enumerated under; `allocation` is the provenance note recording the -/// split that produced both. `Err(RealizationError::Accuracy)` is the -/// fail-closed answer for a composition with no sound rule or one that -/// misses `intent`'s target. -pub(crate) fn construct_summary_with( - expr: &OperatorNode, - intent: &AggIntent, - realization: Realization, - planning_inputs: CandidatePlanningInputs<'_>, - child_target: Option<&AccuracyTarget>, - allocation: Option, -) -> Result, RealizationError> { - let local_target = match allocation.as_ref() { - Some(GuaranteeSource::BudgetAllocation { local_target, .. }) => Some(local_target), - _ => accuracy_target(intent), - }; - let estimator = crate::accuracy::EstimatorAccuracy::new( - planning_inputs.accuracy, - planning_inputs.evidence.estimator_contract(expr), - local_target, - ); - let realization = match realization { - Realization::Sketch(kind) => match estimator.size_params(kind.algorithm()) { - Some(params) => Realization::Sketch(SketchKind::new(kind.algorithm().clone(), params)), - None => Realization::Sketch(kind), - }, - other => other, - }; - if let Some(NonASAPOp::Aggregate { - reduction, child, .. - }) = expr.non_asap() - { - // `bindable_intent` already established the shape: exactly one - // intent, no HAVING. (Multi-intent nodes and HAVING stay logical.) - if bindable_intent(expr).is_some() { - if let Some((family, estimate)) = summary_family(realization) { - let input = realize_physical_summary_input(intent, &family, reduction, child)?; - let candidate = construct_summary_agg( - expr, - reduction, - intent, - input, - family, - estimate, - planning_inputs, - child_target, - allocation, - )?; - if is_snapshot_weighted_topk(intent, child) { - return finish_weighted_topk(candidate, expr, intent); - } - return Ok(candidate); - } - } - } - retain_exact_rc(Rc::new(expr.clone())) -} - -fn finish_weighted_topk( - candidate: Rc, - logical: &OperatorNode, - intent: &AggIntent, -) -> Result, RealizationError> { - let AggIntent::TopK { k, .. } = intent else { - unreachable!() - }; - let Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(groups), - child, - .. - }) = logical.non_asap() - else { - return Err(RealizationError::PhysicalRealization( - "TopK requires explicit grouping", - )); - }; - let schema = child.schema.clone(); - let score = ranking_score_index(child, &schema)?; - let cols = schema - .fields - .iter() - .enumerate() - .map(|(i, field)| { - let source = if i == score { - candidate.schema.fields.len() - 1 - } else { - let matches = candidate - .schema - .fields - .iter() - .enumerate() - .filter(|(_, f)| f.name == field.name && f.dtype == field.dtype) - .map(|(i, _)| i) - .collect::>(); - match matches.as_slice() { - [i] => *i, - _ => { - return Err(RealizationError::PhysicalRealization( - "ambiguous TopK output identity", - )) - } - } - }; - Ok(ProjectItem { - alias: Some(field.name.clone()), - expr: ScalarExpr::Column(source), - }) - }) - .collect::, _>>()?; - let guarantee = candidate.guarantee.clone(); - let projected = Rc::new( - OperatorNode::with_schema( - Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: candidate, - }), - schema.clone(), - ) - .with_guarantee(guarantee.clone()), - ); - let sorted = Rc::new( - OperatorNode::with_schema( - Operator::NonASAP(NonASAPOp::Sort { - keys: vec![SortKey { - expr: ScalarExpr::Column(score), - ascending: false, - nulls_first: false, - }], - partition_by: groups.clone(), - child: projected, - }), - schema.clone(), - ) - .with_guarantee(guarantee.clone()), - ); - let result = Rc::new( - OperatorNode::with_schema( - Operator::NonASAP(NonASAPOp::Limit { - n: Some(*k), - offset: 0, - partition_by: groups.clone(), - child: sorted, - }), - schema, - ) - .with_guarantee(guarantee), - ); - validate_maintained(&result, ExecutionTiming::QueryTime)?; - Ok(result) -} - -fn is_current_series_source(child: &OperatorNode) -> bool { - let source = match child.non_asap() { - Some(NonASAPOp::TimeRange { child, .. }) => child.as_ref(), - _ => child, - }; - matches!(source.non_asap(), Some(NonASAPOp::Scan { - source: asap_types::ir::operator::Source::TimeSeries { .. }, schema, .. - }) if schema.has_promql_series_identity()) -} - -fn is_snapshot_weighted_topk(intent: &AggIntent, child: &OperatorNode) -> bool { - matches!(intent, AggIntent::TopK { .. }) - && (is_current_series_source(child) - || matches!(child.non_asap(), - Some(NonASAPOp::Aggregate { measures, child, .. }) - if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase]) - || (matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - && matches!(child.non_asap(), Some(NonASAPOp::Aggregate { measures, .. }) - if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase]))))) -} - -/// Translate an [`Realization`] into the `(family, needs a -/// SummaryEstimate evaluation)` pair [`construct_summary_agg`] needs, or `None` -/// for `PassThrough` (the caller falls back to [`retain_exact`]). -/// -/// Every family's partial state needs a evaluation to recover a value, except -/// `ExactAggregate` — its partial state *is* the value already, so no -/// estimate step follows it. -fn summary_family(realization: Realization) -> Option<(FieldDataType, bool)> { - Some(match realization { - Realization::ExactAggregate { kind, params } => { - (FieldDataType::ExactAggregate(kind, params), false) - } - Realization::Sketch(kind) => ( - FieldDataType::Sketch(kind, GroupingStrategy::default()), - true, - ), - Realization::Sample { kind, params } => (FieldDataType::Sample(kind, params), true), - Realization::Wavelet { kind, params } => (FieldDataType::Wavelet(kind, params), true), - Realization::StatModel { kind, params } => (FieldDataType::StatModel(kind, params), true), - Realization::PassThrough => return None, - }) -} - -type PhysicalSummaryInputRule = - fn(&AggIntent, &FieldDataType, &Reduction, &Rc) -> PhysicalSummaryInputRuleResult; - -/// Ordered physical-realization rules for realizations that consume more -/// than the immediate logical input. New composite primitives add a rule here -/// instead of adding query- or algorithm-specific branches to -/// `construct_summary_agg`. -const PHYSICAL_SUMMARY_INPUT_RULES: &[PhysicalSummaryInputRule] = &[ - realize_value_frequency_summary_input, - realize_counter_value_summary_input, - realize_current_series_summary_input, - realize_keyed_additive_summary_input, -]; - -fn realize_value_frequency_summary_input( - intent: &AggIntent, - family: &FieldDataType, - _reduction: &Reduction, - child: &Rc, -) -> PhysicalSummaryInputRuleResult { - // Frequency counts hash sample values as items but add one per observation. - // Using the sample as a weight would turn counts into sums and admit signed CMS updates. - if !matches!(family, FieldDataType::Sketch(kind, _) - if kind.algorithm() == &SketchAlgorithm::UnivMon - || (matches!(intent, AggIntent::Count { .. }) - && matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CountSketch))) - { - return PhysicalSummaryInputRuleResult::NotApplicable; - } - let schema = &child.schema; - // One item per observation is a single value stream. `summary_candidates` - // already withholds UnivMon from a distinct-tuple count; refused here too - // so the invariant does not rest on that table alone. - if intent.input_cols().len() > 1 { - return PhysicalSummaryInputRuleResult::Unsupported( - "a value-frequency summary reads a single column", - ); - } - PhysicalSummaryInputRuleResult::Realized(PhysicalSummaryInput { - child: Rc::clone(child), - input: SummaryUpdate { - item: Some(SummaryInputExpr::Column(summarised_column(intent, schema))), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }, - }) -} - -fn realize_physical_summary_input( - intent: &AggIntent, - family: &FieldDataType, - reduction: &Reduction, - child: &Rc, -) -> Result { - for rule in PHYSICAL_SUMMARY_INPUT_RULES { - match rule(intent, family, reduction, child) { - PhysicalSummaryInputRuleResult::NotApplicable => {} - PhysicalSummaryInputRuleResult::Realized(input) => return Ok(input), - PhysicalSummaryInputRuleResult::Unsupported(reason) => { - return Err(RealizationError::PhysicalRealization(reason)); - } - } - } - - let child_schema = &child.schema; - if matches!(intent, AggIntent::TopK { .. }) { - return Err(RealizationError::PhysicalRealization( - "Top-K needs an explicit item identity and additive update input", - )); - } - Ok(PhysicalSummaryInput { - child: Rc::clone(child), - input: SummaryUpdate { - item: None, - weight: summarised_input(intent, child_schema) - .map_err(RealizationError::PhysicalRealization)?, - weight_domain: WeightDomain::UnknownOrSigned, - }, - }) -} - -/// Emit `SummaryAgg` (recursively binding the child), plus the -/// `SummaryEstimate` evaluation when `estimate` is set. -// Retain the exact expression and schema while placing its value production -// on the update path (a node runs when its consumer runs, so beneath a -// maintained summary this value production is ingestion-time work). -// Read-time consumers keep their original shared nodes. -fn maintenance_exact_values(node: Rc) -> Option> { - let operator = match &node.operator { - // These guards can fall back at read time, but cannot recover a parent - // sketch after an invalid value has entered its maintained state. - Operator::NonASAP(NonASAPOp::BinaryOp { operator, .. }) - if operator.checked_finite_division || operator.checked_relative_division => - { - return None; - } - Operator::NonASAP(NonASAPOp::BinaryOp { - lhs, - rhs, - operator, - return_bool, - }) if operator.vector_match.is_none() - && matches!(operator.kind, BinaryOpKind::Arithmetic(_)) - && node - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) => - { - Operator::NonASAP(NonASAPOp::BinaryOp { - lhs: maintenance_exact_values(lhs.clone())?, - rhs: maintenance_exact_values(rhs.clone())?, - operator: operator.clone(), - return_bool: *return_bool, - }) - } - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) - if matches!( - child.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(..), - .. - }) - ) => - { - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { - child: child.clone(), - }) - } - _ => return Some(node), - }; - Some(Rc::new( - OperatorNode::with_schema(operator, node.schema.clone()) - .with_guarantee(node.guarantee.clone()), - )) -} - -#[allow(clippy::too_many_arguments)] -fn construct_summary_agg( - node: &OperatorNode, - reduction: &Reduction, - intent: &AggIntent, - input: PhysicalSummaryInput, - family: FieldDataType, - estimate: bool, - planning_inputs: CandidatePlanningInputs<'_>, - child_target: Option<&AccuracyTarget>, - allocation: Option, -) -> Result, RealizationError> { - // The single canonical pre-ASAP derivation (per-series vs cross-series, - // name overrides) already computes the row shape; binding only retypes - // the summary state column. - let keyed_heap = input.input.item.is_some() - && matches!( - &family, - FieldDataType::Sketch(kind, _) - if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap) - ); - let snapshot_weighted = matches!(node.non_asap(), Some(NonASAPOp::Aggregate { child, .. }) - if is_snapshot_weighted_topk(intent, child)); - let mut family = family; - let score_population = if snapshot_weighted { - let bound = planning_inputs.evidence.topk_max_distinct_items(node); - if bound.is_some_and(|n| n == 0 || n > (1u64 << 53)) { - return Err(RealizationError::PhysicalRealization( - "invalid weighted TopK distinct-item bound", - )); - } - if let (Some(n), FieldDataType::Sketch(kind, grouping)) = (bound, &family) { - let (eps, delta) = accuracy_budget(accuracy_target(intent).expect("TopK target")); - let params = default_size_params( - kind.algorithm().clone(), - intent, - eps, - delta / (2.0 * n as f64), - ); - family = FieldDataType::Sketch( - SketchKind::new(kind.algorithm().clone(), params), - grouping.clone(), - ); - } - bound - } else { - None - }; - let physical_reduction = if snapshot_weighted { - let Some(NonASAPOp::Aggregate { child, .. }) = node.non_asap() else { - unreachable!() - }; - let source = &input.child.schema; - let Reduction::Reduce(keys) = reduction else { - return Err(RealizationError::PhysicalRealization( - "TopK requires explicit partitions", - )); - }; - if keys.is_without() { - return Err(RealizationError::PhysicalRealization( - "TopK requires explicit partitions", - )); - } - let mapped = keys - .iter() - .map(|index| { - let reference = schema_column_ref(child, *index).ok_or( - RealizationError::PhysicalRealization("invalid TopK partition key"), - )?; - let matches = source - .fields - .iter() - .enumerate() - .filter(|(_, column)| column_ref(column) == reference) - .map(|(index, _)| index) - .collect::>(); - match matches.as_slice() { - [index] => Ok(*index), - _ => Err(RealizationError::PhysicalRealization( - "ambiguous TopK partition key", - )), - } - }) - .collect::, _>>()?; - Reduction::by(mapped) - } else if keyed_heap && matches!(reduction, Reduction::PerEntity) { - Reduction::by(vec![]) - } else { - reduction.clone() - }; - let out_schema = &node.schema; - let measures = match node.non_asap() { - Some(NonASAPOp::Aggregate { measures, .. }) => measures.len(), - _ => 1, - }; - let state_idx = summary_col_index(out_schema, reduction, measures); - - let evaluation_schema = if keyed_heap - && matches!(node.non_asap(), Some(NonASAPOp::Aggregate { child, .. }) if is_snapshot_weighted_topk(intent, child)) - { - keyed_heap_evaluation_schema(&input, node)? - } else { - out_schema.clone() - }; - - let summary_input = input.input; - let query = estimate.then(|| { - if snapshot_weighted { - if let FieldDataType::Sketch(kind, _) = &family { - let capacity = match kind.params() { - SketchParams::CmsWithHeap { heap_size, .. } - | SketchParams::CountSketchWithHeap { heap_size, .. } => *heap_size, - _ => unreachable!(), - }; - return PostAsapSketchStatistic::TopK { - k: capacity as usize, - }; - } - } - evaluation(intent, &summary_input) - }); - - let mut state_schema = out_schema.clone(); - if keyed_heap { - let mut state = state_schema.fields[state_idx].clone(); - state.dtype = family.clone(); - // A top-k's output row holds the ranked item at `state_idx`; the - // state column is the heap itself. - if let AggIntent::TopK { k, .. } = intent { - state.name = format!("topk_{k}"); - state.nullable = false; - } - let mut fields = if snapshot_weighted { - evaluation_schema.fields[..reduction.group_keys().map_or(0, |keys| keys.len())].to_vec() - } else { - Vec::new() - }; - fields.push(state); - state_schema = Schema::lifted(fields, None); - } else if let Some(field) = state_schema.fields.get_mut(state_idx) { - field.dtype = family.clone(); - field.nullable = false; - if matches!(&family, FieldDataType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) - { - // State identity is independent of which statistic reads it. - field.name = "univmon".into(); - } else if let (AggIntent::Quantile { .. }, SummaryInputExpr::Column(col)) = - (intent, &summary_input.weight) - { - // The quantile is a evaluation parameter: name the state after the - // column it summarizes, not after the query's output column. - let child_schema = input.child.schema.clone(); - if let Ok(i) = resolve_column_ref(col, &child_schema) { - field.name = child_schema.fields[i].name.clone(); - } - } - } - - let bound_child = realize_child_with( - &input.child, - planning_inputs, - if snapshot_weighted { - Some(&AccuracyTarget::Exact) - } else { - child_target - }, - )?; - let bound_child = if snapshot_weighted && is_current_series_source(&input.child) { - // Explicit snapshot selection prevents historical observations from - // becoming repeated weights in an instant-vector heap. - let root = Rc::new(node.clone()); - let population = crate::pass1::maintained_population::MaintainedPopulationStrategy::new( - std::slice::from_ref(&root), - ) - .candidate(&root) - .ok_or(RealizationError::PhysicalRealization( - "snapshot ranking requires a supported current-series population", - ))?; - let Operator::ASAP(ASAPOp::EvaluatePopulation { child, .. }) = &population.operator else { - return Err(RealizationError::PhysicalRealization( - "missing population evaluation", - )); - }; - Rc::clone(child) - } else if snapshot_weighted { - // Each evaluation's finalized rates feed a fresh summary; rate snapshots - // must never accumulate across evaluations. Query time is only the - // initial layout; a maintained summary's materialization moves it to ingestion. - finalize_query_candidate(bound_child, &input.child)? - } else { - let child = - finalize_exact_accumulator(bound_child, &input.child, ExecutionTiming::IngestionTime)?; - maintenance_exact_values(child).unwrap_or(retain_exact(&input.child)?) - }; - - // ── Guarantee (issue #172) ────────────────────────────────────────── - // Derived *before* the node exists, so an illegal composition is never - // materialized: the local guarantee of this family's evaluation (or exact - // accumulator) composed over the child's, under the operator this - // family applies to the child's values. - let local_target = match allocation.as_ref() { - Some(GuaranteeSource::BudgetAllocation { local_target, .. }) => Some(local_target), - _ => accuracy_target(intent), - }; - let estimator = crate::accuracy::EstimatorAccuracy::new( - planning_inputs.accuracy, - planning_inputs.evidence.estimator_contract(node), - local_target, - ); - let membership_query = if snapshot_weighted { - Some(evaluation(intent, &summary_input)) - } else { - query.clone() - }; - let mut guarantee = compose_guarantee( - &family, - membership_query.as_ref(), - &bound_child, - intent, - &estimator, - planning_inputs.evidence, - allocation, - )?; - - if snapshot_weighted { - use asap_types::ir::properties::{BoundExpr, ProbabilityExpr}; - let target = accuracy_target(intent).expect("TopK target"); - guarantee = if let Some(mut score) = - estimator.local_guarantee(&family, query.as_ref().unwrap()) - { - let count = match score_population { - Some(n) => BoundExpr::Constant { value: n as f64 }, - None => { - score - .provenance - .push(GuaranteeSource::UnavailableStatistic { - statistic: "topk_max_distinct_items".into(), - }); - BoundExpr::Unknown { - statistic: "topk_max_distinct_items".into(), - } - } - }; - score.provenance.push(GuaranteeSource::CompositionStep { - operator: CompositionOperator::ApproximateAggregate, - rule: "simultaneous_score_bounds_over_distinct_partition_item_identities".into(), - }); - score.failure_probability = ProbabilityExpr::Scaled { - count, - inner: Box::new(score.failure_probability), - }; - // #455: missing evidence preserves a logical candidate. Only known - // contributions that already violate the target reject it here. - if !estimator.satisfies(&score.optimistic_floor(), target) { - return Err(RealizationError::PhysicalRealization( - "weighted TopK scores miss accuracy target", - )); - } - let stats = planning_inputs.evidence.propagation_stats( - &CompositionOperator::TopKSelection, - &family, - membership_query.as_ref(), - ); - let mut joint = estimator.propagate( - &CompositionOperator::TopKSelection, - std::slice::from_ref(&score), - None, - &stats, - )?; - joint.failure_probability = ProbabilityExpr::UnionBound { - terms: vec![joint.failure_probability, score.failure_probability], - }; - if !estimator.satisfies(&joint.optimistic_floor(), target) { - return Err(RealizationError::PhysicalRealization( - "weighted TopK joint guarantee misses target", - )); - } - Some(joint) - } else { - None - }; - } - - // `reduction` is carried onto `SummaryAgg` verbatim — not flattened to a - // bare `Vec` — so `SummaryExecutor::find_candidates` can tell - // a genuine empty-`by` reduction apart from a per-entity shape with no - // grouping concept at all (issue #163). `construct_summary_agg` is the - // single place that decides this; nothing downstream re-derives it. - let agg = OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: bound_child, - family, - input: summary_input, - reduction: physical_reduction, - grouping: GroupingStrategy::default(), - filter: None, - }), - state_schema, - ) - .with_guarantee( - // Summary *state* carries no caller-visible guarantee; only a - // finalized value does. An exact accumulator's state is its value. - if estimate { None } else { guarantee.clone() }, - ); - let agg = std::rc::Rc::new(agg); - match query { - // The evaluation: downstream of the estimate the schema is the plain - // pre-ASAP row shape again (the summary-state type does not - // propagate). - Some(query) => Ok(std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: agg, - query, - }), - evaluation_schema, - ) - .with_guarantee(guarantee), - )), - None => Ok(agg), - } -} - -// Heap evaluation rows contain the encoded item identity, subpopulation keys, -// and an estimated score. They never inherit the exact-value producer's schema. -fn keyed_heap_evaluation_schema( - input: &PhysicalSummaryInput, - node: &OperatorNode, -) -> Result { - let source = &input.child.schema; - let mut refs = Vec::new(); - let Some(NonASAPOp::Aggregate { - reduction, child, .. - }) = node.non_asap() - else { - return Err(RealizationError::PhysicalRealization( - "heap evaluation requires an aggregate", - )); - }; - if let Reduction::Reduce(groups) = reduction { - if groups.is_without() { - return Err(RealizationError::PhysicalRealization( - "heap evaluation requires explicit grouping", - )); - } - for index in groups.iter() { - refs.push(schema_column_ref(child, *index).ok_or( - RealizationError::PhysicalRealization("invalid heap partition key"), - )?); - } - } - fn item_refs( - item: &SummaryInputExpr, - schema: &Schema, - refs: &mut Vec, - ) -> Result<(), RealizationError> { - match item { - SummaryInputExpr::Column(column) => refs.push(column.clone()), - SummaryInputExpr::Tuple(items) => { - for item in items { - item_refs(item, schema, refs)?; - } - } - SummaryInputExpr::EntityIdentity(EntityIdentity::PromqlLabelSet { excluding }) => { - if !schema.closed { - return Err(RealizationError::PhysicalRealization( - "dynamic label identity requires an explicit row representation", - )); - } - for (index, column) in schema.fields.iter().enumerate() { - if Some(index) != schema.time_index && column.name != "value" { - let reference = match &column.table { - Some(table) => ColumnRef::Qualified { - table: table.clone(), - name: column.name.clone(), - }, - None => ColumnRef::Named(column.name.clone()), - }; - if !excluding.contains(&reference) { - refs.push(reference); - } - } - } - } - _ => { - return Err(RealizationError::PhysicalRealization( - "unsupported heap item identity", - )) - } - } - Ok(()) - } - item_refs( - input - .input - .item - .as_ref() - .ok_or(RealizationError::PhysicalRealization( - "heap item identity is missing", - ))?, - source, - &mut refs, - )?; - let mut fields = Vec::::new(); - for reference in refs { - let matches: Vec<_> = source - .fields - .iter() - .filter(|column| match &reference { - ColumnRef::Named(name) => &column.name == name, - ColumnRef::Qualified { table, name } => { - column.table.as_ref() == Some(table) && &column.name == name - } - ColumnRef::SampleValue => column.name == "value", - ColumnRef::Wildcard => false, - }) - .collect(); - let [column] = matches.as_slice() else { - return Err(RealizationError::PhysicalRealization( - "heap key must resolve to exactly one source column", - )); - }; - if fields.iter().any(|field| field.name == column.name) || column.name == "__asap_estimate" - { - return Err(RealizationError::PhysicalRealization( - "heap keys must have distinct output names", - )); - } - fields.push(Field::new( - column.name.clone(), - column.dtype.clone(), - column.nullable, - )); - } - if fields.is_empty() { - return Err(RealizationError::PhysicalRealization( - "heap evaluation has no identity columns", - )); - } - fields.push(Field::new( - "__asap_estimate", - FieldDataType::Plain(asap_types::ir::schema::DataType::Float64), - false, - )); - Ok(Schema::lifted(fields, None)) -} - -fn ranking_score_index(logical: &OperatorNode, values: &Schema) -> Result { - if is_current_series_source(logical) { - return values - .fields - .iter() - .position(|field| { - field.name == "value" - && field.dtype - == FieldDataType::Plain(asap_types::ir::schema::DataType::Float64) - }) - .ok_or(RealizationError::PhysicalRealization( - "snapshot ranking requires the sample value column", - )); - } - let Some(NonASAPOp::Aggregate { - reduction, - measures, - .. - }) = logical.non_asap() - else { - return Err(RealizationError::PhysicalRealization( - "ranking requires an explicit aggregate score", - )); - }; - if measures.len() != 1 { - return Err(RealizationError::PhysicalRealization( - "ranking requires exactly one score", - )); - } - let index = match reduction { - Reduction::Reduce(groups) if !groups.is_without() => groups.len(), - Reduction::PerEntity => values - .fields - .iter() - .position(|field| field.name == "value") - .ok_or(RealizationError::PhysicalRealization( - "ranking requires the sample value column", - ))?, - _ => { - return Err(RealizationError::PhysicalRealization( - "ranking requires explicit grouping", - )) - } - }; - if Some(index) == values.time_index - || !values.fields.get(index).is_some_and(|field| { - matches!( - field.dtype, - FieldDataType::Plain( - asap_types::ir::schema::DataType::Int64 - | asap_types::ir::schema::DataType::Float64 - ) - ) - }) - { - return Err(RealizationError::PhysicalRealization( - "ranking score must be numeric", - )); - } - Ok(index) -} - -/// Rebuild a heap from this evaluation's finalized per-series counter values. -/// The rate window is preserved; raw counter samples never become CMS weights. -fn realize_counter_value_summary_input( - intent: &AggIntent, - family: &FieldDataType, - output_reduction: &Reduction, - child: &Rc, -) -> PhysicalSummaryInputRuleResult { - if !matches!(intent, AggIntent::TopK { .. }) - || !matches!(family, FieldDataType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) - || !matches!(child.non_asap(), Some(NonASAPOp::Aggregate { reduction: Reduction::PerEntity, measures, .. }) if matches!(measures.as_slice(), [AggIntent::Rate | AggIntent::Increase])) - { - return PhysicalSummaryInputRuleResult::NotApplicable; - } - let schema = &child.schema; - if !schema.closed { - return PhysicalSummaryInputRuleResult::Unsupported( - "counter ranking needs the complete resolved series identity", - ); - } - let Reduction::Reduce(groups) = output_reduction else { - return PhysicalSummaryInputRuleResult::Unsupported( - "counter ranking requires explicit partitions", - ); - }; - if groups.is_without() { - return PhysicalSummaryInputRuleResult::Unsupported( - "counter ranking requires resolved partitions", - ); - } - // Retain the evaluation timestamp in each returned row. This sketch is a - // snapshot, not an additive history of successive rate evaluations. - let items = schema - .fields - .iter() - .enumerate() - .filter(|(index, column)| column.name != "value" && !groups.contains(index)) - .map(|(index, _)| schema_column_ref(child, index).map(SummaryInputExpr::Column)) - .collect::>>(); - let Some(items) = items.filter(|items| !items.is_empty()) else { - return PhysicalSummaryInputRuleResult::Unsupported("counter ranking has no item columns"); - }; - PhysicalSummaryInputRuleResult::Realized(PhysicalSummaryInput { - child: Rc::clone(child), - input: SummaryUpdate { - item: Some(SummaryInputExpr::Tuple(items)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::ResetAwareCounterDerivative, - }, - }, - }) -} - -/// An instant-vector source has one current value per full series identity. -/// Rebuild the state for each evaluation; historical samples are not updates. -fn realize_current_series_summary_input( - intent: &AggIntent, - family: &FieldDataType, - output_reduction: &Reduction, - child: &Rc, -) -> PhysicalSummaryInputRuleResult { - if !matches!(intent, AggIntent::TopK { .. }) || !is_current_series_source(child) { - return PhysicalSummaryInputRuleResult::NotApplicable; - } - let FieldDataType::Sketch(kind, _) = family else { - return PhysicalSummaryInputRuleResult::NotApplicable; - }; - match kind.algorithm() { - SketchAlgorithm::CountSketchWithHeap => {} - SketchAlgorithm::CmsWithHeap => { - return PhysicalSummaryInputRuleResult::Unsupported( - "current sample values do not prove non-negative CMS weights", - ) - } - _ => return PhysicalSummaryInputRuleResult::NotApplicable, - } - let Reduction::Reduce(groups) = output_reduction else { - return PhysicalSummaryInputRuleResult::Unsupported( - "snapshot ranking requires explicit partitions", - ); - }; - if groups.is_without() { - return PhysicalSummaryInputRuleResult::Unsupported( - "snapshot ranking requires resolved partitions", - ); - } - let schema = &child.schema; - let items = schema - .fields - .iter() - .enumerate() - .filter(|(index, column)| column.name != "value" && !groups.contains(index)) - .map(|(index, _)| schema_column_ref(child, index).map(SummaryInputExpr::Column)) - .collect::>>(); - let Some(items) = items.filter(|items| !items.is_empty()) else { - return PhysicalSummaryInputRuleResult::Unsupported( - "snapshot ranking has no item identity", - ); - }; - PhysicalSummaryInputRuleResult::Realized(PhysicalSummaryInput { - child: Rc::clone(child), - input: SummaryUpdate { - item: Some(SummaryInputExpr::Tuple(items)), - weight: SummaryInputExpr::Column(ColumnRef::SampleValue), - weight_domain: WeightDomain::UnknownOrSigned, - }, - }) -} - -/// The guarantee of the value a `family` node produces over `child` — -/// [`AccuracyModel::propagate`] under the [`CompositionOperator`] this family -/// applies to its child's values — checked against `intent`'s own -/// `AccuracyTarget` whenever the child is approximate (an approximate -/// parent over an exact child is sized to that target by construction and -/// is not re-checked here, so single-layer behavior is unchanged; see -/// [`crate::accuracy`]'s precedence rules). `Ok(None)` is "no error model" -/// (an approximate family the model has no local guarantee for, over an -/// exact child) — unknown, never exact. -fn compose_guarantee( - family: &FieldDataType, - query: Option<&PostAsapSketchStatistic>, - child: &OperatorNode, - intent: &AggIntent, - accuracy: &dyn AccuracyModel, - evidence: &dyn AccuracyEvidenceProvider, - allocation: Option, -) -> Result, AccuracyError> { - let (op, local) = match (family, query) { - (FieldDataType::ExactAggregate(kind, _), _) => { - let op = match kind { - // A row count does not depend on the rows' values: exact - // regardless of the child's own error. - ExactKind::Count => { - return Ok(Some(ResultGuarantee::exact( - "ExactAggregate(Count): row count is independent of input values", - ))) - } - // Counter-reset detection over perturbed values has no finite - // Lipschitz constant — over an approximate child this is a - // deterministic transform with no registered rule. - _ => { - crate::pass1::function_rules::function_rules(intent) - .expect("exact accumulator intents have registered accuracy rules") - .accuracy - } - }; - ( - op, - Some(ResultGuarantee::exact(format!("ExactAggregate({kind:?})"))), - ) - } - (_, Some(query)) => ( - if matches!(query, PostAsapSketchStatistic::TopK { .. }) { - CompositionOperator::TopKSelection - } else { - CompositionOperator::ApproximateAggregate - }, - accuracy.local_guarantee(family, query), - ), - (_, None) => (CompositionOperator::ApproximateAggregate, None), - }; - let Some(input) = child.guarantee.clone() else { - // Shape is constructible, but the child has no accuracy certificate. - return Ok(None); - }; - if local.is_none() { - // No local error model: retain the candidate with unknown accuracy. - // Propagating the input alone would falsely certify the summary. - return Ok(None); - } - let stats = evidence.propagation_stats(&op, family, query); - let mut guarantee = - accuracy.propagate(&op, std::slice::from_ref(&input), local.as_ref(), &stats)?; - if let Some(note) = allocation { - guarantee.provenance.push(note); - } - if let Some(target) = accuracy_target(intent) { - guarantee.provenance.push(GuaranteeSource::AccuracyTarget { - target: target.clone(), - }); - // Check even over an exact input: parameter clamps or a conservative - // confidence conversion can make the tightest available sketch miss - // its requested target. - if !accuracy.satisfies(&guarantee.optimistic_floor(), target) { - return Err(AccuracyError::TargetNotSatisfied { - metric: guarantee.metric, - bound: guarantee.bound.evaluate(), - failure_probability: guarantee.failure_probability.evaluate(), - target: target.clone(), - }); - } - } - Ok(Some(guarantee)) -} - -/// Index of the summary-state column in the aggregate's output schema: -/// cross-series output is `by ++ [agg]` (the column after the keys); -/// a per-series reduction keeps every label and replaces the sample value -/// (named `value` — mirror `per_series_reduction_schema`'s fallback). -/// `reduction` is the caller's already-read `Reduction` (issue #165). -/// `without` output is `kept labels ++ measures`, and its keys are the -/// *excluded* labels, so the state column follows the kept labels instead. -fn summary_col_index(out_schema: &Schema, reduction: &Reduction, measures: usize) -> usize { - match reduction { - Reduction::PerEntity => out_schema - .column_id("value") - .or_else(|| (0..out_schema.fields.len()).find(|&i| Some(i) != out_schema.time_index)) - .unwrap_or(0), - Reduction::Reduce(keys) if keys.is_without() => { - out_schema.fields.len().saturating_sub(measures) - } - Reduction::Reduce(keys) => keys.len(), - } -} - -/// The `SummaryEstimate` evaluation for a summary-bound intent. -fn evaluation(intent: &AggIntent, input: &SummaryUpdate) -> PostAsapSketchStatistic { - match intent { - AggIntent::Quantile { q, .. } => PostAsapSketchStatistic::Quantile { q: *q }, - AggIntent::Cardinality { .. } => PostAsapSketchStatistic::Cardinality, - AggIntent::FrequencyL2 { .. } => PostAsapSketchStatistic::FrequencyL2, - AggIntent::FrequencyEntropy { .. } => PostAsapSketchStatistic::FrequencyEntropy, - AggIntent::TopK { k, .. } => PostAsapSketchStatistic::TopK { k: *k }, - AggIntent::Count { .. } => PostAsapSketchStatistic::PointCount { - key: match &input.weight { - SummaryInputExpr::Column(col) => col.clone(), - SummaryInputExpr::Constant(1.0) => ColumnRef::SampleValue, - _ => unreachable!("point count requires one column"), - }, - value: None, - }, - other => { - unreachable!("no summary realization for {other:?} (realizations_for_intent)") - } - } -} - -// ── SharedSubDAGStrategy ──────────────────────────────────────────────── - -/// Wraps `asap_types::ir::cse::share_common_sub_dags`'s sharing -/// decision as an explicit candidate pair, wherever a [`TargetSubDAG`] -/// already has two or more consumers. -/// -/// This strategy does not decide sharing itself, nor does it discover which -/// nodes are shared — by the time a caller builds a `TargetSubDAG` with -/// `consumer_count >= 2`, `share_common_sub_dags` has already made that -/// (legality-gated, `PartialEq`-checked) call; [`discover_targets`] below -/// discovers real consumer counts across a workload the same way for -/// [`search_workload_with`] (this module's own tests reuse the identical -/// dedup logic to build realistic fixtures — see the module docs' -/// "Non-goals" on why that traversal isn't itself part of this strategy). -/// This strategy only reframes "two or more consumers already share this -/// `Rc`" as the two-way choice a downstream cost model (today, -/// `CostModel::cse_share_decision`) picks between: build once and share, or -/// build independently at each consumer. -pub struct SharedSubDAGStrategy; - -impl ReplacementStrategy for SharedSubDAGStrategy { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - target.consumer_count >= 2 - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - if target.consumer_count < 2 { - return Vec::new(); - } - let count = target.consumer_count; - vec![ - ReplacementSubDAG { - strategy: "SharedSubDAGStrategy", - // The already-interned `Rc` itself: reusing it verbatim *is* - // "build once and share" — no new node to construct. - replacement: Replacement::SubDAG(Rc::clone(target.root)), - provenance: ReplacementProvenance::CseShare, - rationale: format!( - "build once and share: share_common_sub_dags already interned this \ - sub_dag once and reused it across {count} consumers — one build can \ - answer all of them instead of computing it {count} times" - ), - }, - ReplacementSubDAG { - strategy: "SharedSubDAGStrategy", - // A structurally-identical but freshly-allocated `Rc`: same - // value (`PartialEq`), deliberately *not* the same pointer, - // representing "undo the sharing and recompute independently". - replacement: Replacement::SubDAG(Rc::new((**target.root).clone())), - provenance: ReplacementProvenance::CseRecompute, - rationale: format!( - "build independently: undo the sharing share_common_sub_dags found and \ - recompute this sub_dag separately at each of its {count} consumers — \ - worth it only when independence outweighs the shared-maintenance cost, \ - a CostModel's call (e.g. CostModel::cse_share_decision) and not this \ - strategy's" - ), - }, - ] - } -} - -// ── Workload-wide search: TargetSubDAGCandidates / CandidateLogicalASAPDAGs / search_workload ────────── -// -// Merged in from the former `search.rs` (issue #252, part of #33) — see this -// file's own top-level "Workload-wide search" doc section for the full -// design rationale. - -/// A generous, documented backstop against a hypothetically ill-behaved -/// future [`ReplacementStrategy`] (see the module docs' "Termination" -/// section) — not a bound either shipped strategy could ever approach. -/// [`ASAPStrategies`] and [`SharedSubDAGStrategy`] both converge in -/// exactly 2 passes over a fixed target set, regardless of workload size. -pub const MAX_SEARCH_ITERATIONS: usize = 1_000; - -// ── TargetSubDAGCandidates ────────────────────────────────────────────── - -/// Candidates for one distinct [`TargetSubDAG`] (its -/// own `target` `Rc`, keyed by pointer identity in -/// [`CandidateLogicalASAPDAGs`]'s internal map — never re-derived by value) plus every -/// [`ReplacementSubDAG`] alternative any registered [`ReplacementStrategy`] -/// proposed for it. -/// -/// `candidates` is deliberately *not* required to be non-empty — a -/// `TargetSubDAG` no registered strategy has an opinion on still gets a -/// group (with an empty candidate list), so [`CandidateLogicalASAPDAGs`] always has -/// exactly one group per discovered `TargetSubDAG`, not "one group per -/// `TargetSubDAG` something matched". -#[derive(Debug, Clone)] -pub struct TargetSubDAGCandidates { - /// The target sub-DAG this group is for. - pub target: Rc, - /// How many operator-child positions across the whole workload - /// reference this exact `Rc` — see [`discover_targets`]. - pub consumer_count: usize, - /// Every distinct alternative discovered for `target`, in discovery - /// order (not ranked — see `candidate_selection::cost_sorted` for the ranked - /// view). - pub candidates: Vec, - /// Every candidate a strategy considered for `target` but refused on - /// accuracy-legality grounds (issue #172), plus any `candidates` entry - /// the root-target check ([`search_workload_with_targets`]) moved here. - /// Never ranked — `candidate_selection::cost_sorted`/`candidate_selection::global_selection` - /// read only `candidates`, so a `CostModel` cannot resurrect one. - pub rejected: Vec, -} - -impl TargetSubDAGCandidates { - pub(crate) fn new(target: Rc, consumer_count: usize) -> Self { - Self { - target, - consumer_count, - candidates: Vec::new(), - rejected: Vec::new(), - } - } - - /// Add `candidate` unless it's already present (see - /// [`is_duplicate_rewrite`]/[`is_duplicate_summary`] for what "already - /// present" means for each [`Replacement`] variant). Returns whether it - /// was actually added — [`search_workload_with`]'s fixpoint loop uses - /// this to detect when a pass made no progress. - fn add_candidate(&mut self, candidate: ReplacementSubDAG) -> bool { - let is_duplicate = self.candidates.iter().any(|existing| { - match (&existing.replacement, &candidate.replacement) { - (Replacement::SubDAG(existing_rc), Replacement::SubDAG(rc)) - if is_logical_rewrite(existing_rc) && is_logical_rewrite(rc) => - { - is_duplicate_rewrite(existing_rc, rc, &self.target) - } - (Replacement::SubDAG(existing_node), Replacement::SubDAG(node)) => { - is_duplicate_summary(existing_node, node) - } - ( - Replacement::ExactComposition(existing), - Replacement::ExactComposition(candidate), - ) => existing.same_as(candidate), - // Different `Replacement` variants are never the same - // candidate. - _ => false, - } - }); - if is_duplicate { - false - } else { - self.candidates.push(candidate); - true - } - } -} - -/// Are `existing` and `candidate` the same logical-rewrite -/// [`Replacement::SubDAG`] candidate for a group targeting `target`? -/// -/// Structural (`OperatorNode`) value equality alone is *not* enough here: -/// this module's one shipped multi-candidate logical-rewrite source, -/// [`SharedSubDAGStrategy`], deliberately returns **two** candidates that -/// are value-equal to each other (`build once and share` vs. `build -/// independently` — see that strategy's own doc) but represent genuinely -/// different physical choices, distinguished *only* by whether the -/// candidate's `Rc` is the group's own `target` `Rc` (share) or a freshly -/// allocated one (recompute independently) — this IR has no field that -/// records "materialized once and shared", so `Rc` identity against -/// `target` is the only signal that distinction exists in at all. Treating -/// those two as duplicates of each other via pure value equality would -/// silently collapse a real choice into one candidate — the "false-positive -/// dedup is a wrong answer, not a missed optimization" failure mode -/// `cse.rs`'s own "Correctness" section warns about, just one level up from -/// where that module states it. -/// -/// So: two candidates whose "is this the target's own `Rc`?" bit disagrees -/// are never duplicates of each other, full stop. Only when that bit -/// *agrees* does this fall through to the real dedup discipline — -/// [`structural_hash`] as a candidate-narrowing filter, `OperatorNode`'s -/// derived `PartialEq` as the actual decision — protecting against the -/// (currently hypothetical, since neither shipped strategy causes it) -/// case of the exact same alternative being proposed twice. A fresh -/// [`HashCache`] per call: this is a pairwise check between two candidates -/// for one group, not a bottom-up pass over a whole DAG, so there is no -/// wider traversal to amortize the cache across the way `InternTable`'s own -/// use of `structural_hash` does. -fn is_duplicate_rewrite( - existing: &Rc, - candidate: &Rc, - target: &Rc, -) -> bool { - let existing_is_target = Rc::ptr_eq(existing, target); - let candidate_is_target = Rc::ptr_eq(candidate, target); - if existing_is_target != candidate_is_target { - return false; - } - let mut cache = HashCache::new(); - structural_hash(existing, &mut cache) == structural_hash(candidate, &mut cache) - && existing == candidate -} - -/// Are `existing` and `candidate` the same bound-summary -/// [`Replacement::SubDAG`] candidate? -/// -/// A bound summary embeds `SketchParams`/`f64`-bearing accuracy targets and -/// guarantees, so value equality is not a dedup decision this module is -/// willing to make (see [`structural_hash`]'s own doc on `f64` hashing). -/// Per this module's inherited "hash is a filter, `PartialEq` is the -/// decision, no exceptions" rule, there is no real equality check to back a -/// dedup *decision* here — and skipping the check is the only choice that -/// rule permits: never merging two candidates -/// is harmless (at worst, a redundant entry in a group's candidate list), -/// while comparing by some proxy this module can't actually verify (e.g. -/// `Debug` text, or `ReplacementSubDAG::rationale` — documented elsewhere in -/// this crate as prose for a report, "not machine parsing") risks exactly -/// the false-positive merge the rule exists to prevent. Both strategies -/// shipped today already return a structurally distinct candidate for every -/// entry of one `replacements()` call, so this is future-proofing against a -/// hypothetical repeat call, not a gap either strategy's own tests exercise. -fn is_duplicate_summary(_existing: &Rc, _candidate: &Rc) -> bool { - false -} - -// ── CandidateLogicalASAPDAGs ──────────────────────────────────────────────────────────── - -/// The deduped candidate space [`search_workload`]/[`search_workload_with`] -/// discover: one [`TargetSubDAGCandidates`] per distinct `TargetSubDAG` in the -/// (already-CSE'd) workload, plus the workload's own post-CSE roots so a -/// caller can still map a `Root`'s `Id` back to the `Rc` whose -/// group holds its alternatives. Memos are keyed by `*const OperatorNode`. -pub struct CandidateLogicalASAPDAGs { - /// The workload's roots, after the one `share_common_sub_dags` pass - /// [`search_workload_with`] runs up front — the same post-CSE roots - /// every `TargetSubDAG` in `groups` was discovered from. - pub roots: Vec<(Id, Rc)>, - pub(crate) groups: HashMap<*const OperatorNode, TargetSubDAGCandidates>, - /// Discovery order — stable iteration for [`CandidateLogicalASAPDAGs::target_subdag_candidates`]/ - /// `candidate_selection::cost_sorted`, since `HashMap` iteration order isn't. - pub(crate) order: Vec<*const OperatorNode>, - /// Composition proofs are computed with the search model, then retained - /// through costing and DAG assembly so no later default can replace it. - pub(crate) composition_plans: Vec, -} - -/// One exact composition validated during search: `operation` at `target`, -/// over the child candidate `child`, giving `plan`. -pub struct PreparedComposition { - pub target: *const OperatorNode, - pub operation: ExactComposition, - pub child: Rc, - pub plan: Rc, -} - -impl CandidateLogicalASAPDAGs { - fn prepare_compositions( - &mut self, - accuracy: &dyn AccuracyModel, - targets: &HashMap<*const OperatorNode, Vec>, - ) { - self.composition_plans.clear(); - for group in self.groups.values() { - for candidate in &group.candidates { - let Replacement::ExactComposition(operation) = &candidate.replacement else { - continue; - }; - let children: Vec<_> = match operation.placement { - OperationPlacement::Read => self - .groups - .get(&Rc::as_ptr(&operation.child_target)) - .into_iter() - .flat_map(|g| &g.candidates) - .filter_map(|c| match &c.replacement { - Replacement::SubDAG(child) - if !is_logical_rewrite(child) && operation.accepts_child(child) => - { - Some(Rc::clone(child)) - } - _ => None, - }) - .collect(), - OperationPlacement::Maintenance => { - retain_exact(&operation.child_target).into_iter().collect() - } - }; - for child in children { - let Ok(plan) = operation.compose_with_accuracy(Rc::clone(&child), accuracy) - else { - continue; - }; - if let Some(requirements) = targets.get(&Rc::as_ptr(&group.target)) { - if operation.placement == OperationPlacement::Maintenance - || !requirements.iter().all(|target| { - plan.guarantee - .as_ref() - .is_some_and(|g| accuracy.satisfies(g, target)) - }) - { - continue; - } - } - self.composition_plans.push(PreparedComposition { - target: Rc::as_ptr(&group.target), - operation: operation.clone(), - child, - plan, - }); - } - } - } - } -} - -/// DAG candidates assembled from an unpriced search space. -/// This is an internal planning stage: callers must still validate materialization -/// requirements and compile supported physical operators before deployment. -/// The caller supplies a finite expansion budget; exceeding it is an error, -/// never a silently truncated inventory presented as exhaustive. -#[derive(Debug)] -pub struct CandidateDAGInventory { - pub candidates: Vec)>>, - pub rejected_assemblies: Vec, -} - -type CandidateDAGChoice<'a> = (Option<&'a ReplacementSubDAG>, Option>); - -impl CandidateLogicalASAPDAGs { - pub fn enumerate_candidate_dags( - &self, - expansion_limit: usize, - ) -> Result, RealizationError> { - self.enumerate_candidate_roots(&self.roots, expansion_limit) - } - - /// Enumerate one workload root without expanding independent roots' choices. - /// Discovery and composition proofs still come from the shared workload - /// space. Deployment may price combinations lazily; this API does not rank - /// candidates or claim that independently cheapest roots minimize shared cost. - pub fn enumerate_candidate_dags_for_root( - &self, - id: &Id, - expansion_limit: usize, - ) -> Result, RealizationError> { - let roots = self - .roots - .iter() - .filter(|(candidate, _)| candidate == id) - .cloned() - .collect::>(); - if roots.len() != 1 { - return Err(RealizationError::PhysicalRealization( - "candidate enumeration requires one uniquely identified workload root", - )); - } - self.enumerate_candidate_roots(&roots, expansion_limit) - } - - fn enumerate_candidate_roots( - &self, - roots: &[(Id, Rc)], - expansion_limit: usize, - ) -> Result, RealizationError> { - let mut reachable = Vec::new(); - let mut nodes = HashMap::new(); - let mut counts = HashMap::new(); - for (_, root) in roots { - walk(root, &mut reachable, &mut nodes, &mut counts); - } - // Rewrites may introduce descendants absent from the original root. - let mut cursor = 0; - while cursor < reachable.len() { - let ptr = reachable[cursor]; - cursor += 1; - if let Some(group) = self.groups.get(&ptr) { - for candidate in &group.candidates { - if let Replacement::SubDAG(rewritten) = &candidate.replacement { - if is_logical_rewrite(rewritten) { - walk(rewritten, &mut reachable, &mut nodes, &mut counts); - } - } - } - } - } - let order = self - .order - .iter() - .copied() - .filter(|ptr| counts.contains_key(ptr)) - .collect::>(); - // Composition plans carry the proofs established during discovery. - // No cost ranking is consulted while expanding these choices. - let options: Vec>> = order - .iter() - .map(|ptr| { - let group = &self.groups[ptr]; - let mut choices = vec![(None, None)]; - for candidate in &group.candidates { - match &candidate.replacement { - Replacement::ExactComposition(operation) => { - for prepared in &self.composition_plans { - if prepared.target == *ptr - && prepared.operation.placement == operation.placement - && prepared.operation.op == operation.op - && Rc::ptr_eq( - &prepared.operation.child_target, - &operation.child_target, - ) - { - choices - .push((Some(candidate), Some(Rc::clone(&prepared.plan)))); - } - } - } - _ => choices.push((Some(candidate), None)), - } - } - choices - }) - .collect(); - let combinations = options - .iter() - .try_fold(1usize, |n, choices| n.checked_mul(choices.len())) - .filter(|n| *n <= expansion_limit) - .ok_or(RealizationError::PhysicalRealization( - "candidate expansion budget exceeded; no partial inventory returned", - ))?; - let mut inventory = CandidateDAGInventory { - candidates: Vec::new(), - rejected_assemblies: Vec::new(), - }; - // Hash buckets avoid quadratic comparisons across a large workload - // inventory. Equality still decides deduplication, including collisions. - let mut seen = HashMap::>::new(); - for mut ordinal in 0..combinations { - let mut groups = HashMap::new(); - let mut assembled_nodes = HashMap::new(); - for (ptr, choices) in order.iter().zip(&options) { - let (chosen, prepared) = &choices[ordinal % choices.len()]; - ordinal /= choices.len(); - let group = &self.groups[ptr]; - if let Some(node) = prepared { - assembled_nodes.insert(*ptr, Rc::clone(node)); - } - groups.insert( - *ptr, - TargetSubDAGSelection { - target: &group.target, - consumer_count: group.consumer_count, - effective_consumer_count: group.consumer_count, - chosen: *chosen, - }, - ); - } - let assembly = GlobalSelection { - order: order.clone(), - groups, - composition_plans: HashMap::new(), - assembled_nodes: RefCell::new(assembled_nodes), - }; - let roots = roots - .iter() - .map(|(id, root)| { - assembly - .assemble_target(root) - // Exposed query candidates return values. Internal assembly - // still retains accumulator states for sharing and storage. - .and_then(|node| finalize_query_candidate(node, root)) - .map(|node| (id.clone(), node)) - }) - .collect::, _>>(); - match roots { - Ok(roots) => { - let roots = share_common_sub_dags(roots); - use std::hash::{Hash, Hasher}; - let mut hash = std::collections::hash_map::DefaultHasher::new(); - let mut cache = HashCache::new(); - for (_, node) in &roots { - structural_hash(node, &mut cache).hash(&mut hash); - } - let bucket = seen.entry(hash.finish()).or_default(); - if !bucket - .iter() - .any(|&index| inventory.candidates[index] == roots) - { - bucket.push(inventory.candidates.len()); - inventory.candidates.push(roots); - } - } - Err(error) => { - let reason = error.to_string(); - if !inventory.rejected_assemblies.contains(&reason) { - inventory.rejected_assemblies.push(reason); - } - } - } - } - Ok(inventory) - } -} - -impl CandidateLogicalASAPDAGs { - /// One candidate set per discovered target sub-DAG, in discovery order. - pub fn target_subdag_candidates(&self) -> impl Iterator { - self.order.iter().map(move |ptr| &self.groups[ptr]) - } - - /// Every discovered target's candidate set, keyed by target identity. - pub fn groups(&self) -> &HashMap<*const OperatorNode, TargetSubDAGCandidates> { - &self.groups - } - - /// Target identities in discovery order. - pub fn order(&self) -> &[*const OperatorNode] { - &self.order - } - - /// The exact compositions validated during search. - pub fn composition_plans(&self) -> &[PreparedComposition] { - &self.composition_plans - } - - /// How many distinct targets were discovered. - pub fn len(&self) -> usize { - self.groups.len() - } - - /// Whether no targets were discovered at all (an empty workload, or one - /// with no `OperatorNode`s reachable from any root — never true for a - /// non-empty `roots`, since every root is itself a target). - pub fn is_empty(&self) -> bool { - self.groups.is_empty() - } - - /// The candidate set for `target`, if `target`'s own `Rc` is a discovered - /// `TargetSubDAG` (i.e. `Rc::ptr_eq` to some node reachable from - /// `roots`). - pub fn candidates_for_target( - &self, - target: &Rc, - ) -> Option<&TargetSubDAGCandidates> { - self.groups.get(&Rc::as_ptr(target)) - } -} - -/// Find the explicitly-tagged CSE share/recompute pair inside `group`, even -/// when other strategies contributed additional alternatives to the same -/// memo group. Provenance makes these two orthogonal choices identifiable -/// without inferring semantics from pointer or expression shape. -pub fn cse_candidate_pair( - group: &TargetSubDAGCandidates, -) -> Option<(&ReplacementSubDAG, &ReplacementSubDAG)> { - let mut share = None; - let mut recompute = None; - for candidate in &group.candidates { - match candidate.provenance { - ReplacementProvenance::CseShare => { - let Replacement::SubDAG(rc) = &candidate.replacement else { - return None; - }; - if !Rc::ptr_eq(rc, &group.target) || share.replace(candidate).is_some() { - return None; - } - } - ReplacementProvenance::CseRecompute => { - let Replacement::SubDAG(rc) = &candidate.replacement else { - return None; - }; - if Rc::ptr_eq(rc, &group.target) - || rc.as_ref() != group.target.as_ref() - || recompute.replace(candidate).is_some() - { - return None; - } - } - _ => {} - } - } - Some((share?, recompute?)) -} -/// Direct relational-skeleton children and their edge multiplicities. -/// `Concat` is transparent, matching [`walk_children`]'s site scope. -pub fn direct_child_counts(node: &OperatorNode) -> Vec<(*const OperatorNode, usize)> { - fn push(children: &mut Vec<(*const OperatorNode, usize)>, child: &Rc) { - let ptr = Rc::as_ptr(child); - match children.iter_mut().find(|(existing, _)| *existing == ptr) { - Some((_, count)) => *count += 1, - None => children.push((ptr, 1)), - } - } - - fn collect(node: &OperatorNode, children: &mut Vec<(*const OperatorNode, usize)>) { - if let Some(NonASAPOp::Concat { - children: concat_children, - .. - }) = node.non_asap() - { - for c in concat_children { - collect(c, children); - } - return; - } - for child in node.children() { - push(children, child); - } - } - - let mut children = Vec::new(); - collect(node, &mut children); - children -} -// ── GlobalSelection: assemble a DAG from given choices ─────────────────── - -/// One target sub-DAG's chosen candidate and usage counts: the input -/// [`GlobalSelection`] assembles a DAG from. Building a DAG from given -/// choices needs no cost model; whoever makes the choices (enumeration here, -/// or the legacy cost-based selection in plan selection) fills these in. -#[derive(Debug)] -pub struct TargetSubDAGSelection<'a> { - /// The target sub-DAG this selection is for. - pub target: &'a Rc, - /// [`TargetSubDAGCandidates::consumer_count`] — how many operator-child positions - /// directly reference `target`, ignoring every ancestor's own choice. - pub consumer_count: usize, - /// How many times `target`'s computation actually runs once every - /// ancestor's own selected candidate is accounted for. Equal to - /// `consumer_count` unless some ancestor on a path from a root to this - /// site has a [`SharedSubDAGStrategy`] alternative that chose to - /// recompute independently. - pub effective_consumer_count: usize, - /// The candidate chosen for this target, or `None` when no replacement - /// is selected. The candidate set need not be empty: an unproven DDSketch - /// ratio can remain available for backend inspection but be excluded from - /// automatic selection, or costing can prefer raw recomputation. - /// DAG assembly then preserves exact computation at this target where - /// supported, while independently selected children may remain visible. - pub chosen: Option<&'a ReplacementSubDAG>, -} - -/// One [`TargetSubDAGSelection`] per discovered site, in the same discovery -/// order [`CandidateLogicalASAPDAGs::target_subdag_candidates`] uses, plus the -/// DAG assembly over those choices. -#[derive(Debug)] -pub struct GlobalSelection<'a> { - order: Vec<*const OperatorNode>, - groups: HashMap<*const OperatorNode, TargetSubDAGSelection<'a>>, - /// The validated plan of each site whose chosen candidate is a - /// [`Replacement::ExactComposition`]. - composition_plans: HashMap<*const OperatorNode, Rc>, - /// [`Self::assemble_selected_dag`]'s memo — one bound node per target for the - /// life of this selection, so two parents composing over one shared - /// child get the *same* `Rc` (a kept pre-ASAP sub-DAG - /// shared by two parents stays one `Rc` the same way). - assembled_nodes: RefCell>>, -} - -fn normalize_cross_input_equi_predicate( - pred: &Predicate, - left_width: usize, - total_width: usize, -) -> Option { - let ScalarExpr::Compare { - left, - op: asap_types::ir::scalar::CompareOpKind::Eq, - right, - semantics, - } = &pred.0 - else { - return None; - }; - let (ScalarExpr::Column(left_id), ScalarExpr::Column(right_id)) = - (left.as_ref(), right.as_ref()) - else { - return None; - }; - let is_left = |id: ColumnId| id < left_width; - let is_right = |id: ColumnId| left_width <= id && id < total_width; - let (left_id, right_id) = if is_left(*left_id) && is_right(*right_id) { - (*left_id, *right_id) - } else if is_right(*left_id) && is_left(*right_id) { - (*right_id, *left_id) - } else { - return None; - }; - Some(Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(left_id)), - op: asap_types::ir::scalar::CompareOpKind::Eq, - right: Box::new(ScalarExpr::Column(right_id)), - semantics: *semantics, - })) -} - -impl<'a> GlobalSelection<'a> { - /// A selection over `groups`, listed in `order`. `composition_plans` - /// holds the validated plan of every site that chose an exact composition. - pub fn new( - order: Vec<*const OperatorNode>, - groups: HashMap<*const OperatorNode, TargetSubDAGSelection<'a>>, - composition_plans: HashMap<*const OperatorNode, Rc>, - ) -> Self { - Self { - order, - groups, - composition_plans, - assembled_nodes: RefCell::new(HashMap::new()), - } - } - - /// One selection per discovered target sub-DAG, in discovery order. - pub fn target_selections(&self) -> impl Iterator> { - self.order.iter().map(move |ptr| &self.groups[ptr]) - } - - /// The selection for `target`, if `target`'s own `Rc` is a discovered - /// site (i.e. `Rc::ptr_eq` to some node reachable from the workload's - /// roots). - pub fn for_target(&self, target: &Rc) -> Option<&TargetSubDAGSelection<'a>> { - self.groups.get(&Rc::as_ptr(target)) - } - - /// Link this selection's per-site decisions into one data_state-validated - /// post-ASAP DAG rooted at `target` — the one place a committed - /// composition's child *reference* becomes an actual `Rc` - /// edge (issue #171). `None` if `target` is not a discovered site. - /// - /// Per site: a [`Replacement::ExactComposition`] uses its validated - /// operation/child plan, retaining the search model's guarantee; - /// a bound-summary [`Replacement::SubDAG`] is - /// re-linked so its `SummaryAgg` child is the child target's own - /// DAG assembly whenever that is phase-legal beneath maintenance - /// (so a child that chose an `ValueOperationAtIngestionTime` actually ends up under - /// the summary); a logical-rewrite [`Replacement::SubDAG`] is kept - /// as it is (exact); an unmatched site keeps its own operator with each - /// child assembled independently ([`Self::assemble_residual`]). - /// Memoized by target identity, so a shared inner summary is one `Rc` - /// no matter how many roots reach it. - pub fn assemble_selected_dag( - &self, - target: &Rc, - ) -> Result>, RealizationError> { - if !self.groups.contains_key(&Rc::as_ptr(target)) { - return Ok(None); - } - self.assemble_target(target).map(Some) - } - - /// Assemble a complete query result, including an exact-state evaluation when - /// needed. `assemble_selected_dag` also serves internal state frontiers; - /// callers exposing query results must use this boundary instead. - pub fn assemble_selected_query( - &self, - target: &Rc, - ) -> Result>, RealizationError> { - self.assemble_selected_dag(target)? - .map(|node| finalize_query_candidate(node, target)) - .transpose() - } - - fn assemble_target( - &self, - target: &Rc, - ) -> Result, RealizationError> { - let ptr = Rc::as_ptr(target); - if let Some(node) = self.assembled_nodes.borrow().get(&ptr) { - return Ok(Rc::clone(node)); - } - // A selected summary that realizes its inner aggregate, instead of - // hiding it in `KeepPreAsap`, is kept; materialization assignment decides - // whether it runs in precompute or at query time. - let selected_composed_summary = self - .groups - .get(&ptr) - .and_then(|sel| sel.chosen) - .is_some_and(|candidate| { - matches!(&candidate.replacement, - Replacement::SubDAG(node) if matches!(&node.operator, - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) - if child.contains_asap() || !contains_aggregate(child))) - }); - let node = if query_time_nested_sum(target) && !selected_composed_summary { - self.assemble_residual(target)? - } else { - match self - .groups - .get(&ptr) - .and_then(|sel| sel.chosen) - .map(|c| &c.replacement) - { - None => self.assemble_residual(target)?, - Some(Replacement::SubDAG(node)) if node.contains_asap() => { - self.relink_summary(node, target)? - } - Some(Replacement::SubDAG(kept)) => retain_exact(kept)?, - Some(Replacement::ExactComposition(_)) => Rc::clone( - self.composition_plans - .get(&ptr) - .expect("selected compositions have a validated plan"), - ), - } - }; - self.assembled_nodes - .borrow_mut() - .insert(ptr, Rc::clone(&node)); - Ok(node) - } - - /// Keep `target`'s own operator and assemble each child independently, - /// so a selected summary remains visible beneath a relational operator - /// that has no summary realization of its own instead of being - /// swallowed by one opaque kept sub-DAG. Every child that is a - /// discovered target is assembled (and finalized to query-time values); - /// any other child is kept as it is. The guarantee is composed from the - /// assembled children: all exact → exact; exactly one child → that - /// child's guarantee; otherwise unknown. An inner `Join` first has its - /// cross-input equi-predicate normalized; any other join is kept whole. - fn assemble_residual( - &self, - target: &Rc, - ) -> Result, RealizationError> { - if target.children().is_empty() { - // A leaf has nothing to assemble beneath it: keep it as it is. - return retain_exact(target); - } - let mut operator = target.operator.clone(); - if let Operator::NonASAP(NonASAPOp::Join { - left, - right, - kind, - pred, - }) = &mut operator - { - let left_width = left.schema.fields.len(); - let total_width = left_width + right.schema.fields.len(); - let normalized_pred = matches!(kind, JoinKind::Inner) - .then(|| normalize_cross_input_equi_predicate(pred, left_width, total_width)) - .flatten(); - let Some(normalized) = normalized_pred else { - return retain_exact(target); - }; - *pred = normalized; - } - let mut failure = None; - let mut children = Vec::new(); - let operator = operator.map_children(|child| { - if failure.is_some() { - return Rc::clone(child); - } - let assembled = if self.groups.contains_key(&Rc::as_ptr(child)) { - self.assemble_target(child) - .and_then(|node| finalize_query_candidate(node, child)) - } else { - Ok(Rc::clone(child)) - }; - match assembled { - Ok(node) => { - children.push(Rc::clone(&node)); - node - } - Err(error) => { - failure = Some(error); - Rc::clone(child) - } - } - }); - if let Some(error) = failure { - return Err(error); - } - // An operator that computes new values from its input rows has no - // sound accuracy composition over an approximate input (e.g. `max` - // over a quantile evaluation's rank error). Without a selected - // composition such a node stays an exact pre-ASAP sub-DAG; only the - // read-time nested SUM keeps its assembled children. - let computes_values = matches!( - target.non_asap(), - Some( - NonASAPOp::Aggregate { .. } - | NonASAPOp::BinaryOp { .. } - | NonASAPOp::SQLWindowFunc { .. } - ) - ) && !query_time_nested_sum(target); - let approximate_input = children.iter().any(|child| { - !child - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) - }); - if computes_values && approximate_input { - return retain_exact(target); - } - let guarantee = match children.as_slice() { - [child] => child.guarantee.clone(), - children - if children.iter().all(|child| { - child - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact) - }) => - { - Some(ResultGuarantee::exact(format!( - "{} over exact inputs", - target.operator.kind_name() - ))) - } - _ => None, - }; - let node = Rc::new( - OperatorNode::with_schema(operator, target.schema.clone()).with_guarantee(guarantee), - ); - validate_maintained(&node, ExecutionTiming::QueryTime)?; - Ok(node) - } - - /// Re-link a bound summary candidate's `SummaryAgg` child to the - /// child target's own DAG assembly when that is legal beneath - /// maintenance; otherwise keep the candidate exactly as constructed. - fn relink_summary( - &self, - node: &Rc, - target: &Rc, - ) -> Result, RealizationError> { - let Some(NonASAPOp::Aggregate { - child: pre_child, .. - }) = target.non_asap() - else { - return Ok(Rc::clone(node)); - }; - let has_maintenance_operation = self - .groups - .get(&Rc::as_ptr(pre_child)) - .and_then(|selection| selection.chosen) - .is_some_and(|candidate| { - matches!( - &candidate.replacement, - Replacement::ExactComposition(composition) - if composition.placement == OperationPlacement::Maintenance - ) - }); - if !has_maintenance_operation { - return Ok(Rc::clone(node)); - } - let new_child = self.assemble_target(pre_child)?; - Ok(relink_agg_child(node, &new_child)) - } -} - -/// A mergeable outer SUM over a relationally wrapped aggregate is a read-time -/// reduction of the inner summary values. Maintaining the outer SUM directly -/// would hide that inner temporal aggregate inside one kept sub-DAG and lose -/// its independently selected summary. -fn query_time_nested_sum(target: &OperatorNode) -> bool { - let Some(NonASAPOp::Aggregate { - measures, - filters, - having: None, - child, - .. - }) = target.non_asap() - else { - return false; - }; - !any_measure_filtered(filters) - && matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - && contains_aggregate(child) -} - -fn contains_aggregate(expr: &OperatorNode) -> bool { - match expr.non_asap() { - Some(NonASAPOp::Aggregate { .. }) => true, - Some( - NonASAPOp::Project { child, .. } - | NonASAPOp::Filter { child, .. } - | NonASAPOp::Sort { child, .. } - | NonASAPOp::Limit { child, .. }, - ) => contains_aggregate(child), - _ => false, - } -} - -/// Rebuild `node` (a `SummaryAgg`, possibly under a `SummaryEstimate`) with -/// `new_child` as the `SummaryAgg`'s child, if the result still validates -/// as maintained state; otherwise return `node` unchanged. -fn relink_agg_child(node: &Rc, new_child: &Rc) -> Rc { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) => { - let inner = relink_agg_child(summary_input, new_child); - if Rc::ptr_eq(&inner, summary_input) { - return Rc::clone(node); - } - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input: inner, - query: query.clone(), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()), - ) - } - Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - grouping, - filter, - }) => { - if Rc::ptr_eq(child, new_child) { - return Rc::clone(node); - } - let rebuilt = std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: Rc::clone(new_child), - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }), - node.schema.clone(), - ) - .with_guarantee(node.guarantee.clone()), - ); - match validate_maintained(&rebuilt, ExecutionTiming::IngestionTime) { - Ok(_) => rebuilt, - Err(_) => Rc::clone(node), - } - } - _ => Rc::clone(node), - } -} - -// ── default_strategies ────────────────────────────────────────────────── - -/// The context-free strategies [`search_workload`] runs with the built-in -/// accuracy models. Workload-dependent strategies such as -/// [`RollupStrategy`] and [`AccuracyReconciliationStrategy`] (issue #273, -/// cross-consumer accuracy reconciliation for CSE sharing — see that -/// module's own docs) are added by [`search_workload`] after CSE and target -/// discovery, when their sibling context exists. -/// [`crate::pass1::explanation::explain_replacements`] (issue #257) uses -/// this same set (via [`search_workload`]) rather than keeping a second, -/// explanation-specific list to stay in sync with. -/// -/// `AvgToSumOverCountStrategy` is -/// included here (issue #253) even though it's a -/// [`Replacement::Rewrite`]-only strategy — it's context-free (`matches`/`replacements` need nothing beyond -/// the target itself) exactly like [`SharedSubDAGStrategy`], so it belongs -/// in this list rather than being derived per-workload the way -/// [`RollupStrategy`] is. Rewriting `avg` into `sum`/`count` upfront is what -/// lets [`ASAPStrategies`] and [`SharedSubDAGStrategy`] see a -/// mergeable accumulator to sketch or share at all — see that module's own -/// doc comment for why a bare `avg` node otherwise never becomes a -/// [`ReplacementStrategy`] target for anything. -pub fn default_strategies() -> Vec> { - vec![ - Box::new(ASAPStrategies::default()), - Box::new(HydraGroupingStrategy::default()), - Box::new(SharedSubDAGStrategy), - Box::new(crate::pass1::rewrite::AvgToSumOverCountStrategy), - Box::new(ExactCompositionStrategy), - ] -} - -/// Default context-free strategies with typed planning-time accuracy -/// evidence. This is the production counterpart of -/// constructing [`ASAPStrategies::new_with_planning_inputs_and_evidence`] and -/// [`HydraGroupingStrategy::new_with_planning_inputs_and_evidence`] separately. -pub fn default_strategies_with_evidence<'a>( - evidence: &'a dyn AccuracyEvidenceProvider, -) -> Vec> { - vec![ - Box::new(ASAPStrategies::new_with_planning_inputs_and_evidence( - &DEFAULT_ACCURACY_MODEL, - &DEFAULT_ALLOCATOR, - evidence, - )), - Box::new( - HydraGroupingStrategy::new_with_planning_inputs_and_evidence( - &DEFAULT_ACCURACY_MODEL, - &DEFAULT_ALLOCATOR, - evidence, - ), - ), - Box::new(SharedSubDAGStrategy), - Box::new(crate::pass1::rewrite::AvgToSumOverCountStrategy), - Box::new(ExactCompositionStrategy), - ] -} - -// ── search_workload ────────────────────────────────────────────────────── - -/// Search a whole workload's pre-ASAP roots for every candidate replacement -/// [`default_strategies`] can find, deduped into a [`CandidateLogicalASAPDAGs`]. -/// Candidate generation is cost-free; ranking happens in plan selection. Use -/// [`search_workload_with`] to plug in a custom strategy set. -pub fn search_workload(roots: Vec<(Id, Rc)>) -> CandidateLogicalASAPDAGs { - search_workload_with(roots, &default_strategies()) -} - -/// Like [`search_workload`], but with an explicit set of context-free -/// `strategies`. The workload-dependent -/// [`RollupStrategy`] is derived and added automatically after CSE for both -/// entry points, because only this function owns the post-CSE sibling set. -/// -/// Runs [`share_common_sub_dags`] once over `roots` first — so every -/// strategy (and, transitively, every -/// [`crate::pass1::explanation::ReplacementExplanation`] a caller reads off the -/// result) sees the same already-deduplicated DAG — then discovers every -/// `TargetSubDAG` (see [`discover_targets`]) and runs the -/// fixpoint loop the module docs describe, capped at -/// [`MAX_SEARCH_ITERATIONS`] passes (see the module docs' "Termination" -/// section). Deduping candidate plans this way needs no cost model. -pub fn search_workload_with<'s, Id>( - roots: Vec<(Id, Rc)>, - strategies: &[Box], -) -> CandidateLogicalASAPDAGs { - let mut space = search_cse_workload_with(cse_workload(roots), strategies); - space.prepare_compositions(&DefaultAccuracyModel, &HashMap::new()); - space -} - -/// [`search_workload_with`] plus a per-root end-to-end `AccuracyTarget` -/// (issue #172) — the workload's `QueryRequirements.accuracy`, threaded -/// alongside each root. After the search, every root that carries a target -/// has its group's bound-summary [`Replacement::SubDAG`] candidates checked with -/// `accuracy_model`'s [`AccuracyModel::satisfies`]: a candidate whose -/// guarantee is fully known and misses the target is moved from -/// [`TargetSubDAGCandidates::candidates`] to [`TargetSubDAGCandidates::rejected`] *before* -/// `candidate_selection::cost_sorted`/`candidate_selection::global_selection` ever rank the -/// group. A constructible candidate with unknown accuracy remains visible for -/// downstream review under an approximate target, but default whole-plan -/// selection does not commit it. An exact target cannot accept an unknown -/// approximate summary. A kept pre-ASAP candidate is -/// exact and always survives — the raw/pre-ASAP alternative is what an -/// unsatisfiable root keeps. Logical-rewrite [`Replacement::SubDAG`] -/// candidates are not bound values and are left alone; the targets *inside* -/// a rewrite are their own groups. -/// -/// Precedence against per-node `AggIntent.accuracy` is documented in -/// [`crate::accuracy`]'s module docs. -pub fn search_workload_with_targets<'s, Id>( - roots: Vec<(Id, Rc, Option)>, - strategies: &[Box], - accuracy_model: &dyn AccuracyModel, -) -> CandidateLogicalASAPDAGs { - let mut targets = Vec::with_capacity(roots.len()); - let roots = roots - .into_iter() - .map(|(id, root, target)| { - targets.push(target); - (id, root) - }) - .collect(); - let mut space = search_cse_workload_with(cse_workload(roots), strategies); - // `cse_workload` preserves root order, so targets zip by position. - let root_ptrs: Vec<(*const OperatorNode, AccuracyTarget)> = space - .roots - .iter() - .zip(targets) - .filter_map(|((_, root), target)| target.map(|t| (Rc::as_ptr(root), t))) - .collect(); - // Whole-root proposals join the root group before its target check. - for (index, (ptr, target)) in root_ptrs.iter().enumerate() { - if root_ptrs[..index].contains(&(*ptr, target.clone())) { - continue; - } - let group = space.groups.get_mut(ptr).expect("every root has a group"); - let root = Rc::clone(&group.target); - for strategy in strategies { - let name = strategy.name(); - let proposals = strategy.propose_for_root(&root, target); - for mut candidate in proposals.candidates { - candidate.strategy = name; - group.add_candidate(candidate); - } - group - .rejected - .extend(proposals.rejected.into_iter().map(|mut rejection| { - rejection.strategy = name; - rejection - })); - } - } - let mut composition_targets: HashMap<_, Vec<_>> = HashMap::new(); - for (ptr, target) in root_ptrs { - composition_targets - .entry(ptr) - .or_default() - .push(target.clone()); - let Some(group) = space.groups.get_mut(&ptr) else { - continue; - }; - let (legal, illegal): (Vec<_>, Vec<_>) = - group - .candidates - .drain(..) - .partition(|candidate| match &candidate.replacement { - Replacement::SubDAG(node) if is_logical_rewrite(node) => true, - Replacement::SubDAG(node) => node.guarantee.as_ref().map_or_else( - || !matches!(target, AccuracyTarget::Exact), - |g| accuracy_model.satisfies(&g.optimistic_floor(), &target), - ), - // A composition's guarantee depends on the concrete child; - // prepare_compositions checks those pairs after all roots. - Replacement::ExactComposition(_) => true, - }); - group.candidates = legal; - group.rejected.extend(illegal.into_iter().map(|candidate| { - let (metric, bound, failure_probability) = match &candidate.replacement { - Replacement::SubDAG(node) => node - .guarantee - .as_ref() - .map(|g| { - ( - g.metric, - g.bound.evaluate(), - g.failure_probability.evaluate(), - ) - }) - .unwrap_or(( - asap_types::ir::properties::ErrorMetric::AbsoluteValue, - None, - None, - )), - Replacement::ExactComposition(_) => ( - asap_types::ir::properties::ErrorMetric::AbsoluteValue, - None, - None, - ), - }; - RejectedCandidate { - strategy: candidate.strategy, - description: format!("{} (root end-to-end target check)", candidate.rationale), - error: AccuracyError::TargetNotSatisfied { - metric, - bound, - failure_probability, - target: target.clone(), - }, - } - })); - } - space.prepare_compositions(accuracy_model, &composition_targets); - space -} - -/// The strictest accuracy among `siblings` that read the same summary input -/// as `root` — same child, grouping and filters, and the same intent apart -/// from its accuracy (and a quantile's rank, a evaluation parameter) — when -/// stricter than `root`'s own. One summary sized for the strictest consumer -/// serves every sibling: #509's summary-capability rule. -fn strictest_sibling_accuracy( - root: &OperatorNode, - siblings: &[Rc], -) -> Option { - fn approximate(intent: &AggIntent) -> Option<&AccuracyTarget> { - accuracy_target(intent).filter(|accuracy| !matches!(accuracy, AccuracyTarget::Exact)) - } - let Some(NonASAPOp::Aggregate { - reduction, - filters, - child, - .. - }) = root.non_asap() - else { - return None; - }; - let intent = bindable_intent(root)?; - let own = accuracy_budget(approximate(intent)?); - let (mut eps, mut delta) = own; - for sibling in siblings { - let Some(NonASAPOp::Aggregate { - reduction: sibling_reduction, - filters: sibling_filters, - child: sibling_child, - .. - }) = sibling.non_asap() - else { - continue; - }; - let Some(other) = bindable_intent(sibling) else { - continue; - }; - let Some(accuracy) = approximate(other) else { - continue; - }; - let same_intent = match (intent, other) { - (AggIntent::Quantile { col, .. }, AggIntent::Quantile { col: other_col, .. }) => { - col == other_col - } - _ => override_accuracy(intent, accuracy) == *other, - }; - if same_intent - && sibling_reduction == reduction - && sibling_filters == filters - && (Rc::ptr_eq(sibling_child, child) || sibling_child == child) - { - let (sibling_eps, sibling_delta) = accuracy_budget(accuracy); - eps = eps.min(sibling_eps); - delta = delta.min(sibling_delta); - } - } - if (eps, delta) == own { - None - } else if delta == DEFAULT_DELTA { - Some(AccuracyTarget::Epsilon(eps)) - } else { - Some(AccuracyTarget::EpsilonDelta { - epsilon: eps, - delta, - }) - } -} - -fn cse_workload(roots: Vec<(Id, Rc)>) -> Vec<(Id, Rc)> { - share_common_sub_dags(roots) -} - -fn search_cse_workload_with<'s, Id>( - cse_roots: Vec<(Id, Rc)>, - strategies: &[Box], -) -> CandidateLogicalASAPDAGs { - for (_, root) in &cse_roots { - assert!( - !root.contains_asap(), - "search_workload: a workload root already contains an ASAP operator \ - ({}); replacement search takes the front end's pre-ASAP DAG only", - root.operator.kind_name() - ); - } - let mut order = Vec::new(); - let mut nodes = HashMap::new(); - let mut counts: HashMap<*const OperatorNode, usize> = HashMap::new(); - discover_targets(&cse_roots, &mut order, &mut nodes, &mut counts); - let siblings: Vec> = order - .iter() - .filter_map(|ptr| { - let node = &nodes[ptr]; - matches!(node.non_asap(), Some(NonASAPOp::Aggregate { .. })).then(|| Rc::clone(node)) - }) - .collect(); - let rollup_strategy = RollupStrategy::new(&siblings); - let accuracy_reconciliation_strategy = AccuracyReconciliationStrategy::new(&siblings); - let limits: Vec> = order - .iter() - .filter_map(|ptr| { - let node = &nodes[ptr]; - matches!(node.non_asap(), Some(NonASAPOp::Limit { .. })).then(|| Rc::clone(node)) - }) - .collect(); - let topk_reuse_strategy = TopKLimitReuseStrategy::new(&limits); - - let mut groups: HashMap<*const OperatorNode, TargetSubDAGCandidates> = HashMap::new(); - for ptr in &order { - groups.insert( - *ptr, - TargetSubDAGCandidates::new(Rc::clone(&nodes[ptr]), counts[ptr]), - ); - } - - // Round-based frontier: every target is asked exactly once per strategy - // (never re-asked — see the module docs' "Termination" section on why - // that matters for `Replacement::Summary` dedup specifically). A round - // can grow the *next* round's frontier only by a candidate's own - // reachable children exposing a genuinely new, not-yet-known `Rc` — see - // `discover_new_descendant_targets`. - let mut frontier = order.clone(); - let mut rounds = 0usize; - while !frontier.is_empty() { - rounds += 1; - assert!( - rounds <= MAX_SEARCH_ITERATIONS, - "search_workload: fixpoint search did not converge within {MAX_SEARCH_ITERATIONS} \ - rounds — a registered ReplacementStrategy's Replacement::Rewrite candidates keep \ - exposing new, never-before-seen descendant structure every round. \ - ASAPStrategies/SharedSubDAGStrategy never do this (see replacement.rs's \ - module docs' \"Termination\" section); check any custom strategies passed to \ - search_workload_with.", - ); - - let targets_before = order.len(); - for ptr in &frontier { - let (root, consumer_count) = { - let group = &groups[ptr]; - (Rc::clone(&group.target), group.consumer_count) - }; - let strictest = strictest_sibling_accuracy(&root, &siblings); - let mut target = TargetSubDAG::with_consumer_count(&root, consumer_count); - target.strictest_sibling_accuracy = strictest.as_ref(); - - let mut proposed = Vec::new(); - let mut rejected = Vec::new(); - for strategy in strategies { - if strategy.matches(&target) { - let name = strategy.name(); - let proposals = strategy.propose(&target); - proposed.extend(proposals.candidates.into_iter().map(|mut candidate| { - candidate.strategy = name; - candidate - })); - rejected.extend(proposals.rejected.into_iter().map(|mut rejection| { - rejection.strategy = name; - rejection - })); - } - } - if rollup_strategy.matches(&target) { - let name = rollup_strategy.name(); - proposed.extend(rollup_strategy.replacements(&target).into_iter().map( - |mut candidate| { - candidate.strategy = name; - candidate - }, - )); - } - if accuracy_reconciliation_strategy.matches(&target) { - let name = accuracy_reconciliation_strategy.name(); - proposed.extend( - accuracy_reconciliation_strategy - .replacements(&target) - .into_iter() - .map(|mut candidate| { - candidate.strategy = name; - candidate - }), - ); - } - if topk_reuse_strategy.matches(&target) { - let name = topk_reuse_strategy.name(); - proposed.extend(topk_reuse_strategy.replacements(&target).into_iter().map( - |mut candidate| { - candidate.strategy = name; - candidate - }, - )); - } - - for candidate in &proposed { - if let Replacement::SubDAG(rc) = &candidate.replacement { - if is_logical_rewrite(rc) { - discover_new_descendant_targets(rc, &mut order, &mut nodes, &mut counts); - } - } - } - - let group = groups - .get_mut(ptr) - .expect("every discovered target has a group"); - for candidate in proposed { - group.add_candidate(candidate); - } - group.rejected.extend(rejected); - } - - // Any pointer `discover_new_descendant_targets` appended to `order` - // this round is a genuinely new target — give it a group and process - // it next round. Targets already in `groups` are never revisited. - let new_targets = &order[targets_before..]; - for ptr in new_targets { - groups.entry(*ptr).or_insert_with(|| { - TargetSubDAGCandidates::new(Rc::clone(&nodes[ptr]), counts[ptr]) - }); - } - frontier = new_targets.to_vec(); - } - - add_effective_count_cse_candidates(&order, &mut groups); - - CandidateLogicalASAPDAGs { - roots: cse_roots, - groups, - order, - composition_plans: Vec::new(), - } -} - -/// Materialize share/recompute alternatives for descendants whose raw edge -/// count is one but whose effective count can exceed one when a repeated -/// ancestor is recomputed. We only do this when an ordinary repeated group -/// proves that `SharedSubDAGStrategy` is part of this search's strategy set. -fn add_effective_count_cse_candidates( - order: &[*const OperatorNode], - groups: &mut HashMap<*const OperatorNode, TargetSubDAGCandidates>, -) { - let mut possible_children: HashMap<*const OperatorNode, Vec<*const OperatorNode>> = - HashMap::new(); - for ptr in order { - let group = &groups[ptr]; - let children = possible_children.entry(*ptr).or_default(); - for (child, _) in direct_child_counts(&group.target) { - if !children.contains(&child) { - children.push(child); - } - } - for candidate in &group.candidates { - if let Replacement::SubDAG(rewrite) = &candidate.replacement { - if !is_logical_rewrite(rewrite) { - continue; - } - for (child, _) in direct_child_counts(rewrite) { - if !children.contains(&child) { - children.push(child); - } - } - } - } - } - - let mut potentially_repeated = HashSet::new(); - let mut queue = VecDeque::new(); - for ptr in order { - let group = &groups[ptr]; - if group.consumer_count >= 2 && cse_candidate_pair(group).is_some() { - potentially_repeated.insert(*ptr); - queue.push_back(*ptr); - } - } - while let Some(parent) = queue.pop_front() { - if let Some(children) = possible_children.get(&parent) { - for child in children { - if groups.contains_key(child) && potentially_repeated.insert(*child) { - queue.push_back(*child); - } - } - } - } - - for ptr in order { - let group = groups - .get_mut(ptr) - .expect("every discovered site has a group"); - if potentially_repeated.contains(ptr) && cse_candidate_pair(group).is_none() { - let target = Rc::clone(&group.target); - let site = TargetSubDAG::with_consumer_count(&target, 2); - for mut candidate in SharedSubDAGStrategy.replacements(&site) { - candidate.rationale = format!( - "{}: this sub-DAG can become repeated when a repeated ancestor is recomputed; \ - global_selection decides using its effective consumer count", - match candidate.provenance { - ReplacementProvenance::CseShare => "build once and share", - ReplacementProvenance::CseRecompute => "recompute independently", - _ => unreachable!("SharedSubDAGStrategy only emits CSE candidates"), - } - ); - group.add_candidate(candidate); - } - } - } -} - -// ── target discovery ───────────────────────────────────────────────────── - -/// Walk every root's whole DAG, discovering one `TargetSubDAG` per distinct -/// `Rc` and its real `consumer_count` — see the module docs' "Where -/// `TargetSubDAG` discovery comes from" section for the full rationale. -pub(crate) fn discover_targets( - roots: &[(Id, Rc)], - order: &mut Vec<*const OperatorNode>, - nodes: &mut HashMap<*const OperatorNode, Rc>, - counts: &mut HashMap<*const OperatorNode, usize>, -) { - for (_, root) in roots { - walk(root, order, nodes, counts); - } -} - -/// Scan `candidate`'s **children** (deliberately never `candidate`'s own -/// top-level pointer — see the module docs' "Termination" section: a -/// logical rewrite's value is an alternative *for* the target that -/// proposed it, never a new target of its own) for any `Rc` not already -/// known, appending each to `order`/`nodes`/`counts` so -/// [`search_workload_with`]'s next round processes it. A no-op when every -/// child is already known — the case both shipped strategies always produce -/// (see that section). -fn discover_new_descendant_targets( - candidate: &Rc, - order: &mut Vec<*const OperatorNode>, - nodes: &mut HashMap<*const OperatorNode, Rc>, - counts: &mut HashMap<*const OperatorNode, usize>, -) { - walk_children(candidate, order, nodes, counts); -} - -/// Visit `node`: count this occurrence, and — the first time this exact -/// `Rc` is seen — record it as a target and recurse into its children. -fn walk( - node: &Rc, - order: &mut Vec<*const OperatorNode>, - nodes: &mut HashMap<*const OperatorNode, Rc>, - counts: &mut HashMap<*const OperatorNode, usize>, -) { - let ptr = Rc::as_ptr(node); - let already_visited = counts.contains_key(&ptr); - *counts.entry(ptr).or_insert(0) += 1; - if !already_visited { - order.push(ptr); - nodes.insert(ptr, Rc::clone(node)); - walk_children(node, order, nodes, counts); - } -} - -/// `node`'s own operator children ([`OperatorNode::children`]: operator -/// inputs plus the operator nodes its scalar expressions read), the same -/// scope `asap_types::ir::cse::share_common_sub_dags` itself uses and -/// `tests::count_consumers` mirrors for its own fixtures. `Concat` is -/// transparent: its branches are walked in place of it. -fn walk_children( - node: &OperatorNode, - order: &mut Vec<*const OperatorNode>, - nodes: &mut HashMap<*const OperatorNode, Rc>, - counts: &mut HashMap<*const OperatorNode, usize>, -) { - if let Some(NonASAPOp::Concat { children, .. }) = node.non_asap() { - for c in children { - walk_children(c, order, nodes, counts); - } - return; - } - for child in node.children() { - walk(child, order, nodes, counts); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::accuracy::PropagationStats; - use crate::test_support::{agg, agg_per_entity, lower_promql, maintained, metric_scan, timed}; - use asap_types::ir::operator::operator_properties::{Reduction as ReductionTy, Source}; - use asap_types::ir::operator::{ - agg_is_exact, default_cardinality, default_quantile, MathFunc, TimeFunc, - }; - use asap_types::ir::schema::{DataType, Field, Schema as SchemaTy}; - use asap_types::ir::Predicate; - use asap_types::ir::TimeRangeKind; - - use asap_types::types::AccuracyTarget; - use std::collections::HashMap; - - // Candidate shape without execution timing: what is computed, not where. - fn timing_free_shape(node: &Rc) -> serde_json::Value { - fn strip(value: &mut serde_json::Value) { - match value { - serde_json::Value::Object(fields) => { - fields.remove("timing"); - fields.values_mut().for_each(strip); - } - serde_json::Value::Array(values) => values.iter_mut().for_each(strip), - _ => {} - } - } - let mut shape = serde_json::to_value( - asap_types::ir::physical_export::compile_physical_asap_dag(&timed(node)).unwrap(), - ) - .unwrap(); - strip(&mut shape); - shape - } - - // Rate inventories never offer two candidates that differ only in timing. - #[test] - fn rate_candidate_inventories_have_no_timing_only_duplicates() { - for (query, accuracy) in [ - ("sum by(job)(rate(m[1m]))", AccuracyTarget::Exact), - ("topk by(job)(2, rate(m[1m]))", AccuracyTarget::Epsilon(0.1)), - ] { - let root = lower_promql(query, accuracy); - let inventory = search_workload(vec![(0usize, root)]) - .enumerate_candidate_dags(4096) - .unwrap(); - let shapes = inventory - .candidates - .iter() - .map(|forest| timing_free_shape(&forest[0].1)) - .collect::>(); - for (i, shape) in shapes.iter().enumerate() { - assert!(!shapes[..i].contains(shape), "{query}: duplicate {i}"); - } - } - } - - // Grouped Sum over Rate evaluations stays a summary state in the inventory, - // so materialization assignment can place it in precompute or at query time. - #[test] - fn grouped_rate_sum_inventory_keeps_sum_state_for_materialization_placement() { - let root = lower_promql("sum by(job)(rate(m[1m]))", AccuracyTarget::Exact); - let inventory = search_workload(vec![(0usize, root)]) - .enumerate_candidate_dags(4096) - .unwrap(); - let is_exact = |node: &OperatorNode, kind: ExactKind| { - matches!(&node.operator, Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::ExactAggregate(k, _), .. - }) if *k == kind) - }; - assert!(inventory.candidates.iter().any(|forest| { - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: sum }) = &forest[0].1.operator else { - return false; - }; - let Operator::ASAP(ASAPOp::SummaryAgg { child: rate, .. }) = &sum.operator else { - return false; - }; - is_exact(sum, ExactKind::Sum) - && matches!(&rate.operator, Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) if is_exact(child, ExactKind::Rate)) - })); - } - - // Every exposed query result has a evaluation; internal accumulator frontiers stay states. - #[test] - fn query_candidate_roots_do_not_leak_exact_accumulator_state() { - for query in [ - "sum by(job)(rate(m[1m]))", - "sum by(job)(m)", - "sum_over_time(m[1m])", - ] { - let root = lower_promql(query, AccuracyTarget::Exact); - let space = search_workload(vec![(0usize, root.clone())]); - let inventory = space.enumerate_candidate_dags(4096).unwrap(); - assert!(!inventory.candidates.is_empty()); - let strategy = ASAPStrategies::default(); - for candidate in strategy.propose(&TargetSubDAG::new(&root)).candidates { - if let Replacement::SubDAG(node) = candidate.replacement { - let output = finalize_query_candidate(node, &root).unwrap(); - assert!( - output - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_))), - "direct candidate {query} leaks state" - ); - } - } - for node in inventory.candidates.iter().map(|forest| &forest[0].1) { - assert!( - node.schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_))), - "{query}: query root leaks state: {:?}", - node.schema - ); - } - } - } - - #[test] - fn unpriced_inventory_retains_quantile_families_and_raw_execution() { - let query = agg(vec![2], default_quantile(0.9), metric_scan(&["job"])); - let space = search_workload(vec![(0usize, query)]); - let inventory = space.enumerate_candidate_dags(4096).unwrap(); - let roots = inventory - .candidates - .iter() - .map(|forest| format!("{:?}", forest[0].1)) - .collect::>(); - assert!(roots.iter().any(|root| root.contains("Kll"))); - assert!(roots.iter().any(|root| root.contains("DDSketch"))); - assert!(inventory - .candidates - .iter() - .any(|forest| !forest[0].1.contains_asap())); - } - - // Independent roots must not require materializing their Cartesian product. - #[test] - fn root_inventory_preserves_choices_without_workload_cartesian_expansion() { - let roots = (0..24usize) - .map(|id| { - ( - id, - agg( - vec![2], - default_quantile((id + 1) as f64 / 25.0), - metric_scan(&["job"]), - ), - ) - }) - .collect(); - let space = search_workload(roots); - assert!(space.enumerate_candidate_dags(4096).is_err()); - for id in 0..24 { - let inventory = space.enumerate_candidate_dags_for_root(&id, 4096).unwrap(); - assert!(inventory - .candidates - .iter() - .all(|forest| forest.len() == 1 && forest[0].0 == id)); - let descriptions = inventory - .candidates - .iter() - .map(|forest| format!("{:?}", forest[0].1)) - .collect::>(); - assert!(descriptions.iter().any(|node| node.contains("Kll"))); - assert!(descriptions.iter().any(|node| node.contains("DDSketch"))); - assert!(inventory - .candidates - .iter() - .any(|forest| !forest[0].1.contains_asap())); - } - assert!(space.enumerate_candidate_dags_for_root(&24, 4096).is_err()); - assert!(space.enumerate_candidate_dags_for_root(&0, 0).is_err()); - } - - // Factoring changes enumeration, not the set of root computations. - #[test] - fn root_inventory_matches_projection_of_exhaustive_workload_inventory() { - let roots = (0..2usize) - .map(|id| { - ( - id, - agg( - vec![2], - default_quantile(0.5 + id as f64 * 0.4), - metric_scan(&["job"]), - ), - ) - }) - .collect(); - let space = search_workload(roots); - let full = space.enumerate_candidate_dags(4096).unwrap(); - for id in 0..2 { - let inventory = space.enumerate_candidate_dags_for_root(&id, 4096).unwrap(); - for forest in &full.candidates { - let node = &forest.iter().find(|(root, _)| *root == id).unwrap().1; - assert!(inventory.candidates.iter().any(|one| &one[0].1 == node)); - } - for one in &inventory.candidates { - assert!(full.candidates.iter().any(|forest| forest - .iter() - .any(|(root, node)| *root == id && node == &one[0].1))); - } - } - } - - #[test] - fn inventory_budget_never_returns_a_silent_partial_search() { - let query = agg(vec![2], default_quantile(0.9), metric_scan(&["job"])); - let space = search_workload(vec![(0usize, query)]); - assert!(space.enumerate_candidate_dags(0).is_err()); - } - - // Finite samples can overflow a sum although their native average is finite. - #[test] - fn temporal_average_requires_finite_division_guard() { - let root = lower_promql("avg_over_time(a[5m])", AccuracyTarget::Exact); - let candidates = ASAPStrategies::default().replacements(&TargetSubDAG::new(&root)); - let operator = candidates - .iter() - .find_map(|c| match &c.replacement { - Replacement::SubDAG(node) => match &node.operator { - Operator::NonASAP(NonASAPOp::BinaryOp { operator, .. }) => Some(operator), - _ => None, - }, - _ => None, - }) - .expect("maintained average candidate"); - assert!(operator.checked_finite_division); - assert!( - crate::pass1::rewrite::SemanticEquivalentRewriteStrategy - .replacements(&TargetSubDAG::new(&root)) - .is_empty(), - "an unconditional pre-ASAP rewrite would bypass the runtime guard" - ); - } - - // Approximate requests also admit exact temporal ranking candidates. - #[test] - fn approximate_temporal_topk_admits_exact_maintained_values() { - let root = lower_promql( - "topk by(job)(1,count_over_time(a[5m]))", - AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - ); - let planning_inputs = CandidatePlanningInputs::with_default_accuracy(); - let node = exact_topk_over_temporal_values(&root, planning_inputs) - .unwrap() - .expect("exact ranking is legal for an approximate request"); - assert!(node.guarantee.as_ref().unwrap().is_exact()); - crate::test_support::time_and_export(&node).unwrap(); - } - - // Exact Top-K consumes the Planner's maintained temporal values. - #[test] - fn exact_temporal_topk_has_a_maintained_value_candidate() { - for query in [ - "topk(5, sum_over_time(a[5m]))", - "topk by(job)(5, count_over_time(a[5m]))", - ] { - let root = lower_promql(query, AccuracyTarget::Exact); - let planning_inputs = CandidatePlanningInputs::with_default_accuracy(); - let node = exact_topk_over_temporal_values(&root, planning_inputs) - .unwrap() - .expect("exact Top-K candidate"); - assert!(node.guarantee.as_ref().unwrap().is_exact()); - let Operator::NonASAP(NonASAPOp::Limit { - child: sorted, - n, - offset, - partition_by, - }) = &node.operator - else { - panic!("temporal TopK must compose Sort and Limit"); - }; - assert_eq!((*n, *offset), (Some(5), 0)); - let Operator::NonASAP(NonASAPOp::Sort { - keys, - partition_by: sort_groups, - child: values, - }) = &sorted.operator - else { - panic!("Limit must consume sorted temporal values"); - }; - assert_eq!(sort_groups, partition_by); - assert_eq!( - partition_by.keys().len(), - usize::from(query.contains("by(job)")) - ); - assert_eq!(keys.len(), 1); - assert!(!keys[0].ascending); - assert_eq!(node.schema, values.schema); - crate::test_support::time_and_export(&node).unwrap(); - } - } - - // A bounded exact mean can share the relative division proof with a quantile. - #[test] - fn bounded_mean_quantile_ratio_is_certified() { - struct Domain; - impl AccuracyEvidenceProvider for Domain { - fn quantile_input_domain( - &self, - _: &OperatorNode, - ) -> Option { - Some(crate::accuracy::QuantileInputDomain { - lower: 1.0, - upper: 1000.0, - max_samples: 10000, - contract: "finite test population".into(), - }) - } - } - let target = AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }; - let inputs = CandidatePlanningInputs { - evidence: &Domain, - ..CandidatePlanningInputs::with_default_accuracy() - }; - for query in [ - "avg_over_time(a[5m]) / quantile_over_time(0.5,a[5m])", - "quantile_over_time(0.5,a[5m]) / avg_over_time(a[5m])", - ] { - let root = lower_promql(query, target.clone()); - let node = realize_binary(&root, inputs, Some(&target)) - .unwrap() - .expect("bounded ratio candidate"); - assert!(DefaultAccuracyModel.satisfies(node.guarantee.as_ref().unwrap(), &target)); - } - } - - // Missing domain proof permits an uncertified direct quantile ratio only. - #[test] - fn quantile_ratio_without_input_proof_has_no_root_guarantee() { - let target = AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }; - let root = lower_promql( - "quantile_over_time(0.5,a[5m]) / quantile_over_time(0.9,a[5m])", - target.clone(), - ); - let planning_inputs = CandidatePlanningInputs::with_default_accuracy(); - let candidate = realize_binary(&root, planning_inputs, Some(&target)) - .unwrap() - .expect("direct quantile ratio candidate"); - assert!(candidate.guarantee.is_none()); - - let other = lower_promql( - "avg_over_time(a[5m]) / quantile_over_time(0.5,a[5m])", - target.clone(), - ); - assert!(realize_binary(&other, planning_inputs, Some(&target)) - .unwrap() - .is_none()); - } - - fn eps(e: f64) -> AccuracyTarget { - AccuracyTarget::Epsilon(e) - } - - // ── realizations_for_intent / sizing ─────────────────────────────── - - /// The most-preferred `Realization` — `realizations_for_intent(intent, - /// &DefaultCostModel)`'s head — for tests that only care about the - /// default pick, not the full candidate list. - fn preferred(intent: &AggIntent) -> Realization { - realizations_for_intent(intent) - .into_iter() - .next() - .expect("every intent has at least one Realization") - } - - /// Shorthand for asserting the realization *category*. - #[derive(Debug, PartialEq)] - enum Cat { - Sketch(SketchAlgorithm), - Acc(ExactKind), - Pass, - } - - fn cat(intent: &AggIntent) -> Cat { - match preferred(intent) { - Realization::ExactAggregate { kind, .. } => Cat::Acc(kind), - Realization::Sketch(kind) => Cat::Sketch(kind.algorithm().clone()), - Realization::PassThrough => Cat::Pass, - other => { - panic!("this coverage matrix expects only Exact/Sketch/PassThrough, got {other:?}") - } - } - } - - /// The `AggIntent → SummaryKind` coverage matrix (issue #98): every intent - /// variant maps to a sketch, an exact accumulator, or an explicit - /// pass-through. `realizations_for_intent`'s match is exhaustive, so a - /// new variant cannot compile without a decision; this matrix pins what - /// each decision *is* (its preferred/first candidate). - #[test] - fn agg_intent_to_summary_kind_coverage_matrix() { - use AggIntent as A; - use Cat::*; - use ExactKind as E; - use SketchAlgorithm as K; - let matrix: Vec<(A, Cat)> = vec![ - // approximate-capable, at an ε target → sketch - (default_quantile(0.99), Sketch(K::Kll)), - (default_cardinality(), Sketch(K::Hll)), - ( - A::Cardinality { - cols: vec![0, 1], - accuracy: eps(0.01), - }, - Sketch(K::Hll), - ), - ( - A::Count { - accuracy: eps(0.01), - }, - Sketch(K::Cms), - ), - ( - A::TopK { - k: 10, - accuracy: eps(0.01), - }, - Sketch(K::CmsWithHeap), - ), - // the same intents at Exact → exact realization - ( - A::Quantile { - col: None, - q: 0.5, - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - ( - A::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - ( - A::Cardinality { - cols: vec![0, 1], - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - ( - A::Count { - accuracy: AccuracyTarget::Exact, - }, - Acc(E::Count), - ), - ( - A::TopK { - k: 10, - accuracy: AccuracyTarget::Exact, - }, - Pass, - ), - // exact mergeable accumulators - (A::Sum { col: None }, Acc(E::Sum)), - (A::Min { col: None }, Acc(E::Min)), - (A::Max { col: None }, Acc(E::Max)), - (A::Rate, Acc(E::Rate)), - (A::IRate, Acc(E::IRate)), - (A::Increase, Acc(E::Increase)), - // exact but non-mergeable → pass-through - (A::Avg { col: None }, Pass), - ( - A::StdDev { - col: None, - population: false, - }, - Pass, - ), - ( - A::Variance { - col: None, - population: true, - }, - Pass, - ), - // classic-bucket histogram_quantile is not re-sketchable (#79) - (A::HistogramQuantile { q: 0.99, le: 0 }, Pass), - // counter-derivative / range-vector functions (#44) - (A::Changes, Pass), - (A::Delta, Pass), - (A::IDelta, Pass), - (A::Deriv, Pass), - (A::Resets, Pass), - (A::PredictLinear { seconds: 60.0 }, Pass), - ( - A::DoubleExpSmoothing { - smoothing: 0.5, - trend: 0.5, - }, - Pass, - ), - // native-histogram accessors (#43) - (A::HistogramCount, Pass), - (A::HistogramSum, Pass), - (A::HistogramAvg, Pass), - (A::HistogramStdDev, Pass), - (A::HistogramStdVar, Pass), - ( - A::HistogramFraction { - lower: 0.0, - upper: 1.0, - }, - Pass, - ), - // per-sample transforms (#45, #46) + presence (#47) - (A::Math(MathFunc::Abs), Pass), - (A::TimeFn(TimeFunc::Hour), Pass), - (A::Absent, Pass), - (A::AbsentOverTime, Pass), - (A::PresentOverTime, Pass), - // extended aggregations (#49) - (A::Group, Pass), - (A::CountValues { label: "v".into() }, Pass), - // additional range reducers (#51) - (A::LastOverTime, Pass), - (A::FirstOverTime, Pass), - (A::MadOverTime, Pass), - (A::TsOfMinOverTime, Pass), - (A::TsOfMaxOverTime, Pass), - (A::TsOfFirstOverTime, Pass), - (A::TsOfLastOverTime, Pass), - ]; - for (intent, expected) in &matrix { - assert_eq!(&cat(intent), expected, "realization for {intent:?}"); - } - // Every accumulator pick is mergeable; every sketch pick is on a - // genuinely approximate target (the `agg_is_*` helpers stay truthful). - for (intent, expected) in &matrix { - if let Cat::Acc(_) = expected { - assert!(agg_is_mergeable(intent), "{intent:?}"); - } - if let Cat::Sketch(_) = expected { - assert!( - !agg_is_exact(intent) || matches!(intent, AggIntent::Count { .. }), - "{intent:?} sketches only under an approximate target" - ); - } - } - } - - // Correlation must never acquire a single-input sketch or scalar accumulator. - #[test] - fn pearson_corr_keeps_exact_paired_input() { - let intent = AggIntent::PearsonCorr { left: 0, right: 1 }; - assert!(matches!( - realizations_for_intent(&intent).as_slice(), - [Realization::PassThrough] - )); - assert!(summary_candidates(&intent).is_empty()); - } - - #[test] - fn accuracy_target_drives_the_boundary() { - // Same intent, three targets → three different decisions. - let exact = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }; - assert_eq!(preferred(&exact), Realization::PassThrough); - - let approx = default_quantile(0.99); // ε = 0.01 - assert_eq!( - preferred(&approx), - Realization::Sketch(SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { k: 269 }, - )) - ); - - let looser = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: eps(0.05), - }; - assert_eq!( - preferred(&looser), - Realization::Sketch(SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { k: 52 }, - )) - ); - } - - #[test] - fn default_cardinality_sizes_hll_to_its_rse_magnitude() { - assert_eq!( - preferred(&default_cardinality()), - Realization::Sketch(SketchKind::new( - SketchAlgorithm::Hll, - SketchParams::Hll { precision: 14 }, - )) - ); - } - - // Exact counting remains a legal candidate under an approximate target. - #[test] - fn approximate_count_includes_exact_accumulator_candidate() { - let intent = AggIntent::Count { - accuracy: eps(0.01), - }; - assert!(realizations_for_intent(&intent) - .iter() - .any(|candidate| matches!( - candidate, - Realization::ExactAggregate { - kind: ExactKind::Count, - .. - } - ))); - } - - #[test] - fn epsilon_delta_sizes_cms_depth() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.001, - delta: 0.001, - }, - }; - assert_eq!( - preferred(&intent), - Realization::Sketch(SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 2719, - depth: 7 - }, // ⌈e/0.001⌉, ⌈ln 1000⌉ - )) - ); - // Epsilon-only falls back to DEFAULT_DELTA → depth 5. - let intent = AggIntent::Count { - accuracy: eps(0.001), - }; - assert_eq!( - preferred(&intent), - Realization::Sketch(SketchKind::new( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: 2719, - depth: 5 - }, - )) - ); - } - - #[test] - fn topk_heap_capacity_respects_accuracy_and_output_count() { - let intent = AggIntent::TopK { - k: 25, - accuracy: eps(0.01), - }; - match preferred(&intent) { - Realization::Sketch(kind) if kind.algorithm() == &SketchAlgorithm::CmsWithHeap => { - let SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } = kind.params() - else { - unreachable!("SketchKind validates CmsWithHeap params") - }; - assert_eq!(*heap_size, 100); - assert_eq!(*width, 272); // ⌈e/0.01⌉ - assert_eq!(*depth, 5); - } - other => panic!("expected CmsWithHeap, got {other:?}"), - } - } - - #[test] - fn candidate_lists_match_the_issue_map() { - assert_eq!( - summary_candidates(&default_quantile(0.5)), - &[SketchAlgorithm::Kll, SketchAlgorithm::DDSketch] - ); - assert_eq!( - summary_candidates(&default_cardinality()), - &[ - SketchAlgorithm::Hll, - SketchAlgorithm::Theta, - SketchAlgorithm::Kmv, - SketchAlgorithm::UnivMon - ] - ); - assert_eq!( - summary_candidates(&AggIntent::TopK { - k: 5, - accuracy: eps(0.01) - }), - &[ - SketchAlgorithm::CmsWithHeap, - SketchAlgorithm::CountSketchWithHeap - ] - ); - assert_eq!( - summary_candidates(&AggIntent::Count { - accuracy: eps(0.01) - }), - &[ - SketchAlgorithm::Cms, - SketchAlgorithm::CountSketch, - SketchAlgorithm::UnivMon - ] - ); - assert!(summary_candidates(&AggIntent::Rate).is_empty()); - } - - #[test] - fn realizations_for_intent_enumerates_every_candidate_ranked() { - // Quantile's candidate list is [Kll, DDSketch] — realizations_for_intent - // must return both, ranked with the DefaultCostModel's preferred - // (Kll) first. - let kinds: Vec = realizations_for_intent(&default_quantile(0.99)) - .into_iter() - .map(|realization| match realization { - Realization::Sketch(kind) => kind.algorithm().clone(), - other => panic!("expected Sketch, got {other:?}"), - }) - .collect(); - assert_eq!(kinds, vec![SketchAlgorithm::Kll, SketchAlgorithm::DDSketch]); - } - - #[test] - fn degenerate_epsilon_saturates_to_tightest_params() { - let intent = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: eps(0.0), - }; - assert_eq!( - preferred(&intent), - Realization::Sketch(SketchKind::new( - SketchAlgorithm::Kll, - SketchParams::Kll { k: 65_535 }, - )) - ); - } - - // ── posterior_aware_size_params (issue #239, integration point 2) ────── - - fn count_intent(e: f64) -> AggIntent { - AggIntent::Count { accuracy: eps(e) } - } - - #[test] - fn posterior_aware_sizing_shrinks_width_under_stated_assumption() { - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); - let relaxed = posterior_aware_size_params( - SketchAlgorithm::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: 0.5, - }, - ); - match (worst_case, relaxed) { - ( - SketchParams::Cms { - width: w0, - depth: d0, - }, - SketchParams::Cms { - width: w1, - depth: d1, - }, - ) => { - assert!( - w1 < w0, - "expected relaxed width {w1} to be strictly smaller than worst-case {w0}" - ); - assert_eq!(d0, d1, "depth must be unaffected by width_relaxation"); - } - other => panic!("expected Cms/Cms pair, got {other:?}"), - } - } - - #[test] - fn posterior_aware_sizing_at_full_relaxation_matches_worst_case() { - // width_relaxation = 1.0 must reproduce default_size_params exactly - // — the "no risk taken" boundary. - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); - let relaxed = posterior_aware_size_params( - SketchAlgorithm::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: 1.0, - }, - ); - assert_eq!(worst_case, relaxed); - } - - #[test] - fn posterior_aware_sizing_invalid_relaxation_falls_back_to_worst_case() { - let intent = count_intent(0.01); - let worst_case = default_size_params(SketchAlgorithm::Cms, &intent, 0.01, 0.01); - for bad in [0.0, -0.5, 1.5, f64::NAN, f64::INFINITY] { - let relaxed = posterior_aware_size_params( - SketchAlgorithm::Cms, - &intent, - 0.01, - 0.01, - ExpectedCaseSizing { - width_relaxation: bad, - }, - ); - assert_eq!( - worst_case, relaxed, - "width_relaxation={bad} should fall back to the worst-case width" - ); - } - } - - #[test] - fn posterior_aware_sizing_does_not_apply_cms_l1_relaxation_to_count_sketch() { - let cms_heap_intent = AggIntent::TopK { - k: 7, - accuracy: eps(0.01), - }; - let assumption = ExpectedCaseSizing { - width_relaxation: 0.25, - }; - // CountSketch - assert_eq!( - posterior_aware_size_params( - SketchAlgorithm::CountSketch, - &count_intent(0.01), - 0.01, - 0.01, - assumption - ), - default_size_params( - SketchAlgorithm::CountSketch, - &count_intent(0.01), - 0.01, - 0.01 - ), - ); - // CmsWithHeap / CountSketchWithHeap carry k through untouched. - match posterior_aware_size_params( - SketchAlgorithm::CmsWithHeap, - &cms_heap_intent, - 0.01, - 0.01, - assumption, - ) { - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - } => { - assert_eq!(width, 68); - assert_eq!(depth, 5); - assert_eq!(heap_size, 100); - } - other => panic!("expected CmsWithHeap, got {other:?}"), - } - } - - #[test] - fn posterior_aware_sizing_leaves_non_cms_kinds_unchanged() { - // Kll/Hll/etc. have no width_relaxation concept — must be byte-for- - // byte identical to default_size_params. - let intent = default_quantile(0.99); - let assumption = ExpectedCaseSizing { - width_relaxation: 0.1, - }; - assert_eq!( - posterior_aware_size_params(SketchAlgorithm::Kll, &intent, 0.01, 0.01, assumption), - default_size_params(SketchAlgorithm::Kll, &intent, 0.01, 0.01), - ); - } - - #[test] - fn default_size_params_unchanged_by_new_function_existing() { - // Regression pin: default_size_params's own worst-case behavior for - // existing callers must be untouched by adding - // posterior_aware_size_params alongside it. - assert_eq!( - default_size_params(SketchAlgorithm::Cms, &count_intent(0.001), 0.001, 0.001), - SketchParams::Cms { - width: 2719, - depth: 7 - }, - ); - } - - // ── ASAPStrategies / SharedSubDAGStrategy fixtures ─────────── - - // ── ASAPStrategies ───────────────────────────────────────────── - - #[test] - fn matches_a_bindable_aggregate() { - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - assert!(ASAPStrategies::default().matches(&target)); - } - - #[test] - fn does_not_match_a_multi_intent_or_having_aggregate() { - let strategy = ASAPStrategies::default(); - - let multi = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: ReductionTy::by(vec![2]), - measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: metric_scan(&["job"]), - })) - .unwrap(); - let target = TargetSubDAG::new(&multi); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - - let having_q = crate::test_support::aggregate( - ReductionTy::by(vec![2]), - vec![default_quantile(0.99)], - vec![], - Some(asap_types::ir::Predicate(ScalarExpr::Literal( - asap_types::ir::scalar::ScalarValue::Boolean(true), - ))), - metric_scan(&["job"]), - ); - let target = TargetSubDAG::new(&having_q); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn does_not_match_a_non_aggregate_node() { - let scan = metric_scan(&["job"]); - let target = TargetSubDAG::new(&scan); - assert!(!ASAPStrategies::default().matches(&target)); - assert!(ASAPStrategies::default().replacements(&target).is_empty()); - } - - #[test] - fn approximate_quantile_enumerates_every_summary_candidate() { - // Quantile's candidate list is [Kll, DDSketch] (summary_candidates) — - // every entry must come back as its own bound summary candidate, - // not just Kll (the CostModel-ranked head realizations_for_intent commits to). - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let replacements = ASAPStrategies::default().replacements(&target); - assert_eq!( - replacements.len(), - 2, - "expected 2 candidates, got {replacements:?}" - ); - - let kinds: Vec = replacements - .iter() - .map(|r| match &r.replacement { - Replacement::SubDAG(node) => summary_family_algorithm(node), - Replacement::ExactComposition(_) => { - panic!("expected a Summary replacement") - } - }) - .collect(); - assert!(kinds.contains(&SketchAlgorithm::Kll), "{kinds:?}"); - assert!(kinds.contains(&SketchAlgorithm::DDSketch), "{kinds:?}"); - assert!( - replacements.iter().all(|r| !r.rationale.is_empty()), - "every candidate must carry a rationale" - ); - } - - #[test] - fn cardinality_epsilon_delta_keeps_unknown_accuracy_candidates() { - let q = agg(vec![2], default_cardinality(), metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let replacements = ASAPStrategies::default().replacements(&target); - let kinds: Vec = replacements - .iter() - .map(|r| match &r.replacement { - Replacement::SubDAG(node) => summary_family_algorithm(node), - Replacement::ExactComposition(_) => { - panic!("expected a Summary replacement") - } - }) - .collect(); - assert_eq!( - kinds, - vec![ - SketchAlgorithm::Hll, - SketchAlgorithm::Theta, - SketchAlgorithm::Kmv, - SketchAlgorithm::UnivMon, - ] - ); - - let q = agg( - vec![2], - AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }, - metric_scan(&["job"]), - ); - let kinds: Vec<_> = ASAPStrategies::default() - .replacements(&TargetSubDAG::new(&q)) - .iter() - .map(|r| match &r.replacement { - Replacement::SubDAG(node) => summary_family_algorithm(node), - Replacement::ExactComposition(_) => { - panic!("expected a Summary replacement") - } - }) - .collect(); - assert_eq!( - kinds, - vec![ - SketchAlgorithm::Hll, - SketchAlgorithm::Theta, - SketchAlgorithm::Kmv, - SketchAlgorithm::UnivMon, - ] - ); - } - - #[test] - fn exact_accuracy_target_yields_exactly_one_pass_through_candidate() { - // Exact quantile has no sketch candidate at all — realizations_for_intent - // produces PassThrough, the only option, so exactly one candidate. - let intent = AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }; - let q = agg(vec![2], intent, metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let replacements = ASAPStrategies::default().replacements(&target); - assert_eq!(replacements.len(), 1, "{replacements:?}"); - assert!(matches!( - &replacements[0].replacement, - Replacement::SubDAG(node) if !node.contains_asap() - )); - assert!(replacements[0].rationale.contains("only realization")); - } - - #[test] - fn exact_mergeable_intent_yields_exactly_one_accumulator_candidate() { - let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - let replacements = ASAPStrategies::default().replacements(&target); - assert_eq!(replacements.len(), 1, "{replacements:?}"); - assert!(matches!( - &replacements[0].replacement, - Replacement::SubDAG(node) if matches!( - node.operator, - Operator::ASAP(ASAPOp::SummaryAgg { .. }) - ) - )); - } - - /// Constructing the outer target's candidates never leaks its algorithm - /// choice into the nested aggregate. Existing approximate composition - /// remains governed by the accuracy model, independently of #171's exact - /// value-operation candidates. - #[test] - fn enumerating_the_targets_candidates_does_not_leak_into_a_nested_aggregate() { - // outer: quantile(0.99, ...) over inner: quantile(0.5, m) — both - // Quantile, so both share the [Kll, DDSketch] candidate list. - // - // Rank-over-rank has no registered rule in `DefaultAccuracyModel` - // (issue #172 — see `approximate_over_approximate_is_rejected_by_default`), - // so this test injects `RankAdditiveModel` to admit the composition - // and keep exercising the per-node enumeration property it is about. - let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["job"])); - let outer = agg(vec![], default_quantile(0.99), inner); - let target = TargetSubDAG::new(&outer); - let replacements = - ASAPStrategies::new_with_planning_inputs(&RankAdditiveModel, &EqualSplitAllocator) - .replacements(&target); - - assert_eq!(replacements.len(), 2, "{replacements:?}"); - assert!(replacements.iter().all(|candidate| { - matches!(&candidate.replacement, Replacement::SubDAG(n) if n.contains_asap()) - })); - // The inner target is still independently enumerated, and nothing - // about the outer target's choice reaches it. - let space = search_workload(vec![("q", Rc::clone(&outer))]); - let Some(NonASAPOp::Aggregate { child, .. }) = space.roots[0].1.non_asap() else { - unreachable!() - }; - let inner_group = space - .candidates_for_target(child) - .expect("inner quantile is a target"); - let inner_kinds: Vec = inner_group - .candidates - .iter() - .filter_map(|c| match &c.replacement { - Replacement::SubDAG(node) if node.contains_asap() => { - Some(summary_family_algorithm(node)) - } - _ => None, - }) - .collect(); - assert_eq!( - inner_kinds, - vec![SketchAlgorithm::Kll, SketchAlgorithm::DDSketch], - "the nested inner aggregate keeps its own candidates" - ); - } - - /// The `FieldDataType`'s committed `SketchAlgorithm`, from the top - /// `SummaryAgg` reachable under a (possibly `SummaryEstimate`-wrapped) - /// bound root. - fn summary_family_algorithm(node: &OperatorNode) -> SketchAlgorithm { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - summary_family_algorithm(summary_input) - } - Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) => match family { - asap_types::ir::schema::FieldDataType::Sketch(kind, _) => kind.algorithm().clone(), - other => panic!("expected a Sketch family, got {other:?}"), - }, - other => panic!("expected SummaryAgg/SummaryEstimate, got {other:?}"), - } - } - - // ── SharedSubDAGStrategy ──────────────────────────────────────────── - - #[test] - fn does_not_match_a_single_consumer_target() { - let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - assert_eq!(target.consumer_count, 1); - assert!(!SharedSubDAGStrategy.matches(&target)); - assert!(SharedSubDAGStrategy.replacements(&target).is_empty()); - } - - #[test] - fn two_or_more_consumers_yields_the_share_vs_independent_pair() { - let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let target = TargetSubDAG::with_consumer_count(&q, 2); - assert!(SharedSubDAGStrategy.matches(&target)); - - let replacements = SharedSubDAGStrategy.replacements(&target); - assert_eq!(replacements.len(), 2, "{replacements:?}"); - - let shared = match &replacements[0].replacement { - Replacement::SubDAG(rc) => rc, - other => panic!("expected a Rewrite replacement, got {other:?}"), - }; - assert!( - Rc::ptr_eq(shared, &q), - "the 'build once and share' candidate must be the same Rc as the target" - ); - assert!(replacements[0].rationale.contains("build once and share")); - - let independent = match &replacements[1].replacement { - Replacement::SubDAG(rc) => rc, - other => panic!("expected a Rewrite replacement, got {other:?}"), - }; - assert!( - !Rc::ptr_eq(independent, &q), - "the 'build independently' candidate must be a distinct Rc from the target" - ); - assert_eq!( - **independent, *q, - "the 'build independently' candidate must still be structurally identical" - ); - assert!(replacements[1].rationale.contains("build independently")); - } - - #[test] - fn three_consumers_are_reported_verbatim_in_both_rationales() { - let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let target = TargetSubDAG::with_consumer_count(&q, 3); - let replacements = SharedSubDAGStrategy.replacements(&target); - assert!(replacements[0].rationale.contains('3')); - assert!(replacements[1].rationale.contains('3')); - } - - /// Builds realistic multi-consumer `TargetSubDAG`s the same way this - /// module's own [`discover_targets`]/`walk` does: dedup by `Rc::as_ptr`, - /// walking only the relational-skeleton operator children - /// `asap_types::ir::cse::share_common_sub_dags` itself scopes to, - /// so a shared node nested below another shared node is only ever - /// counted at the highest (maximal) point sharing starts. Test-only: - /// this module deliberately does not ship a workload-wide discovery - /// pass of its own (see the module docs' "Non-goals"). - fn count_consumers(roots: &[Rc]) -> HashMap<*const OperatorNode, usize> { - fn walk(node: &Rc, counts: &mut HashMap<*const OperatorNode, usize>) { - let ptr = Rc::as_ptr(node); - let already_visited = counts.contains_key(&ptr); - *counts.entry(ptr).or_insert(0) += 1; - if !already_visited { - walk_children(node, counts); - } - } - fn walk_children(node: &OperatorNode, counts: &mut HashMap<*const OperatorNode, usize>) { - if let Some(NonASAPOp::Concat { children, .. }) = node.non_asap() { - for c in children { - walk_children(c, counts); - } - return; - } - for child in node.children() { - walk(child, counts); - } - } - - let mut counts = HashMap::new(); - for root in roots { - walk(root, &mut counts); - } - counts - } - - #[test] - fn realistic_cse_output_produces_a_two_consumer_target() { - // Two workload roots that `share_common_sub_dags` collapses onto one - // Rc (mirrors `explanation`'s and `cse`'s own fixtures): a grouped - // Sum aggregate over the same scan, built independently at each root. - let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let shared = asap_types::ir::cse::share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!(Rc::ptr_eq(ra, rb), "fixture sanity: the two roots merged"); - - let roots: Vec> = shared.into_iter().map(|(_, rc)| rc).collect(); - let counts = count_consumers(&roots); - let count = counts[&Rc::as_ptr(&roots[0])]; - assert_eq!(count, 2); - - let target = TargetSubDAG::with_consumer_count(&roots[0], count); - assert!(SharedSubDAGStrategy.matches(&target)); - assert_eq!(SharedSubDAGStrategy.replacements(&target).len(), 2); - } - - // ── search_workload / CandidateLogicalASAPDAGs / TargetSubDAGCandidates (merged from search.rs) ── - // - // Reuses this test module's own `metric_scan`/`agg` fixture helpers - // above (identical to `search.rs`'s own copies, which are dropped here - // to avoid a duplicate-definition collision now that both test modules - // share one file) and `count_consumers` above (which mirrors - // `discover_targets`' own real, non-test traversal for these fixtures). - - // ── discovery + MEMO shape ─────────────────────────────────────────── - - #[test] - fn single_bindable_aggregate_keeps_unprovable_hydra_candidates() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }; - let root = agg(vec![2], intent, metric_scan(&["job"])); - let space = search_workload(vec![("q", root)]); - - // One group for the Aggregate, one for its Scan child. - assert_eq!(space.len(), 2); - - let agg_group = space - .target_subdag_candidates() - .find(|g| matches!(g.target.non_asap(), Some(NonASAPOp::Aggregate { .. }))) - .expect("an Aggregate group must be discovered"); - assert_eq!(agg_group.consumer_count, 1); - assert_eq!( - agg_group.candidates.len(), - 6, - "Hydra candidates with unknown evidence remain available: {:?}", - agg_group.candidates - ); - assert!(agg_group - .candidates - .iter() - .all(|c| matches!(&c.replacement, Replacement::SubDAG(n) if n.contains_asap()))); - assert_eq!( - agg_group - .candidates - .iter() - .filter(|candidate| { - let Replacement::SubDAG(node) = &candidate.replacement else { - return false; - }; - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = - &node.operator - else { - return false; - }; - matches!( - &summary_input.operator, - Operator::ASAP(ASAPOp::SummaryAgg { - grouping: GroupingStrategy::SharedMultiSubpopulation { .. }, - .. - }) - ) - }) - .count(), - 2, - "Hydra candidates remain visible with symbolic shared-grid error" - ); - assert_eq!( - agg_group - .candidates - .iter() - .filter(|candidate| candidate.has_missing_accuracy_evidence()) - .count(), - 2 - ); - let scan_group = space - .target_subdag_candidates() - .find(|g| matches!(g.target.non_asap(), Some(NonASAPOp::Scan { .. }))) - .expect("a Scan group must be discovered"); - assert_eq!(scan_group.consumer_count, 1); - assert!( - scan_group.candidates.is_empty(), - "no strategy matches a bare Scan" - ); - } - - #[test] - fn cardinality_group_keeps_all_four_candidates() { - let root = agg(vec![2], default_cardinality(), metric_scan(&["job"])); - let space = search_workload(vec![("q", root)]); - let agg_group = space - .target_subdag_candidates() - .find(|g| matches!(g.target.non_asap(), Some(NonASAPOp::Aggregate { .. }))) - .unwrap(); - assert_eq!(agg_group.candidates.len(), 4); - assert!(agg_group.candidates.iter().any(|candidate| matches!( - &candidate.replacement, - Replacement::SubDAG(node) if node.guarantee.is_none() - && candidate.has_missing_accuracy_evidence() - ))); - - let root = agg(vec![2], default_cardinality(), metric_scan(&["job"])); - let targeted = search_workload_with_targets( - vec![( - "q", - root, - Some(AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }), - )], - &default_strategies(), - &DefaultAccuracyModel, - ); - let target = &targeted.roots[0].1; - assert!(targeted - .candidates_for_target(target) - .unwrap() - .candidates - .iter() - .any(|candidate| matches!( - &candidate.replacement, - Replacement::SubDAG(node) if node.guarantee.is_none() - && candidate.has_missing_accuracy_evidence() - ))); - - let exact_target = search_workload_with_targets( - vec![( - "q", - agg(vec![2], default_cardinality(), metric_scan(&["job"])), - Some(AccuracyTarget::Exact), - )], - &default_strategies(), - &DefaultAccuracyModel, - ); - assert!(exact_target - .candidates_for_target(&exact_target.roots[0].1) - .unwrap() - .candidates - .iter() - .all(|candidate| !candidate.has_missing_accuracy_evidence())); - } - - #[test] - fn shared_aggregate_across_two_roots_gets_both_strategies_candidates() { - // Two independently-built, structurally identical Sum aggregates: - // share_common_sub_dags (run inside search_workload) collapses them - // onto one Rc with consumer_count 2, so this single group should - // carry ASAPStrategies's one ExactAggregate candidate *and* - // SharedSubDAGStrategy's share-vs-recompute pair. - let a = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let b = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let space = search_workload(vec![("a", a), ("b", b)]); - - // roots[0] and roots[1] must have merged onto the same Rc. - assert!(Rc::ptr_eq(&space.roots[0].1, &space.roots[1].1)); - - let group = space.candidates_for_target(&space.roots[0].1).unwrap(); - assert_eq!(group.consumer_count, 2); - assert_eq!( - group.candidates.len(), - 3, - "1 ExactAggregate Summary + 2 Rewrite (share/recompute): {:?}", - group.candidates - ); - - // Old `Replacement::Summary` ↔ a `Subtree` containing an ASAP node; - // old `Replacement::Rewrite` ↔ a pure pre-ASAP `Subtree`. - let summary_count = group - .candidates - .iter() - .filter(|c| matches!(&c.replacement, Replacement::SubDAG(n) if n.contains_asap())) - .count(); - let rewrite_count = group - .candidates - .iter() - .filter(|c| matches!(&c.replacement, Replacement::SubDAG(n) if !n.contains_asap())) - .count(); - assert_eq!(summary_count, 1); - assert_eq!(rewrite_count, 2); - - // The two Rewrite candidates must NOT have collapsed into one - // (the "false-positive dedup" failure mode `is_duplicate_rewrite` - // exists to prevent). - let one_is_the_target = group.candidates.iter().any( - |c| matches!(&c.replacement, Replacement::SubDAG(rc) if Rc::ptr_eq(rc, &group.target)), - ); - let one_is_not = group.candidates.iter().any( - |c| matches!(&c.replacement, Replacement::SubDAG(rc) if !Rc::ptr_eq(rc, &group.target)), - ); - assert!(one_is_the_target && one_is_not); - } - - #[test] - fn nested_shared_sub_dag_below_an_unshared_parent_is_still_discovered() { - // A shared grouped Aggregate nested under two *different*, - // unshared Filter parents — real consumer_count must come from - // walking the whole DAG, not just root-level pointer identity - // (a naive whole-root-only consumer-count pass would miss this; - // this module's discover_targets must not). - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::Predicate; - - let shared = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - // Different predicates so the two Filter *parents* stay distinct - // (don't themselves merge under CSE) — only their shared `child` - // should collapse onto one `Rc`. - let root_a = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Int64(1))), - child: Rc::clone(&shared), - })) - .unwrap(); - let root_b = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Int64(2))), - child: Rc::clone(&shared), - })) - .unwrap(); - - let space = search_workload(vec![("a", root_a), ("b", root_b)]); - assert_eq!( - space.len(), - 4, - "2 distinct Filters + 1 shared Aggregate + 1 shared Scan" - ); - - // `share_common_sub_dags` re-clones+re-interns anything that already - // had more than one owner going in (see `cse.rs`'s own doc on - // `intern_child`'s clone-fallback path) — so the post-CSE shared - // node is a *fresh* Rc, structurally equal to (but not the same - // pointer as) the pre-search `shared` variable. Recover it from the - // post-CSE root's own `child` field instead of the stale `shared` - // handle. - let Some(NonASAPOp::Filter { - child: post_cse_shared_a, - .. - }) = space.roots[0].1.non_asap() - else { - panic!("expected a Filter root"); - }; - let Some(NonASAPOp::Filter { - child: post_cse_shared_b, - .. - }) = space.roots[1].1.non_asap() - else { - panic!("expected a Filter root"); - }; - assert!( - Rc::ptr_eq(post_cse_shared_a, post_cse_shared_b), - "fixture sanity: the two Filters' children must still merge" - ); - let post_cse_shared = post_cse_shared_a; - let group = space - .candidates_for_target(post_cse_shared) - .expect("shared node must be a discovered target"); - assert_eq!(group.consumer_count, 2); - assert!( - SharedSubDAGStrategy.matches(&TargetSubDAG::with_consumer_count( - post_cse_shared, - group.consumer_count - )) - ); - } - - // ── dedup ──────────────────────────────────────────────────────────── - - #[test] - fn add_candidate_rejects_a_true_rewrite_duplicate() { - // SharedSubDAGStrategy's `Replacement::Rewrite` candidates are - // real `OperatorNode` values with `PartialEq`, so `add_candidate` can - // (and must) actually reject a genuine repeat — unlike the - // `Replacement::Summary` case (see the test below). - let root = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let mut group = TargetSubDAGCandidates::new(Rc::clone(&root), 2); - let target = TargetSubDAG::with_consumer_count(&root, 2); - let mut inserted = 0; - for candidate in SharedSubDAGStrategy.replacements(&target) { - if group.add_candidate(candidate) { - inserted += 1; - } - } - assert_eq!(inserted, 2, "share + recompute-independently candidates"); - - // Re-adding the identical candidate list must add nothing new: the - // "share" candidate is literally the same Rc as before, and the - // "recompute independently" candidate is a fresh Rc but - // structurally identical value, both already covered by - // `is_duplicate_rewrite`. - let mut re_inserted = 0; - for candidate in SharedSubDAGStrategy.replacements(&target) { - if group.add_candidate(candidate) { - re_inserted += 1; - } - } - assert_eq!( - re_inserted, 0, - "re-proposing the same Rewrite candidates must not grow the group" - ); - assert_eq!(group.candidates.len(), 2); - } - - #[test] - fn add_candidate_never_dedups_summary_candidates() { - // Documented, deliberate consequence of `is_duplicate_summary` - // refusing value equality on `f64`-bearing summaries: re-proposing - // the same `Replacement::Summary` candidates DOES grow the group — - // this module refuses to guess at an equality check it can't back - // with a real `PartialEq`. `search_workload_with` never actually - // does this in practice (every target is asked exactly once — see the - // module docs' "Termination" section), so this test exists to pin - // the documented behavior, not to endorse calling `replacements` - // twice for the same target. - let root = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let mut group = TargetSubDAGCandidates::new(Rc::clone(&root), 1); - let strategy = ASAPStrategies::default(); - let target = TargetSubDAG::new(&root); - for candidate in strategy.replacements(&target) { - group.add_candidate(candidate); - } - assert_eq!(group.candidates.len(), 2); - - for candidate in strategy.replacements(&target) { - group.add_candidate(candidate); - } - assert_eq!( - group.candidates.len(), - 4, - "Summary candidates are never deduped by this module — see is_duplicate_summary" - ); - } - - #[test] - fn is_duplicate_rewrite_never_merges_share_with_recompute() { - let target = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let share = Rc::clone(&target); - let recompute = Rc::new((*target).clone()); - assert!(!Rc::ptr_eq(&share, &recompute)); - assert_eq!( - *share, *recompute, - "fixture sanity: same value, different Rc" - ); - assert!(!is_duplicate_rewrite(&share, &recompute, &target)); - assert!(!is_duplicate_rewrite(&recompute, &share, &target)); - } - - #[test] - fn is_duplicate_rewrite_catches_a_real_repeat() { - let target = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let first_recompute = Rc::new((*target).clone()); - let second_recompute = Rc::new((*target).clone()); - assert!(!Rc::ptr_eq(&first_recompute, &second_recompute)); - assert!(is_duplicate_rewrite( - &first_recompute, - &second_recompute, - &target - )); - } - - // ── termination ────────────────────────────────────────────────────── - - #[test] - fn default_strategies_converge_without_hitting_the_iteration_cap() { - // A workload exercising both strategies at once; if this test - // completes at all, the fixpoint converged well under - // MAX_SEARCH_ITERATIONS (both strategies are idempotent — see the - // module docs — so this always converges in exactly 2 passes). - let a = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let b = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let space = search_workload(vec![("a", a), ("b", b)]); - assert!(!space.is_empty()); - } - - /// A deliberately ill-behaved [`ReplacementStrategy`]: every call to - /// `replacements` wraps `target` in two `Filter` layers — the outer one - /// (ignored by target discovery — see [`discover_new_descendant_targets`]) - /// and an inner one carrying a monotonically-increasing counter, so the - /// inner layer is a **brand-new, never-before-seen `Rc` every call**. - /// Each round, `search_workload_with` discovers that inner layer as a - /// new target, processes it next round (this strategy matches - /// everything), and gets handed *another* fresh inner layer — the - /// frontier never empties, exactly the failure mode - /// [`MAX_SEARCH_ITERATIONS`] exists to catch. - struct AlwaysGrowingStrategy { - next: std::cell::Cell, - } - - impl ReplacementStrategy for AlwaysGrowingStrategy { - fn matches(&self, _target: &TargetSubDAG<'_>) -> bool { - true - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - let n = self.next.get(); - self.next.set(n + 1); - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::Predicate; - let fresh_inner_layer = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Int64(n))), - child: Rc::clone(target.root), - })) - .unwrap(); - let outer_wrapper = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), - child: fresh_inner_layer, - })) - .unwrap(); - vec![ReplacementSubDAG { - strategy: "AlwaysGrowingStrategy", - replacement: Replacement::SubDAG(outer_wrapper), - provenance: ReplacementProvenance::LogicalRewrite, - rationale: format!("pathological candidate #{n}"), - }] - } - } - - #[test] - #[should_panic(expected = "did not converge")] - fn a_pathologically_growing_strategy_trips_the_iteration_cap() { - let root = metric_scan(&["job"]); - let strategies: Vec> = vec![Box::new(AlwaysGrowingStrategy { - next: std::cell::Cell::new(0), - })]; - let _ = search_workload_with(vec![("q", root)], &strategies); - } - // ── realize_child / retain_exact: end-to-end single-target realization ── - // - // Moved from the former `bind.rs` (issue #251): `bind.rs`'s own - // workload-wide orchestration (`implement_workload`/ - // `implement_workload_with`) was deleted. Current whole-workload logical - // selection uses `candidate_selection::global_selection`; these tests exercise - // `construct_summary_agg`'s schema derivation end to end through - // `realize_child` — production logic that still lives in this module — - // so they move here rather than disappear. Unlike `bind.rs` (an - // external caller that had to reconstruct the rank-and-take-first - // pattern by hand since `realize_child` is `pub(crate)`), these tests - // call `realize_child` directly. - - fn field<'a>(schema: &'a Schema, name: &str) -> &'a Field { - schema - .fields - .iter() - .find(|f| f.name == name) - .unwrap_or_else(|| panic!("no field {name:?} in {schema:?}")) - } - - fn realize(expr: &OperatorNode) -> Result, RealizationError> { - realize_child(&Rc::new(expr.clone())) - } - - #[test] - fn quantile_realizes_kll_wrapped_in_estimate() { - // quantile by (job) (m) at ε=0.01 → Estimate(Quantile) over - // SummaryAgg(Kll{k:269}) over the kept Scan. job = col 2. - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - let root = realize(&q).unwrap(); - - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &root.operator - else { - panic!("expected SummaryEstimate root, got {:?}", root.operator); - }; - assert!(matches!(query, PostAsapSketchStatistic::Quantile { q } if *q == 0.99)); - // Estimate edge: plain row shape — group key + Float64 answer. - assert_eq!( - field(&root.schema, "quantile_0_99").dtype, - FieldDataType::Plain(DataType::Float64) - ); - assert_eq!( - field(&root.schema, "job").dtype, - FieldDataType::Plain(DataType::Utf8) - ); - - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - reduction, - .. - }) = &summary_input.operator - else { - panic!("expected SummaryAgg, got {:?}", summary_input.operator); - }; - assert_eq!( - family, - &FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }), - GroupingStrategy::default() - ) - ); - assert_eq!(input, &SummaryUpdate::column(ColumnRef::SampleValue)); - assert_eq!(reduction, &ReductionTy::by(vec![2])); - // SummaryAgg edge: the state column, named after its input, carries - // the committed family. - assert_eq!( - field(&summary_input.schema, "value").dtype, - FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 269 }), - GroupingStrategy::default() - ) - ); - // The kept pre-ASAP leaf is the Scan node itself (no wrapper). - assert!(matches!(child.non_asap(), Some(NonASAPOp::Scan { .. }))); - assert!(!child.contains_asap()); - } - - #[test] - fn extension_intent_stays_logical_by_default() { - // Core has no realization for a deployment-specific `Extension` - // intent, so it stays `PassThrough`. - let intent = AggIntent::Extension { - ext_kind: "frequency".to_string(), - payload: serde_json::json!({ "item": "checkout" }), - }; - let q = agg(vec![], intent, metric_scan(&[])); - let root = realize(&q).unwrap(); - assert!(!root.contains_asap()); - } - - #[test] - fn exact_sum_realizes_accumulator_without_estimate() { - let q = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let root = realize(&q).unwrap(); - let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &root.operator else { - panic!( - "expected bare SummaryAgg (no estimate), got {:?}", - root.operator - ); - }; - assert_eq!( - family, - &FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - assert_eq!( - field(&root.schema, "sum").dtype, - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - } - - #[test] - fn per_series_rate_keeps_labels_and_retypes_value() { - // rate(m[5m]) — per-series: every label survives; the sample value - // column becomes the Rate accumulator state. - use std::time::Duration; - let q = agg_per_entity( - AggIntent::Rate, - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeRange { - kind: TimeRangeKind::Range, - range: Duration::from_secs(300), - child: metric_scan(&["job"]), - })) - .unwrap(), - ); - let root = realize(&q).unwrap(); - let Operator::ASAP(ASAPOp::SummaryAgg { family, .. }) = &root.operator else { - panic!("expected SummaryAgg, got {:?}", root.operator); - }; - assert_eq!( - family, - &FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) - ); - assert_eq!( - root.schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect::>(), - vec!["ts", "value", "job"], - ); - assert_eq!( - field(&root.schema, "value").dtype, - FieldDataType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) - ); - assert_eq!(root.schema.time_index, Some(0)); - } - - /// Issue #163, case 1: a bare per-series range function (e.g. - /// `quantile_over_time(...)`) realizes to `SummaryAgg { reduction: - /// PerEntity, .. }` — proving the pre-ASAP `Reduction` this crate - /// already computes (issue #165) is carried onto the post-ASAP node - /// verbatim, not flattened back into an ambiguous bare `Vec`. - #[test] - fn bare_per_series_aggregate_realizes_summary_agg_with_per_entity_reduction() { - use std::time::Duration; - let q = agg_per_entity( - default_quantile(0.99), - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::TimeRange { - kind: TimeRangeKind::Range, - range: Duration::from_secs(10), - child: metric_scan(&["job"]), - })) - .unwrap(), - ); - let root = realize(&q).unwrap(); - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { - panic!("expected estimate root, got {:?}", root.operator); - }; - let Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) = &summary_input.operator else { - panic!("expected SummaryAgg, got {:?}", summary_input.operator); - }; - assert_eq!(reduction, &ReductionTy::PerEntity); - } - - /// Issue #163, case 2: an aggregation operator explicitly invoked with - /// no grouping keys realizes to `SummaryAgg { - /// reduction: Reduce(vec![]), .. }` — byte-identical `by: []` to the - /// previous test at the old `Vec` shape; `reduction` is what - /// tells them apart now. - #[test] - fn explicit_empty_by_aggregate_realizes_summary_agg_with_reduce_reduction() { - let intent = AggIntent::Cardinality { - cols: vec![], - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let q = agg(vec![], intent, metric_scan(&["job"])); - let root = realize(&q).unwrap(); - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { - panic!("expected estimate root, got {:?}", root.operator); - }; - let Operator::ASAP(ASAPOp::SummaryAgg { reduction, .. }) = &summary_input.operator else { - panic!("expected SummaryAgg, got {:?}", summary_input.operator); - }; - assert_eq!(reduction, &ReductionTy::by(vec![])); - } - - #[test] - fn nested_aggregates_realize_per_node() { - // quantile(0.9, sum by (job) (m)) — the realization decision - // fires per node over the nested DAG: KLL over an exact Sum - // accumulator. - let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let outer = agg(vec![], default_quantile(0.9), inner); - // Timing is not stored during realization: time the candidate under - // a maintained materialization assignment to read the maintenance boundary. - let root = maintained(&realize(&outer).unwrap()); - - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { - panic!("expected estimate root, got {:?}", root.operator); - }; - let Operator::ASAP(ASAPOp::SummaryAgg { child, family, .. }) = &summary_input.operator - else { - panic!( - "expected outer SummaryAgg, got {:?}", - summary_input.operator - ); - }; - assert!(matches!( - family, - FieldDataType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::Kll - )); - assert_eq!(child.timing, Some(ExecutionTiming::IngestionTime)); - let Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child }) = &child.operator else { - panic!("expected explicit maintenance evaluation"); - }; - let Operator::ASAP(ASAPOp::SummaryAgg { - family: inner_family, - child: leaf, - .. - }) = &child.operator - else { - panic!("expected inner SummaryAgg, got {:?}", child.operator); - }; - assert_eq!( - inner_family, - &FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) - ); - assert!(!leaf.contains_asap()); - } - - /// Issue #115: the summary is built over the intent's own input columns. - /// Before `Cardinality`/`Quantile` carried them, `summarised_input` always - /// fell through to `ColumnRef::SampleValue`, so an HLL was built over the - /// wrong column for every SQL `COUNT(DISTINCT c)`. A distinct-tuple count - /// hashes the whole tuple as one item — feeding the sketch a single leg - /// would report single-column cardinality instead. - #[test] - fn sketch_realizes_over_the_intents_input_columns() { - // `metric_scan(&["job"])` → columns [ts=0, value=1, job=2]. - let column = |name: &str| SummaryInputExpr::Column(ColumnRef::Named(name.into())); - let cases = [ - (vec![2], column("job")), - (vec![1], column("value")), - // PromQL convention: no column ⇒ the synthetic sample value. - (vec![], SummaryInputExpr::Column(ColumnRef::SampleValue)), - ( - vec![1, 2], - SummaryInputExpr::Tuple(vec![column("value"), column("job")]), - ), - ]; - for (cols, want) in cases { - let intent = AggIntent::Cardinality { - cols: cols.clone(), - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let root = realize(&agg(vec![0], intent, metric_scan(&["job"]))).unwrap(); - let bound = find_summary_input(&root) - .unwrap_or_else(|| panic!("expected a SummaryAgg for cols={cols:?}")); - assert_eq!(bound, want, "wrong summarised input for cols={cols:?}"); - } - } - - /// The update expression of the first `SummaryAgg` in the tree. - fn find_summary_input(node: &OperatorNode) -> Option { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) if input.item.is_none() => { - Some(input.weight.clone()) - } - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - find_summary_input(summary_input) - } - _ => None, - } - } - - #[test] - fn pass_through_intents_stay_logical() { - // avg is exact but non-mergeable; histogram_quantile (classic - // buckets, #79) is never sketchable; exact quantile is exact by - // decree. All three stay whole logical sub-DAGs. - for intent in [ - AggIntent::Avg { col: None }, - AggIntent::HistogramQuantile { q: 0.99, le: 0 }, - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Exact, - }, - ] { - let q = agg(vec![2], intent.clone(), metric_scan(&["job"])); - let root = realize(&q).unwrap(); - // Kept pass-through: the pre-ASAP node itself, not a wrapper. - assert!( - !root.contains_asap() && root.operator == q.operator && root.schema == q.schema, - "expected kept pre-ASAP passthrough for {intent:?}" - ); - } - } - - #[test] - fn logical_parent_subsumes_bindable_child() { - // Filter over a bindable quantile: a kept non-ASAP sub-DAG has no - // summary children, so the conservative fallback keeps the whole sub-DAG - // logical. - use asap_types::ir::scalar::{CompareOpKind, ScalarValue}; - use asap_types::ir::Predicate; - let q = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Filter { - pred: Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(0)), - op: CompareOpKind::Gt, - right: Box::new(ScalarExpr::Literal(ScalarValue::Float64(0.5))), - semantics: asap_types::ir::ExprSemantics::Promql, - }), - child: agg(vec![], default_quantile(0.99), metric_scan(&[])), - })) - .unwrap(); - let root = realize(&q).unwrap(); - assert!( - !root.contains_asap() && root.operator == q.operator && root.schema == q.schema, - "expected the whole Filter sub_dag kept pre-ASAP" - ); - } - - #[test] - fn having_and_multi_intent_stay_logical() { - use asap_types::ir::scalar::ScalarValue; - use asap_types::ir::Predicate; - let q = crate::test_support::aggregate( - ReductionTy::by(vec![2]), - vec![default_quantile(0.99)], - vec![], - Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true)))), - metric_scan(&["job"]), - ); - assert!(!realize(&q).unwrap().contains_asap()); - - let multi = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: ReductionTy::by(vec![2]), - measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: metric_scan(&["job"]), - })) - .unwrap(); - assert!(!realize(&multi).unwrap().contains_asap()); - } - - // No binding rule applies a per-measure `FILTER` (#466), so the - // aggregate is retained exactly rather than bound to a summary. - #[test] - fn filtered_measure_stays_logical() { - use asap_types::ir::scalar::ScalarValue; - let mut q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - if let Operator::NonASAP(NonASAPOp::Aggregate { filters, .. }) = - &mut Rc::make_mut(&mut q).operator - { - *filters = vec![Some(Predicate(ScalarExpr::Literal(ScalarValue::Boolean( - true, - ))))]; - } - assert!(bindable_intent(&q).is_none()); - assert!(!realize(&q).unwrap().contains_asap()); - } - - #[test] - fn unsupported_direct_topk_without_margin_evidence_falls_back_to_pre_asap() { - let q = agg( - vec![2], - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["job"]), - ); - let root = realize(&q).unwrap(); - assert!(!root.contains_asap()); - } - - #[test] - fn count_ranked_topk_without_margin_evidence_keeps_an_uncertified_candidate() { - let inner = agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["job"]), - ); - let root = agg( - vec![], - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - inner, - ); - let proposals = ASAPStrategies::default().replacements(&TargetSubDAG::new(&root)); - assert!(!proposals.is_empty()); - assert!(proposals.iter().any(|candidate| matches!( - &candidate.replacement, - Replacement::SubDAG(node) if node.guarantee.as_ref().is_some_and(|g| - g.bound.evaluate().is_none() - && g.failure_probability.evaluate().is_none()) - ))); - } - - struct SeparatedTopKEvidence; - - impl AccuracyEvidenceProvider for SeparatedTopKEvidence { - fn propagation_stats( - &self, - op: &CompositionOperator, - _family: &FieldDataType, - _query: Option<&PostAsapSketchStatistic>, - ) -> PropagationStats { - if matches!(op, CompositionOperator::TopKSelection) { - PropagationStats { - topk_selected_lower_bound: Some(101.0), - topk_excluded_upper_bound: Some(100.0), - topk_interval_failure_probability: Some(0.005), - ..Default::default() - } - } else { - PropagationStats::default() - } - } - } - - #[test] - fn topk_margin_evidence_is_consumed_by_candidate_construction() { - let inner = agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }, - metric_scan(&["job"]), - ); - let q = agg( - vec![], - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }, - inner, - ); - let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &SeparatedTopKEvidence, - ); - let replacements = strategy.replacements(&TargetSubDAG::new(&q)); - assert!(!replacements.is_empty()); - assert!(replacements.iter().all(|candidate| matches!( - &candidate.replacement, - Replacement::SubDAG(node) - if node.guarantee.as_ref().is_some_and(|g| - g.metric == ErrorMetric::TopKMembership - && g.failure_probability.evaluate() == Some(0.005)) - ))); - } - - #[test] - fn count_ranked_topk_fuses_to_one_global_heap_sketch() { - let inner = agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["service"]), - ); - let outer = agg( - vec![], - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - inner, - ); - let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &SeparatedTopKEvidence, - ); - let candidates = strategy.replacements(&TargetSubDAG::new(&outer)); - let node = candidates - .iter() - .find_map(|candidate| match &candidate.replacement { - Replacement::SubDAG(node) if candidate.rationale.contains("CmsWithHeap") => { - Some(node) - } - _ => None, - }) - .expect("CmsWithHeap candidate"); - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &node.operator - else { - panic!("expected Top-K evaluation") - }; - assert!(matches!(query, PostAsapSketchStatistic::TopK { k: 10 })); - let Operator::ASAP(ASAPOp::SummaryAgg { - child, - family, - input, - .. - }) = &summary_input.operator - else { - panic!("expected fused summary aggregation") - }; - assert!(matches!( - input.item.as_ref(), - Some(SummaryInputExpr::Column(ColumnRef::Named(name))) if name == "service" - )); - assert_eq!(input.weight, SummaryInputExpr::Constant(1.0)); - assert_eq!( - input.weight_domain, - WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - } - ); - assert!(matches!( - family, - FieldDataType::Sketch(kind, _) - if kind.algorithm() == &SketchAlgorithm::CmsWithHeap - )); - assert!(!child.contains_asap()); - } - - #[test] - fn sum_ranked_topk_fuses_to_one_value_weighted_heap_sketch() { - let inner = agg( - vec![2], - AggIntent::Sum { col: None }, - metric_scan(&["service"]), - ); - let outer = agg( - vec![], - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - inner, - ); - let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &SeparatedTopKEvidence, - ); - let candidates = strategy.replacements(&TargetSubDAG::new(&outer)); - assert!( - candidates.iter().all(|candidate| { - !candidate.rationale.contains("CmsWithHeap") - || candidate.rationale.contains("CountSketchWithHeap") - }), - "generic weighted Sum has no non-negative proof and must reject CMS" - ); - let node = candidates - .iter() - .find_map(|candidate| match &candidate.replacement { - Replacement::SubDAG(node) - if candidate.rationale.contains("CountSketchWithHeap") => - { - Some(node) - } - _ => None, - }) - .expect("CountSketchWithHeap candidate"); - let Operator::ASAP(ASAPOp::SummaryEstimate { - summary_input, - query, - }) = &node.operator - else { - panic!("expected Top-K evaluation") - }; - assert!(matches!(query, PostAsapSketchStatistic::TopK { k: 5 })); - let Operator::ASAP(ASAPOp::SummaryAgg { child, input, .. }) = &summary_input.operator - else { - panic!("expected fused summary aggregation") - }; - assert!(matches!( - input.item.as_ref(), - Some(SummaryInputExpr::Column(ColumnRef::Named(name))) if name == "service" - )); - assert_eq!( - input.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ); - assert!(!child.contains_asap()); - } - - #[test] - fn temporal_per_entity_topk_uses_series_identity_and_sample_value() { - let inner = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: ReductionTy::PerEntity, - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::TimeRange { - kind: TimeRangeKind::Range, - range: std::time::Duration::from_secs(60), - child: metric_scan(&["service"]), - }, - )) - .unwrap(), - })) - .unwrap(); - let outer = agg( - vec![2], - AggIntent::TopK { - k: 5, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - inner, - ); - let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &SeparatedTopKEvidence, - ); - let candidates = strategy.replacements(&TargetSubDAG::new(&outer)); - let input = candidates.iter().find_map(|candidate| { - let Replacement::SubDAG(node) = &candidate.replacement else { - return None; - }; - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &node.operator - else { - return None; - }; - let Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) = &summary_input.operator else { - return None; - }; - input.item.is_some().then_some(input) - }); - let input = input.expect("keyed summary input"); - assert_eq!( - input.item.as_ref(), - Some(&SummaryInputExpr::EntityIdentity( - EntityIdentity::PromqlLabelSet { - excluding: vec![ColumnRef::Named("service".into())] - } - )) - ); - assert_eq!( - input.weight, - SummaryInputExpr::Column(ColumnRef::SampleValue) - ); - } - - #[test] - fn multidimensional_ranked_item_is_a_structured_tuple() { - let inner = agg( - vec![2, 3], - AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["service", "region"]), - ); - let outer = agg( - vec![], - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - inner, - ); - let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &SeparatedTopKEvidence, - ); - - let update = strategy - .replacements(&TargetSubDAG::new(&outer)) - .into_iter() - .find_map(|candidate| match candidate.replacement { - Replacement::SubDAG(node) => match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - match &summary_input.operator { - Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) => Some(input.clone()), - _ => None, - } - } - _ => None, - }, - _ => None, - }) - .expect("tuple-keyed summary candidate"); - assert!(matches!( - update.item, - Some(SummaryInputExpr::Tuple(ref items)) - if items == &[ - SummaryInputExpr::Column(ColumnRef::Named("service".into())), - SummaryInputExpr::Column(ColumnRef::Named("region".into())), - ] - )); - assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); - } - - #[test] - fn subpopulation_columns_are_not_part_of_the_ranked_item_tuple() { - let inner = agg( - vec![2, 3, 4], - AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["service", "method", "region"]), - ); - // The inner aggregate outputs its grouping keys first, so column 2 is - // `region`. Each region is a separate Top-K subpopulation. - let outer = agg( - vec![2], - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - inner, - ); - let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &SeparatedTopKEvidence, - ); - let (update, reduction) = strategy - .replacements(&TargetSubDAG::new(&outer)) - .into_iter() - .find_map(|candidate| match candidate.replacement { - Replacement::SubDAG(node) => match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - match &summary_input.operator { - Operator::ASAP(ASAPOp::SummaryAgg { - input, reduction, .. - }) => Some((input.clone(), reduction.clone())), - _ => None, - } - } - _ => None, - }, - _ => None, - }) - .expect("subpopulation-aware summary candidate"); - assert_eq!(reduction, ReductionTy::by(vec![2])); - assert!(matches!( - update.item, - Some(SummaryInputExpr::Tuple(ref items)) - if items == &[ - SummaryInputExpr::Column(ColumnRef::Named("service".into())), - SummaryInputExpr::Column(ColumnRef::Named("method".into())), - ] - )); - assert_eq!(update.weight, SummaryInputExpr::Constant(1.0)); - } - - #[test] - fn sql_reducer_resolves_named_input_column() { - // SUM(bytes) over a tabular scan: `col` resolves positionally to the - // named column, not the PromQL sample value. - let scan = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: SchemaTy { - fields: vec![ - Field::plain("host", DataType::Utf8, false), - Field::plain("bytes", DataType::Int64, false), - ], - time_index: None, - unique_keys: vec![], - closed: true, - }, - })) - .unwrap(); - let q = agg(vec![0], AggIntent::Sum { col: Some(1) }, scan); - let root = realize(&q).unwrap(); - let Operator::ASAP(ASAPOp::SummaryAgg { input, .. }) = &root.operator else { - panic!("expected SummaryAgg, got {:?}", root.operator); - }; - let SummaryInputExpr::Column(col) = &input.weight else { - panic!("expected observation column") - }; - assert_eq!(col, &ColumnRef::Named("bytes".into())); - } - - // ── Accuracy guarantees and fail-closed composition (issue #172) ───── - - use asap_types::ir::properties::ErrorMetric; - - /// A test-only `AccuracyModel` that *registers* a rule the default - /// deliberately lacks — a sketch over rank-bounded inputs composes - /// additively, keeping the outer sketch's own metric — so the - /// composition/allocation machinery can be exercised end to end. - /// Everything else delegates to `DefaultAccuracyModel`. - struct RankAdditiveModel; - - impl AccuracyModel for RankAdditiveModel { - fn local_guarantee( - &self, - family: &FieldDataType, - query: &PostAsapSketchStatistic, - ) -> Option { - DefaultAccuracyModel.local_guarantee(family, query) - } - - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - stats: &PropagationStats, - ) -> Result { - let rank = |g: &ResultGuarantee| g.is_exact() || g.metric == ErrorMetric::Rank; - if let (CompositionOperator::ApproximateAggregate, true, Some(local)) = - (op, inputs.iter().all(rank), local) - { - let relabel = |g: &ResultGuarantee| ResultGuarantee { - metric: ErrorMetric::AbsoluteValue, - ..g.clone() - }; - let inputs: Vec<_> = inputs.iter().map(relabel).collect(); - let mut out = - DefaultAccuracyModel.propagate(op, &inputs, Some(&relabel(local)), stats)?; - out.metric = local.metric; - return Ok(out); - } - DefaultAccuracyModel.propagate(op, inputs, local, stats) - } - - fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { - DefaultAccuracyModel.satisfies(guarantee, target) - } - } - - fn quantile_eps(q: f64, eps: f64) -> AggIntent { - AggIntent::Quantile { - col: None, - q, - accuracy: AccuracyTarget::Epsilon(eps), - } - } - - fn summary_child(node: &OperatorNode) -> &Rc { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - summary_child(summary_input) - } - Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) => child, - other => panic!("expected a SummaryAgg, got {other:?}"), - } - } - - #[test] - fn approximate_over_approximate_is_rejected_by_default_not_treated_as_exact() { - // quantile(0.99, quantile by (job) (0.5, m)): rank over rank — no - // registered rule, so every outer sketch candidate is refused with a - // typed reason and the raw/pre-ASAP alternative is what remains. - let inner = agg(vec![2], default_quantile(0.5), metric_scan(&["job"])); - let outer = agg(vec![], default_quantile(0.99), inner); - let proposals = ASAPStrategies::default().propose(&TargetSubDAG::new(&outer)); - assert!( - proposals.candidates.is_empty(), - "no outer sketch may be proposed over an approximate child without a rule: {:?}", - proposals.candidates - ); - // Every attempt — the as-declared composition and the equal-split - // re-sizing, for each of KLL/DDSketch — is refused for the same - // typed reason: no rule, whatever the budget. - assert_eq!(proposals.rejected.len(), 4, "{:?}", proposals.rejected); - for rejection in &proposals.rejected { - assert!( - matches!( - &rejection.error, - AccuracyError::UnsupportedComposition { input_metrics, .. } - if input_metrics == &vec![ErrorMetric::Rank] - ), - "{:?}", - rejection.error - ); - } - // Fallback keeps the whole sub-DAG pre-ASAP — executed exactly. - let realized = realize_child(&outer).unwrap(); - assert!(!realized.contains_asap()); - assert!(realized - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact)); - - // Cross-metric: a quantile over a cardinality estimate. - let inner = agg(vec![2], default_cardinality(), metric_scan(&["job"])); - let outer = agg(vec![], default_quantile(0.99), inner); - let proposals = ASAPStrategies::default().propose(&TargetSubDAG::new(&outer)); - assert!(proposals.candidates.is_empty()); - assert!(proposals.rejected.iter().all(|r| matches!( - &r.error, - AccuracyError::UnsupportedComposition { input_metrics, .. } - if input_metrics == &vec![ErrorMetric::Cardinality] - ))); - } - - #[test] - fn exact_child_contributes_zero_error() { - // quantile(0.9, sum by (job) (m)): KLL over an exact Sum accumulator - // — the evaluation's guarantee is exactly KLL's own local guarantee. - let inner = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["job"])); - let outer = agg(vec![], default_quantile(0.9), inner); - let root = realize(&outer).unwrap(); - let guarantee = root - .guarantee - .as_ref() - .expect("a evaluation carries a guarantee"); - assert_eq!(guarantee.metric, ErrorMetric::Rank); - assert_eq!( - guarantee.bound.evaluate(), - Some(crate::accuracy::estimators::kll::kll_rank_error_99(269)) - ); - assert_eq!(guarantee.approximate_layer_count(), 1); - assert!(guarantee.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::ChildGuarantee { guarantee, .. } if guarantee.is_exact() - ))); - assert!(guarantee.provenance.iter().any(|s| matches!( - s, - GuaranteeSource::CompositionStep { rule, .. } if rule == "exact_input" - ))); - // The sketch *state* node carries no guarantee; the exact - // accumulator's state is its value and does. - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &root.operator else { - panic!() - }; - assert!(summary_input.guarantee.is_none()); - assert!(summary_child(&root) - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact)); - } - - #[test] - fn exact_sum_can_consume_an_approximate_evaluation() { - // sum(count_distinct by (job) (m)) is an outer exact summary over - // the inner HLL evaluation. Both summary levels remain explicit. - let inner = agg(vec![2], default_cardinality(), metric_scan(&["job"])); - let outer = agg(vec![], AggIntent::Sum { col: None }, inner); - let root = realize(&outer).unwrap(); - let Operator::ASAP(ASAPOp::SummaryAgg { child, .. }) = &root.operator else { - panic!("outer exact sum should remain a SummaryAgg") - }; - assert!(matches!( - child.operator, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - )); - assert!(root.guarantee.is_some()); - - // count(...) over the same child is exact: a row count does not - // depend on the rows' values. - let inner = agg(vec![2], default_cardinality(), metric_scan(&["job"])); - let outer = agg( - vec![], - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - inner, - ); - let root = realize(&outer).unwrap(); - assert!(root - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact)); - } - - #[test] - fn equal_split_allocation_supports_nested_summary_evaluations() { - // A registered rank-additive rule and valid budget split make both - // summary levels explicit while preserving the composed guarantee. - let inner = agg(vec![2], quantile_eps(0.5, 0.1), metric_scan(&["job"])); - let outer = agg(vec![], quantile_eps(0.99, 0.1), inner); - let strategy = - ASAPStrategies::new_with_planning_inputs(&RankAdditiveModel, &EqualSplitAllocator); - let proposals = strategy.propose(&TargetSubDAG::new(&outer)); - - assert!(!proposals.candidates.is_empty()); - assert!(proposals.candidates.iter().all(|candidate| { - let Replacement::SubDAG(node) = &candidate.replacement else { - return false; - }; - matches!( - node.operator, - Operator::ASAP(ASAPOp::SummaryEstimate { .. }) - ) && node.guarantee.as_ref().is_some_and(|guarantee| { - DefaultAccuracyModel.satisfies(guarantee, &AccuracyTarget::Epsilon(0.1)) - }) - })); - } - - #[test] - fn root_target_check_removes_candidates_before_cost_ranking() { - let q = agg(vec![2], default_quantile(0.99), metric_scan(&["job"])); - // A root target tighter than the node's own ε=0.01: every sketch - // candidate misses it and is moved to `rejected`; nothing is left - // for the cost model to rank. - let space = search_workload_with_targets( - vec![("q", Rc::clone(&q), Some(AccuracyTarget::Epsilon(0.001)))], - &default_strategies(), - &DefaultAccuracyModel, - ); - let root = &space.roots[0].1; - let group = space.candidates_for_target(root).unwrap(); - assert!(group - .candidates - .iter() - .all(|c| matches!(&c.replacement, Replacement::SubDAG(n) if !n.contains_asap()))); - assert!(group.rejected.iter().all(|r| matches!( - r.error, - AccuracyError::TargetNotSatisfied { target: AccuracyTarget::Epsilon(e), .. } if e == 0.001 - ))); - assert!(group.rejected.len() >= 2); - - // A root target the node's own sizing meets keeps every candidate. - let space = search_workload_with_targets( - vec![("q", Rc::clone(&q), Some(AccuracyTarget::Epsilon(0.01)))], - &default_strategies(), - &DefaultAccuracyModel, - ); - let group = space.candidates_for_target(&space.roots[0].1).unwrap(); - assert!(group - .candidates - .iter() - .any(|c| matches!(&c.replacement, Replacement::SubDAG(n) if n.contains_asap()))); - - // An `Exact` root target admits only exact candidates. - let space = search_workload_with_targets( - vec![("q", Rc::clone(&q), Some(AccuracyTarget::Exact))], - &default_strategies(), - &DefaultAccuracyModel, - ); - let group = space.candidates_for_target(&space.roots[0].1).unwrap(); - assert!(group.candidates.iter().all(|c| match &c.replacement { - Replacement::SubDAG(node) if node.contains_asap() => node - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::is_exact), - Replacement::SubDAG(_) => true, - Replacement::ExactComposition(_) => false, - })); - } - - #[test] - fn topk_accuracy_target_keeps_uncertified_membership() { - let inner = agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["job"]), - ); - let q = agg( - vec![], - AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - inner, - ); - let space = search_workload_with_targets( - vec![("q", Rc::clone(&q), Some(AccuracyTarget::Epsilon(0.01)))], - &default_strategies(), - &DefaultAccuracyModel, - ); - let group = space.candidates_for_target(&space.roots[0].1).unwrap(); - - assert!(group.candidates.iter().any(|candidate| matches!( - &candidate.replacement, - Replacement::SubDAG(node) if node.guarantee.as_ref().is_some_and(ResultGuarantee::has_unknown) - ))); - let candidate = group - .candidates - .iter() - .find(|candidate| candidate.has_missing_accuracy_evidence()) - .unwrap(); - let Replacement::SubDAG(node) = &candidate.replacement else { - unreachable!() - }; - assert!(node - .guarantee - .as_ref() - .is_some_and(ResultGuarantee::has_unknown)); - } - // Source evidence alone must enable Planner-owned sizing and certification. - #[test] - fn scoped_hll_evidence_sizes_and_certifies_without_a_deployment_model() { - use crate::accuracy::EstimatorContract; - struct SourceEvidence { - expression: OperatorNode, - max_distinct: u32, - } - impl AccuracyEvidenceProvider for SourceEvidence { - fn estimator_contract(&self, expression: &OperatorNode) -> Option { - (expression == &self.expression).then_some(EstimatorContract::ClassicHll { - max_distinct_per_evaluation: self.max_distinct, - }) - } - } - let target = AccuracyTarget::EpsilonDelta { - epsilon: 0.05, - delta: 0.01, - }; - let root = agg( - vec![], - AggIntent::Cardinality { - cols: vec![], - accuracy: target.clone(), - }, - metric_scan(&[]), - ); - let evidence = SourceEvidence { - expression: (*root).clone(), - max_distinct: 128, - }; - let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &evidence, - ); - let candidates = strategy.replacements(&TargetSubDAG::new(&root)); - let hll = candidates - .iter() - .find_map(|candidate| match &candidate.replacement { - Replacement::SubDAG(node) - if summary_family_algorithm(node) == SketchAlgorithm::Hll => - { - Some(node) - } - _ => None, - }) - .expect("HLL candidate"); - assert!(DefaultAccuracyModel - .satisfies(hll.guarantee.as_ref().expect("HLL confidence"), &target)); - let Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) = &hll.operator else { - panic!("evaluation") - }; - let Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) = &summary_input.operator - else { - panic!("HLL state") - }; - let expected = crate::accuracy::estimators::hll::ClassicHllConfidence::new(128, 0.05) - .unwrap() - .precision(0.01) - .unwrap(); - assert_eq!( - kind.params(), - &SketchParams::Hll { - precision: expected - } - ); - let absent = ASAPStrategies::default().replacements(&TargetSubDAG::new(&root)); - assert!(!absent.iter().any(|candidate| matches!(&candidate.replacement, Replacement::SubDAG(node) - if summary_family_algorithm(node) == SketchAlgorithm::Hll && node.guarantee.as_ref().is_some_and(|g| DefaultAccuracyModel.satisfies(g, &target))))); - // Invalid contracts, infeasible targets and evidence for another source - // must never authorize a confidence-bearing HLL candidate. - for (max_distinct, delta, wrong_scope) in [ - (0, 0.01, false), - (4097, 0.01, false), - (128, 1e-12, false), - (128, 0.01, true), - ] { - let target = AccuracyTarget::EpsilonDelta { - epsilon: 0.05, - delta, - }; - let query = agg( - vec![], - AggIntent::Cardinality { - cols: vec![], - accuracy: target.clone(), - }, - metric_scan(&[]), - ); - let evidence = SourceEvidence { - expression: if wrong_scope { - (*metric_scan(&["other"])).clone() - } else { - (*query).clone() - }, - max_distinct, - }; - let strategy = ASAPStrategies::new_with_planning_inputs_and_evidence( - &DefaultAccuracyModel, - &EqualSplitAllocator, - &evidence, - ); - assert!(!strategy.replacements(&TargetSubDAG::new(&query)).iter().any(|candidate| - matches!(&candidate.replacement, Replacement::SubDAG(node) - if summary_family_algorithm(node) == SketchAlgorithm::Hll && node.guarantee.as_ref().is_some_and(|g| DefaultAccuracyModel.satisfies(g, &target))))); - } - } - - // A numeric group key must not be mistaken for the ranked aggregate score. - #[test] - fn ranking_uses_aggregate_output_position_not_first_numeric_column() { - let logical = agg(vec![2], AggIntent::Sum { col: None }, metric_scan(&["id"])); - let mut values = logical.schema.clone(); - values.fields[0].dtype = FieldDataType::Plain(DataType::Int64); - assert_eq!(ranking_score_index(&logical, &values).unwrap(), 1); - } - // A heap's key schema is derived from its encoded item, not all label columns. - #[test] - fn heap_evaluation_preserves_numeric_item_identity() { - let mut schema = metric_scan(&["id", "description"]).schema.clone(); - schema.fields[2].dtype = FieldDataType::Plain(DataType::Int64); - let raw = crate::test_support::scan("m", schema); - let node = agg( - vec![], - AggIntent::TopK { - k: 2, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - agg(vec![2], AggIntent::Sum { col: None }, raw.clone()), - ); - let input = PhysicalSummaryInput { - child: raw, - input: SummaryUpdate { - item: Some(SummaryInputExpr::Column(ColumnRef::Named("id".into()))), - weight: SummaryInputExpr::Constant(1.0), - weight_domain: WeightDomain::NonNegative { - proof: NonNegativeWeightProof::UnitCount, - }, - }, - }; - let schema = keyed_heap_evaluation_schema(&input, &node).unwrap(); - assert_eq!( - schema - .fields - .iter() - .map(|f| f.name.as_str()) - .collect::>(), - vec!["id", "__asap_estimate"] - ); - assert_eq!( - schema.fields[0].dtype, - FieldDataType::Plain(DataType::Int64) - ); - } - - // Every SummaryAgg a strategy proposes derives coverage of its whole - // source: an unrestricted selection over a definition reading that source. - #[test] - fn proposed_summary_states_cover_their_whole_source() { - let root = agg( - vec![], - AggIntent::Quantile { - q: 0.9, - col: None, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - metric_scan(&["job"]), - ); - let source = Source::TimeSeries { metric: "m".into() }; - let proposals = ASAPStrategies::default().propose(&TargetSubDAG::new(&root)); - let states: Vec<_> = proposals - .candidates - .iter() - .filter_map(|candidate| match &candidate.replacement { - Replacement::SubDAG(node) => Some(node), - _ => None, - }) - .flat_map(OperatorNode::reachable) - .filter(|node| matches!(node.asap(), Some(ASAPOp::SummaryAgg { .. }))) - .collect(); - assert!(!states.is_empty()); - for state in states { - let coverage = state.coverage().expect("summary state has coverage"); - assert_eq!(coverage.selection, [Default::default()]); - let reads = OperatorNode::reachable(&coverage.definition) - .into_iter() - .filter_map(|node| match node.non_asap() { - Some(NonASAPOp::Scan { source, .. }) => Some(source.clone()), - _ => None, - }) - .collect::>(); - assert_eq!(reads, std::slice::from_ref(&source)); - } - } - - fn equi_pred(left: ColumnId, right: ColumnId) -> Predicate { - Predicate(ScalarExpr::Compare { - left: Box::new(ScalarExpr::Column(left)), - op: asap_types::ir::scalar::CompareOpKind::Eq, - right: Box::new(ScalarExpr::Column(right)), - semantics: asap_types::ir::ExprSemantics::Sql, - }) - } - - #[test] - fn relational_join_predicate_requires_and_normalizes_cross_input_columns() { - let forward = normalize_cross_input_equi_predicate(&equi_pred(1, 3), 2, 4) - .expect("left-to-right equality"); - let reverse = normalize_cross_input_equi_predicate(&equi_pred(3, 1), 2, 4) - .expect("right-to-left equality"); - assert_eq!(forward, reverse, "reverse equality must be canonicalized"); - assert!(normalize_cross_input_equi_predicate(&equi_pred(0, 1), 2, 4).is_none()); - assert!(normalize_cross_input_equi_predicate(&equi_pred(0, 4), 2, 4).is_none()); - } - - // A value projection cannot consume an opaque exact accumulator edge. - #[test] - fn residual_projection_finalizes_selected_exact_state() { - let inner = agg(vec![], AggIntent::Sum { col: None }, metric_scan(&[])); - let root = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols: vec![ProjectItem { - expr: ScalarExpr::Column(0), - alias: Some("result".into()), - }], - qualifier: None, - child: inner.clone(), - })) - .unwrap(); - let space = search_workload_with_targets( - vec![("q", root.clone(), Some(AccuracyTarget::Exact))], - &default_strategies(), - &DefaultAccuracyModel, - ); - // No site has a chosen candidate, so the root is assembled as a residual. - let groups = space - .target_subdag_candidates() - .map(|group| { - ( - Rc::as_ptr(&group.target), - TargetSubDAGSelection { - target: &group.target, - consumer_count: group.consumer_count, - effective_consumer_count: group.consumer_count, - chosen: None, - }, - ) - }) - .collect(); - let selected = GlobalSelection::new(space.order.clone(), groups, HashMap::new()); - // CSE re-interns the workload, so the space's root/child `Rc`s are not - // the fixture's. Assembly only assembles children that are discovered - // targets, so seed the memo under the space's own child pointer. - let root = Rc::clone(&space.roots[0].1); - let Some(NonASAPOp::Project { child: inner, .. }) = root.non_asap() else { - unreachable!() - }; - assert!(space.candidates_for_target(inner).is_some()); - selected - .assembled_nodes - .borrow_mut() - .insert(Rc::as_ptr(inner), realize(inner.as_ref()).unwrap()); - let node = selected.assemble_target(&root).unwrap(); - let Operator::NonASAP(NonASAPOp::Project { child, .. }) = &node.operator else { - panic!("expected Project"); - }; - assert!(matches!( - child.operator, - Operator::ASAP(ASAPOp::FinalizeExactAccumulator { .. }) - )); - assert!(child - .schema - .fields - .iter() - .all(|field| matches!(field.dtype, FieldDataType::Plain(_)))); - } -} diff --git a/crates/logical-optimizer/src/pass1/rewrite.rs b/crates/logical-optimizer/src/pass1/rewrite.rs deleted file mode 100644 index d8a1afaa..00000000 --- a/crates/logical-optimizer/src/pass1/rewrite.rs +++ /dev/null @@ -1,926 +0,0 @@ -//! [`SemanticEquivalentRewriteStrategy`] — algebraic, semantic-equivalent -//! query rewriting, -//! the third bullet in `docs/design_docs/asap_aware_mapping.md`'s "Degrees of freedom" -//! section: "Semantic-equivalent rewriting (e.g. `avg` → `sum`/`count`) to -//! increase how often the [sharing/sketch] optimizations above apply" -//! (issue #253, part of #33, per Peilin's #33 comment). -//! -//! ## Why `avg` needs this and `sum`/`count` don't -//! -//! [`replacement::realizations_for_intent`] dispatches `AggIntent::Avg` -//! straight to `Realization::PassThrough` — see that module's own -//! comment on why: `Avg`/`StdDev`/`Variance` "need richer partial state" -//! than a bare sketch/exact accumulator gives, so there is no summary -//! realization for a bare `avg` node to bind to at all. A logical `avg` -//! node therefore can never be a [`SharedSubDAGStrategy`] target either: -//! CSE-style sharing needs *some* mergeable accumulator underneath, and -//! `PassThrough` has none. -//! -//! `Sum` and `Count` are both ordinary mergeable accumulators -//! (`agg_is_mergeable`) — exactly the shape [`SharedSubDAGStrategy`] and a -//! future sketch-family search already know how to reuse across a -//! workload. Rewriting `Aggregate{ measures: [Avg{col}], .. }` into two -//! independent single-measure `Sum` and `Count` aggregates, divided with a -//! `BinaryOp`, computes the same result but *reshapes* it into targets other -//! strategies can bind and share independently. This module only performs -//! that reshaping — see "Non-goals" below for why it does not also decide -//! whether the reshaping is worth it. -//! -//! ## Scope -//! -//! Ordinary `by(...)` averages use a schema-preserving projection. Temporal -//! Float64 temporal averages require a typed finite-division guard; their -//! sum/count components are never exported as an unconditional logical rewrite. -//! -//! - **`without(...)` grouping** leaves an `Aggregate`'s own output schema -//! *open* (`closed: false`, see `without_output_schema`), while the -//! `Project` this strategy always wraps the rewrite in forces -//! `closed: true` (see `NonASAPOp::output_schema`'s `Project` arm). Under -//! `without(...)` the rewritten form's `closed` flag would silently flip -//! relative to the original — exactly the kind of schema drift this -//! module exists to avoid. -//! -//! Both are follow-ups (issue #253 itself scopes to "the concrete case in -//! Peilin's comment"), not correctness bugs in what ships here — a node -//! outside this scope simply doesn't `match`, the same "safe but -//! uninformative" fallback [`ASAPStrategies`]/[`SharedSubDAGStrategy`] -//! already use for shapes they don't have an opinion on. -//! -//! ## Non-goals (mirrors [`replacement`]'s own discipline) -//! -//! **No "is it worth it" heuristic.** An earlier draft of this idea needed a -//! manual before/after-CSE cost comparison to decide whether rewriting -//! helps. With `ReplacementStrategy`'s exhaustive-candidate shape in place, -//! that's unnecessary: this strategy just reports the rewritten form as one -//! more [`ReplacementSubDAG`] alongside whatever else applies to the same -//! target. A future cost-based search (issue #252) is what decides whether -//! the rewritten form is actually worth picking, by letting the original -//! and rewritten forms compete on cost — not this strategy. - -use asap_types::ir::operator::non_asap::any_measure_filtered; -use std::rc::Rc; - -use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::{BinaryOpKind, Reduction}; -use asap_types::ir::scalar::ArithmeticOpKind; -use asap_types::ir::schema::{ColumnId, DataType}; -use asap_types::ir::{BinaryOperator, NonASAPOp, OperatorNode, ProjectItem, ScalarExpr}; - -use asap_types::types::AccuracyTarget; - -use crate::pass1::replacement::{ - Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, -}; - -/// The shape [`AvgToSumOverCountStrategy`] rewrites: a single `Avg{col}` -/// measure, no `HAVING`, grouped with an ordinary `by(...)` reduction (see -/// the module docs' "Scope" for why `without(...)`/`PerEntity` are -/// excluded). Returns the grouping key count and the summed column so -/// [`build_rewrite`] doesn't have to re-match. -/// `a / b` with PromQL arithmetic semantics and no vector matching. -fn arithmetic( - op: ArithmeticOpKind, - lhs: Rc, - rhs: Rc, -) -> Option> { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::BinaryOp { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(op), - vector_match: None, - }, - return_bool: false, - lhs, - rhs, - })) - .ok() -} - -fn avg_rewrite_target(node: &OperatorNode) -> Option<(usize, Option)> { - let Some(NonASAPOp::Aggregate { - reduction, - measures, - filters, - having: None, - child, - .. - }) = node.non_asap() - else { - return None; - }; - if any_measure_filtered(filters) { - return None; - } - let Reduction::Reduce(by) = reduction else { - return None; - }; - if by.is_without() { - return None; - } - let [AggIntent::Avg { col }] = measures.as_slice() else { - return None; - }; - // `AggIntent::Count` represents COUNT(*), not COUNT(col). AVG(col) can - // therefore be decomposed through it only when the averaged input is - // provably non-null; otherwise NULL rows would incorrectly contribute to - // the denominator. - let input_schema = &child.schema; - let value_col = col - .or_else(|| input_schema.column_id("value")) - .or_else(|| (0..input_schema.fields.len()).find(|i| !by.contains(i)))?; - if input_schema.fields.get(value_col)?.nullable { - return None; - } - Some((by.keys().len(), *col)) -} - -/// Build the rewritten `Project{ cast(sum) } / Aggregate{ Count }` DAG for -/// `root`, or `None` if `root` isn't [`avg_rewrite_target`]'s shape. `Sum` and -/// `Count` deliberately live in separate, single-measure aggregates so the -/// replacement fixpoint discovers each as an independently bindable target. -/// -/// The `Project`'s leading `by.len()` items are bare `Column(i)` -/// pass-throughs of the grouping keys — identical in name/type to the -/// original `Avg` aggregate's own leading columns, since both aggregates -/// share the same `reduction`/`child` and only differ in `measures` -/// (`aggregate_output_schema`'s grouping-column derivation never looks at -/// `measures` at all). The final item recomputes `sum / count`, casting the -/// numerator to `Float64` before division, and aliases it to the original -/// `avg` column's own name — matching -/// [`AggIntent::Avg::output_column`]'s `(name, Float64, nullable: false)` -/// exactly regardless of the summed column's own type (integer division -/// would otherwise silently reappear whenever the input column is itself -/// integer-typed: `Sum`'s output type tracks its input, `Count`'s is always -/// `Int64`, and `ScalarExpr::scalar_type`'s own `Arithmetic` type inference -/// types a `Div` of two `Int64` operands as `Int64` — the explicit operand -/// `Cast` is what keeps both the division and rewritten `avg` column -/// `Float64` the way the original always was, not an incidental extra step). -// These are conditional physical components, never an unconditional Rewrite. -// The caller must attach the finite-division execution guard before admission. -pub(crate) fn temporal_average_components(root: &Rc) -> Option> { - let Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures, - filters, - child, - having: None, - .. - }) = root.non_asap() - else { - return None; - }; - if any_measure_filtered(filters) { - return None; - } - let [AggIntent::Avg { col }] = measures.as_slice() else { - return None; - }; - if !matches!(child.non_asap(), Some(NonASAPOp::TimeRange { .. })) { - return None; - } - let schema = &child.schema; - let value = schema - .fields - .get(col.or_else(|| schema.column_id("value"))?)?; - if value.nullable || value.dtype != DataType::Float64 { - return None; - } - let aggregate = |intent| { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(child), - })) - .ok() - }; - let rewritten = arithmetic( - ArithmeticOpKind::Div, - aggregate(AggIntent::Sum { col: *col })?, - aggregate(AggIntent::Count { - accuracy: AccuracyTarget::Exact, - })?, - )?; - (root.schema == rewritten.schema).then_some(rewritten) -} - -fn build_rewrite(root: &Rc) -> Option> { - let (group_count, col) = avg_rewrite_target(root)?; - let Some(NonASAPOp::Aggregate { - reduction, - output_names, - child, - .. - }) = root.non_asap() - else { - unreachable!("avg_rewrite_target already confirmed an Aggregate shape"); - }; - - // The original `avg` column's own name: `output_names[0]` if the - // producing front end overrode it (SQL threading DataFusion's own - // generated name — see `NonASAPOp::Aggregate::output_names`'s docs), - // else `AggIntent::Avg`'s synthetic default. Either way this is the - // *only* thing about the original output column this rewrite needs to - // reproduce — `AggIntent::Avg::output_column`'s `(Float64, nullable: - // false)` half is already reproduced structurally (see `build_rewrite`'s - // own doc comment) rather than looked up here. - let avg_name = output_names - .first() - .filter(|s| !s.is_empty()) - .cloned() - .unwrap_or_else(|| "avg".to_string()); - - let sum_agg = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: reduction.clone(), - measures: vec![AggIntent::Sum { col }], - output_names: Vec::new(), - filters: vec![], - having: None, - child: Rc::clone(child), - })) - .ok()?; - let count_agg = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: reduction.clone(), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }], - output_names: Vec::new(), - filters: vec![], - having: None, - child: Rc::clone(child), - })) - .ok()?; - - let sum_idx = group_count; - let mut cols: Vec = (0..group_count) - .map(|i| ProjectItem { - alias: None, - expr: ScalarExpr::Column(i), - }) - .collect(); - cols.push(ProjectItem { - alias: Some(avg_name), - expr: ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Column(sum_idx)), - to: DataType::Float64, - try_cast: false, - }, - }); - - let float_sum = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: sum_agg, - })) - .ok()?; - arithmetic(ArithmeticOpKind::Div, float_sum, count_agg) -} - -/// Compose adjacent per-entity and cross-entity accumulators when their -/// algebra, rather than a query-language spelling, proves equivalence. -pub(crate) fn composed_aggregate_rewrite(root: &Rc) -> Option> { - let original_schema = &root.schema; - let Some(NonASAPOp::Aggregate { - reduction: outer_reduction @ Reduction::Reduce(_), - measures: outer_measures, - output_names, - filters: outer_filters, - having: None, - child, - }) = root.non_asap() - else { - return None; - }; - let Some(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: inner_measures, - filters: inner_filters, - having: None, - child: inner_child, - .. - }) = child.non_asap() - else { - return None; - }; - if any_measure_filtered(outer_filters) || any_measure_filtered(inner_filters) { - return None; - } - let ([outer], [inner]) = (outer_measures.as_slice(), inner_measures.as_slice()) else { - return None; - }; - let composed = match (outer, inner) { - (AggIntent::Sum { col: None }, AggIntent::Sum { .. }) - | (AggIntent::Sum { col: None }, AggIntent::Count { .. }) - | (AggIntent::Min { col: None }, AggIntent::Min { .. }) - | (AggIntent::Max { col: None }, AggIntent::Max { .. }) => inner.clone(), - _ => return None, - }; - let aggregate = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: outer_reduction.clone(), - measures: vec![composed], - output_names: output_names.clone(), - filters: vec![], - having: None, - child: Rc::clone(inner_child), - })) - .ok()?; - - // The outer Sum sees PromQL's Float64 sample value, whereas the composed - // Count accumulator is Int64. Keep the original observable type. - let rewritten = if matches!( - (outer, inner), - (AggIntent::Sum { col: None }, AggIntent::Count { .. }) - ) { - let Reduction::Reduce(by) = outer_reduction else { - unreachable!() - }; - if by.is_without() { - return None; - } - let mut cols: Vec = (0..by.keys().len()) - .map(|i| ProjectItem { - alias: None, - expr: ScalarExpr::Column(i), - }) - .collect(); - cols.push(ProjectItem { - // PromQL deliberately supplies an empty output-name override. Use - // the already-derived caller-visible name instead of allowing - // Project to invent `col_N` and then failing schema equality. - alias: original_schema - .fields - .last() - .map(|column| column.name.clone()), - expr: ScalarExpr::Cast { - expr: Box::new(ScalarExpr::Column(by.keys().len())), - to: DataType::Float64, - try_cast: false, - }, - }); - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Project { - cols, - qualifier: None, - child: aggregate, - })) - .ok()? - } else { - aggregate - }; - - // Positional keys, aliases, types, and nullability are part of the rule's - // contract. A future schema change therefore disables rather than widens - // the rewrite. - (*original_schema == rewritten.schema).then_some(rewritten) -} - -/// Rewrites `Aggregate{ measures: [Avg{col}], .. }` into the semantically -/// equivalent pair of single-measure `Sum` and `Count` aggregates divided by -/// a `BinaryOp` — see the module docs for why keeping the accumulators in -/// separate relational nodes lets later strategies reach them independently. -/// -/// A unit struct: unlike [`ASAPStrategies`], this strategy doesn't -/// bind anything (its one [`Replacement`] is always [`Replacement::Rewrite`], -/// never [`Replacement::Summary`]) and so has no `CostModel` -/// to hold a reference to — the same "no state needed" shape -/// [`SharedSubDAGStrategy`] already has. -#[derive(Debug, Default, Clone, Copy)] -pub struct SemanticEquivalentRewriteStrategy; - -/// Backward-compatible name for callers that registered the original, narrower -/// average rewrite. It now denotes the same semantic-rewrite strategy. -pub use SemanticEquivalentRewriteStrategy as AvgToSumOverCountStrategy; - -impl ReplacementStrategy for SemanticEquivalentRewriteStrategy { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - avg_rewrite_target(target.root).is_some() - || composed_aggregate_rewrite(target.root).is_some() - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - if let Some(rewritten) = composed_aggregate_rewrite(target.root) { - return vec![ReplacementSubDAG { - strategy: "SemanticEquivalentRewriteStrategy", - replacement: Replacement::SubDAG(rewritten), - provenance: crate::pass1::replacement::ReplacementProvenance::LogicalRewrite, - rationale: "compose compatible per-entity and cross-entity accumulators using their algebraic intent while preserving the original output schema".into(), - }]; - } - let Some(rewritten) = build_rewrite(target.root) else { - return Vec::new(); - }; - vec![ReplacementSubDAG { - strategy: "AvgToSumOverCountStrategy", - replacement: Replacement::SubDAG(rewritten), - provenance: crate::pass1::replacement::ReplacementProvenance::LogicalRewrite, - rationale: - "avg has no summary realization at all (replacement::realizations_for_intent \ - dispatches it to PassThrough) and so can never share or sketch; \ - rewriting it into sum/count under the same grouping — re-divided back \ - into the original avg column by a wrapping Project — computes the same \ - result from two ordinary mergeable accumulators SharedSubDAGStrategy \ - (and a future sketch-family search) can actually reuse across the \ - workload" - .to_string(), - }] - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::lower_promql; - use asap_types::ir::operator::operator_properties::Source; - use asap_types::ir::schema::{Field, Schema}; - use asap_types::types::AccuracyTarget; - use std::time::Duration; - - use crate::test_support::metric_scan; - use asap_types::ir::TimeRangeKind; - - fn avg_agg( - by: Vec, - col: Option, - child: Rc, - ) -> Rc { - avg_agg_with(by, col, vec![], None, child) - } - - fn avg_agg_with( - by: Vec, - col: Option, - output_names: Vec, - having: Option, - child: Rc, - ) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![AggIntent::Avg { col }], - output_names, - filters: vec![], - having, - child, - })) - .unwrap() - } - - // Temporal averages expose two single-measure children without closing labels. - #[test] - fn temporal_average_components_preserves_schema_and_exposes_sum_count() { - let root = lower_promql("avg_over_time(a{job=\"api\"}[5m])", AccuracyTarget::Exact); - assert!(SemanticEquivalentRewriteStrategy - .replacements(&TargetSubDAG::new(&root)) - .is_empty()); - let rewritten = - temporal_average_components(&root).expect("conditional sum/count components"); - assert_eq!(root.schema.clone(), rewritten.schema.clone()); - assert!(matches!( - rewritten.non_asap(), - Some(NonASAPOp::BinaryOp { .. }) - )); - } - - // ── matches ────────────────────────────────────────────────────────── - - #[test] - fn matches_a_bare_avg_aggregate() { - let q = avg_agg(vec![], None, metric_scan(&[])); - let target = TargetSubDAG::new(&q); - assert!(AvgToSumOverCountStrategy.matches(&target)); - } - - #[test] - fn matches_a_grouped_avg_aggregate() { - let q = avg_agg(vec![2], None, metric_scan(&["job"])); - let target = TargetSubDAG::new(&q); - assert!(AvgToSumOverCountStrategy.matches(&target)); - } - - #[test] - fn does_not_match_a_multi_measure_aggregate() { - let q = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Sum { col: None }, AggIntent::Avg { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: metric_scan(&["job"]), - })) - .unwrap(); - let target = TargetSubDAG::new(&q); - assert!(!AvgToSumOverCountStrategy.matches(&target)); - assert!(AvgToSumOverCountStrategy.replacements(&target).is_empty()); - } - - #[test] - fn does_not_match_a_having_bearing_avg_aggregate() { - let q = avg_agg_with( - vec![2], - None, - vec![], - Some(asap_types::ir::Predicate(ScalarExpr::Literal( - asap_types::ir::scalar::ScalarValue::Boolean(true), - ))), - metric_scan(&["job"]), - ); - let target = TargetSubDAG::new(&q); - assert!(!AvgToSumOverCountStrategy.matches(&target)); - assert!(AvgToSumOverCountStrategy.replacements(&target).is_empty()); - } - - #[test] - fn does_not_match_a_non_avg_intent() { - for intent in [ - AggIntent::Sum { col: None }, - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - AggIntent::Min { col: None }, - ] { - let q = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![intent.clone()], - output_names: vec![], - filters: vec![], - having: None, - child: metric_scan(&["job"]), - })) - .unwrap(); - let target = TargetSubDAG::new(&q); - assert!( - !AvgToSumOverCountStrategy.matches(&target), - "expected no match for {intent:?}" - ); - assert!(AvgToSumOverCountStrategy.replacements(&target).is_empty()); - } - } - - #[test] - fn does_not_match_a_without_grouped_avg_aggregate() { - let q = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::Reduce( - asap_types::ir::operator::operator_properties::GroupKeys::without(vec![2]), - ), - measures: vec![AggIntent::Avg { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: metric_scan(&["job"]), - })) - .unwrap(); - let target = TargetSubDAG::new(&q); - assert!(!AvgToSumOverCountStrategy.matches(&target)); - assert!(AvgToSumOverCountStrategy.replacements(&target).is_empty()); - } - - #[test] - fn does_not_match_a_per_entity_avg_aggregate() { - let q = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Avg { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: metric_scan(&[]), - })) - .unwrap(); - let target = TargetSubDAG::new(&q); - assert!(!AvgToSumOverCountStrategy.matches(&target)); - assert!(AvgToSumOverCountStrategy.replacements(&target).is_empty()); - } - - #[test] - fn does_not_match_a_non_aggregate_node() { - let scan = metric_scan(&["job"]); - let target = TargetSubDAG::new(&scan); - assert!(!AvgToSumOverCountStrategy.matches(&target)); - assert!(AvgToSumOverCountStrategy.replacements(&target).is_empty()); - } - - // ── replacements / schema round-trip ───────────────────────────────── - - /// The rewritten DAG must keep `Sum` and `Count` in separate aggregates, - /// and its `output_schema()` must equal the original `Avg` aggregate's. - #[test] - fn avg_rewrites_and_schema_matches_exactly_when_ungrouped() { - let original = avg_agg(vec![], None, metric_scan(&[])); - let original_rc = Rc::clone(&original); - let target = TargetSubDAG::new(&original_rc); - - let replacements = AvgToSumOverCountStrategy.replacements(&target); - assert_eq!(replacements.len(), 1, "{replacements:?}"); - assert!(!replacements[0].rationale.is_empty()); - - let rewritten = match &replacements[0].replacement { - Replacement::SubDAG(rc) => rc, - other => panic!("expected a Rewrite replacement, got {other:?}"), - }; - let Some(NonASAPOp::BinaryOp { lhs, rhs, .. }) = rewritten.non_asap() else { - panic!("expected sum/count BinaryOp, got {rewritten:?}"); - }; - let Some(NonASAPOp::Project { child: sum, .. }) = lhs.non_asap() else { - panic!("expected cast Project above Sum, got {lhs:?}"); - }; - assert!(matches!(sum.non_asap(), - Some(NonASAPOp::Aggregate { measures, .. }) - if matches!(measures.as_slice(), [AggIntent::Sum { col: None }]) - )); - assert!(matches!(rhs.non_asap(), - Some(NonASAPOp::Aggregate { measures, .. }) - if matches!(measures.as_slice(), [AggIntent::Count { accuracy: AccuracyTarget::Exact }]) - )); - - let original_schema = original.schema.clone(); - let rewritten_schema = rewritten.schema.clone(); - assert_eq!( - original_schema, rewritten_schema, - "the rewritten DAG must report exactly the same output schema as the original avg" - ); - } - - /// A named (SQL-style) output override survives the rewrite: the - /// `Project`'s final column is re-aliased to `output_names[0]`, not the - /// synthetic `"avg"` default. - #[test] - fn preserves_an_explicit_output_name_override() { - let q = avg_agg_with( - vec![], - None, - vec!["avg_latency".to_string()], - None, - metric_scan(&[]), - ); - let original_schema = q.schema.clone(); - let target = TargetSubDAG::new(&q); - - let replacements = AvgToSumOverCountStrategy.replacements(&target); - let rewritten = match &replacements[0].replacement { - Replacement::SubDAG(rc) => rc, - other => panic!("expected a Rewrite replacement, got {other:?}"), - }; - let rewritten_schema = rewritten.schema.clone(); - assert_eq!(original_schema, rewritten_schema); - assert_eq!(rewritten_schema.fields[0].name, "avg_latency"); - } - - /// A grouped rewrite must preserve the aggregate's grouping-key metadata; - /// CSE and roll-up legality both depend on it. - #[test] - fn grouped_avg_rewrite_preserves_the_whole_schema() { - let original = avg_agg(vec![2], None, metric_scan(&["job"])); - let original_schema = original.schema.clone(); - let original_rc = Rc::clone(&original); - let target = TargetSubDAG::new(&original_rc); - - let replacements = AvgToSumOverCountStrategy.replacements(&target); - let rewritten = match &replacements[0].replacement { - Replacement::SubDAG(rc) => rc, - other => panic!("expected a Rewrite replacement, got {other:?}"), - }; - let rewritten_schema = rewritten.schema.clone(); - - assert_eq!(rewritten_schema, original_schema); - } - - #[test] - fn default_search_discovers_bindable_sum_and_count_targets() { - let root = avg_agg(vec![2], None, metric_scan(&["job"])); - let space = crate::pass1::replacement::search_workload(vec![("avg", Rc::clone(&root))]); - - let avg_group = space - .candidates_for_target(&space.roots[0].1) - .expect("avg group"); - assert!(avg_group.candidates.iter().any(|candidate| { - candidate.provenance == crate::pass1::replacement::ReplacementProvenance::LogicalRewrite - })); - - let mut found_sum = false; - let mut found_count = false; - for group in space.target_subdag_candidates() { - let Some(NonASAPOp::Aggregate { measures, .. }) = group.target.non_asap() else { - continue; - }; - let expected = matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - || matches!( - measures.as_slice(), - [AggIntent::Count { - accuracy: AccuracyTarget::Exact - }] - ); - if !expected { - continue; - } - assert!( - group - .candidates - .iter() - .any(|candidate| matches!(&candidate.replacement, - Replacement::SubDAG(node) if node.contains_asap())), - "rewritten accumulator must be independently bindable: {measures:?}" - ); - found_sum |= matches!(measures.as_slice(), [AggIntent::Sum { .. }]); - found_count |= matches!( - measures.as_slice(), - [AggIntent::Count { - accuracy: AccuracyTarget::Exact - }] - ); - } - assert!(found_sum && found_count); - } - - #[test] - fn works_with_a_bound_column_not_just_the_sample_value() { - let mut schema_cols = vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("job", DataType::Utf8, true), - Field::plain("bytes", DataType::Int64, false), - ]; - let child = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: { - let cols = std::mem::take(&mut schema_cols); - Schema::with_time_index(cols, 0, vec![]) - }, - })) - .unwrap(); - let original = avg_agg(vec![1], Some(2), child); - let original_schema = original.schema.clone(); - let original_rc = Rc::clone(&original); - let target = TargetSubDAG::new(&original_rc); - - let replacements = AvgToSumOverCountStrategy.replacements(&target); - let rewritten = match &replacements[0].replacement { - Replacement::SubDAG(rc) => rc, - other => panic!("expected a Rewrite replacement, got {other:?}"), - }; - let rewritten_schema = rewritten.schema.clone(); - - // The whole reason for the explicit `Cast` in `build_rewrite`: an - // `Int64` input column (`bytes`) makes `Sum`'s own output `Int64` - // too, and a bare (uncast) `Int64 / Int64` would type the avg - // column `Int64` — this assertion is what would catch that - // regression. - assert_eq!(rewritten_schema.fields, original_schema.fields); - assert_eq!( - rewritten_schema.fields.last().unwrap().dtype, - DataType::Float64 - ); - - let Some(NonASAPOp::BinaryOp { lhs, .. }) = rewritten.non_asap() else { - panic!("expected sum/count BinaryOp"); - }; - let Some(NonASAPOp::Project { cols, .. }) = lhs.non_asap() else { - panic!("expected cast Project above Sum"); - }; - assert!(matches!( - &cols.last().unwrap().expr, - ScalarExpr::Cast { - to: DataType::Float64, - .. - } - )); - } - - #[test] - fn does_not_rewrite_avg_of_a_nullable_column_via_count_star() { - let child = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("latency", DataType::Float64, true), - ], - 0, - vec![], - ), - })) - .unwrap(); - let q = avg_agg(vec![], Some(2), child); - let target = TargetSubDAG::new(&q); - - assert!(!AvgToSumOverCountStrategy.matches(&target)); - assert!(AvgToSumOverCountStrategy.replacements(&target).is_empty()); - } - - fn nested_aggregate(outer: AggIntent, inner: AggIntent) -> Rc { - let temporal = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![inner], - output_names: vec![], - filters: vec![], - having: None, - child: OperatorNode::new_shared(asap_types::ir::Operator::NonASAP( - NonASAPOp::TimeRange { - kind: TimeRangeKind::Range, - range: Duration::from_secs(300), - child: metric_scan(&["service"]), - }, - )) - .unwrap(), - })) - .unwrap(); - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![outer], - // Match the PromQL front end: an empty entry selects the intent's - // canonical output name rather than an explicit alias. - output_names: vec![String::new()], - filters: vec![], - having: None, - child: temporal, - })) - .unwrap() - } - - #[test] - fn semantic_rewrite_composes_every_supported_aggregate_pair() { - let exact_count = AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }; - for (outer, inner) in [ - (AggIntent::Sum { col: None }, AggIntent::Sum { col: None }), - (AggIntent::Sum { col: None }, exact_count), - (AggIntent::Min { col: None }, AggIntent::Min { col: None }), - (AggIntent::Max { col: None }, AggIntent::Max { col: None }), - ] { - let expected = inner.clone(); - let original = nested_aggregate(outer, inner); - let candidates = - SemanticEquivalentRewriteStrategy.replacements(&TargetSubDAG::new(&original)); - let [candidate] = candidates.as_slice() else { - panic!("supported pair should produce exactly one rewrite") - }; - let Replacement::SubDAG(rewritten) = &candidate.replacement else { - panic!("expected a logical rewrite") - }; - assert_eq!(original.schema.clone(), rewritten.schema.clone()); - let aggregate = match rewritten.non_asap() { - Some(NonASAPOp::Aggregate { .. }) => rewritten.as_ref(), - Some(NonASAPOp::Project { child, .. }) => child.as_ref(), - other => panic!("expected Aggregate or cast Project, got {other:?}"), - }; - let Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - measures, - child, - .. - }) = aggregate.non_asap() - else { - panic!("expected composed cross-entity aggregate") - }; - assert_eq!(by.keys(), &[2]); - assert_eq!(measures, &[expected]); - assert!(matches!(child.non_asap(), - Some(NonASAPOp::TimeRange { range, child, .. }) - if *range == Duration::from_secs(300) - && matches!(child.non_asap(), Some(NonASAPOp::Scan { .. })) - )); - } - } - - #[test] - fn semantic_rewrite_rejects_non_composable_aggregate_pairs() { - for (outer, inner) in [ - (AggIntent::Sum { col: None }, AggIntent::Min { col: None }), - (AggIntent::Avg { col: None }, AggIntent::Sum { col: None }), - (AggIntent::Min { col: None }, AggIntent::Max { col: None }), - ] { - let original = nested_aggregate(outer, inner); - assert!(composed_aggregate_rewrite(&original).is_none()); - } - } - - #[test] - fn default_search_discovers_promql_shaped_sum_count_composition() { - let root = nested_aggregate( - AggIntent::Sum { col: None }, - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - ); - let space = crate::pass1::replacement::search_workload(vec![("sum-count", root)]); - let root = &space.roots[0].1; - let group = space.candidates_for_target(root).expect("root memo group"); - let candidate = group - .candidates - .iter() - .find(|candidate| candidate.strategy == "SemanticEquivalentRewriteStrategy") - .expect("default search should run semantic rewrites"); - let Replacement::SubDAG(rewritten) = &candidate.replacement else { - panic!("expected logical rewrite") - }; - assert_eq!(rewritten.schema.clone().fields[1].name, "sum"); - } -} diff --git a/crates/logical-optimizer/src/pass1/rollup.rs b/crates/logical-optimizer/src/pass1/rollup.rs deleted file mode 100644 index 5b8bb59e..00000000 --- a/crates/logical-optimizer/src/pass1/rollup.rs +++ /dev/null @@ -1,894 +0,0 @@ -//! [`RollupStrategy`] — group-by-lattice roll-up reuse (fine-to-coarse) as a -//! [`ReplacementStrategy`] (issue #254, part of #33). -//! -//! ## The optimization: AHA-style roll-up, not independent subpopulations -//! -//! `docs/asap_aware_mapping.md`'s "Degrees of freedom" section names this -//! axis directly: **"AHA vs treating hierarchical subpopulations -//! independently."** Two `Aggregate` nodes over the same source, grouped at -//! different granularities (e.g. `by (job)` and `by (job, region)`), are -//! today two *independent* passes over the source. But when the coarser -//! grouping's keys are a subset of the finer grouping's keys, the coarser -//! answer is derivable from the finer one *without ever touching the raw -//! source again* — the same "roll up the group-by lattice" reuse a SQL -//! `ROLLUP`/OLAP-cube engine already exploits between grouping levels of one -//! `GROUP BY ROLLUP(...)` (see `frontend-sql`'s own `ROLLUP` lowering -//! tests), generalized here to two *independently written* aggregates in a -//! workload rather than one multi-level `ROLLUP` clause. -//! -//! This is exactly Peilin's catalog entry (issue #33's comment thread): -//! "Rolling up aggregations on a fine-grained group by to get a -//! coarse-grained group by (like AHA)," alongside "CSE across aggregations, -//! and group by key management" — this strategy is the *cross-aggregate* -//! sibling of `ir::cse::share_common_sub_dags`'s *identical*-sub-DAG -//! sharing: CSE shares two structurally-*equal* aggregates onto one `Rc`; -//! this strategy relates two structurally-*different* (differently grouped) -//! aggregates over the same shared source. -//! -//! ## Why re-aggregating the finer side needs a *combinator*, not literally -//! "the same measure" reapplied -//! -//! Re-deriving `by (job)` from `by (job, region)` means computing, for each -//! `job`, "the combination of every `(job, region)` partial result for that -//! `job`." Whether that combination is *the same operator reapplied* depends -//! on the operator: -//! -//! - `Sum`/`Min`/`Max` are **self-combining**: a sum of sums is a sum, a min -//! of mins is a min. Reapplying the identical `AggIntent` over the finer -//! side's own output column is correct. -//! - Exact `Count` is **not** self-combining: re-`Count`ing the finer side's own -//! output rows counts the number of *finer groups* per coarser group (how -//! many distinct `region`s a `job` has), not the original row count. The -//! correct combinator is `Sum` — `count(A ∪ B) = count(A) + count(B)`, the -//! same "`COUNT(*)` over a pre-aggregated summary table becomes -//! `SUM(count)`" rule every OLAP rollup/cube/materialized-view-matching -//! implementation applies. Approximate `Count` is excluded because summing -//! finalized per-finer-group estimates does not preserve the coarser error -//! target. `Increase` mirrors exact count's additive combination (a -//! counter's total increase across sub-windows is additive), so it combines -//! via `Sum` too. -//! - `Rate` (`increase / duration`) has **no** valid self- or sum-combinator -//! here, so it is deliberately left unhandled (see [`rollup_combinator`]). -//! In practice this never matters: `Rate`/`Increase` are constructed with -//! `Reduction::PerEntity` (per-series, no `by` at all — see -//! `AggIntent::is_per_series` and `asap_frontend_promql::promql::reduction_for`), -//! never `Reduction::Reduce`, so they never carry a `by` set for this -//! strategy to compare in the first place; `PerEntity` nodes never match -//! ([`bindable_grouped_aggregate`] returns `None` for them). `Increase`'s -//! entry in [`rollup_combinator`] is a defensive completion of the match -//! (exhaustive-over-the-mergeable-vocabulary, "no silent fallthrough"), -//! not a case expected to fire today. -//! -//! [`rollup_combinator`] is the single place this substitution is decided — -//! [`is_legal_rollup_source`] (the standalone legality predicate issue #254 -//! asks for) consults it, and so does this module's `replacements`, so the -//! two can never disagree about which intents are eligible. -//! -//! ## `ColumnId` comparability — only sound for identical child IR -//! -//! A `ColumnId` is a *position* into a specific `Schema` (`crates/types/src/ir/schema/mod.rs`'s -//! own doc: "the same edge, the same schema, the same positional numbering"). -//! Comparing the coarser aggregate's `by` positions against the finer -//! aggregate's `by` positions is only meaningful when both aggregates have -//! pointer-identical or `PartialEq`-equal children, including equal schemas. -//! Thus both `by` lists index the same shape. The equality fallback matters -//! for scans that CSE conservatively declines to alias because they have no -//! declared unique key. Structurally different sources remain out of scope: -//! this module never reconciles `ColumnId`s across distinct schemas. -//! -//! ## Non-goals (tracked separately, not attempted here — same split -//! `replacement.rs`'s own module docs draw for `SharedSubDAGStrategy`'s -//! `consumer_count`) -//! -//! - **No sibling discovery inside this strategy.** Finding every aggregate -//! across a workload is not a per-target operation. The workload search -//! performs that traversal after CSE, then constructs [`RollupStrategy`] -//! with the discovered aggregate set. Direct/custom callers supply their -//! own set through [`RollupStrategy::new`]. -//! - **No materialized roll-up operator.** Actually building a pre-aggregated -//! summary/scan leaf at execution time is separate, larger work outside -//! `asap-logical-optimizer`'s scope (see issue #254's own "Non-goal" section) -//! — this module only constructs the pre-ASAP `NonASAPOp::Aggregate` -//! rewrite; a `CostModel`/search engine decides whether to prefer it. -//! - **No cross-schema reconciliation** (see "`ColumnId` comparability" -//! above) and **no `without(...)` grouping support** — `without`'s kept -//! set is runtime-open (never enumerable at plan time, per -//! `GroupKeys`'s own doc), so there is no fixed `ColumnId` set to compare -//! against a superset/subset relationship at all; [`is_legal_rollup_source`] -//! declines both directions. - -use asap_types::ir::operator::non_asap::any_measure_filtered; -use std::collections::HashSet; -use std::rc::Rc; - -use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::{GroupKeys, Reduction}; -use asap_types::ir::schema::{ColumnId, Schema}; -use asap_types::ir::{NonASAPOp, OperatorNode}; - -use asap_types::types::AccuracyTarget; - -use crate::pass1::replacement::{ - Replacement, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, -}; - -/// The `(by, intent, child)` shape this strategy operates on: a single -/// measure, no `HAVING` — the same bindable shape -/// [`crate::pass1::replacement::ASAPStrategies`] requires (see that module's -/// private `bindable_intent`) — **plus** a genuine [`Reduction::Reduce`] -/// grouping to compare (not [`Reduction::PerEntity`], which has no `by` set -/// at all). `None` for anything else, including a multi-measure or `HAVING` -/// aggregate, a non-`Aggregate` node, or a `PerEntity` reduction. -fn bindable_grouped_aggregate( - node: &OperatorNode, -) -> Option<(&GroupKeys, &AggIntent, &Rc)> { - let Some(NonASAPOp::Aggregate { - reduction, - measures, - filters, - having, - child, - .. - }) = node.non_asap() - else { - return None; - }; - let ([intent], None) = (measures.as_slice(), having) else { - return None; - }; - if any_measure_filtered(filters) { - return None; - } - let Reduction::Reduce(by) = reduction else { - return None; - }; - Some((by, intent, child)) -} - -/// The `AggIntent` to combine `intent`'s partial results with, when -/// re-aggregating over an already-computed finer aggregate's own measure -/// column at position `finer_measure_col` — see the module docs' "Why -/// re-aggregating the finer side needs a *combinator*" section for the -/// reasoning behind each arm. `None` for any intent this module does not -/// (yet) know a correct combinator for — including every intent -/// `agg_is_mergeable` permits but this module doesn't specifically handle -/// (`Rate`, and everything outside the `Sum`/`Count`/`Min`/`Max`/`Increase` -/// vocabulary `agg_is_mergeable`'s own doc names) — so `is_legal_rollup_source` -/// (which calls this) is *strictly narrower* than `agg_is_mergeable` alone, -/// deliberately: `agg_is_mergeable` answers "does *some* partial-state merge -/// exist", not "is self- or sum-recombination the right one," and this -/// module only ever proposes a rewrite it can construct correctly. -fn rollup_combinator(intent: &AggIntent, finer_measure_col: ColumnId) -> Option { - match intent { - // Self-combining: reapplying the identical operator over the finer - // side's own output column is correct unchanged. - AggIntent::Sum { .. } => Some(AggIntent::Sum { - col: Some(finer_measure_col), - }), - AggIntent::Min { .. } => Some(AggIntent::Min { - col: Some(finer_measure_col), - }), - AggIntent::Max { .. } => Some(AggIntent::Max { - col: Some(finer_measure_col), - }), - // Not self-combining — see the module docs. Both merge by addition - // over the finer side's own output column instead. - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - } - | AggIntent::Increase => Some(AggIntent::Sum { - col: Some(finer_measure_col), - }), - _ => None, - } -} - -/// **The standalone legality predicate issue #254 asks for**: is `finer`'s -/// grouping a legal roll-up source for `coarser`'s grouping, given that both -/// nodes compute an aggregation intent over the same shared child? -/// -/// Issue #256 (`GroupingStrategy`/Hydra axis, built in parallel from the -/// same base) is expected to call this exact function before assuming a -/// Hydra-backed aggregate composes with a roll-up — it is named and -/// exported for that reason, not left as private inline logic in `matches`/ -/// `replacements`. -/// -/// Legal iff, **in this order**: -/// -/// 1. `finer_intent == coarser_intent` — the two aggregates compute the -/// identical intent + column (verified here, not just assumed by the -/// caller, so this function is a complete, self-contained answer on its -/// own). -/// 2. [`rollup_combinator`] knows how to re-derive `finer_intent` from a -/// pre-aggregated column at all (excludes `Avg`/`StdDev`/`Variance` — -/// never `agg_is_mergeable` — and also excludes every `agg_is_mergeable` -/// intent this module doesn't specifically handle, e.g. `Rate`). -/// 3. Neither grouping is a `without(...)` exclusion grouping — `without`'s -/// kept set is runtime-open, so there is no fixed `ColumnId` set to -/// compare a superset/subset relationship against (see the module docs). -/// 4. `finer_output_schema` (the finer aggregate's own *output* schema, not -/// the shared child's) carries a provable unique key -/// ([`Schema::has_unique_key`]) — **the exact legality gate -/// `ir::cse::share_common_sub_dags` already applies to its own -/// sharing decisions**, reused verbatim here rather than re-invented: -/// `share_common_sub_dags`'s own doc ("Legality: gated by -/// `Schema::unique_keys`") states a producer's output is only safely -/// reusable across consumers when its row identity is provably stable — -/// exactly the property re-aggregating over `finer` as if it were a -/// fresh source requires. -/// 5. `coarser_by` is a **strict, proper** subset of `finer_by` (same -/// `ColumnId`s, finer strictly more of them) — an *equal* `by` is -/// `SharedSubDAGStrategy`'s CSE-sharing question, not a roll-up, so -/// equality is deliberately excluded here, not treated as a degenerate -/// roll-up. -pub fn is_legal_rollup_source( - finer_by: &GroupKeys, - finer_output_schema: &Schema, - finer_intent: &AggIntent, - coarser_by: &GroupKeys, - coarser_intent: &AggIntent, -) -> bool { - if finer_intent != coarser_intent { - return false; - } - if rollup_combinator(finer_intent, 0).is_none() { - return false; - } - if finer_by.is_without() || coarser_by.is_without() { - return false; - } - if !finer_output_schema.has_unique_key() { - return false; - } - is_strict_column_superset(finer_by.keys(), coarser_by.keys()) -} - -/// Whether `finer` is a strict, proper superset of `coarser` — every -/// `ColumnId` in `coarser` also appears in `finer`, and `finer` has more of -/// them (an equal-length or shorter `finer` can never be a proper -/// superset, so the length check alone rules out equality without a set -/// comparison). -fn is_strict_column_superset(finer: &[ColumnId], coarser: &[ColumnId]) -> bool { - let finer_set: HashSet<&ColumnId> = finer.iter().collect(); - let coarser_set: HashSet<&ColumnId> = coarser.iter().collect(); - if finer_set.len() <= coarser_set.len() { - return false; - } - coarser_set.is_subset(&finer_set) -} - -/// Wraps the group-by-lattice roll-up reuse (issue #254, part of #33) as a -/// [`ReplacementStrategy`]: given a coarser `Aggregate` [`TargetSubDAG`], -/// finds every already-known sibling `Aggregate` that has an identical child -/// IR and is a legal, strictly finer roll-up source for it (per -/// [`is_legal_rollup_source`]), and proposes replacing the target with a -/// re-aggregation over that sibling instead of the shared raw source. -/// -/// `siblings` is **caller-supplied, not discovered here** — see the module -/// docs' "Non-goals" on why finding the full sibling set across a workload -/// is a workload-wide traversal this strategy does not own. -pub struct RollupStrategy { - siblings: Vec>, -} - -impl RollupStrategy { - /// A strategy that owns clones of every node in `siblings` and considers - /// each as a candidate roll-up source (or target) — typically the full set of `Aggregate` - /// nodes a workload-wide discovery pass (issue #252) already found - /// sharing at least one child `Rc` with something else. - pub fn new(siblings: &[Rc]) -> Self { - Self { - siblings: siblings.to_vec(), - } - } - - /// Every sibling that is a legal, strictly finer roll-up source for - /// `target` — shared between `matches` and `replacements` so the two - /// can never disagree about which siblings qualify. - fn finer_sources(&self, target: &TargetSubDAG<'_>) -> Vec<&Rc> { - let Some((coarser_by, coarser_intent, coarser_child)) = - bindable_grouped_aggregate(target.root) - else { - return Vec::new(); - }; - - self.siblings - .iter() - .filter(|&candidate| { - if Rc::ptr_eq(candidate, target.root) { - return false; - } - let Some((finer_by, finer_intent, finer_child)) = - bindable_grouped_aggregate(candidate) - else { - return false; - }; - if !Rc::ptr_eq(finer_child, coarser_child) && finer_child != coarser_child { - return false; - } - is_legal_rollup_source( - finer_by, - &candidate.schema, - finer_intent, - coarser_by, - coarser_intent, - ) - }) - .collect() - } -} - -impl ReplacementStrategy for RollupStrategy { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - !self.finer_sources(target).is_empty() - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - let Some((coarser_by, coarser_intent, _)) = bindable_grouped_aggregate(target.root) else { - return Vec::new(); - }; - let Some(NonASAPOp::Aggregate { output_names, .. }) = target.root.non_asap() else { - unreachable!("bindable_grouped_aggregate already confirmed Aggregate"); - }; - self.finer_sources(target) - .into_iter() - .filter_map(|finer| build_rollup(finer, coarser_by, coarser_intent, output_names)) - .collect() - } -} - -/// Build the coarser replacement: a new `NonASAPOp::Aggregate` grouped by -/// `coarser_by`'s columns (repositioned into `finer`'s own output schema — -/// see below), computing `rollup_combinator(intent, ..)` over `finer`'s own -/// measure column, with `child = finer` instead of the original shared -/// source. -/// -/// `coarser_by`'s `ColumnId`s are positions into the *shared child's* -/// schema (the same schema `finer_by`'s `ColumnId`s index into — see the -/// module docs' "`ColumnId` comparability" section). `finer`'s own output -/// schema is a *different* schema (`finer_by`'s columns, in order, followed -/// by its one measure column — `aggregate_output_schema`'s `by ++ measures` -/// shape), so each of `coarser_by`'s columns must be translated from its -/// position in the shared child to its position in `finer`'s output: the -/// index its `ColumnId` occupies within `finer_by`'s own ordered list. -fn build_rollup( - finer: &Rc, - coarser_by: &GroupKeys, - intent: &AggIntent, - output_names: &[String], -) -> Option { - let (finer_by, _, _) = bindable_grouped_aggregate(finer)?; - // `finer`'s own single measure sits right after its `by` columns in its - // output schema (`aggregate_output_schema`'s `by ++ measures` layout). - let finer_measure_col: ColumnId = finer_by.len(); - let combinator = rollup_combinator(intent, finer_measure_col)?; - - let remapped_by: Vec = coarser_by - .keys() - .iter() - .map(|id| finer_by.keys().iter().position(|f| f == id)) - .collect::>>()?; - - let rewritten = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(remapped_by), - measures: vec![combinator], - output_names: output_names.to_vec(), - filters: vec![], - having: None, - child: Rc::clone(finer), - })) - .ok()?; - - Some(ReplacementSubDAG { - strategy: "RollupStrategy", - replacement: Replacement::SubDAG(rewritten), - provenance: crate::pass1::replacement::ReplacementProvenance::LogicalRewrite, - rationale: format!( - "rolls up from the finer Aggregate grouped by {:?} (a strict superset of this \ - node's own {:?} grouping over the same shared source) instead of an independent \ - pass over the raw source — both compute {intent:?} over the same input, and the \ - finer side has a provable unique key (Schema::has_unique_key)", - finer_by.keys(), - coarser_by.keys(), - ), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::operator::operator_properties::Source; - use asap_types::ir::schema::{DataType, Field}; - use asap_types::types::AccuracyTarget; - - /// `[ts(0), value(1), job(2), region(3)]`. - fn metric_scan() -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - Field::plain("region", DataType::Utf8, true), - ], - 0, - vec![], - ), - })) - .unwrap() - } - - fn agg(by: Vec, intent: AggIntent, child: &Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(child), - })) - .unwrap() - } - - fn without_agg( - excluded: Vec, - intent: AggIntent, - child: &Rc, - ) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(excluded)), - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(child), - })) - .unwrap() - } - - // ── is_legal_rollup_source (the standalone predicate) ─────────────── - - #[test] - fn predicate_accepts_a_strict_superset_over_a_mergeable_intent_with_a_unique_key() { - let finer_schema = Schema::with_time_index(vec![], 0, vec![vec![0, 1]]); - assert!(is_legal_rollup_source( - &GroupKeys::by(vec![2, 3]), - &finer_schema, - &AggIntent::Sum { col: None }, - &GroupKeys::by(vec![2]), - &AggIntent::Sum { col: None }, - )); - } - - #[test] - fn predicate_rejects_mismatched_intents() { - let finer_schema = Schema::with_time_index(vec![], 0, vec![vec![0, 1]]); - assert!(!is_legal_rollup_source( - &GroupKeys::by(vec![2, 3]), - &finer_schema, - &AggIntent::Sum { col: None }, - &GroupKeys::by(vec![2]), - &AggIntent::Sum { col: Some(9) }, - )); - } - - #[test] - fn predicate_rejects_a_non_mergeable_intent() { - let finer_schema = Schema::with_time_index(vec![], 0, vec![vec![0, 1]]); - assert!(!is_legal_rollup_source( - &GroupKeys::by(vec![2, 3]), - &finer_schema, - &AggIntent::Avg { col: None }, - &GroupKeys::by(vec![2]), - &AggIntent::Avg { col: None }, - )); - } - - #[test] - fn predicate_rejects_an_intent_with_no_known_combinator() { - // Rate is `agg_is_mergeable` but this module has no correct - // self-/sum-combinator for it (see `rollup_combinator`'s doc). - let finer_schema = Schema::with_time_index(vec![], 0, vec![vec![0, 1]]); - assert!(!is_legal_rollup_source( - &GroupKeys::by(vec![2, 3]), - &finer_schema, - &AggIntent::Rate, - &GroupKeys::by(vec![2]), - &AggIntent::Rate, - )); - } - - #[test] - fn predicate_rejects_a_finer_side_with_no_unique_key() { - // A schema with an empty `unique_keys` — as if `finer` were, e.g., a - // `without(...)`-grouped or otherwise non-hoistable aggregate — even - // though the `by` sets themselves are a clean strict superset. - let no_unique_key_schema = Schema::new(vec![]); - assert!(!is_legal_rollup_source( - &GroupKeys::by(vec![2, 3]), - &no_unique_key_schema, - &AggIntent::Sum { col: None }, - &GroupKeys::by(vec![2]), - &AggIntent::Sum { col: None }, - )); - } - - #[test] - fn predicate_rejects_equal_by_sets() { - // Equality is `SharedSubDAGStrategy`'s question, not a roll-up. - let finer_schema = Schema::with_time_index(vec![], 0, vec![vec![0]]); - assert!(!is_legal_rollup_source( - &GroupKeys::by(vec![2]), - &finer_schema, - &AggIntent::Sum { col: None }, - &GroupKeys::by(vec![2]), - &AggIntent::Sum { col: None }, - )); - } - - #[test] - fn predicate_does_not_treat_duplicate_keys_as_a_strict_superset() { - let finer_schema = Schema::with_time_index(vec![], 0, vec![vec![0, 1]]); - assert!(!is_legal_rollup_source( - &GroupKeys::by(vec![2, 3, 3]), - &finer_schema, - &AggIntent::Sum { col: None }, - &GroupKeys::by(vec![2, 3]), - &AggIntent::Sum { col: None }, - )); - } - - #[test] - fn predicate_rejects_unrelated_by_sets() { - let finer_schema = Schema::with_time_index(vec![], 0, vec![vec![0, 1]]); - assert!(!is_legal_rollup_source( - &GroupKeys::by(vec![3, 4]), - &finer_schema, - &AggIntent::Sum { col: None }, - &GroupKeys::by(vec![2]), - &AggIntent::Sum { col: None }, - )); - } - - #[test] - fn predicate_rejects_a_without_grouping_on_either_side() { - let finer_schema = Schema::with_time_index(vec![], 0, vec![vec![0, 1]]); - assert!(!is_legal_rollup_source( - &GroupKeys::without(vec![2, 3]), - &finer_schema, - &AggIntent::Sum { col: None }, - &GroupKeys::by(vec![2]), - &AggIntent::Sum { col: None }, - )); - assert!(!is_legal_rollup_source( - &GroupKeys::by(vec![2, 3]), - &finer_schema, - &AggIntent::Sum { col: None }, - &GroupKeys::without(vec![2]), - &AggIntent::Sum { col: None }, - )); - } - - // ── RollupStrategy ──────────────────────────────────────────────────── - - #[test] - fn superset_by_over_identical_mergeable_intent_and_shared_child_rolls_up() { - let scan = metric_scan(); - let fine = agg(vec![2, 3], AggIntent::Sum { col: Some(1) }, &scan); - let coarse = agg(vec![2], AggIntent::Sum { col: Some(1) }, &scan); - - let siblings = vec![Rc::clone(&fine), Rc::clone(&coarse)]; - let strategy = RollupStrategy::new(&siblings); - let target = TargetSubDAG::new(&coarse); - - assert!(strategy.matches(&target)); - let replacements = strategy.replacements(&target); - assert_eq!(replacements.len(), 1, "{replacements:?}"); - - let Replacement::SubDAG(rewritten) = &replacements[0].replacement else { - panic!("expected a Rewrite replacement"); - }; - let Some(NonASAPOp::Aggregate { - reduction, - measures, - child, - having, - .. - }) = rewritten.non_asap() - else { - panic!("expected an Aggregate rewrite, got {rewritten:?}"); - }; - assert!(having.is_none()); - assert!( - Rc::ptr_eq(child, &fine), - "child must be the finer aggregate" - ); - assert_eq!( - reduction.expect_reduce(), - &vec![0usize], - "coarser's `job` column (position 2 in the shared source) is the finer \ - aggregate's own column 0" - ); - assert_eq!( - measures, - &vec![AggIntent::Sum { col: Some(2) }], - "Sum is self-combining: re-Sum over the finer aggregate's own Sum output \ - column (position 2, right after its two `by` keys, job and region)" - ); - assert!(!replacements[0].rationale.is_empty()); - assert!(replacements[0].rationale.contains("finer")); - } - - #[test] - fn count_rolls_up_via_sum_not_count() { - // Count is not self-combining (see the module docs) — the rewritten - // measure must be Sum over the finer Count's own output column, not - // Count reapplied. - let scan = metric_scan(); - let fine = agg( - vec![2, 3], - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - &scan, - ); - let coarse = agg( - vec![2], - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - &scan, - ); - - let siblings = vec![Rc::clone(&fine), Rc::clone(&coarse)]; - let strategy = RollupStrategy::new(&siblings); - let target = TargetSubDAG::new(&coarse); - - let replacements = strategy.replacements(&target); - assert_eq!(replacements.len(), 1, "{replacements:?}"); - let Replacement::SubDAG(rewritten) = &replacements[0].replacement else { - panic!("expected a Rewrite replacement"); - }; - let Some(NonASAPOp::Aggregate { measures, .. }) = rewritten.non_asap() else { - panic!("expected an Aggregate rewrite"); - }; - assert_eq!( - measures, - &vec![AggIntent::Sum { col: Some(2) }], - "Count's own output column sits at position 2, right after its two `by` keys" - ); - } - - #[test] - fn approximate_count_does_not_roll_up_via_sum() { - let scan = metric_scan(); - let intent = AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let fine = agg(vec![2, 3], intent.clone(), &scan); - let coarse = agg(vec![2], intent, &scan); - let siblings = vec![Rc::clone(&fine), Rc::clone(&coarse)]; - let strategy = RollupStrategy::new(&siblings); - let target = TargetSubDAG::new(&coarse); - - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn default_workload_search_adds_rollup_for_two_query_workload() { - let fine_scan = metric_scan(); - let coarse_scan = metric_scan(); - let fine = agg(vec![2, 3], AggIntent::Sum { col: Some(1) }, &fine_scan); - let coarse = agg(vec![2], AggIntent::Sum { col: Some(1) }, &coarse_scan); - - let space = - crate::pass1::replacement::search_workload(vec![("fine", fine), ("coarse", coarse)]); - let coarse_group = space - .target_subdag_candidates() - .find(|group| { - matches!(group.target.non_asap(), - Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - .. - }) if by.keys() == [2] - ) - }) - .expect("coarser aggregate group"); - - let rewrite = coarse_group - .candidates - .iter() - .find_map(|candidate| match &candidate.replacement { - // Old `Replacement::Rewrite`: a pure pre-ASAP sub-DAG. - Replacement::SubDAG(rewrite) if !rewrite.contains_asap() => Some(rewrite), - Replacement::SubDAG(_) | Replacement::ExactComposition(_) => None, - }) - .expect("default search must include the roll-up rewrite"); - let Some(NonASAPOp::Aggregate { child, .. }) = rewrite.non_asap() else { - panic!("expected aggregate rewrite, got {rewrite:?}"); - }; - assert!(matches!(child.non_asap(), - Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - .. - }) if by.keys() == [2, 3] - )); - } - - #[test] - fn workload_search_does_not_roll_up_approximate_count() { - let intent = AggIntent::Count { - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let fine = agg(vec![2, 3], intent.clone(), &metric_scan()); - let coarse = agg(vec![2], intent, &metric_scan()); - let space = - crate::pass1::replacement::search_workload(vec![("fine", fine), ("coarse", coarse)]); - let coarse_group = space - .target_subdag_candidates() - .find(|group| { - matches!(group.target.non_asap(), - Some(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(by), - .. - }) if by.keys() == [2] - ) - }) - .expect("coarser aggregate group"); - - assert!(coarse_group - .candidates - .iter() - .all(|candidate| !matches!(&candidate.replacement, - Replacement::SubDAG(rewrite) if !rewrite.contains_asap()))); - } - - #[test] - fn rollup_preserves_the_coarser_output_name() { - let scan = metric_scan(); - let fine = agg(vec![2, 3], AggIntent::Sum { col: Some(1) }, &scan); - let coarse = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Sum { col: Some(1) }], - output_names: vec!["total_requests".into()], - filters: vec![], - having: None, - child: Rc::clone(&scan), - })) - .unwrap(); - let original_schema = coarse.schema.clone(); - - let siblings = vec![Rc::clone(&fine), Rc::clone(&coarse)]; - let strategy = RollupStrategy::new(&siblings); - let replacements = strategy.replacements(&TargetSubDAG::new(&coarse)); - let Replacement::SubDAG(rewritten) = &replacements[0].replacement else { - panic!("expected a Rewrite replacement"); - }; - - assert_eq!(rewritten.schema.clone(), original_schema); - let Some(NonASAPOp::Aggregate { output_names, .. }) = rewritten.non_asap() else { - unreachable!(); - }; - assert_eq!(output_names, &vec!["total_requests".to_string()]); - } - - #[test] - fn non_mergeable_intent_does_not_roll_up() { - let scan = metric_scan(); - let fine = agg(vec![2, 3], AggIntent::Avg { col: Some(1) }, &scan); - let coarse = agg(vec![2], AggIntent::Avg { col: Some(1) }, &scan); - - let siblings = vec![Rc::clone(&fine), Rc::clone(&coarse)]; - let strategy = RollupStrategy::new(&siblings); - let target = TargetSubDAG::new(&coarse); - - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn no_unique_key_on_the_finer_side_does_not_roll_up() { - // `without(...)` groupings never carry a provable unique key - // (`without_output_schema` always reports `unique_keys: []`) even - // though its `keys()` (the *excluded* positions here) happens to be - // numerically a superset of the coarser side's *kept* positions — - // `is_legal_rollup_source` rejects any `without` grouping outright, - // and would reject on the missing unique key regardless. - let scan = metric_scan(); - let fine = without_agg(vec![2, 3], AggIntent::Sum { col: Some(1) }, &scan); - let coarse = agg(vec![2], AggIntent::Sum { col: Some(1) }, &scan); - - assert!( - !fine.schema.clone().has_unique_key(), - "fixture sanity: a without(...) aggregate has no provable unique key" - ); - - let siblings = vec![Rc::clone(&fine), Rc::clone(&coarse)]; - let strategy = RollupStrategy::new(&siblings); - let target = TargetSubDAG::new(&coarse); - - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } - - #[test] - fn unrelated_by_sets_do_not_roll_up() { - // Neither `[job]` nor `[region]` is a superset of the other. - let scan = metric_scan(); - let a = agg(vec![2], AggIntent::Sum { col: Some(1) }, &scan); - let b = agg(vec![3], AggIntent::Sum { col: Some(1) }, &scan); - - let siblings = vec![Rc::clone(&a), Rc::clone(&b)]; - let strategy = RollupStrategy::new(&siblings); - - let target_a = TargetSubDAG::new(&a); - assert!(!strategy.matches(&target_a)); - assert!(strategy.replacements(&target_a).is_empty()); - - let target_b = TargetSubDAG::new(&b); - assert!(!strategy.matches(&target_b)); - assert!(strategy.replacements(&target_b).is_empty()); - } - - #[test] - fn equal_by_sets_do_not_roll_up() { - // Equal groupings are `SharedSubDAGStrategy`'s CSE-sharing - // question (build once and share, or build independently) — a - // roll-up requires a *strict* superset, not equality. - let scan = metric_scan(); - let a = agg(vec![2], AggIntent::Sum { col: Some(1) }, &scan); - let b = agg(vec![2], AggIntent::Sum { col: Some(1) }, &scan); - - let siblings = vec![Rc::clone(&a), Rc::clone(&b)]; - let strategy = RollupStrategy::new(&siblings); - let target = TargetSubDAG::new(&a); - assert!(!strategy.matches(&target)); - } - - #[test] - fn structurally_identical_children_roll_up_without_cse_aliasing() { - // Scans without unique keys are deliberately not pointer-aliased by - // CSE. Structural equality still proves identical schemas and makes - // the two aggregates' positional ColumnIds comparable. - let fine = agg(vec![2, 3], AggIntent::Sum { col: Some(1) }, &metric_scan()); - let coarse = agg(vec![2], AggIntent::Sum { col: Some(1) }, &metric_scan()); - - let siblings = vec![Rc::clone(&fine), Rc::clone(&coarse)]; - let strategy = RollupStrategy::new(&siblings); - let target = TargetSubDAG::new(&coarse); - assert!(strategy.matches(&target)); - assert_eq!(strategy.replacements(&target).len(), 1); - } - - #[test] - fn does_not_match_a_multi_measure_or_having_aggregate() { - let scan = metric_scan(); - let fine = agg(vec![2, 3], AggIntent::Sum { col: Some(1) }, &scan); - let multi = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![ - AggIntent::Sum { col: Some(1) }, - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - ], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(&scan), - })) - .unwrap(); - - let siblings = vec![Rc::clone(&fine), Rc::clone(&multi)]; - let strategy = RollupStrategy::new(&siblings); - let target = TargetSubDAG::new(&multi); - assert!(!strategy.matches(&target)); - assert!(strategy.replacements(&target).is_empty()); - } -} diff --git a/crates/logical-optimizer/src/pass2/mod.rs b/crates/logical-optimizer/src/pass2/mod.rs index 09f37094..5c9d354d 100644 --- a/crates/logical-optimizer/src/pass2/mod.rs +++ b/crates/logical-optimizer/src/pass2/mod.rs @@ -6,7 +6,5 @@ //! ([`window_composition`]). pub mod identical_expressions; -pub mod reconciliation; pub mod summary_capability; -pub mod topk_reuse; pub mod window_composition; diff --git a/crates/logical-optimizer/src/pass2/reconciliation.rs b/crates/logical-optimizer/src/pass2/reconciliation.rs deleted file mode 100644 index 36ad637f..00000000 --- a/crates/logical-optimizer/src/pass2/reconciliation.rs +++ /dev/null @@ -1,732 +0,0 @@ -//! [`AccuracyReconciliationStrategy`] — cross-consumer accuracy -//! reconciliation for CSE sharing (issue #273, part of #33). -//! -//! ## The gap this closes -//! -//! `asap_types::ir::cse::share_common_sub_dags` (pre-ASAP CSE) only -//! ever merges two sub-DAGs that are *exactly* [`PartialEq`]-equal, -//! including their [`AggIntent`]'s `accuracy: AccuracyTarget` field. Two -//! otherwise-identical aggregates that differ *only* in how tight an -//! accuracy bound they ask for — `quantile(0.99, x)` at `epsilon=0.01` for -//! one consumer, the same `quantile(0.99, x)` at `epsilon=0.05` for -//! another — are therefore never the same `Rc`, never collapse into one -//! [`crate::pass1::replacement::TargetSubDAGCandidates`], and [`crate::pass1::replacement::SharedSubDAGStrategy`] -//! never even gets a `TargetSubDAG` with `consumer_count >= 2` to propose -//! sharing for. This crate would build two entirely independent sketches -//! for what is conceptually one computation, even though a single sketch -//! built to the tighter of the two bounds would answer both. -//! -//! This module is **additive**, not a relaxation of `share_common_sub_dags` -//! itself: `accuracy` still participates in exact structural equality -//! everywhere else in this crate (correctness elsewhere — e.g. a downstream -//! consumer that pattern-matches on a specific `AccuracyTarget` — depends on -//! that). What this module adds is a *second*, narrower notion of "close -//! enough to share" that sits entirely inside the [`ReplacementStrategy`] -//! extension point: one more candidate a `cost_model::CostModel` -//! may or may not prefer, never a forced rewrite and never a change to what -//! `share_common_sub_dags` itself merges. -//! -//! ## What counts as a "near-duplicate", and why -//! -//! Two `NonASAPOp::Aggregate` nodes are accuracy-near-duplicates here iff, -//! **in this order**: -//! -//! 1. Both are the same bindable shape [`crate::pass1::replacement::ASAPStrategies`] -//! itself targets — a single measure, no `HAVING` (`bindable_intent`'s own -//! scope) — **and** that one measure is one of the four accuracy-bearing -//! [`AggIntent`] variants ([`crate::pass1::realization::accuracy_target`]'s own -//! scope: `Count` / `Quantile` / `Cardinality` / `TopK`). Every other -//! intent has no `AccuracyTarget` to reconcile in the first place. -//! 2. Same `reduction` (grouping), same `output_names`, and the same shared -//! `child` (`Rc::ptr_eq`, or value-equal for two independently-built but -//! identical sub-DAGs CSE conservatively declined to alias) — the same -//! "identical everything else" bar [`crate::pass1::rollup::RollupStrategy`] and -//! [`crate::pass2::topk_reuse::TopKLimitReuseStrategy`] already hold their own -//! sibling-reuse candidates to. -//! 3. The one measure is identical **except** for `accuracy` — same variant, -//! same `col`/`q`/`k` (see [`same_intent_except_accuracy`]). -//! 4. Neither side's `accuracy` is [`AccuracyTarget::Exact`] (see -//! [`dominates`]'s doc for why exact accuracy is excluded rather than -//! trivially "always tightest"). -//! 5. The tighter candidate's own **output** schema carries a provable -//! unique key (`Schema::has_unique_key`) — the exact legality gate -//! `share_common_sub_dags` itself applies (see `cse.rs`'s "Legality" -//! section) and [`crate::pass1::rollup::RollupStrategy::is_legal_rollup_source`] -//! already reuses verbatim for the identical reason: a producer's output -//! is only safely reusable across a second, independent consumer when -//! its row identity is provably stable across reads. A global or -//! `without(...)`-grouped aggregate reports no unique key, so it is never -//! proposed as a reconciliation *source* (it may still be a looser -//! *target* reading from something else that does carry one). -//! -//! ## Safety of tightening: why reading the tighter build is always sound -//! -//! [`crate::pass1::realization::accuracy_budget`] resolves *every* `AccuracyTarget` -//! (`Epsilon`/`EpsilonDelta`) to the literal `(eps, delta)` pair -//! `realizations_for_intent`'s `sketch_realizations` feeds into the -//! analytical sizing — the same numbers `default_size_params`' -//! `kll_k` / `cms_width` / `cms_depth` / `hll_precision` / `kmv_k` / DDSketch's -//! own `alpha == eps` invert. Every shipped formula is monotonic in its -//! input. So a sketch satisfying budget `(e1, d1)` -//! also satisfies any -//! requirement `(e2, d2)` with `e1 <= e2 && d1 <= d2` — [`dominates`]'s exact -//! check — regardless of which of `Epsilon`/`EpsilonDelta` either side is -//! spelled as, because both resolve through the identical `accuracy_budget` -//! mapping before sizing ever happens. Combined with "same [`AggIntent`] -//! variant apart from accuracy" (point 3 above), every algorithm the tighter -//! sibling may select answers the identical aggregate query and must satisfy -//! the tighter budget; the two groups need not independently choose the same -//! algorithm. That is the whole safety argument this module leans on: -//! **never** a claim that some -//! `AccuracyTarget` is "close enough" by fuzzy/heuristic similarity, always -//! a literal Pareto-domination check on the exact numbers a build would be -//! sized with. -//! -//! `AccuracyTarget::Exact` is deliberately excluded from both sides (see -//! [`dominates`]): `realizations_for_intent` routes it to `exact_realization` -//! instead of `sketch_realizations` — a different `Realization` family -//! entirely, not just a tighter budget within the same one — so "build once -//! at the tighter of the two" doesn't mean the same thing there. Reconciling -//! an exact consumer with an approximate one is a different, larger question -//! (does an exact accumulator ever make sense to share with a sketch -//! consumer, cost-wise?) this module leaves alone rather than answers -//! speculatively. -//! -//! Cross-shape comparison (`Epsilon` vs. `EpsilonDelta`) is *not* a design -//! question left open here the way the issue's own "Why this wasn't done in -//! #259" section worried about — `accuracy_budget` already commits both -//! shapes to concrete `(eps, delta)` numbers today (an `Epsilon(e)` resolves -//! to `(e, DEFAULT_DELTA)`), so [`dominates`] compares those resolved numbers -//! directly rather than inventing a second, shape-aware ordering. -//! -//! ## Never forced -//! -//! Like every [`ReplacementStrategy`], this only ever *proposes* — the -//! looser-accuracy consumer's own independently-sized candidate (from -//! [`crate::pass1::replacement::ASAPStrategies`]) stays in its -//! [`crate::pass1::replacement::TargetSubDAGCandidates`] right alongside this strategy's -//! "read the tighter sibling instead" [`Replacement::Rewrite`] candidate; -//! `cost_model::CostModel`-driven ranking picks between them; -//! nothing here removes or filters the independent candidate. -//! -//! ## Costing this candidate shape: a dedicated arm, not a reused one -//! -//! This strategy's candidates carry their own -//! [`crate::pass1::replacement::ReplacementProvenance::AccuracyReconciliation`] -//! rather than reusing `LogicalRewrite` -//! ([`crate::pass1::rollup::RollupStrategy`]/[`crate::pass2::topk_reuse::TopKLimitReuseStrategy`]'s -//! tag), because it needs its own cost treatment in -//! `cost_model::DefaultCostModel::estimate_cost`, not just its own -//! label. Every other `Replacement::Rewrite` shape that reaches -//! `estimate_cost` (`SharedSubDAGStrategy`'s `CseRecompute`, `Rollup`'s and -//! `TopKLimitReuse`'s `LogicalRewrite`) really does rebuild `target` from a -//! different source, so pricing it as "one `cse_recompute_cost` of `target` -//! itself, per consumer" is the right shape of cost. This strategy's -//! candidate never rebuilds `target` at all — it reads `rc` (the tighter -//! sibling), which — per this module's own safety argument — is a sub-DAG -//! this crate is already going to build regardless of whether `target` -//! reads from it too. Pricing it with the same "rebuild `target`, once per -//! consumer" formula would charge it for work it never does, and — because -//! that formula scales with `target.consumer_count` — makes the candidate -//! *more* expensive exactly when sharing would help *more* (more of -//! `target`'s own consumers piggybacking on one already-necessary build): -//! the literal inversion this module's tests -//! (`estimate_cost_does_not_scale_with_the_readers_own_consumer_count`) -//! pin against. `estimate_cost` instead prices this shape as a -//! `cost_model::CostModel::cse_shared_maintenance_cost` read -//! against `rc`'s **own** bound summary — the same order-of-magnitude, -//! per-family cost `SharedSubDAGStrategy`'s own `CseShare` candidate is -//! priced with, reflecting "one more reference into a structure that's -//! already being maintained" rather than "build a whole new one." -//! -//! `candidate_selection::global_selection` treats this rewrite as a cross-group edge: -//! selecting it increments `rc`'s own `effective_consumer_count`, then lets -//! that sibling group propagate the uses through its selected implementation. -//! Accuracy edges are directed strictly from looser to tighter budgets, so -//! they cannot cycle among themselves; both near-duplicates also have the -//! same structural child, so adding the edge preserves the reference DAG's -//! parent-before-child topological ordering. - -use asap_types::ir::operator::non_asap::any_measure_filtered; -use std::cmp::Ordering; -use std::rc::Rc; - -use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::operator::operator_properties::Reduction; -use asap_types::ir::{NonASAPOp, OperatorNode}; -use asap_types::types::AccuracyTarget; - -use crate::pass1::realization::{accuracy_budget, accuracy_target, dominates}; -use crate::pass1::replacement::{ - Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, -}; - -/// `bindable_accuracy_aggregate`'s return shape: `(reduction, intent, -/// accuracy, output_names, child)` — factored into its own alias per -/// `clippy::type_complexity`, not a semantic distinction. -type BindableAccuracyAggregate<'a> = ( - &'a Reduction, - &'a AggIntent, - &'a AccuracyTarget, - &'a [String], - &'a Rc, -); - -/// The `(reduction, intent, accuracy, output_names, child)` shape this -/// module operates on: the same single-measure, no-`HAVING` bindable shape -/// [`crate::pass1::replacement::ASAPStrategies`] targets (see that -/// module's private `bindable_intent`), further narrowed to a measure whose -/// intent actually carries an [`AccuracyTarget`] -/// ([`crate::pass1::realization::accuracy_target`]'s own scope: `Count` / -/// `Quantile` / `Cardinality` / `TopK`). `None` for anything else, including -/// a multi-measure or `HAVING` aggregate, a non-`Aggregate` node, or an -/// accuracy-free intent (`Sum`, `Avg`, …). -fn bindable_accuracy_aggregate(node: &OperatorNode) -> Option> { - let Some(NonASAPOp::Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - }) = node.non_asap() - else { - return None; - }; - let ([intent], None) = (measures.as_slice(), having) else { - return None; - }; - if any_measure_filtered(filters) { - return None; - } - let accuracy = accuracy_target(intent)?; - Some((reduction, intent, accuracy, output_names.as_slice(), child)) -} - -/// Are `a` and `b` the identical [`AggIntent`] apart from `accuracy` — -/// same variant, same `col`/`q`/`k`/ranking basis? Only ever called with two -/// accuracy-bearing intents (both `bindable_accuracy_aggregate`-gated -/// first), but written as a full match rather than assuming that, the same -/// defensive-completeness style [`crate::pass1::replacement::describe_intent`] -/// uses for its own non-exhaustive `AggIntent` match. -fn same_intent_except_accuracy(a: &AggIntent, b: &AggIntent) -> bool { - match (a, b) { - (AggIntent::Count { .. }, AggIntent::Count { .. }) => true, - ( - AggIntent::Quantile { col: c1, q: q1, .. }, - AggIntent::Quantile { col: c2, q: q2, .. }, - ) => c1 == c2 && q1 == q2, - (AggIntent::TopK { k: k1, .. }, AggIntent::TopK { k: k2, .. }) => k1 == k2, - (AggIntent::Cardinality { cols: c1, .. }, AggIntent::Cardinality { cols: c2, .. }) => { - c1 == c2 - } - _ => false, - } -} - -/// `dominates(a, b)` and the two resolved budgets aren't equal — the -/// **strict** ordering [`AccuracyReconciliationStrategy`] actually needs. -/// Without strictness, two `AccuracyTarget`s that resolve to the identical -/// `(eps, delta)` budget via different spellings (e.g. `Epsilon(0.01)` vs. -/// `EpsilonDelta { epsilon: 0.01, delta: DEFAULT_DELTA }`) would each -/// `dominates` the other, and this module would propose "read the sibling -/// instead" both ways for zero actual benefit — a degenerate tie, not a real -/// choice. Requiring strictness means only a build that is genuinely tighter -/// somewhere ever gets proposed as a replacement for a looser one. -fn strictly_tighter(a: &AccuracyTarget, b: &AccuracyTarget) -> bool { - dominates(a, b) && accuracy_budget(a) != accuracy_budget(b) -} - -/// Reconciles near-duplicate [`AggIntent`]s that differ only in their -/// [`AccuracyTarget`] (issue #273, part of #33): given a looser-accuracy -/// [`TargetSubDAG`], finds every already-known sibling `Aggregate` that -/// computes the identical intent over the identical input at a strictly -/// tighter accuracy, and proposes reading that sibling's own (to-be-built) -/// result instead of building an independent, looser copy — "build once at -/// the tightest of the group's accuracy requirements, all consumers read -/// from it," ranked by `cost_model::CostModel` like any other -/// candidate, never forced. See the module docs for the full design. -/// -/// `siblings` is **caller-supplied, not discovered here** — the identical -/// "workload-wide discovery isn't this strategy's job" split -/// [`crate::pass1::rollup::RollupStrategy`] and [`crate::pass2::topk_reuse::TopKLimitReuseStrategy`] -/// already draw; [`crate::pass1::replacement::search_workload_with`] constructs -/// this strategy from the same post-CSE `Aggregate` sibling set it already -/// builds for `RollupStrategy`. -pub struct AccuracyReconciliationStrategy { - siblings: Vec>, -} - -impl AccuracyReconciliationStrategy { - /// A strategy that owns clones of every node in `siblings` and considers - /// each as a candidate tighter-accuracy source (or looser-accuracy - /// target) — typically the full set of `Aggregate` nodes a workload-wide - /// discovery pass already found. - pub fn new(siblings: &[Rc]) -> Self { - Self { - siblings: siblings.to_vec(), - } - } - - /// Every sibling that is a legal, strictly-tighter accuracy source for - /// `target` — shared between `matches` and `replacements` so the two can - /// never disagree about which siblings qualify. Sorted tightest-first - /// (arbitrary but deterministic order over the exhaustive candidate - /// list — this strategy makes no claim about which tighter source a - /// `CostModel` should prefer). - /// - /// Also requires the candidate's own *output* schema to carry a provable - /// unique key ([`Schema::has_unique_key`]) — the exact legality gate - /// `ir::cse::share_common_sub_dags` already applies to its own - /// sharing decisions, and [`crate::pass1::rollup::RollupStrategy`] already - /// reuses verbatim for the identical reason (see that module's - /// `is_legal_rollup_source` doc, point 4): a producer's output is only - /// safely reusable across a second, independent consumer when its row - /// identity is provably stable across reads. Without this, a global or - /// `without(...)`-grouped candidate (whose own `Reduction::by(vec![])` - /// reports no unique key — see `cse.rs`'s "Legality" section) would get - /// proposed for reconciliation even though nothing guarantees a second - /// read of it lines up row-for-row with the first. - fn tighter_sources<'a>(&'a self, target: &TargetSubDAG<'_>) -> Vec<&'a Rc> { - let Some((target_reduction, target_intent, target_accuracy, target_names, target_child)) = - bindable_accuracy_aggregate(target.root) - else { - return Vec::new(); - }; - - let mut sources: Vec<&Rc> = self - .siblings - .iter() - .filter(|candidate| { - if Rc::ptr_eq(candidate, target.root) { - return false; - } - let Some((reduction, intent, accuracy, names, child)) = - bindable_accuracy_aggregate(candidate) - else { - return false; - }; - reduction == target_reduction - && names == target_names - && (Rc::ptr_eq(child, target_child) || child == target_child) - && same_intent_except_accuracy(intent, target_intent) - && strictly_tighter(accuracy, target_accuracy) - && candidate.schema.has_unique_key() - }) - .collect(); - sources.sort_by(|a, b| { - let (.., a_accuracy, _, _) = - bindable_accuracy_aggregate(a).expect("filter above already confirmed this shape"); - let (.., b_accuracy, _, _) = - bindable_accuracy_aggregate(b).expect("filter above already confirmed this shape"); - accuracy_budget(a_accuracy) - .partial_cmp(&accuracy_budget(b_accuracy)) - .unwrap_or(Ordering::Equal) - }); - sources - } -} - -impl ReplacementStrategy for AccuracyReconciliationStrategy { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - !self.tighter_sources(target).is_empty() - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - let Some((.., target_accuracy, _, _)) = bindable_accuracy_aggregate(target.root) else { - return Vec::new(); - }; - self.tighter_sources(target) - .into_iter() - .map(|source| { - let (.., source_accuracy, _, _) = bindable_accuracy_aggregate(source) - .expect("tighter_sources only returns bindable_accuracy_aggregate matches"); - ReplacementSubDAG { - strategy: self.name(), - replacement: Replacement::SubDAG(Rc::clone(source)), - provenance: ReplacementProvenance::AccuracyReconciliation, - rationale: format!( - "reuses a near-duplicate sibling aggregate — identical intent and grouping \ - over the same shared input, differing only in AccuracyTarget — built to a \ - strictly tighter accuracy ({source_accuracy:?} dominates this node's own \ - {target_accuracy:?} on both eps and delta) instead of an independent, \ - looser-accuracy copy; every shipped sketch sizing formula is monotonic in \ - (eps, delta), so a build sized to the tighter bound always satisfies this \ - consumer's own looser one too — see AccuracyReconciliationStrategy's module \ - docs for the full argument" - ), - } - }) - .collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::ir::cse::share_common_sub_dags; - use asap_types::ir::operator::operator_properties::{GroupKeys, Source}; - use asap_types::ir::schema::{ColumnId, DataType, Field, Schema}; - - /// `[ts(0), value(1), job(2)]`. - /// A unique-keyed scan (`[ts]`) so `share_common_sub_dags` is actually - /// willing to hoist it — see `Schema::has_unique_key`/`cse.rs`'s own - /// "Legality" section: a producer with no provable unique key is always - /// inserted fresh, never hoisted, regardless of structural equality. - fn metric_scan() -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("job", DataType::Utf8, true), - ], - 0, - vec![vec![0]], - ), - })) - .unwrap() - } - - fn agg(by: Vec, intent: AggIntent, child: &Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::by(by), - measures: vec![intent], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(child), - })) - .unwrap() - } - - fn quantile(q: f64, accuracy: AccuracyTarget, child: &Rc) -> Rc { - agg( - vec![2], - AggIntent::Quantile { - col: None, - q, - accuracy, - }, - child, - ) - } - - /// A globally-grouped (`by(vec![])`) quantile — `aggregate_output_schema` - /// reports no unique key for an empty `by` (see `aggregate_schema.rs`'s own - /// `unique_keys = if by.is_empty() || has_count_values { vec![] } else - /// { .. }`). - fn ungrouped_quantile( - q: f64, - accuracy: AccuracyTarget, - child: &Rc, - ) -> Rc { - agg( - vec![], - AggIntent::Quantile { - col: None, - q, - accuracy, - }, - child, - ) - } - - /// A `without(...)`-grouped quantile — never carries a provable unique - /// key regardless of the excluded set (mirrors `rollup.rs`'s own - /// `without_agg` test helper). - fn without_quantile( - q: f64, - accuracy: AccuracyTarget, - excluded: Vec, - child: &Rc, - ) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(excluded)), - measures: vec![AggIntent::Quantile { - col: None, - q, - accuracy, - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::clone(child), - })) - .unwrap() - } - - // ── dominates / strictly_tighter ───────────────────────────────────── - - #[test] - fn a_smaller_epsilon_dominates_a_larger_one() { - assert!(dominates( - &AccuracyTarget::Epsilon(0.01), - &AccuracyTarget::Epsilon(0.05) - )); - assert!(!dominates( - &AccuracyTarget::Epsilon(0.05), - &AccuracyTarget::Epsilon(0.01) - )); - } - - #[test] - fn epsilon_delta_needs_both_dimensions_at_least_as_tight() { - // Smaller epsilon but larger delta: neither Pareto-dominates. - let a = AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.1, - }; - let b = AccuracyTarget::EpsilonDelta { - epsilon: 0.05, - delta: 0.01, - }; - assert!(!dominates(&a, &b)); - assert!(!dominates(&b, &a)); - } - - #[test] - fn exact_never_dominates_or_is_dominated() { - assert!(!dominates( - &AccuracyTarget::Exact, - &AccuracyTarget::Epsilon(0.5) - )); - assert!(!dominates( - &AccuracyTarget::Epsilon(0.5), - &AccuracyTarget::Exact - )); - assert!(!dominates(&AccuracyTarget::Exact, &AccuracyTarget::Exact)); - } - - #[test] - fn equal_resolved_budgets_are_not_strictly_tighter_either_way() { - // Same numeric (eps, delta) budget, different AccuracyTarget spelling. - let epsilon_only = AccuracyTarget::Epsilon(0.01); - let equivalent_epsilon_delta = AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: crate::pass1::realization::DEFAULT_DELTA, - }; - assert!(dominates(&epsilon_only, &equivalent_epsilon_delta)); - assert!(dominates(&equivalent_epsilon_delta, &epsilon_only)); - assert!(!strictly_tighter(&epsilon_only, &equivalent_epsilon_delta)); - assert!(!strictly_tighter(&equivalent_epsilon_delta, &epsilon_only)); - } - - // ── the issue's own scenario: quantile(0.99, x) at eps=0.01 vs eps=0.05 ─ - - #[test] - fn looser_quantile_proposes_reading_the_tighter_sibling() { - let scan = metric_scan(); - let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - - let strategy = AccuracyReconciliationStrategy::new(&[Rc::clone(&tight), Rc::clone(&loose)]); - - assert!(strategy.matches(&TargetSubDAG::new(&loose))); - let replacements = strategy.replacements(&TargetSubDAG::new(&loose)); - assert_eq!(replacements.len(), 1); - let Replacement::SubDAG(rc) = &replacements[0].replacement else { - panic!("expected a Rewrite candidate"); - }; - assert!(Rc::ptr_eq(rc, &tight)); - assert_eq!( - replacements[0].provenance, - ReplacementProvenance::AccuracyReconciliation - ); - - // The tighter side has nothing looser to read from: no candidate. - assert!(!strategy.matches(&TargetSubDAG::new(&tight))); - assert!(strategy.replacements(&TargetSubDAG::new(&tight)).is_empty()); - } - - #[test] - fn end_to_end_search_workload_proposes_the_reconciliation_candidate() { - // The full search_workload_with entry point, not just the strategy in - // isolation — proves this is actually wired into the round loop - // alongside RollupStrategy/TopKLimitReuseStrategy. - let scan = metric_scan(); - let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let loose = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - - let space = - crate::pass1::replacement::search_workload(vec![("tight", tight), ("loose", loose)]); - - let loose_root = &space.roots[1].1; - let loose_group = space - .candidates_for_target(loose_root) - .expect("loose consumer's own target has a group"); - assert!( - loose_group.candidates.iter().any(|candidate| { - candidate.strategy == "AccuracyReconciliationStrategy" - && matches!(candidate.replacement, Replacement::SubDAG(_)) - }), - "expected an AccuracyReconciliationStrategy candidate for the looser consumer, got: \ - {:?}", - loose_group - .candidates - .iter() - .map(|c| c.strategy) - .collect::>() - ); - } - - #[test] - fn different_columns_are_not_near_duplicates() { - let scan = metric_scan(); - let a = agg( - vec![2], - AggIntent::Quantile { - col: Some(1), - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.01), - }, - &scan, - ); - let b = agg( - vec![2], - AggIntent::Quantile { - col: Some(0), - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.05), - }, - &scan, - ); - let strategy = AccuracyReconciliationStrategy::new(&[Rc::clone(&a), Rc::clone(&b)]); - assert!(!strategy.matches(&TargetSubDAG::new(&b))); - } - - #[test] - fn topk_intents_differing_only_in_accuracy_are_near_duplicates() { - let tight = AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.01), - }; - let loose = AggIntent::TopK { - k: 10, - accuracy: AccuracyTarget::Epsilon(0.02), - }; - assert!(same_intent_except_accuracy(&tight, &loose)); - } - - #[test] - fn different_grouping_is_not_a_near_duplicate() { - let scan = metric_scan(); - let tight = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let loose = agg( - vec![], - AggIntent::Quantile { - col: None, - q: 0.99, - accuracy: AccuracyTarget::Epsilon(0.05), - }, - &scan, - ); - let strategy = AccuracyReconciliationStrategy::new(&[Rc::clone(&tight), Rc::clone(&loose)]); - assert!(!strategy.matches(&TargetSubDAG::new(&loose))); - } - - #[test] - fn exact_accuracy_is_never_reconciled() { - let scan = metric_scan(); - let exact = quantile(0.99, AccuracyTarget::Exact, &scan); - let approx = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - let strategy = - AccuracyReconciliationStrategy::new(&[Rc::clone(&exact), Rc::clone(&approx)]); - assert!(!strategy.matches(&TargetSubDAG::new(&approx))); - assert!(!strategy.matches(&TargetSubDAG::new(&exact))); - } - - // ── exact structural equality / share_common_sub_dags is unchanged ──── - - #[test] - fn share_common_subdags_still_never_merges_differing_accuracy() { - // The additive guarantee this issue explicitly must not violate: - // pre-ASAP CSE's own exact-equality merge stays exact. Two - // aggregates differing only in `accuracy` must come back as two - // distinct `Rc`s, not one shared `Rc` — AccuracyReconciliationStrategy - // is the *only* place cross-accuracy sharing gets proposed, never - // `share_common_sub_dags` itself. - let scan = metric_scan(); - let a = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let b = quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - - let roots = share_common_sub_dags(vec![("a", a), ("b", b)]); - assert!( - !Rc::ptr_eq(&roots[0].1, &roots[1].1), - "share_common_sub_dags must not merge aggregates with different AccuracyTarget" - ); - assert_ne!( - roots[0].1, roots[1].1, - "the two aggregates really are structurally different (accuracy differs)" - ); - - // The identical scan child, though, is still shared exactly as - // before — this module changes nothing about that. - let Some(NonASAPOp::Aggregate { child: child_a, .. }) = roots[0].1.non_asap() else { - panic!("expected an Aggregate root"); - }; - let Some(NonASAPOp::Aggregate { child: child_b, .. }) = roots[1].1.non_asap() else { - panic!("expected an Aggregate root"); - }; - assert!(Rc::ptr_eq(child_a, child_b)); - } - - #[test] - fn identical_accuracy_still_merges_via_ordinary_cse() { - // Sanity check the fixture itself: truly identical aggregates - // (same accuracy too) still merge via share_common_sub_dags's own - // exact equality — unrelated to this module, but pins the contrast - // with the test above. - let scan = metric_scan(); - let a = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let b = quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - - let roots = share_common_sub_dags(vec![("a", a), ("b", b)]); - assert!(Rc::ptr_eq(&roots[0].1, &roots[1].1)); - } - - // ── legality gate: the tighter source needs a provable unique key ───── - - #[test] - fn a_globally_grouped_tighter_sibling_with_no_unique_key_is_not_a_source() { - // `by(vec![])` (global aggregation) reports no unique key - // (`aggregate_output_schema`'s own `unique_keys = if by.is_empty() .. - // { vec![] } ..`) — the same legality gate - // `share_common_sub_dags`/`RollupStrategy` apply, which this - // strategy must not bypass (module docs, point 5). - let scan = metric_scan(); - let tight = ungrouped_quantile(0.99, AccuracyTarget::Epsilon(0.01), &scan); - let loose = ungrouped_quantile(0.99, AccuracyTarget::Epsilon(0.05), &scan); - - assert!( - !tight.schema.clone().has_unique_key(), - "fixture sanity: a globally-grouped aggregate has no provable unique key" - ); - - let strategy = AccuracyReconciliationStrategy::new(&[Rc::clone(&tight), Rc::clone(&loose)]); - assert!(!strategy.matches(&TargetSubDAG::new(&loose))); - assert!(strategy.replacements(&TargetSubDAG::new(&loose)).is_empty()); - } - - #[test] - fn a_without_grouped_tighter_sibling_with_no_unique_key_is_not_a_source() { - // Mirrors RollupStrategy's own - // `no_unique_key_on_the_finer_side_does_not_roll_up`: a - // `without(...)` grouping never carries a provable unique key, - // regardless of the excluded set. - let scan = metric_scan(); - let tight = without_quantile(0.99, AccuracyTarget::Epsilon(0.01), vec![2], &scan); - let loose = without_quantile(0.99, AccuracyTarget::Epsilon(0.05), vec![2], &scan); - - assert!( - !tight.schema.clone().has_unique_key(), - "fixture sanity: a without(...) aggregate has no provable unique key" - ); - - let strategy = AccuracyReconciliationStrategy::new(&[Rc::clone(&tight), Rc::clone(&loose)]); - assert!(!strategy.matches(&TargetSubDAG::new(&loose))); - assert!(strategy.replacements(&TargetSubDAG::new(&loose)).is_empty()); - } -} diff --git a/crates/logical-optimizer/src/pass2/topk_reuse.rs b/crates/logical-optimizer/src/pass2/topk_reuse.rs deleted file mode 100644 index f58c35d7..00000000 --- a/crates/logical-optimizer/src/pass2/topk_reuse.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Workload-aware reuse between compatible ordered limits. -//! -//! If two queries rank the same input and request top-k results with -//! `small_k < large_k`, the smaller result is exactly the first `small_k` -//! rows of the larger result. This strategy therefore replaces -//! `Limit(small_k, Sort(X))` with `Limit(small_k, Limit(large_k, Sort(X)))`. - -use std::rc::Rc; - -use asap_types::ir::{NonASAPOp, OperatorNode}; - -use crate::pass1::replacement::{ - Replacement, ReplacementProvenance, ReplacementStrategy, ReplacementSubDAG, TargetSubDAG, -}; - -/// Derives a smaller top-k result from a compatible larger top-k sibling. -pub struct TopKLimitReuseStrategy { - limits: Vec>, -} - -impl TopKLimitReuseStrategy { - pub fn new(limits: &[Rc]) -> Self { - Self { - limits: limits.to_vec(), - } - } - - fn larger_sources<'a>(&'a self, target: &TargetSubDAG<'_>) -> Vec<&'a Rc> { - let Some(NonASAPOp::Limit { - n: Some(target_n), - offset: 0, - child: target_child, - .. - }) = target.root.non_asap() - else { - return Vec::new(); - }; - - let mut sources: Vec<_> = self - .limits - .iter() - .filter(|candidate| { - if Rc::ptr_eq(candidate, target.root) { - return false; - } - let Some(NonASAPOp::Limit { - n: Some(n), - offset: 0, - child, - .. - }) = candidate.non_asap() - else { - return false; - }; - n > target_n - && (Rc::ptr_eq(child, target_child) || child.as_ref() == target_child.as_ref()) - }) - .collect(); - // Prefer the smallest sufficient materialized top-k when several - // larger siblings are available. - sources.sort_by_key(|source| match source.non_asap() { - Some(NonASAPOp::Limit { n: Some(n), .. }) => *n, - _ => unreachable!(), - }); - sources - } -} - -impl ReplacementStrategy for TopKLimitReuseStrategy { - fn matches(&self, target: &TargetSubDAG<'_>) -> bool { - !self.larger_sources(target).is_empty() - } - - fn replacements(&self, target: &TargetSubDAG<'_>) -> Vec { - let Some(NonASAPOp::Limit { - n: Some(target_n), - offset: 0, - partition_by, - .. - }) = target.root.non_asap() - else { - return Vec::new(); - }; - - self.larger_sources(target) - .into_iter() - .filter_map(|source| { - let source_n = match source.non_asap() { - Some(NonASAPOp::Limit { n: Some(n), .. }) => *n, - _ => unreachable!(), - }; - let rewritten = OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Limit { - n: Some(*target_n), - offset: 0, - partition_by: partition_by.clone(), - child: Rc::clone(source), - })) - .ok()?; - Some(ReplacementSubDAG { - strategy: "TopKLimitReuseStrategy", - replacement: Replacement::SubDAG(rewritten), - provenance: ReplacementProvenance::LogicalRewrite, - rationale: format!( - "derives top-{target_n} from the compatible shared top-{source_n} result; both rank the identical input with the same ordering" - ), - }) - }) - .collect() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::scan; - use asap_types::ir::operator::operator_properties::GroupKeys; - use asap_types::ir::schema::Schema; - - fn scan_named(metric: &str) -> Rc { - scan(metric, Schema::with_time_index(vec![], 0, vec![])) - } - - fn limit(n: usize, offset: usize, child: Rc) -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Limit { - n: Some(n), - offset, - partition_by: GroupKeys::none(), - child, - })) - .unwrap() - } - - #[test] - fn smaller_limit_reuses_larger_compatible_limit() { - let child = scan_named("m"); - let small = limit(5, 0, Rc::clone(&child)); - let large = limit(10, 0, child); - let strategy = TopKLimitReuseStrategy::new(&[Rc::clone(&small), Rc::clone(&large)]); - let replacements = strategy.replacements(&TargetSubDAG::new(&small)); - assert_eq!(replacements.len(), 1); - let Replacement::SubDAG(rewrite) = &replacements[0].replacement else { - panic!() - }; - let Some(NonASAPOp::Limit { - n: Some(5), child, .. - }) = rewrite.non_asap() - else { - panic!() - }; - assert!(Rc::ptr_eq(child, &large)); - } - - #[test] - fn offset_or_different_input_is_not_reused() { - let a = scan_named("a"); - let b = scan_named("b"); - let small = limit(5, 0, a); - let large = limit(10, 0, b); - let offset = limit(20, 1, scan_named("a")); - let strategy = TopKLimitReuseStrategy::new(&[Rc::clone(&small), large, offset]); - assert!(!strategy.matches(&TargetSubDAG::new(&small))); - } -} diff --git a/crates/plan-selection/src/cost/cost_model.rs b/crates/plan-selection/src/cost/cost_model.rs deleted file mode 100644 index 783c1363..00000000 --- a/crates/plan-selection/src/cost/cost_model.rs +++ /dev/null @@ -1,1302 +0,0 @@ -//! Cost model interface (issues #6, #33). -//! -//! `asap-plan` deliberately has no cost model *implementation* of its own — -//! ranking candidate summaries by real cost (bandwidth budget, memory -//! footprint, site count, observed drift, workload-level CSE credit, …) -//! needs knowledge this crate doesn't have and shouldn't acquire: the crate -//! doc's layering invariant is that `asap-plan` depends only on [`asap_ir`], -//! never on a runtime or a deployment model. What it *can* own is the -//! interface every deployment's cost model plugs into, so selection over the -//! candidates [`replacement`] generates has exactly one extension point. -//! Candidate generation itself does not consult a cost model. -//! -//! [`CostModel::rank_candidates`] is scoped to the approximate-**sketch** -//! family specifically (it takes [`SketchAlgorithm`]s) — `asap_sketch` also has -//! sibling families for sampling-based, wavelet-transform, and fitted -//! statistical-model summaries -//! ([`asap_types::ir::schema::SamplingKind`]/…/[`asap_types::ir::schema::StatModelKind`]), -//! each with its own `(Kind, Params)` pair, deliberately *not* folded into -//! this trait: no core `AggIntent` picks one of those families today, so -//! there is no ranking decision for this trait to own yet. Should a family -//! other than `Sketch` ever need its own ranking, it gets its own trait -//! method rather than overloading this one across incompatible `Kind` types. -//! -//! ## CSE sharing (issue #237, #223 stage 4) -//! -//! [`CseCandidate`]/[`ShareDecision`]/[`CostModel::cse_share_decision`] below -//! decide whether a CSE-detected shared sub-DAG -//! ([`asap_types::ir::cse::share_common_sub_dags`], issue #223 stages -//! 1-2, PR #235) is actually worth sharing, via a real Volcano/Cascades-style -//! cost comparison rather than a fixed rule. See -//! `docs/design_docs/cse-cost-model-decision.md` for the full design discussion (why -//! cost-based, why not a full plan-search engine, the layering constraint -//! that forces detection to stay cost-agnostic). -//! `cost_sorted` -//! (via [`asap_logical_optimizer::pass1::replacement`]'s own `cse_preference`) and -//! [`DefaultCostModel::estimate_cost`] are this crate's own callers. - -use std::rc::Rc; - -use asap_logical_optimizer::pass1::exact_composition::ExactOperation; -use asap_types::ir::operator::agg_intent::AggIntent; -use asap_types::ir::schema::{ - FieldDataType, GroupingStrategy, HydraParams, SketchAlgorithm, SketchParams, -}; -use asap_types::ir::{ASAPOp, Operator, OperatorNode}; - -use crate::cost::recurrence::{ - self, CostRate, EvaluationRate, Horizon, RecurrenceCostExplanation, RecurrenceError, - RecurrenceProfile, -}; -use asap_logical_optimizer::pass1::exact_composition::{ExactComposition, OperationPlacement}; -use asap_logical_optimizer::pass1::replacement::{ - realize_child, Replacement, ReplacementProvenance, ReplacementSubDAG, TargetSubDAG, -}; - -// ── Recurring-cost vocabulary for mixed exact/summary plans (issue #171) ── - -pub use asap_types::cost::CostUnit; - -/// Who produced a set of [`ExactCompositionCostInputs`], and under which -/// model version — carried into every composed decision's explanation and -/// DAG export so a reviewer can tell a deployment's measured numbers from -/// a placeholder. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct CostProvenance { - /// The cost model's own name (e.g. `"DefaultCostModel"`). - pub model: String, - /// The model's own version string, whatever scheme it uses. - pub version: String, -} - -/// Which mixed-execution shapes the downstream runtime can actually -/// execute (issue #171). [`asap_logical_optimizer::pass1::exact_composition::ExactCompositionStrategy`] -/// proposes an `ValueOperationAtQueryTime` candidate only when -/// `query_time` is set, and an `ValueOperationAtIngestionTime` candidate only -/// when `ingestion_time` is — a runtime that cannot run an exact -/// operator on the update path must never be handed one. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct ValueOperationCapabilities { - /// The runtime can apply an exact operator to summary evaluations at - /// query evaluation time. - pub query_time: bool, - /// The runtime can apply an exact row transform on the update path, - /// feeding its output into maintained summary state. - pub ingestion_time: bool, -} - -impl ValueOperationCapabilities { - /// Neither shape supported. - pub const NONE: Self = Self { - query_time: false, - ingestion_time: false, - }; - /// Both shapes supported. - pub const ALL: Self = Self { - query_time: true, - ingestion_time: true, - }; - - pub fn supports(self, placement: OperationPlacement) -> bool { - match placement { - OperationPlacement::Read => self.query_time, - OperationPlacement::Maintenance => self.ingestion_time, - } - } -} - -/// What [`CostModel::exact_composition_cost_inputs`] is asked about: one -/// composed alternative at one site, paired with the concrete summary it -/// composes with. -#[derive(Debug, Clone, Copy)] -pub struct ExactCompositionCostRequest<'a> { - /// The target the composed candidate replaces. - pub target: &'a OperatorNode, - /// The composition itself — placement, operator, child target. - pub composition: &'a ExactComposition, - /// For [`OperationPlacement::Read`]: the child target's *selected* - /// summary evaluation candidate the exact operator consumes. For - /// [`OperationPlacement::Maintenance`]: the maintained summary *above* the - /// transform that consumes its output (the `SummaryAgg` this transform - /// feeds). Either way, the summary whose maintenance/read cost the - /// formula charges. - pub summary: &'a OperatorNode, - /// How many times this site actually runs once ancestors' own choices - /// are accounted for (see `candidate_selection::global_selection`). - pub effective_consumer_count: usize, -} - -/// Every input the issue #171 cost formulas need, each individually -/// optional: **an unknown stays `None` — never a zero** — so a formula -/// with a missing input yields no rate at all rather than a spuriously -/// cheap one, and global selection then keeps the conservative -/// keep-as-is behavior. A deployment model that wants defaults supplies -/// them explicitly by overriding [`CostModel::exact_composition_cost_inputs`]. -#[derive(Debug, Clone, PartialEq)] -pub struct ExactCompositionCostInputs { - /// Exact operator cost per row it processes — per evaluation row for a - /// read-time operation, per input row for an maintenance-time operation. - pub exact_cost_per_row: Option, - /// Rows the exact operator consumes per evaluation (read-time operation) or - /// per update (transform). - pub expected_input_rows: Option, - /// Rows the exact operator emits per evaluation/update. - pub expected_output_rows: Option, - /// Cost of one update to the composed-with summary's maintained state. - pub summary_maintenance_cost_per_update: Option, - /// Cost of one evaluation of that summary at evaluation time. - pub summary_read_cost: Option, - /// Update (ingest) events per second reaching this site. - pub update_rate: Option, - /// Evaluations per second across every consumer of this site. - pub evaluation_rate: Option, - /// Cost of one full raw recompute of the target from pre-ASAP data — - /// the kept-query baseline's per-evaluation cost. - pub raw_recompute_cost: Option, - /// Recurring formulas require `CostUnitsPerSecond`; totals yield no rate. - pub unit: CostUnit, - pub provenance: CostProvenance, -} - -impl ExactCompositionCostInputs { - /// Every input unknown, attributed to `provenance` — what a model that - /// has no statistics for a site returns. - pub fn unknown(provenance: CostProvenance) -> Self { - Self { - exact_cost_per_row: None, - expected_input_rows: None, - expected_output_rows: None, - summary_maintenance_cost_per_update: None, - summary_read_cost: None, - update_rate: None, - evaluation_rate: None, - raw_recompute_cost: None, - unit: CostUnit::CostUnitsPerSecond, - provenance, - } - } - - /// The rate for whichever composition placement is requested — - /// [`read_operation_plan_cost_rate`] or [`maintenance_operation_plan_cost_rate`]. - pub fn composed_plan_cost_rate(&self, placement: OperationPlacement) -> Option { - match placement { - OperationPlacement::Read => read_operation_plan_cost_rate(self), - OperationPlacement::Maintenance => maintenance_operation_plan_cost_rate(self), - } - } -} - -/// Outer exact read-time operation over a maintained summary: -/// -/// ```text -/// read_operation_plan_cost_rate = -/// update_rate * summary_maintenance_cost_per_update -/// + evaluation_rate * (summary_read_cost -/// + output_rows_per_eval * exact_read-time operation_cost_per_row) -/// ``` -/// -/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. -pub fn read_operation_plan_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { - if inputs.unit != CostUnit::CostUnitsPerSecond { - return None; - } - let maintenance = inputs.update_rate? * inputs.summary_maintenance_cost_per_update?; - let per_eval = - inputs.summary_read_cost? + inputs.expected_output_rows? * inputs.exact_cost_per_row?; - let evaluation = inputs.evaluation_rate?.0 * per_eval; - finite_rate(maintenance + evaluation) -} - -/// Outer maintained summary over an exact maintenance-time operation: -/// -/// ```text -/// maintenance_operation_plan_cost_rate = -/// update_rate * (exact_function_cost_per_input_row -/// + summary_maintenance_cost_per_update) -/// + evaluation_rate * summary_read_cost -/// ``` -/// -/// `None` if any input is unknown — see [`ExactCompositionCostInputs`]. -pub fn maintenance_operation_plan_cost_rate( - inputs: &ExactCompositionCostInputs, -) -> Option { - if inputs.unit != CostUnit::CostUnitsPerSecond { - return None; - } - let per_update = inputs.exact_cost_per_row? + inputs.summary_maintenance_cost_per_update?; - let maintenance = inputs.update_rate? * per_update; - let evaluation = inputs.evaluation_rate?.0 * inputs.summary_read_cost?; - finite_rate(maintenance + evaluation) -} - -/// The raw/pre-ASAP fallback baseline: -/// -/// ```text -/// raw_recompute_cost_rate = evaluation_rate * raw_recompute_cost -/// ``` -/// -/// `None` if either input is unknown — see [`ExactCompositionCostInputs`]. -pub fn raw_recompute_cost_rate(inputs: &ExactCompositionCostInputs) -> Option { - if inputs.unit != CostUnit::CostUnitsPerSecond { - return None; - } - finite_rate(inputs.evaluation_rate?.0 * inputs.raw_recompute_cost?) -} - -fn finite_rate(units_per_second: f64) -> Option { - units_per_second - .is_finite() - .then_some(CostRate(units_per_second)) -} - -/// A CSE-detected, legality-gated shared sub-DAG with two or more consumers -/// — the unit [`CostModel::cse_share_decision`] decides over. Built by -/// `cost_sorted` -/// (via [`asap_logical_optimizer::pass1::replacement`]'s own `cse_preference`) the first time it -/// needs a representative bound node for a sub-DAG that -/// [`asap_types::ir::cse::share_common_sub_dags`] already collapsed -/// onto one `Rc` for two or more workload roots. See -/// `docs/design_docs/cse-cost-model-decision.md`. -pub struct CseCandidate<'a> { - /// The shared sub-DAG itself. - pub sub_dag: &'a Rc, - /// The node this sub-DAG bound to — gives the cost model the - /// concrete `FieldDataType`/`(kind, params)` actually at stake, not - /// just the logical shape. - pub bound_summary: &'a OperatorNode, - /// How many workload roots reference this exact shared sub-DAG, counted - /// once up front over the whole workload (always >= 2 — a candidate is - /// only ever constructed for an actually-shared sub-DAG). - pub consumer_count: usize, -} - -/// A cost estimate produced by a [`CostModel`] hook. A newtype around `f64` -/// rather than a bare `f64` return type, so a future cost dimension (e.g. -/// separate CPU/memory/network estimates, once a deployment actually needs -/// to compare along more than one axis) can be added as a field here -/// without changing every hook's signature a second time. Today it's still -/// a single unitless scalar — the same magnitude convention -/// [`default_cse_recompute_cost`]/[`default_cse_shared_maintenance_cost`] -/// already used as bare `f64`s, just wrapped. -#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] -pub struct Cost(pub f64); - -impl Cost { - /// The cost of an operation that costs nothing at all. - pub const ZERO: Cost = Cost(0.0); -} - -impl std::fmt::Display for Cost { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl std::ops::Add for Cost { - type Output = Cost; - fn add(self, rhs: Cost) -> Cost { - Cost(self.0 + rhs.0) - } -} - -impl std::ops::Mul for Cost { - type Output = Cost; - fn mul(self, rhs: usize) -> Cost { - Cost(self.0 * rhs as f64) - } -} - -/// The decision [`CostModel::cse_share_decision`] returns for one -/// [`CseCandidate`]. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ShareDecision { - /// Reuse one bound node across every consumer. - Share, - /// Bind each occurrence independently — the shared-maintenance cost - /// isn't worth it for this candidate. - RecomputeIndependently, -} - -/// Default [`CostModel::cse_recompute_cost`]: a structural-size proxy — the -/// number of *unique* nodes in `sub-DAG`'s DAG -/// ([`asap_types::ir::cse::dag_node_count`], the same module this -/// candidate's sharing was detected in). Deliberately **not** a raw -/// `serde_json` serialization length: after CSE, `sub-DAG` is generally a -/// DAG, not a tree (a `CseCandidate` only exists because something got -/// shared), and a naive full serialization re-serializes — over-counts — -/// any descendant `sub-DAG` already shares internally, once per parent -/// that references it, instead of once for the whole DAG. `dag_node_count` -/// dedupes by `Rc` pointer identity, so it charges each unique node's -/// contribution exactly once regardless of how many places within -/// `sub-DAG` reference it. Cheap to compute (one pass, no serialization), -/// and still scales with real structural complexity — a genuinely tiny -/// leaf costs little to recompute, a deep multi-join sub-DAG costs a lot. -/// A deployment with real per-row/per-update cost knowledge should -/// override [`CostModel::cse_recompute_cost`] instead of relying on this. -pub fn default_cse_recompute_cost(sub_dag: &Rc) -> Cost { - Cost(asap_types::ir::cse::dag_node_count(sub_dag) as f64) -} - -/// Default [`CostModel::cse_shared_maintenance_cost`]: a small -/// per-[`FieldDataType`] weight, scaled to the same order of magnitude -/// as [`default_cse_recompute_cost`]'s typical output (a small node -/// count, not a byte length), reflecting that families differ in how -/// expensive they are to keep *continuously updated* for the life of a -/// workload — an exact accumulator is the cheapest (an O(1) merge), -/// sketches/samples cost more (a whole data structure to update per new -/// row), wavelets/fitted models cost the most (coefficient/parameter -/// maintenance). These weights are illustrative, not measured — a -/// deployment with real memory/update-cost numbers should override -/// [`CostModel::cse_shared_maintenance_cost`] instead of relying on this -/// table. -pub fn default_cse_shared_maintenance_cost(family: &FieldDataType) -> Cost { - const UNIT: f64 = 1.0; - let weight = match family { - FieldDataType::Plain(_) => 1.0, - FieldDataType::ExactAggregate(..) => 1.0, - FieldDataType::Sketch(..) => 3.0, - FieldDataType::Sample(..) => 3.0, - FieldDataType::Wavelet(..) => 5.0, - FieldDataType::StatModel(..) => 6.0, - }; - Cost(weight * UNIT) -} - -/// Selection-time preferences and costs over the candidates Stage 1 -/// generates. -/// -/// [`replacement::summary_candidates`] returns every algorithm that *can* answer an -/// intent, in an arbitrary static preference order (issue #98's "one home" -/// for the candidate set), and candidate generation keeps that order. A -/// `CostModel` re-orders the candidates when they are selected, under real, -/// deployment-specific cost knowledge this crate has no way to know about. -pub trait CostModel { - /// Whether [`Self::candidate_cost`] prices a complete physical - /// alternative, including its raw baseline, rather than a local - /// heuristic for one memo-group node. - /// - /// Complete-plan models make every CSE alternative available to final - /// ranking. Choosing a share/recompute arm first through the legacy - /// structural hooks would discard a physical alternative before its - /// evidence-backed cost was compared. - fn candidate_cost_covers_complete_plan(&self) -> bool { - false - } - - /// Opt into historical qualitative ranking/CSE decisions when no numeric - /// candidate cost exists. The safe default leaves an uncosted candidate - /// unselected; a model with an intentional non-numeric policy overrides - /// this to `true`. - fn allow_uncosted_legacy_selection(&self) -> bool { - false - } - - /// Candidate-level cost availability for final selection. A non-finite or - /// negative estimate is unknown/invalid, never an available cost. - fn candidate_cost( - &self, - candidate: &ReplacementSubDAG, - target: &TargetSubDAG<'_>, - ) -> Option { - let value = self.estimate_cost(candidate, target); - (value.is_finite() && value >= 0.0).then_some(Cost(value)) - } - - /// Rank `candidates` (as returned by - /// [`summary_candidates`](asap_logical_optimizer::pass1::realization::summary_candidates)) for - /// `intent`, best choice first. - /// - /// Implementations MAY reorder freely, but MUST return exactly the input - /// candidates: no additions, removals, or duplicates. Semantic legality - /// belongs to replacement generation; cost availability is reported - /// separately by `candidate_cost`. Letting this hook filter would violate - /// [`ReplacementStrategy`]'s exhaustive, never-prune contract. This - /// invariant is checked at every production call site. - /// - /// [`ReplacementStrategy`]: asap_logical_optimizer::pass1::replacement::ReplacementStrategy - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec; - - /// Estimated number of distinct subpopulations produced by `target`'s - /// grouping keys. `None` means the deployment has no cardinality estimate; - /// grouping alternatives remain legal but keep their discovery order. - fn estimated_subpopulation_count(&self, _target: &OperatorNode) -> Option { - None - } - - /// Comparable memory-state cost for a sketch grouping candidate. The - /// default compares `N` independent inner sketches against the complete - /// shared Hydra grid, using [`Self::estimated_subpopulation_count`]. - fn grouping_state_cost( - &self, - candidate: &ReplacementSubDAG, - target: &TargetSubDAG<'_>, - ) -> Option { - let Replacement::SubDAG(node) = &candidate.replacement else { - return None; - }; - let (kind, grouping) = sketch_state(node)?; - let inner = sketch_state_units(kind.params()); - let units = match grouping { - GroupingStrategy::PerSubpopulationInstance => { - inner * self.estimated_subpopulation_count(target.root)? as f64 - } - GroupingStrategy::SharedMultiSubpopulation { params, .. } => { - inner * hydra_grid_cells(params) - } - }; - Some(Cost(units)) - } - - /// Estimate the one-time cost of recomputing `candidate.sub-DAG` - /// independently at a single use site. Default: - /// [`default_cse_recompute_cost`] (a structural-size proxy). See - /// `docs/design_docs/cse-cost-model-decision.md`. - fn cse_recompute_cost(&self, candidate: &CseCandidate) -> Cost { - default_cse_recompute_cost(candidate.sub_dag) - } - - /// Estimate the cost of maintaining `candidate.bound_summary` as one - /// continuously-updated shared summary for the life of the workload. - /// Default: [`default_cse_shared_maintenance_cost`] (a per-family - /// weight table), applied to whichever field of - /// `candidate.bound_summary`'s output schema actually carries summary - /// state (falls back to the cheapest, `Plain`, weight if none does — - /// e.g. `bound_summary` is a kept non-ASAP sub-DAG with nothing - /// summary-shaped to maintain). See `docs/design_docs/cse-cost-model-decision.md`. - fn cse_shared_maintenance_cost(&self, candidate: &CseCandidate) -> Cost { - let family = candidate - .bound_summary - .schema - .fields - .iter() - .map(|f| &f.dtype) - .find(|dtype| !matches!(dtype, FieldDataType::Plain(_))) - .cloned() - .unwrap_or(FieldDataType::Plain( - asap_types::ir::schema::DataType::Float64, - )); - default_cse_shared_maintenance_cost(&family) - } - - /// Decide whether to reuse one shared node across every - /// consumer of `candidate`, or bind each occurrence independently — a - /// Volcano/Cascades-style cost comparison (issue #237, #223 stage 4; see - /// `docs/design_docs/cse-cost-model-decision.md`): share iff the estimated cost of - /// maintaining one shared summary is no greater than the estimated total - /// cost of recomputing it independently everywhere it's used. - /// - /// The default body composes [`cse_recompute_cost`](Self::cse_recompute_cost) - /// and [`cse_shared_maintenance_cost`](Self::cse_shared_maintenance_cost) - /// — a deployment with real cost knowledge should override those two - /// (keeping this comparison), or override this method directly for a - /// wholly different policy. - fn cse_share_decision(&self, candidate: &CseCandidate) -> ShareDecision { - let recompute_total = self.cse_recompute_cost(candidate) * candidate.consumer_count; - let shared = self.cse_shared_maintenance_cost(candidate); - if shared <= recompute_total { - ShareDecision::Share - } else { - ShareDecision::RecomputeIndependently - } - } - - // ── Recurrence-aware costing (issue #287) ─────────────────────────── - // - // See `crate::cost::recurrence`'s module docs for the full cost model - // (`maintained_cost_rate`/`recompute_cost_rate` formulas, units, - // provenance of every new input). The three hooks below are the - // per-update-event/per-read/per-recomputation cost primitives that - // formula is built from; `cse_share_decision_with_recurrence` is the - // composed decision, mirroring how `cse_share_decision` above composes - // `cse_recompute_cost`/`cse_shared_maintenance_cost`. - - /// Cost of maintaining `candidate`'s bound summary for a single ingest - /// update event. Units: cost units per update — the - /// `maintenance_cost_per_update` term of `maintained_cost_rate` - /// (`crate::cost::recurrence`), where it is multiplied by an `UpdateRate` in - /// **Hz** (`update_rate * maintenance_cost_per_update`). - /// - /// Default: a small nominal constant, `Cost(0.01)` — deliberately - /// **not** derived from - /// [`cse_shared_maintenance_cost`](Self::cse_shared_maintenance_cost)'s - /// per-family weight table. That table's values (~1-6) are calibrated - /// against [`cse_recompute_cost`](Self::cse_recompute_cost)'s - /// structural-size proxy for a *life-of-the-workload*, one-time - /// maintenance magnitude — multiplying them by a real ingest rate (even - /// a modest one, e.g. 100 events/s) inflates `maintained_cost_rate` far - /// past any realistic `recompute_cost_rate`, making `Share` - /// unreachable regardless of how infrequently the summary is actually - /// read (issue #287 review). `Cost(0.01)` — one order of magnitude - /// below [`summary_read_cost`](Self::summary_read_cost)'s own nominal - /// default — reflects only that an incremental per-event update is - /// normally far cheaper than a full read or recompute, not a measured - /// ratio; a deployment with a real per-update cost (e.g. observed - /// sketch-insert latency) should override this instead of relying on - /// this placeholder. - fn maintenance_cost_per_update(&self, _candidate: &CseCandidate) -> Cost { - Cost(0.01) - } - - /// Cost of one read against `candidate`'s already-maintained summary. - /// Units: cost units per read — the `summary_read_cost` term of - /// `maintained_cost_rate`. Default: `Cost(1.0)`, a nominal unit read — - /// illustrative, like every other numeric default in this trait; a - /// deployment with a real read-path cost should override this. - fn summary_read_cost(&self, _candidate: &CseCandidate) -> Cost { - Cost(1.0) - } - - /// Cost of recomputing `candidate.sub-DAG` once, from the pre-ASAP/raw - /// path. Units: cost units per recomputation — the `raw_recompute_cost` - /// term of `recompute_cost_rate`. Default: delegates to - /// [`cse_recompute_cost`](Self::cse_recompute_cost) (the same - /// structural-size proxy `cse_share_decision` already uses). - fn raw_recompute_cost(&self, candidate: &CseCandidate) -> Cost { - self.cse_recompute_cost(candidate) - } - - /// The one-time cost of materializing `candidate`'s bound summary for - /// the *first* time — before any read or ingest-driven update charges - /// anything. Units: cost units (a one-time [`Cost`], not a rate). - /// - /// This is what makes a purely (or mostly) one-shot comparison - /// economically sound: without a build cost, "maintained" looked free - /// to construct, so `Share` won unconditionally for any number of - /// one-shot consumers, no matter how few (issue #287 review, bug 1). - /// With it, a single one-shot consumer never benefits from sharing - /// (build + one read costs more than one direct recompute), while many - /// one-shot consumers still amortize the fixed build cost across their - /// reads, same as before. - /// - /// Default: delegates to - /// [`raw_recompute_cost`](Self::raw_recompute_cost) — materializing a - /// summary for the first time costs about as much as computing its - /// answer once from raw, since there's no delta history yet to apply - /// incrementally. A deployment with a distinct measured "cold build" - /// cost should override this instead. - fn summary_build_cost(&self, candidate: &CseCandidate) -> Cost { - self.raw_recompute_cost(candidate) - } - - /// The recurrence-aware counterpart to - /// [`cse_share_decision`](Self::cse_share_decision): the same - /// `Share`/`RecomputeIndependently` choice, weighted by how *often* - /// `candidate`'s consumers actually run (`recurrence`) instead of only - /// how many structurally exist (`candidate.consumer_count`). See - /// `crate::cost::recurrence`'s module docs for the full design. - /// - /// - `recurrence.is_empty()` (no [`RepeatingEntry`]/[`DataWorkload`]-derived - /// metadata available): delegates to - /// [`cse_share_decision`](Self::cse_share_decision), preserving - /// today's structural-consumer-count behavior exactly — issue #287's - /// "preserve existing behavior when recurrence metadata is - /// unavailable" requirement. - /// - Otherwise: compares `maintained_cost_rate` against - /// `recompute_cost_rate` (both cost units/second). If - /// `recurrence.one_shot_consumers > 0` alongside any recurring rate - /// (mixed one-shot + repeating work), `horizon` MUST be `Some` — - /// `Err(RecurrenceError::MissingHorizon)` otherwise, per "the cost - /// model must not silently combine rate-valued and one-shot costs". - /// With no one-shot consumers, `horizon` is optional (comparing bare - /// rates is equivalent to comparing `rate * H` for any fixed `H > 0`). - /// - /// [`RepeatingEntry`]: asap_types::workload::RepeatingEntry - /// [`DataWorkload`]: asap_types::workload::DataWorkload - fn cse_share_decision_with_recurrence( - &self, - candidate: &CseCandidate, - recurrence: &RecurrenceProfile, - horizon: Option, - ) -> Result { - recurrence::decide(self, candidate, recurrence, horizon) - } - - /// Estimate a comparable, numeric cost for one already-constructed - /// [`ReplacementSubDAG`] candidate at `target` — a real `f64`, not just a - /// relative rank, meant for a caller that wants to *display* "candidate A - /// costs ≈ X, candidate B costs ≈ Y" (e.g. a DAG-visualization view built - /// on `cost_sorted`), - /// not just order candidates against each other — that ordering job - /// already belongs to [`rank_candidates`](Self::rank_candidates) (for a - /// [`ASAPStrategies`](asap_logical_optimizer::pass1::replacement::ASAPStrategies) - /// group) and [`cse_share_decision`](Self::cse_share_decision) (for a - /// [`SharedSubDAGStrategy`](asap_logical_optimizer::pass1::replacement::SharedSubDAGStrategy) - /// group). - /// - /// One method covers both candidate shapes this crate ships: - /// `candidate.replacement`'s [`Replacement::SubDAG`] from a summary - /// realization (a `ASAPStrategies` candidate — the bound node is - /// right there, nothing to reconstruct) and the same arm from a rewrite - /// (a `SharedSubDAGStrategy` share-vs-recompute candidate — no bound - /// summary of its own, since sharing is a decision about a target - /// already bound some other way; a representative binding is recovered - /// from `target` itself). `target` is threaded through explicitly - /// (rather than only ever the target embedded in `candidate` — there - /// isn't one for a rewrite) so both arms have the `consumer_count` - /// context a cost estimate needs to be meaningful. - /// - /// Default: **not a real cost model** — always returns `f64::NAN`. - /// `f64::partial_cmp` against `NAN` is always `None`, so a caller that - /// forgot to check whether its `CostModel` actually overrides this can't - /// silently treat the placeholder as a real comparison. A deployment - /// that wants numeric costs exposed should override this method; - /// [`DefaultCostModel`] does, reusing - /// [`cse_recompute_cost`](Self::cse_recompute_cost)/ - /// [`cse_shared_maintenance_cost`](Self::cse_shared_maintenance_cost) — - /// the same arithmetic that already backs `cse_share_decision` — rather - /// than inventing a second, drifting cost formula. - fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { - let _ = (candidate, target); - f64::NAN - } - - /// Physical feasibility evidence for a complete summary candidate. - /// `None` defers admission to physical/deployment compilation; `Some(false)` - /// excludes the candidate without changing its computation or parameters. - fn summary_support_evidence(&self, _summary: &OperatorNode) -> Option { - None - } - - /// Which mixed exact/summary execution shapes the downstream runtime - /// advertises (issue #171). Gates candidate *generation* in - /// [`asap_logical_optimizer::pass1::exact_composition::ExactCompositionStrategy`]: a shape the - /// runtime can't execute is never proposed, so it can't be selected - /// either. - /// - /// Default: [`ValueOperationCapabilities::ALL`]. This describes shapes - /// worth exploring, not proof that a runtime implements them. The - /// [`Self::value_operation_support_evidence`] hook gates selection; - /// a deployment whose runtime lacks a shape can narrow this hook. - fn value_operation_capabilities(&self) -> ValueOperationCapabilities { - ValueOperationCapabilities::ALL - } - - /// Whether the runtime implements this concrete function at this - /// placement. Deployments override this definition-level hook when - /// support differs between functions; the default delegates to the - /// coarse placement capability for backward compatibility. - fn supports_value_operation( - &self, - _operation: &ExactOperation, - placement: OperationPlacement, - ) -> bool { - self.value_operation_capabilities().supports(placement) - } - - /// Runtime support evidence for a mixed exact/summary operation. `None` - /// means that the logical shape is possible but runtime support has not - /// been established. The legacy boolean hook still rules out explicit - /// `false`; implementations that can prove support override this method - /// with `Some(true)`. - fn value_operation_support_evidence( - &self, - operation: &ExactOperation, - placement: OperationPlacement, - ) -> Option { - (!self.supports_value_operation(operation, placement)).then_some(false) - } - - /// The statistics the issue #171 recurring-cost formulas need for one - /// composed alternative — see [`ExactCompositionCostInputs`] for each - /// input and [`read_operation_plan_cost_rate`]/ - /// [`maintenance_operation_plan_cost_rate`]/[`raw_recompute_cost_rate`] for how - /// they combine. One structured hook rather than eight scalar ones, so - /// a deployment answers them all from one place (and can attach its own - /// [`CostProvenance`]). - /// - /// Default: every input unknown ([`ExactCompositionCostInputs::unknown`]) - /// — unknown is never zero, and with no rate derivable - /// `candidate_selection::global_selection` keeps the conservative keep-as-is - /// behavior for the site. A deployment that wants defaults must supply - /// them here explicitly. - fn exact_composition_cost_inputs( - &self, - request: &ExactCompositionCostRequest<'_>, - ) -> ExactCompositionCostInputs { - let _ = request; - ExactCompositionCostInputs::unknown(CostProvenance { - model: "CostModel::exact_composition_cost_inputs (default)".into(), - version: "unknown".into(), - }) - } -} - -fn sketch_state( - node: &OperatorNode, -) -> Option<(&asap_types::ir::schema::SketchKind, &GroupingStrategy)> { - match &node.operator { - Operator::ASAP(ASAPOp::SummaryEstimate { summary_input, .. }) => { - sketch_state(summary_input) - } - Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, grouping), - .. - }) => Some((kind, grouping)), - _ => None, - } -} - -fn sketch_state_units(params: &SketchParams) -> f64 { - match params { - SketchParams::Cms { width, depth } - | SketchParams::CountSketch { width, depth } - | SketchParams::CmsWithHeap { width, depth, .. } - | SketchParams::CountSketchWithHeap { width, depth, .. } => { - f64::from(*width) * f64::from(*depth) - } - _ => 1.0, - } -} - -fn hydra_grid_cells(params: &HydraParams) -> f64 { - match params { - HydraParams::HydraKll { shared_buckets, .. } => f64::from(*shared_buckets), - HydraParams::HydraCms { - shared_rows, - shared_columns, - .. - } - | HydraParams::HydraCountSketch { - shared_rows, - shared_columns, - .. - } => f64::from(*shared_rows) * f64::from(*shared_columns), - } -} - -/// The default cost model: preserves [`summary_candidates`]'s built-in static -/// order. -/// -/// [`summary_candidates`]: asap_logical_optimizer::pass1::realization::summary_candidates -pub struct DefaultCostModel; - -impl CostModel for DefaultCostModel { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - - /// Real numbers, reusing [`CostModel::cse_recompute_cost`]/ - /// [`CostModel::cse_shared_maintenance_cost`] — the same arithmetic - /// `cse_share_decision`'s default body already composes — rather than a - /// second formula: - /// - /// - A [`ReplacementProvenance::SummaryRealization`] candidate (a - /// `ASAPStrategies` binding): `cse_recompute_cost` (the one-time - /// structural cost of building `target` at all) plus - /// `cse_shared_maintenance_cost` of the candidate's own bound family - /// (a pricier family — a sketch over an exact accumulator, say — - /// costs more here, consistent with the per-family weighting - /// [`default_cse_shared_maintenance_cost`] already orders candidates - /// by). - /// - Any other [`Replacement::SubDAG`] (a logical rewrite or a CSE - /// share/recompute candidate): recovers one - /// representative bound node for `target` via `realize_child` (the same - /// rank-and-take-first helper `replacement::realize_child` reuses for the - /// identical need), then charges - /// `cse_shared_maintenance_cost` for the candidate that shares - /// `target`'s own `Rc` (`Rc::ptr_eq`), or `cse_recompute_cost * - /// consumer_count` for the one that doesn't — the same two terms - /// `cse_share_decision` already compares against each other. `NaN` - /// only if `target` itself can't be bound at all (schema derivation - /// failed) — never expected for a target that's already part of a - /// legitimate workload DAG. - /// - /// **Exception**: a [`ReplacementProvenance::AccuracyReconciliation`] - /// candidate (issue #273) never rebuilds `target` — it reads a - /// sibling `rc` that this crate builds regardless — so it gets its - /// own arm: `cse_shared_maintenance_cost` against `rc`'s **own** bound - /// summary (a "read", not a "rebuild `target` per consumer") instead - /// of the `cse_recompute_cost * consumer_count` formula the other - /// `Rewrite` shapes fall through to. See - /// `accuracy_reconciliation.rs`'s own "Costing this candidate shape" - /// module docs for why that formula would otherwise misprice it (in - /// the wrong direction, worse the more consumers would actually - /// benefit from sharing). - fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { - let consumer_count = target.consumer_count.max(1); - match &candidate.replacement { - Replacement::SubDAG(node) - if candidate.provenance == ReplacementProvenance::SummaryRealization => - { - let cse = CseCandidate { - sub_dag: target.root, - bound_summary: node, - consumer_count, - }; - (self.cse_recompute_cost(&cse) + self.cse_shared_maintenance_cost(&cse)).0 - } - Replacement::SubDAG(rc) - if candidate.provenance == ReplacementProvenance::AccuracyReconciliation => - { - let Ok(sibling_bound) = realize_child(rc) else { - return f64::NAN; - }; - let cse = CseCandidate { - sub_dag: rc, - bound_summary: &sibling_bound, - // One additional reference into `rc`'s own (already - // necessary) build, from this one consumer's - // perspective — not `target`'s own `consumer_count`, - // which would conflate the reader-side multiplicity - // with a maintenance metric that's `rc`'s own group's - // concern, not this candidate's. - consumer_count: 1, - }; - self.cse_shared_maintenance_cost(&cse).0 - } - Replacement::SubDAG(rc) => { - let Ok(bound) = realize_child(target.root) else { - return f64::NAN; - }; - let cse = CseCandidate { - sub_dag: target.root, - bound_summary: &bound, - consumer_count, - }; - if Rc::ptr_eq(rc, target.root) { - self.cse_shared_maintenance_cost(&cse).0 - } else { - (self.cse_recompute_cost(&cse) * consumer_count).0 - } - } - // A composed candidate is costed in cost-units-per-second by - // `candidate_selection::global_selection` against the child decision it - // is committed with — a different unit from this structural - // estimate, and unknowable here without that child. `NaN` - // keeps it from ever out-ranking a real estimate by accident. - Replacement::ExactComposition(_) => f64::NAN, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_logical_optimizer::pass1::realization::summary_candidates; - use asap_types::ir::operator::agg_intent::default_cardinality; - - #[test] - fn default_cost_model_preserves_static_order() { - let intent = default_cardinality(); - let candidates = summary_candidates(&intent); - assert_eq!( - DefaultCostModel.rank_candidates(&intent, candidates), - candidates.to_vec() - ); - } - - // ── Recurring-cost formulas (issue #171) ───────────────────────────── - - fn known_inputs() -> ExactCompositionCostInputs { - ExactCompositionCostInputs { - exact_cost_per_row: Some(0.1), - expected_input_rows: Some(50.0), - expected_output_rows: Some(10.0), - summary_maintenance_cost_per_update: Some(0.01), - summary_read_cost: Some(1.0), - update_rate: Some(100.0), - evaluation_rate: Some(EvaluationRate(2.0)), - raw_recompute_cost: Some(100.0), - unit: CostUnit::CostUnitsPerSecond, - provenance: CostProvenance { - model: "test".into(), - version: "1".into(), - }, - } - } - - #[test] - fn composition_formulas_match_the_issue_definitions() { - let inputs = known_inputs(); - // 100 * 0.01 + 2 * (1 + 10 * 0.1) = 1 + 4 = 5 - assert_eq!(read_operation_plan_cost_rate(&inputs).unwrap().0, 5.0); - // 100 * (0.1 + 0.01) + 2 * 1 = 11 + 2 = 13 - assert!((maintenance_operation_plan_cost_rate(&inputs).unwrap().0 - 13.0).abs() < 1e-9); - // 2 * 100 - assert_eq!(raw_recompute_cost_rate(&inputs).unwrap().0, 200.0); - assert_eq!( - crate::cost::recurrence::total_cost(CostRate(5.0), Horizon(10.0), Cost(3.0)), - Cost(53.0) - ); - } - - /// Consolidation preserves type identity and rejects totals in recurring formulas. - #[test] - fn recurring_formulas_require_the_shared_rate_unit() { - let mut inputs = known_inputs(); - let shared: asap_types::cost::CostUnit = inputs.unit; - assert_eq!(shared.as_str(), "cost_units_per_second"); - inputs.unit = asap_types::cost::CostUnit::CostUnits; - assert_eq!(read_operation_plan_cost_rate(&inputs), None); - assert_eq!(maintenance_operation_plan_cost_rate(&inputs), None); - assert_eq!(raw_recompute_cost_rate(&inputs), None); - } - - #[test] - fn a_missing_input_yields_no_rate_not_zero() { - let mut inputs = known_inputs(); - inputs.summary_maintenance_cost_per_update = None; - assert_eq!(read_operation_plan_cost_rate(&inputs), None); - assert_eq!(maintenance_operation_plan_cost_rate(&inputs), None); - // The baseline doesn't need maintenance and is still known. - assert!(raw_recompute_cost_rate(&inputs).is_some()); - let unknown = ExactCompositionCostInputs::unknown(known_inputs().provenance); - assert_eq!(raw_recompute_cost_rate(&unknown), None); - } - - #[test] - fn default_model_has_potential_shapes_but_unknown_runtime_support() { - assert_eq!( - DefaultCostModel.value_operation_capabilities(), - ValueOperationCapabilities::ALL - ); - assert_eq!( - DefaultCostModel.value_operation_support_evidence( - &ExactOperation::Aggregate { - reduction: asap_types::ir::operator::operator_properties::Reduction::by(vec![]), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - filters: vec![], - having: None, - }, - OperationPlacement::Read, - ), - None - ); - assert!(ValueOperationCapabilities::NONE - .supports(OperationPlacement::Read) - .not()); - } - - trait Not { - fn not(self) -> bool; - } - impl Not for bool { - fn not(self) -> bool { - !self - } - } - - // ── CSE sharing (issue #237, #223 stage 4) ────────────────────────── - - use asap_types::ir::operator::operator_properties::Source; - use asap_types::ir::schema::DataType; - use asap_types::ir::schema::{ - ExactKind, ExactParams, Field, GroupingStrategy, Schema, SketchKind, - }; - use asap_types::ir::{NonASAPOp, Predicate, ScalarExpr}; - - fn scan() -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap() - } - - /// A `SummaryAgg` directly over the kept `scan()` sub-DAG. - fn summary_node(family: FieldDataType) -> Rc { - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: scan(), - family: family.clone(), - input: asap_types::ir::schema::SummaryUpdate::column( - asap_types::ir::scalar::ColumnRef::Named("value".into()), - ), - reduction: asap_types::ir::operator::operator_properties::Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(None), - ) - } - - #[test] - fn default_recompute_cost_is_positive_and_grows_with_structural_size() { - let leaf = scan(); - let nested = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Dedup { - cols: vec![0], - child: Rc::clone(&leaf), - })) - .unwrap(); - assert!(default_cse_recompute_cost(&leaf) > Cost::ZERO); - assert!(default_cse_recompute_cost(&nested) > default_cse_recompute_cost(&leaf)); - } - - /// The DAG-awareness this proxy exists for: a sub-DAG that internally - /// re-references one shared descendant (e.g. after single-query CSE, - /// `x op x` collapsing both branches onto one `Rc`) must cost the same - /// as if that descendant only appeared once — not double, the way a - /// naive per-path size measure (a full serialization, or an - /// identity-blind recursive walk) would count it. - #[test] - fn default_recompute_cost_does_not_double_count_an_internally_shared_descendant() { - use asap_types::ir::operator::operator_properties::JoinKind; - use asap_types::ir::scalar::ScalarValue; - - let true_pred = || Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))); - let shared_leaf = scan(); - let no_sharing = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Join { - kind: JoinKind::Inner, - pred: true_pred(), - left: scan(), - right: scan(), - })) - .unwrap(); - let with_sharing = - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Join { - kind: JoinKind::Inner, - pred: true_pred(), - left: Rc::clone(&shared_leaf), - right: Rc::clone(&shared_leaf), - })) - .unwrap(); - assert_eq!( - default_cse_recompute_cost(&no_sharing), - Cost(3.0), - "no sharing: Join + 2 independent Scans = 3 unique nodes" - ); - assert_eq!( - default_cse_recompute_cost(&with_sharing), - Cost(2.0), - "internal sharing: Join + 1 shared Scan (referenced twice) = \ - 2 unique nodes, not 3 — a per-path size measure would \ - wrongly charge for the shared Scan twice" - ); - } - - #[test] - fn default_shared_maintenance_cost_orders_families_cheapest_to_priciest() { - let exact = default_cse_shared_maintenance_cost(&FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let sketch = default_cse_shared_maintenance_cost(&FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 12 }), - GroupingStrategy::default(), - )); - assert!( - exact < sketch, - "an exact accumulator should be cheaper to keep continuously updated \ - than a sketch: exact={exact}, sketch={sketch}" - ); - } - - #[test] - fn cse_share_decision_shares_when_recompute_dominates_maintenance() { - let candidate = CseCandidate { - sub_dag: &scan(), - bound_summary: &summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )), - // Many consumers of a cheap accumulator: recompute_total should - // dominate the fixed maintenance cost. - consumer_count: 1000, - }; - assert_eq!( - DefaultCostModel.cse_share_decision(&candidate), - ShareDecision::Share - ); - } - - #[test] - fn cse_share_decision_recomputes_when_maintenance_dominates_recompute() { - let candidate = CseCandidate { - sub_dag: &scan(), - bound_summary: &summary_node(FieldDataType::StatModel( - asap_types::ir::schema::StatModelKind::Parametric, - asap_types::ir::schema::StatModelParams::Parametric { - family: "gaussian_mixture".into(), - }, - )), - // A single, cheap-to-recompute leaf (scan() alone is 1 DAG - // node, recompute_total = 1) against an expensive-to-maintain - // family (StatModel, maintenance cost 6.0): maintenance should - // dominate. - consumer_count: 1, - }; - assert_eq!( - DefaultCostModel.cse_share_decision(&candidate), - ShareDecision::RecomputeIndependently - ); - } - - #[test] - fn cse_share_decision_default_body_composes_the_two_cost_hooks() { - struct AlwaysExpensiveToRecompute; - impl CostModel for AlwaysExpensiveToRecompute { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - fn cse_recompute_cost(&self, _candidate: &CseCandidate) -> Cost { - Cost(1e9) - } - } - - // Even the priciest family should lose to an overridden recompute - // cost this large, confirming `cse_share_decision`'s default body - // actually calls through to the overridable hooks rather than - // hardcoding a comparison against its own defaults. - let candidate = CseCandidate { - sub_dag: &scan(), - bound_summary: &summary_node(FieldDataType::StatModel( - asap_types::ir::schema::StatModelKind::Parametric, - asap_types::ir::schema::StatModelParams::Parametric { - family: "gaussian_mixture".into(), - }, - )), - consumer_count: 2, - }; - assert_eq!( - AlwaysExpensiveToRecompute.cse_share_decision(&candidate), - ShareDecision::Share - ); - } - - // ── estimate_cost ──────────────────────────────────────────────────── - - /// The trait's default `estimate_cost` body is an explicit placeholder, - /// not a real cost model — a `CostModel` that only overrides - /// `rank_candidates` (the minimum required to implement the trait) must - /// still get `f64::NAN` back, never a value that looks like a real - /// estimate. - #[test] - fn estimate_cost_default_body_is_a_nan_placeholder() { - struct RankOnly; - impl CostModel for RankOnly { - fn rank_candidates( - &self, - _intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - } - - let root = scan(); - let target = TargetSubDAG::new(&root); - let candidate = ReplacementSubDAG { - strategy: "TestStrategy", - replacement: Replacement::SubDAG(summary_node(FieldDataType::Plain( - asap_types::ir::schema::DataType::Float64, - ))), - provenance: asap_logical_optimizer::pass1::replacement::ReplacementProvenance::SummaryRealization, - rationale: "whatever".into(), - }; - assert!(RankOnly.estimate_cost(&candidate, &target).is_nan()); - } - - /// `DefaultCostModel::estimate_cost` for a summary-rooted [`Replacement::SubDAG`] - /// candidate reuses [`default_cse_shared_maintenance_cost`]'s own - /// per-family ordering: a candidate bound to a cheap-to-maintain family - /// (an exact accumulator) must cost less than one bound to an - /// expensive-to-maintain family (a fitted statistical model), same - /// target either way — consistent with - /// `default_shared_maintenance_cost_orders_families_cheapest_to_priciest` - /// above. - #[test] - fn estimate_cost_for_summary_orders_candidates_by_family_cheapest_to_priciest() { - let root = scan(); - let target = TargetSubDAG::new(&root); - - let cheap = ReplacementSubDAG { - strategy: "TestStrategy", - replacement: Replacement::SubDAG(summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - ))), - provenance: asap_logical_optimizer::pass1::replacement::ReplacementProvenance::SummaryRealization, - rationale: "exact accumulator".into(), - }; - let pricey = ReplacementSubDAG { - strategy: "TestStrategy", - replacement: Replacement::SubDAG(summary_node(FieldDataType::StatModel( - asap_types::ir::schema::StatModelKind::Parametric, - asap_types::ir::schema::StatModelParams::Parametric { - family: "gaussian_mixture".into(), - }, - ))), - provenance: asap_logical_optimizer::pass1::replacement::ReplacementProvenance::SummaryRealization, - rationale: "fitted statistical model".into(), - }; - - let cheap_cost = DefaultCostModel.estimate_cost(&cheap, &target); - let pricey_cost = DefaultCostModel.estimate_cost(&pricey, &target); - assert!( - cheap_cost.is_finite() && pricey_cost.is_finite(), - "cheap={cheap_cost}, pricey={pricey_cost}" - ); - assert!( - cheap_cost < pricey_cost, - "an ExactAggregate candidate should cost less than a StatModel one: \ - exact={cheap_cost}, stat_model={pricey_cost}" - ); - } - - /// `DefaultCostModel::estimate_cost` for a relational [`Replacement::SubDAG`] pair - /// (the `SharedSubDAGStrategy` share-vs-recompute shape) agrees with - /// what `cse_share_decision` would already pick for the same target: with - /// many consumers of a cheap-to-recompute leaf, the "share" candidate - /// (the target's own `Rc`) must cost less than the "recompute - /// independently" one (a fresh `Rc`) — mirrors - /// `cse_share_decision_shares_when_recompute_dominates_maintenance` - /// above, through `estimate_cost` instead of `cse_share_decision` - /// directly. - #[test] - fn estimate_cost_for_rewrite_prefers_sharing_when_recompute_dominates_maintenance() { - let target_root = scan(); - let target = TargetSubDAG::with_consumer_count(&target_root, 20); - - let share = ReplacementSubDAG { - strategy: "TestStrategy", - replacement: Replacement::SubDAG(Rc::clone(&target_root)), - provenance: asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseShare, - rationale: "build once and share".into(), - }; - let recompute = ReplacementSubDAG { - strategy: "TestStrategy", - replacement: Replacement::SubDAG(Rc::new((*target_root).clone())), - provenance: - asap_logical_optimizer::pass1::replacement::ReplacementProvenance::CseRecompute, - rationale: "build independently".into(), - }; - - let share_cost = DefaultCostModel.estimate_cost(&share, &target); - let recompute_cost = DefaultCostModel.estimate_cost(&recompute, &target); - assert!( - share_cost.is_finite() && recompute_cost.is_finite(), - "share={share_cost}, recompute={recompute_cost}" - ); - assert!( - share_cost < recompute_cost, - "with 20 consumers of a cheap-to-recompute leaf, sharing should cost less: \ - share={share_cost}, recompute={recompute_cost}" - ); - } -} diff --git a/crates/plan-selection/src/cost/empirical_cost.rs b/crates/plan-selection/src/cost/empirical_cost.rs deleted file mode 100644 index c81904e7..00000000 --- a/crates/plan-selection/src/cost/empirical_cost.rs +++ /dev/null @@ -1,638 +0,0 @@ -//! Offline sketch-bench evidence. Measurements describe a particular dataset, -//! configuration and environment; they are neither runtime feedback nor proofs -//! of an accuracy guarantee. CPU quantities are nanoseconds, never CPU operations. - -use asap_types::ir::operator::AggIntent; -use asap_types::ir::schema::{SketchAlgorithm, SketchParams}; -use serde::{Deserialize, Serialize}; - -use crate::cost::cost_model::{CostModel, DefaultCostModel}; -use asap_logical_optimizer::pass1::realization::{ - accuracy_budget, accuracy_target, default_size_params, -}; -use asap_logical_optimizer::pass1::replacement::{ReplacementSubDAG, TargetSubDAG}; - -pub const EVIDENCE_SCHEMA_VERSION: u32 = 1; -pub const EVIDENCE_MODEL_VERSION: &str = "empirical-update-cpu-v1"; - -pub use crate::cost::empirical_resources::ResourceMeasurements; -pub use asap_types::workload::resources::Measurement; - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct DistributionDescriptor { - pub id: String, - pub family: String, - pub sample_count: u64, - pub distinct_count: Option, - /// Generator parameters or trace identity/checksum, including sampling rules. - pub parameters: serde_json::Value, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EnvironmentDescriptor { - pub id: String, - pub cpu: String, - pub os: String, - pub runtime: String, - pub implementation: String, - pub implementation_version: String, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct MeasurementProvenance { - pub command: String, - pub dataset: String, - pub source_revision: String, - pub repetitions: u32, -} - -/// Observed error on offline ground truth. No confidence or formal guarantee is -/// inferred from these statistics; `metric` defines the meaning of mean/max. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct OfflineError { - pub metric: String, - pub mean: Option, - pub max: Option, - pub trials: u32, - pub ground_truth_method: String, - pub query: serde_json::Value, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct OfflineMeasurement { - pub id: String, - pub algorithm: SketchAlgorithm, - pub params: SketchParams, - pub distribution: DistributionDescriptor, - pub environment: EnvironmentDescriptor, - pub measured_at_unix_seconds: u64, - pub valid_until_unix_seconds: u64, - pub provenance: MeasurementProvenance, - pub metrics: ResourceMeasurements, - pub error: Option, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EvidenceArtifact { - pub schema_version: u32, - pub benchmark_version: String, - /// Identity of the normalization/cost interpretation, independent of JSON - /// layout and of the sketch implementation's source revision. - pub model_version: String, - pub records: Vec, -} - -/// The caller explicitly chooses the offline applicability context. Matching -/// all descriptors prevents reusing costs solely because a distribution name -/// or hardware label happens to agree. Time is injected for reproducibility. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct EvidenceContext { - pub distribution: DistributionDescriptor, - pub environment: EnvironmentDescriptor, - pub now_unix_seconds: u64, -} - -#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] -pub enum EvidenceError { - #[error("unsupported offline evidence schema version {0}")] - UnsupportedVersion(u32), - #[error("invalid offline evidence: {0}")] - Invalid(String), - #[error("no measurement for algorithm and exact configuration")] - MissingConfiguration, - #[error("offline distribution or environment is incompatible")] - IncompatibleContext, - #[error("offline measurement is expired or dated in the future")] - Stale, - #[error("multiple applicable measurements; select an unambiguous artifact")] - Ambiguous, -} - -pub struct EmpiricalEvidenceProvider { - artifact: EvidenceArtifact, - context: EvidenceContext, -} - -impl EmpiricalEvidenceProvider { - pub fn new( - artifact: EvidenceArtifact, - context: EvidenceContext, - ) -> Result { - artifact.validate()?; - if artifact.model_version != EVIDENCE_MODEL_VERSION { - return invalid("unsupported offline cost model version"); - } - validate_context(&context.distribution, &context.environment)?; - Ok(Self { artifact, context }) - } - - pub fn artifact(&self) -> &EvidenceArtifact { - &self.artifact - } - pub fn context(&self) -> &EvidenceContext { - &self.context - } - - /// Never interpolates between configurations, distributions, or machines. - /// The returned row includes complete provenance for user explanations. - pub fn lookup( - &self, - algorithm: &SketchAlgorithm, - params: &SketchParams, - ) -> Result<&OfflineMeasurement, EvidenceError> { - let configurations: Vec<_> = self - .artifact - .records - .iter() - .filter(|r| &r.algorithm == algorithm && &r.params == params) - .collect(); - if configurations.is_empty() { - return Err(EvidenceError::MissingConfiguration); - } - let compatible: Vec<_> = configurations - .into_iter() - .filter(|r| { - r.distribution == self.context.distribution - && r.environment == self.context.environment - }) - .collect(); - if compatible.is_empty() { - return Err(EvidenceError::IncompatibleContext); - } - let mut valid = compatible.into_iter().filter(|r| { - r.measured_at_unix_seconds <= self.context.now_unix_seconds - && self.context.now_unix_seconds <= r.valid_until_unix_seconds - }); - let first = valid.next().ok_or(EvidenceError::Stale)?; - if valid.next().is_some() { - return Err(EvidenceError::Ambiguous); - } - Ok(first) - } - - /// Reorders only a fully measured comparison, preserving all candidates. - /// For deployment-specific sizing call `lookup` with those exact params. - pub fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - eps: f64, - delta: f64, - ) -> Vec { - let costs: Option> = candidates - .iter() - .map(|algorithm| { - let params = default_size_params(algorithm.clone(), intent, eps, delta); - self.lookup(algorithm, ¶ms) - .ok()? - .metrics - .resources - .cpu - .update_cpu_ns - .as_ref() - .map(|m| (algorithm.clone(), m.value)) - }) - .collect(); - let Some(mut costs) = costs else { - return candidates.to_vec(); - }; - costs.sort_by(|a, b| a.1.total_cmp(&b.1)); - costs.into_iter().map(|(algorithm, _)| algorithm).collect() - } -} - -/// Standalone adapter for the existing planner boundary. Empirical data changes -/// candidate discovery order; final structural scores retain their documented -/// default meaning. Complete plan benefit estimates require downstream raw and -/// summary physical evidence and are deliberately not invented here. -pub struct EmpiricalCostModel { - pub provider: EmpiricalEvidenceProvider, -} - -impl EmpiricalCostModel { - pub fn new(provider: EmpiricalEvidenceProvider) -> Self { - Self { provider } - } -} - -impl CostModel for EmpiricalCostModel { - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - let Some(accuracy) = accuracy_target(intent) else { - return candidates.to_vec(); - }; - let (eps, delta) = accuracy_budget(accuracy); - self.provider - .rank_candidates(intent, candidates, eps, delta) - } - - // The default size_params formula retains its formal guarantee. Observed - // offline error is insufficient evidence to shrink a sketch safely. - fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { - DefaultCostModel.estimate_cost(candidate, target) - } -} - -impl EvidenceArtifact { - pub fn validate(&self) -> Result<(), EvidenceError> { - if self.schema_version != EVIDENCE_SCHEMA_VERSION { - return Err(EvidenceError::UnsupportedVersion(self.schema_version)); - } - if self.benchmark_version.trim().is_empty() || self.model_version.trim().is_empty() { - return invalid("missing benchmark or model version"); - } - let mut ids = std::collections::HashSet::new(); - for row in &self.records { - if row.id.trim().is_empty() || !ids.insert(&row.id) { - return invalid("empty or duplicate record id"); - } - validate_context(&row.distribution, &row.environment)?; - let p = &row.provenance; - if [&p.command, &p.dataset, &p.source_revision] - .iter() - .any(|s| s.trim().is_empty()) - || p.repetitions == 0 - { - return invalid("missing measurement provenance"); - } - if row.measured_at_unix_seconds > row.valid_until_unix_seconds { - return invalid("reversed validity interval"); - } - if !valid_params(&row.algorithm, &row.params) { - return invalid("invalid or mismatched sketch parameters"); - } - let m = &row.metrics.resources; - for measurement in [ - &m.cpu.build_cpu_ns, - &m.cpu.update_cpu_ns, - &m.cpu.merge_cpu_ns, - &m.cpu.prepare_cpu_ns, - &m.cpu.read_cpu_ns, - &m.retained_memory_bytes, - &m.peak_memory_bytes, - &m.serialized_bytes, - &m.disk_bytes, - &m.scan_bytes, - ] - .into_iter() - .flatten() - { - if !nonnegative(measurement.value) - || measurement.samples == 0 - || measurement.stddev.is_some_and(|v| !nonnegative(v)) - { - return invalid("invalid measurement or uncertainty"); - } - } - if let Some(error) = &row.error { - if error.metric.trim().is_empty() - || error.ground_truth_method.trim().is_empty() - || error.trials == 0 - || [error.mean, error.max] - .into_iter() - .flatten() - .any(|v| !nonnegative(v)) - { - return invalid("invalid offline error evidence"); - } - } - } - Ok(()) - } -} - -fn invalid(message: &str) -> Result { - Err(EvidenceError::Invalid(message.into())) -} -fn nonnegative(value: f64) -> bool { - value.is_finite() && value >= 0.0 -} - -fn validate_context( - distribution: &DistributionDescriptor, - environment: &EnvironmentDescriptor, -) -> Result<(), EvidenceError> { - if [ - &distribution.id, - &distribution.family, - &environment.id, - &environment.cpu, - &environment.os, - &environment.runtime, - &environment.implementation, - &environment.implementation_version, - ] - .iter() - .any(|s| s.trim().is_empty()) - { - return invalid("missing distribution or environment identity"); - } - if distribution.sample_count == 0 - || distribution - .distinct_count - .is_some_and(|n| n > distribution.sample_count) - || !distribution.parameters.is_object() - { - return invalid("invalid distribution descriptors"); - } - Ok(()) -} - -fn valid_params(algorithm: &SketchAlgorithm, params: &SketchParams) -> bool { - match (algorithm, params) { - ( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - }, - ) => *heap_size > 0 && *sketch_rows > 0 && *sketch_cols > 0 && (1..=64).contains(layers), - (SketchAlgorithm::Kll, SketchParams::Kll { k }) - | (SketchAlgorithm::Kmv, SketchParams::Kmv { k }) - | (SketchAlgorithm::Theta, SketchParams::Theta { k }) => *k > 0, - (SketchAlgorithm::Hll, SketchParams::Hll { precision }) => (4..=18).contains(precision), - (SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }) => { - alpha.is_finite() && *alpha > 0.0 && *alpha < 1.0 - } - (SketchAlgorithm::Cms, SketchParams::Cms { width, depth }) - | (SketchAlgorithm::CountSketch, SketchParams::CountSketch { width, depth }) => { - *width > 0 && *depth > 0 - } - ( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width, - depth, - heap_size, - }, - ) - | ( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width, - depth, - heap_size, - }, - ) => *width > 0 && *depth > 0 && *heap_size > 0, - _ => false, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use asap_types::types::AccuracyTarget; - - /// The documented synthetic wire-format example remains importable and - /// explicitly identifiable as a test fixture. - #[test] - fn checked_in_synthetic_example_is_valid() { - let artifact: EvidenceArtifact = serde_json::from_str(include_str!( - "../../tests/data/offline-evidence-synthetic.json" - )) - .unwrap(); - artifact.validate().unwrap(); - assert!(artifact.records[0] - .environment - .implementation - .contains("SYNTHETIC")); - let schema: serde_json::Value = serde_json::from_str(include_str!( - "../../../../docs/develop_docs/offline-sketch-evidence.schema.json" - )) - .unwrap(); - assert_eq!( - schema["properties"]["schema_version"]["const"], - artifact.schema_version - ); - assert_eq!( - schema["properties"]["model_version"]["const"], - EVIDENCE_MODEL_VERSION - ); - } - - /// Newly shared optional dimensions receive the same numeric validation. - #[test] - fn optional_prepare_and_scan_measurements_are_validated() { - for prepare in [true, false] { - let (mut artifact, _, _) = fixture(); - let resources = &mut artifact.records[0].metrics.resources; - let field = if prepare { - &mut resources.cpu.prepare_cpu_ns - } else { - &mut resources.scan_bytes - }; - *field = Some(Measurement { - value: -1.0, - stddev: None, - samples: 1, - method: None, - }); - assert!(artifact.validate().is_err()); - } - } - - fn fixture() -> (EvidenceArtifact, EvidenceContext, AggIntent) { - let distribution = DistributionDescriptor { - id: "unit-test-uniform".into(), - family: "uniform".into(), - sample_count: 1000, - distinct_count: Some(100), - parameters: serde_json::json!({"seed": 7}), - }; - let environment = EnvironmentDescriptor { - id: "unit-test-machine".into(), - cpu: "test CPU".into(), - os: "test OS".into(), - runtime: "test runtime".into(), - implementation: "synthetic test fixture".into(), - implementation_version: "test-v1".into(), - }; - let intent = AggIntent::Count { - accuracy: AccuracyTarget::EpsilonDelta { - epsilon: 0.01, - delta: 0.01, - }, - }; - let records = [ - (SketchAlgorithm::Cms, 20.0), - (SketchAlgorithm::CountSketch, 10.0), - (SketchAlgorithm::UnivMon, 100.0), - ] - .into_iter() - .map(|(algorithm, cost)| OfflineMeasurement { - id: format!("test-{algorithm:?}"), - params: default_size_params(algorithm.clone(), &intent, 0.01, 0.01), - algorithm, - distribution: distribution.clone(), - environment: environment.clone(), - measured_at_unix_seconds: 100, - valid_until_unix_seconds: 200, - provenance: MeasurementProvenance { - command: "unit test fixture; not a measured benchmark".into(), - dataset: "synthetic fixture".into(), - source_revision: "test".into(), - repetitions: 3, - }, - metrics: ResourceMeasurements { - resources: asap_types::workload::resources::MeasuredResources { - cpu: asap_types::workload::resources::MeasuredCpu { - update_cpu_ns: Some(Measurement { - value: cost, - stddev: Some(1.0), - samples: 3, - method: None, - }), - ..Default::default() - }, - ..Default::default() - }, - }, - error: Some(OfflineError { - metric: "mean absolute relative frequency error".into(), - mean: Some(0.001), - max: None, - trials: 3, - ground_truth_method: "test exact counter fixture".into(), - query: serde_json::json!({"kind":"point_frequency"}), - }), - }) - .collect(); - ( - EvidenceArtifact { - schema_version: EVIDENCE_SCHEMA_VERSION, - benchmark_version: "test-fixture-v1".into(), - model_version: "empirical-update-cpu-v1".into(), - records, - }, - EvidenceContext { - distribution, - environment, - now_unix_seconds: 150, - }, - intent, - ) - } - - /// Missing, mismatched and expired evidence preserve the original ranking; - /// a measurement from another configuration is never extrapolated. - #[test] - fn unavailable_evidence_falls_back_with_specific_reasons() { - let (artifact, context, intent) = fixture(); - let candidates = vec![SketchAlgorithm::Cms, SketchAlgorithm::CountSketch]; - for (mut evidence, mut request, expected) in [ - ( - artifact.clone(), - context.clone(), - EvidenceError::MissingConfiguration, - ), - ( - artifact.clone(), - context.clone(), - EvidenceError::IncompatibleContext, - ), - (artifact.clone(), context.clone(), EvidenceError::Stale), - ] { - match expected { - EvidenceError::MissingConfiguration => { - evidence.records.remove(1); - } - EvidenceError::IncompatibleContext => { - request.distribution.parameters = serde_json::json!({"seed":8}); - } - EvidenceError::Stale => { - request.now_unix_seconds = 201; - } - _ => unreachable!(), - } - let provider = EmpiricalEvidenceProvider::new(evidence, request).unwrap(); - assert_eq!( - provider - .lookup(&artifact.records[1].algorithm, &artifact.records[1].params) - .unwrap_err(), - expected - ); - assert_eq!( - provider.rank_candidates(&intent, &candidates, 0.01, 0.01), - candidates - ); - } - let provider = EmpiricalEvidenceProvider::new(artifact, context).unwrap(); - assert_eq!( - provider.rank_candidates(&intent, &candidates, 0.001, 0.01), - candidates - ); - } - - /// Null remains unknown across serialization; zero is accepted only as an - /// explicit valid measurement, and no point-frequency error becomes a bound. - #[test] - fn serialization_preserves_unknown_zero_and_provenance() { - let (mut artifact, context, _) = fixture(); - artifact.records[0].metrics.resources.disk_bytes = Some(Measurement { - value: 0.0, - stddev: None, - samples: 1, - method: None, - }); - let decoded: EvidenceArtifact = - serde_json::from_str(&serde_json::to_string(&artifact).unwrap()).unwrap(); - let provider = EmpiricalEvidenceProvider::new(decoded, context).unwrap(); - let row = provider - .lookup(&artifact.records[0].algorithm, &artifact.records[0].params) - .unwrap(); - assert!(row.metrics.resources.peak_memory_bytes.is_none()); - assert_eq!( - row.metrics.resources.disk_bytes.as_ref().unwrap().value, - 0.0 - ); - assert_eq!(row.provenance, artifact.records[0].provenance); - assert_eq!(row.error.as_ref().unwrap().query["kind"], "point_frequency"); - } - - /// Malformed values, schema versions and ambiguous live records cannot - /// silently become plausible costs. - #[test] - fn invalid_and_ambiguous_artifacts_are_rejected() { - let (artifact, context, _) = fixture(); - let mut bad = artifact.clone(); - bad.schema_version = 2; - assert_eq!(bad.validate(), Err(EvidenceError::UnsupportedVersion(2))); - let mut bad = artifact.clone(); - bad.records[0] - .metrics - .resources - .cpu - .update_cpu_ns - .as_mut() - .unwrap() - .value = f64::NAN; - assert!(bad.validate().is_err()); - let mut bad = artifact.clone(); - bad.records[0].params = SketchParams::Hll { precision: 14 }; - assert!(bad.validate().is_err()); - let mut bad = artifact.clone(); - bad.records[0].provenance.repetitions = 0; - assert!(bad.validate().is_err()); - let mut duplicate = artifact.records[0].clone(); - duplicate.id = "another-live-measurement".into(); - let mut ambiguous = artifact.clone(); - ambiguous.records.push(duplicate); - let provider = EmpiricalEvidenceProvider::new(ambiguous, context).unwrap(); - assert_eq!( - provider.lookup(&artifact.records[0].algorithm, &artifact.records[0].params), - Err(EvidenceError::Ambiguous) - ); - } -} diff --git a/crates/plan-selection/src/cost/empirical_resources.rs b/crates/plan-selection/src/cost/empirical_resources.rs deleted file mode 100644 index 63bf9077..00000000 --- a/crates/plan-selection/src/cost/empirical_resources.rs +++ /dev/null @@ -1,202 +0,0 @@ -//! Measured resource payloads and compatibility with the v1 benchmark wire format. -//! -//! Physical dimensions live in `asap_types::workload::resources`; flat wire structs below -//! exist only to keep archived artifacts readable and preserve their field names. - -use asap_types::workload::resources::PhysicalResources; -use serde::{Deserialize, Serialize}; - -pub use asap_types::workload::resources::{MeasuredCpu, MeasuredResources, Measurement}; - -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] -#[serde(from = "SketchWire", into = "SketchWire")] -pub struct ResourceMeasurements { - pub resources: MeasuredResources, -} - -#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] -#[serde(from = "ExactWire", into = "ExactWire")] -pub struct ExactResourceMeasurements { - pub resources: MeasuredResources, -} - -// Keep the archived flat v1 schema at the serialization boundary only. New -// optional dimensions are omitted when absent, so legacy snapshots round-trip. -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct SketchWire { - build_cpu_ns: Option, - update_cpu_ns: Option, - merge_cpu_ns: Option, - read_cpu_ns: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - prepare_cpu_ns: Option, - retained_bytes: Option, - peak_bytes: Option, - serialized_bytes: Option, - disk_bytes: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - scan_bytes: Option, -} - -#[derive(Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -struct ExactWire { - empty_build_cpu_ns: Option, - update_cpu_ns: Option, - prepare_cpu_ns: Option, - read_cpu_ns: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - merge_cpu_ns: Option, - retained_bytes: Option, - peak_bytes: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - serialized_bytes: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - disk_bytes: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - scan_bytes: Option, -} - -impl From for ResourceMeasurements { - fn from(w: SketchWire) -> Self { - Self { - resources: PhysicalResources { - cpu: MeasuredCpu { - build_cpu_ns: w.build_cpu_ns, - update_cpu_ns: w.update_cpu_ns, - merge_cpu_ns: w.merge_cpu_ns, - prepare_cpu_ns: w.prepare_cpu_ns, - read_cpu_ns: w.read_cpu_ns, - }, - retained_memory_bytes: w.retained_bytes, - peak_memory_bytes: w.peak_bytes, - serialized_bytes: w.serialized_bytes, - disk_bytes: w.disk_bytes, - scan_bytes: w.scan_bytes, - }, - } - } -} - -impl From for SketchWire { - fn from(value: ResourceMeasurements) -> Self { - let r = value.resources; - Self { - build_cpu_ns: r.cpu.build_cpu_ns, - update_cpu_ns: r.cpu.update_cpu_ns, - merge_cpu_ns: r.cpu.merge_cpu_ns, - prepare_cpu_ns: r.cpu.prepare_cpu_ns, - read_cpu_ns: r.cpu.read_cpu_ns, - retained_bytes: r.retained_memory_bytes, - peak_bytes: r.peak_memory_bytes, - serialized_bytes: r.serialized_bytes, - disk_bytes: r.disk_bytes, - scan_bytes: r.scan_bytes, - } - } -} - -impl From for ExactResourceMeasurements { - fn from(w: ExactWire) -> Self { - Self { - resources: PhysicalResources { - cpu: MeasuredCpu { - build_cpu_ns: w.empty_build_cpu_ns, - update_cpu_ns: w.update_cpu_ns, - merge_cpu_ns: w.merge_cpu_ns, - prepare_cpu_ns: w.prepare_cpu_ns, - read_cpu_ns: w.read_cpu_ns, - }, - retained_memory_bytes: w.retained_bytes, - peak_memory_bytes: w.peak_bytes, - serialized_bytes: w.serialized_bytes, - disk_bytes: w.disk_bytes, - scan_bytes: w.scan_bytes, - }, - } - } -} - -impl From for ExactWire { - fn from(value: ExactResourceMeasurements) -> Self { - let r = value.resources; - Self { - empty_build_cpu_ns: r.cpu.build_cpu_ns, - update_cpu_ns: r.cpu.update_cpu_ns, - merge_cpu_ns: r.cpu.merge_cpu_ns, - prepare_cpu_ns: r.cpu.prepare_cpu_ns, - read_cpu_ns: r.cpu.read_cpu_ns, - retained_bytes: r.retained_memory_bytes, - peak_bytes: r.peak_memory_bytes, - serialized_bytes: r.serialized_bytes, - disk_bytes: r.disk_bytes, - scan_bytes: r.scan_bytes, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn measured(value: f64) -> Option { - Some(Measurement { - value, - stddev: Some(0.5), - samples: 5, - method: Some("test only".into()), - }) - } - - /// The same resource dimensions retain uncertainty through either wire adapter. - #[test] - fn shared_resources_round_trip_without_losing_dimensions() { - let resources = PhysicalResources { - cpu: MeasuredCpu { - build_cpu_ns: measured(1.0), - update_cpu_ns: measured(2.0), - merge_cpu_ns: measured(3.0), - prepare_cpu_ns: measured(4.0), - read_cpu_ns: measured(5.0), - }, - peak_memory_bytes: measured(100.0), - retained_memory_bytes: measured(70.0), - scan_bytes: measured(200.0), - serialized_bytes: measured(40.0), - disk_bytes: measured(4096.0), - }; - let sketch = ResourceMeasurements { - resources: resources.clone(), - }; - let exact = ExactResourceMeasurements { resources }; - assert_eq!( - serde_json::from_value::(serde_json::to_value(&sketch).unwrap()) - .unwrap(), - sketch - ); - assert_eq!( - serde_json::from_value::( - serde_json::to_value(&exact).unwrap() - ) - .unwrap(), - exact - ); - } - - /// Archived names remain accepted, but physical fields use canonical names. - #[test] - fn legacy_fields_map_to_shared_resources() { - let value = serde_json::json!({"empty_build_cpu_ns": measured(3.0), "peak_bytes": measured(100.0), "retained_bytes": measured(70.0)}); - let exact: ExactResourceMeasurements = serde_json::from_value(value).unwrap(); - assert_eq!(exact.resources.cpu.build_cpu_ns, measured(3.0)); - assert_eq!(exact.resources.peak_memory_bytes, measured(100.0)); - assert_eq!(exact.resources.retained_memory_bytes, measured(70.0)); - assert!(exact.resources.scan_bytes.is_none()); - assert!(exact.resources.disk_bytes.is_none()); - assert!( - serde_json::from_value::(serde_json::json!({"cpu_ops": 42})) - .is_err() - ); - } -} diff --git a/crates/plan-selection/src/cost/mod.rs b/crates/plan-selection/src/cost/mod.rs index 6c1f427b..b2654223 100644 --- a/crates/plan-selection/src/cost/mod.rs +++ b/crates/plan-selection/src/cost/mod.rs @@ -1,15 +1,10 @@ -//! Stage 3 pricing: the [`CostModel`](cost_model::CostModel) trait every -//! deployment's cost-based selection plugs into, with the built-in -//! [`DefaultCostModel`](cost_model::DefaultCostModel); recurring and one-shot -//! cost rates ([`recurrence`]); analytical and evidence-based pricing -//! ([`analytical_cost`], [`empirical_cost`]); and -//! the physical lowering and storage I/O profiles they price -//! ([`query_physical_lowering`], [`storage_io`]). +//! Stage 3 pricing: analytical operator costs ([`analytical_cost`]) over +//! edge statistics ([`physical_operator_statistics`]), evaluation rates from +//! query recurrence ([`recurrence`]), and the physical lowering and storage +//! I/O profiles a deployment can price ([`query_physical_lowering`], +//! [`storage_io`], [`physical_handoff_cost`]). pub mod analytical_cost; -pub mod cost_model; -pub mod empirical_cost; -pub mod empirical_resources; pub mod physical_handoff_cost; pub mod physical_operator_statistics; pub mod query_physical_lowering; diff --git a/crates/plan-selection/src/cost/recurrence.rs b/crates/plan-selection/src/cost/recurrence.rs index 9bb8392f..b4329625 100644 --- a/crates/plan-selection/src/cost/recurrence.rs +++ b/crates/plan-selection/src/cost/recurrence.rs @@ -1,270 +1,29 @@ -//! Recurrence-aware cost context (issue #287). -//! -//! ASAPPlanner already models recurring-workload metadata -//! ([`asap_types::workload::RepeatingEntry`]) and ingest-rate metadata -//! ([`asap_types::workload::DataWorkload`]), but until this module -//! neither reached [`CostModel`]'s CSE share-vs-recompute decision -//! ([`CostModel::cse_share_decision`]): that decision only ever compared a -//! *structural* consumer count (how many workload locations reference a -//! shared sub-DAG) against a flat per-family maintenance weight — it had no -//! notion of how *often* those consumers actually run. -//! -//! This module adds that notion as a generic cost context, not a scheduler: -//! no Prometheus rule-group semantics, no execution loop, no temporal-pane -//! boundary reasoning (see the module's own "Out of scope" list mirrored -//! from the issue). -//! -//! ## Units — every new cost input names its own unit explicitly -//! -//! | Type | Unit | Meaning | -//! |---|---|---| -//! | [`UpdateRate`] | Hz (updates/second) | how often the *raw* data underlying a maintained summary changes (ingest rate) | -//! | [`EvaluationRate`] | Hz (evaluations/second) | how often a target is *read* — `sum(1 / query_interval_i)` over every repeating consumer | -//! | [`CostRate`] | cost units / second | a steady-state cost rate — never comparable to a bare [`Cost`](crate::cost::cost_model::Cost) without going through [`total_cost`] | -//! | [`Horizon`] | seconds | the explicit evaluation window a caller supplies to compare a rate-valued cost against a one-shot cost | -//! -//! `Cost` (bare, from [`crate::cost::cost_model`]) stays a one-time, unitless -//! magnitude — exactly what it was before this module existed, preserved -//! for [`CostModel::cse_share_decision`] and everything else that already -//! uses it. `CostRate` is a *new*, distinct type specifically so a rate and -//! a one-shot cost can never be added directly (no `impl Add for -//! CostRate`, and vice versa) — the compiler enforces the issue's "must not -//! silently combine rate-valued and one-shot costs" requirement; [`total_cost`] -//! is the one sanctioned way to combine them, and it takes an explicit -//! [`Horizon`] to do it. -//! -//! ## Cost semantics (from the issue) -//! -//! For a maintained summary: -//! -//! ```text -//! maintained_cost_rate = -//! update_rate * maintenance_cost_per_update -//! + evaluation_rate * summary_read_cost -//! ``` -//! -//! For recomputation from the pre-ASAP/raw path: -//! -//! ```text -//! recompute_cost_rate = evaluation_rate * raw_recompute_cost -//! ``` -//! -//! For a summary shared by multiple repeating consumers with intervals -//! `t1..tn`: -//! -//! ```text -//! evaluation_rate = sum(1 / query_interval_i) -//! ``` -//! ([`evaluation_rate_of`].) -//! -//! Repetition amortizes a maintained summary across more reads; it does -//! *not* reduce the physical maintenance work caused by ingest updates — -//! that's why `update_rate` and `evaluation_rate` are two separate terms in -//! `maintained_cost_rate` above, never folded into one. -//! -//! One-shot consumers are represented separately from a steady-state rate -//! ([`RecurrenceProfile::one_shot_consumers`]). Comparing a mix of one-shot -//! and repeating work requires an explicit evaluation horizon `H`: -//! -//! ```text -//! total_cost(H) = recurring_cost_rate * H + one_shot_cost -//! ``` -//! ([`total_cost`].) -//! -//! ## Provenance of each new cost input -//! -//! - [`EvaluationRate`]: derived from [`asap_types::workload::RepeatingEntry::demand`] -//! values of every repeating consumer reaching a target (via -//! [`evaluation_rate_of`], or `recurrence_profiles` -//! for a whole workload). A one-shot ([`asap_types::workload::BatchEntry`]) -//! consumer contributes to [`RecurrenceProfile::one_shot_consumers`] -//! instead, never to this rate. -//! - [`UpdateRate`]: read from workload-level -//! [`asap_types::workload::DataWorkload::ingestion_rate`] via -//! [`update_rate_from_data_workload`]. Missing evidence remains unknown; -//! data at rest is not assigned a fabricated update rate. -//! - `maintenance_cost_per_update` / `summary_read_cost` / -//! `raw_recompute_cost`: [`CostModel`] hooks (defaults documented on the -//! trait itself, in `cost_model.rs`) — illustrative placeholders, like -//! every other default in that trait; a deployment with real numbers -//! overrides them. -//! -//! ## Preserving existing behavior when recurrence metadata is unavailable -//! -//! [`RecurrenceProfile::is_empty`] is `true` exactly when a caller supplied -//! no [`RepeatingEntry`](asap_types::workload::RepeatingEntry)/ -//! [`DataWorkload`](asap_types::workload::DataWorkload)-derived -//! information at all (no evaluation rate, no update rate, zero recorded -//! one-shot consumers — [`RecurrenceProfile::EMPTY`], its `Default`). -//! [`CostModel::cse_share_decision_with_recurrence`]'s default body checks -//! this first and, when true, delegates to -//! [`CostModel::cse_share_decision`] byte-for-byte — the existing, -//! structural-consumer-count decision this module never has to touch or -//! second-guess when there's nothing new to feed it. +//! Query recurrence as an evaluation rate (issue #287): Stage 3 charges a +//! query-time node per evaluation, at the summed rate of the repeating roots +//! that reach it. -use std::fmt; - -use asap_types::workload::{DataWorkload, RepetitionInterval}; - -use crate::cost::cost_model::{Cost, CostModel, CseCandidate, ShareDecision}; - -// ── Units ──────────────────────────────────────────────────────────────── - -/// How often the *raw* data underlying a maintained summary changes — -/// ingest/update events per second (Hz). See the module docs' provenance -/// table: normally read from [`DataWorkload::ingestion_rate`] via -/// [`update_rate_from_data_workload`]. -#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] -pub struct UpdateRate(pub f64); +use asap_types::workload::RepetitionInterval; /// How often a target is *evaluated* by its consumers — evaluations per /// second (Hz). For a summary shared by repeating consumers with intervals -/// `t1..tn`, `sum(1 / t_i)` ([`evaluation_rate_of`]); a one-shot consumer -/// never contributes to this rate (see [`RecurrenceProfile::one_shot_consumers`]). +/// `t1..tn`, `sum(1 / t_i)` ([`evaluation_rate_of`]). #[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] pub struct EvaluationRate(pub f64); -/// A steady-state cost rate: cost units per second. Deliberately a -/// different type from [`Cost`] (a one-time, unitless magnitude) — there is -/// no `impl Add for CostRate` on purpose, so a rate and a one-shot -/// cost can never be combined except explicitly, through [`total_cost`]. -#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] -pub struct CostRate(pub f64); - -impl CostRate { - /// A cost rate of exactly zero (no ongoing cost at all). - pub const ZERO: CostRate = CostRate(0.0); -} - -impl fmt::Display for CostRate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}/s", self.0) - } -} - -impl std::ops::Add for CostRate { - type Output = CostRate; - fn add(self, rhs: CostRate) -> CostRate { - CostRate(self.0 + rhs.0) - } -} - -/// An explicit evaluation horizon, in seconds — the only input that lets a -/// [`CostRate`] be combined with a one-shot [`Cost`] (via [`total_cost`]). -#[derive(Debug, Clone, Copy, PartialEq, PartialOrd)] -pub struct Horizon(pub f64); - -/// The one sanctioned way to combine a steady-state [`CostRate`] with a -/// one-shot [`Cost`]: `rate * horizon + one_shot`. There is no other path -/// in this module (or in [`CostModel`]) that adds a `CostRate` to a `Cost` -/// — every other cost value stays in exactly one of the two units, -/// enforced by the type system, satisfying issue #287's "the cost model -/// must not silently combine rate-valued and one-shot costs" requirement. -pub fn total_cost(rate: CostRate, horizon: Horizon, one_shot: Cost) -> Cost { - Cost(rate.0 * horizon.0 + one_shot.0) -} - -// ── Errors ─────────────────────────────────────────────────────────────── - -/// Errors from building or applying recurrence-aware cost inputs. +/// Errors from deriving an evaluation rate. #[derive(Debug, Clone, Copy, PartialEq, thiserror::Error)] pub enum RecurrenceError { - /// A [`RepetitionInterval`] of zero (or, in principle, any non-positive - /// value — `RepetitionInterval` is `u32`-backed so only zero is - /// representable) was supplied. A zero interval has no finite rate - /// (`1 / 0`), so it cannot contribute to an [`EvaluationRate`]. + /// A [`RepetitionInterval`] of zero was supplied (`RepetitionInterval` is + /// `u32`-backed, so zero is the only non-positive value). A zero interval + /// has no finite rate (`1 / 0`), so it cannot contribute to an + /// [`EvaluationRate`]. #[error( "invalid RepetitionInterval({0:?}ms): a repeating query's interval must be > 0 to \ contribute a finite evaluation rate" )] InvalidInterval(RepetitionInterval), - /// A comparison mixed one-shot and repeating work - /// ([`RecurrenceProfile::one_shot_consumers`] > 0 alongside a non-empty - /// [`RecurrenceProfile::evaluation_rate`] or [`RecurrenceProfile::update_rate`]) - /// without an explicit [`Horizon`] to combine them — see [`total_cost`] - /// and the module docs' "Cost semantics" section. - #[error( - "comparison mixes one-shot and repeating work but no evaluation horizon was supplied; \ - an explicit Horizon is required to combine a CostRate with a one-shot Cost (see \ - recurrence::total_cost)" - )] - MissingHorizon, - /// An [`UpdateRate`] that isn't finite and non-negative (NaN, infinite, - /// or negative) was supplied — such a value would silently corrupt - /// every downstream comparison (a NaN rate makes every `<=`/`>` - /// comparison `false`, which [`decide`] would otherwise read as "always - /// recompute" with no diagnostic at all). Validated the same way - /// [`evaluation_rate_of`] validates a zero [`RepetitionInterval`]. - #[error( - "invalid UpdateRate({0:?}Hz): an update rate must be finite and >= 0 to contribute a \ - well-defined maintained_cost_rate" - )] - InvalidUpdateRate(UpdateRate), - #[error("invalid EvaluationRate({0:?}Hz): an evaluation rate must be finite and >= 0")] - InvalidEvaluationRate(EvaluationRate), - #[error(transparent)] - InvalidWorkload(#[from] asap_types::workload::WorkloadError), - #[error("workload entry index {index} is out of bounds for {entry_count} entries")] - InvalidWorkloadEntry { index: usize, entry_count: usize }, - /// A [`Horizon`] that isn't finite and strictly positive (NaN, - /// infinite, zero, or negative) was supplied — a non-positive or - /// infinite horizon would silently drop or invert the recurring - /// `CostRate` term in [`total_cost`], exactly the "silently combine" - /// outcome [`MissingHorizon`](Self::MissingHorizon) exists to prevent. - #[error( - "invalid Horizon({0:?}s): an evaluation horizon must be finite and > 0 to combine a \ - CostRate with a one-shot Cost without distorting the comparison" - )] - InvalidHorizon(Horizon), - /// `recurrence_profiles` was called - /// with a `root_recurrence` slice whose length doesn't match the - /// `CandidateLogicalASAPDAGs`'s own root count — a caller error, but recoverable - /// (this method's whole signature promises a `Result`, so this is - /// reported the same way every other input-validation failure is, - /// never a panic). - #[error( - "recurrence_profiles: root_recurrence must have one entry per root, in the same order \ - CandidateLogicalASAPDAGs::roots is in (got {got} entries for {expected} roots)" - )] - RootCountMismatch { - /// `CandidateLogicalASAPDAGs::roots.len()`. - expected: usize, - /// `root_recurrence.len()`. - got: usize, - }, } -/// Reject a non-finite or negative [`UpdateRate`] — the same validation -/// discipline [`evaluation_rate_of`] applies to each [`RepetitionInterval`], -/// applied at every point an `UpdateRate` enters a [`RecurrenceProfile`] -/// ([`RecurrenceProfile::with_update_rate`], -/// [`update_rate_from_data_workload`], -/// `recurrence_profiles`'s own parameter) -/// *and*, as a backstop that can't be bypassed by constructing a -/// `RecurrenceProfile` via its public fields directly, inside [`decide`] -/// itself before any comparison uses it. -pub fn validate_update_rate(rate: UpdateRate) -> Result { - if rate.0.is_finite() && rate.0 >= 0.0 { - Ok(rate) - } else { - Err(RecurrenceError::InvalidUpdateRate(rate)) - } -} - -/// Reject a non-finite or non-positive [`Horizon`] — validated inside -/// [`decide`] wherever a caller-supplied `horizon` is used, so `Horizon(0.0)` -/// or a negative horizon can't silently zero out or invert the recurring -/// `CostRate` term in [`total_cost`]. -pub fn validate_horizon(horizon: Horizon) -> Result { - if horizon.0.is_finite() && horizon.0 > 0.0 { - Ok(horizon) - } else { - Err(RecurrenceError::InvalidHorizon(horizon)) - } -} - -// ── Aggregation ────────────────────────────────────────────────────────── - /// `sum(1 / interval_i)`, converted from milliseconds /// ([`RepetitionInterval`]'s own unit) to Hz, over every repeating /// consumer's interval. `Ok(None)` when `intervals` is empty — "no @@ -288,351 +47,9 @@ where Ok(any.then_some(EvaluationRate(total_hz))) } -/// Read an [`UpdateRate`] from workload-level [`DataWorkload`] evidence. -/// Missing evidence remains `None`; a present non-finite or negative rate is -/// rejected rather than propagated into a [`RecurrenceProfile`]. -pub fn update_rate_from_data_workload( - workload: &DataWorkload, -) -> Result, RecurrenceError> { - workload - .ingestion_rate - .value - .map(|rate| validate_update_rate(UpdateRate(rate.0))) - .transpose() -} - -// ── RecurrenceProfile ──────────────────────────────────────────────────── - -/// Aggregated recurrence context for one [`CseCandidate`]'s shared target: -/// how fast it's evaluated, how many one-shot consumers reference it -/// separately, and how fast its underlying raw data updates. -/// -/// [`RecurrenceProfile::EMPTY`] (also its `Default`) represents "no -/// recurrence metadata available at all" — the case -/// [`CostModel::cse_share_decision_with_recurrence`]'s default body -/// recognizes via [`is_empty`](Self::is_empty) and treats as "preserve -/// existing (structural) behavior". -#[derive(Debug, Clone, Copy, PartialEq, Default)] -pub struct RecurrenceProfile { - /// `sum(1 / interval_i)` over every repeating consumer of this target, - /// in Hz. `None` when no repeating consumer references it. - pub evaluation_rate: Option, - /// How many one-shot (batch) consumers reference this target, - /// independent of `evaluation_rate` — see the module docs' "Cost - /// semantics" section on why these can't be merged into one rate. - pub one_shot_consumers: usize, - /// The ingest/update rate of the raw data this target (if maintained) - /// would be kept up to date against. `None` when no - /// [`DataWorkload::ingestion_rate`] evidence was available. - pub update_rate: Option, -} - -impl RecurrenceProfile { - /// No recurrence metadata at all: no evaluation rate, no one-shot - /// consumers, no update rate. - pub const EMPTY: RecurrenceProfile = RecurrenceProfile { - evaluation_rate: None, - one_shot_consumers: 0, - update_rate: None, - }; - - /// Whether this profile carries no recurrence information at all — the - /// "missing metadata" case [`CostModel::cse_share_decision_with_recurrence`] - /// falls back on. - pub fn is_empty(&self) -> bool { - self.evaluation_rate.is_none() && self.one_shot_consumers == 0 && self.update_rate.is_none() - } - - /// Build a profile purely from a set of repeating consumers' intervals - /// (no one-shot consumers, no update rate — attach those with - /// [`with_one_shot_consumers`](Self::with_one_shot_consumers)/ - /// [`with_update_rate`](Self::with_update_rate)). - pub fn from_repeating_intervals( - intervals: impl IntoIterator, - ) -> Result { - Ok(Self { - evaluation_rate: evaluation_rate_of(intervals)?, - ..Self::EMPTY - }) - } - - /// Attach a count of one-shot consumers. - pub fn with_one_shot_consumers(mut self, one_shot_consumers: usize) -> Self { - self.one_shot_consumers = one_shot_consumers; - self - } - - /// Attach an ingest/update rate (from [`update_rate_from_data_workload`] - /// or a deployment-specific measurement). Validated via - /// [`validate_update_rate`] — rejects a NaN, infinite, or negative rate - /// rather than silently storing it. - pub fn with_update_rate(mut self, update_rate: UpdateRate) -> Result { - self.update_rate = Some(validate_update_rate(update_rate)?); - Ok(self) - } -} - -/// How one workload root recurs — the opaque per-root tag -/// `recurrence_profiles` threads down to -/// every target reachable from that root. Mirrors -/// [`asap_types::workload::QueryWorkload`]'s own `query_batch` (one-shot) -/// vs. `repeating_queries` (an interval each) split, but at the -/// already-opaque `Id` granularity `search_workload`'s callers already use -/// — this crate needs no more of a caller's own query identity than "which -/// of these two recurrence kinds is this root". One-time roots always use a -/// count, so there is no second spelling for the common count-of-one case. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum RootRecurrence { - /// A declared number of one-time invocations for this root. - OneShotCount(usize), - /// A repeated root normalized to evaluations per second. - Repeating(EvaluationRate), - /// No reliable recurrence evidence was supplied. It contributes no read - /// count or evaluation rate, but remains distinct from zero demand. - Unknown, -} - -// ── Explanation ────────────────────────────────────────────────────────── - -/// The full evaluation [`CostModel::cse_share_decision_with_recurrence`] -/// returns: which alternative was selected, both compared cost rates -/// (and, when a [`Horizon`] was supplied, both compared totals), every -/// input that went into them, their units, and provenance — meant to be -/// both machine-consumable (e.g. by issue #286's DAG-viewer cost/benefit -/// annotations) and human-readable (via its [`fmt::Display`] impl). -#[derive(Debug, Clone, PartialEq)] -pub struct RecurrenceCostExplanation { - /// The selected alternative. - pub decision: ShareDecision, - /// `maintained_cost_rate` — cost units/second — as defined in the - /// module docs' "Cost semantics" section. `None` on the structural - /// fallback path (`RecurrenceProfile::is_empty()`): no rate was - /// computed at all there (the decision came from - /// [`CostModel::cse_share_decision`] instead), so this is "not - /// computed", deliberately distinct from a real, computed rate of - /// exactly zero. - pub maintained_cost_rate: Option, - /// `recompute_cost_rate` — cost units/second. `None` on the same - /// structural fallback path, for the same reason. - pub recompute_cost_rate: Option, - /// `total_cost(horizon)` for the maintained alternative, when `horizon` - /// is `Some`. - pub maintained_total: Option, - /// `total_cost(horizon)` for the recompute alternative, when `horizon` - /// is `Some`. - pub recompute_total: Option, - /// The horizon the totals above were computed over, if any. - pub horizon: Option, - /// The [`UpdateRate`] input used, if any. - pub update_rate: Option, - /// The [`EvaluationRate`] input used, if any. - pub evaluation_rate: Option, - /// The one-shot consumer count input used. - pub one_shot_consumers: usize, - /// `maintenance_cost_per_update` — cost units/update. `None` on the - /// structural fallback path (not computed there). - pub maintenance_cost_per_update: Option, - /// `summary_read_cost` — cost units/read. `None` on the structural - /// fallback path. - pub summary_read_cost: Option, - /// `raw_recompute_cost` — cost units/recomputation. `None` on the - /// structural fallback path. - pub raw_recompute_cost: Option, - /// `summary_build_cost` — one-time cost of materializing the maintained - /// summary. `None` on the structural fallback path. - pub summary_build_cost: Option, - /// Human-readable provenance: which model/path produced this - /// explanation (e.g. "recurrence-aware: " or the - /// structural fallback note when `RecurrenceProfile::is_empty()`). - pub provenance: String, -} - -impl fmt::Display for RecurrenceCostExplanation { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - writeln!(f, "decision: {:?} ({})", self.decision, self.provenance)?; - match (self.maintained_cost_rate, self.recompute_cost_rate) { - (Some(maintained), Some(recompute)) => { - writeln!( - f, - " maintained_cost_rate = {} (update_rate={:?}Hz * \ - maintenance_cost_per_update={:?} + evaluation_rate={:?}Hz * \ - summary_read_cost={:?})", - maintained, - self.update_rate.map(|r| r.0), - self.maintenance_cost_per_update, - self.evaluation_rate.map(|r| r.0), - self.summary_read_cost, - )?; - writeln!( - f, - " recompute_cost_rate = {} (evaluation_rate={:?}Hz * \ - raw_recompute_cost={:?})", - recompute, - self.evaluation_rate.map(|r| r.0), - self.raw_recompute_cost, - )?; - } - _ => { - writeln!( - f, - " maintained_cost_rate / recompute_cost_rate: not computed (structural \ - fallback — no recurrence metadata was supplied)" - )?; - } - } - writeln!(f, " one_shot_consumers = {}", self.one_shot_consumers)?; - if let Some(h) = self.horizon { - writeln!( - f, - " horizon = {}s; summary_build_cost = {:?}; maintained_total = {:?}, \ - recompute_total = {:?}", - h.0, self.summary_build_cost, self.maintained_total, self.recompute_total - )?; - } - Ok(()) - } -} - -/// [`CostModel::cse_share_decision_with_recurrence`]'s default body — see -/// that method's own doc for the decision rule; kept as a free function so -/// the logic exists exactly once regardless of how many `CostModel` -/// implementors inherit the default. -pub(crate) fn decide( - cost_model: &C, - candidate: &CseCandidate, - recurrence: &RecurrenceProfile, - horizon: Option, -) -> Result { - // Backstop validation: a `RecurrenceProfile`/`Horizon` may have reached - // here via a direct struct literal (bypassing `with_update_rate`'s own - // check) or a caller-supplied `horizon` argument — this is the one - // choke point every path funnels through before a value actually enters - // a comparison, so it's validated here regardless of how it arrived. - if let Some(rate) = recurrence.update_rate { - validate_update_rate(rate)?; - } - if let Some(h) = horizon { - validate_horizon(h)?; - } - - if recurrence.is_empty() { - let decision = cost_model.cse_share_decision(candidate); - return Ok(RecurrenceCostExplanation { - decision, - maintained_cost_rate: None, - recompute_cost_rate: None, - maintained_total: None, - recompute_total: None, - horizon: None, - update_rate: None, - evaluation_rate: None, - one_shot_consumers: 0, - maintenance_cost_per_update: None, - summary_read_cost: None, - raw_recompute_cost: None, - summary_build_cost: None, - provenance: "structural fallback: no recurrence metadata supplied — delegated to \ - CostModel::cse_share_decision (consumer_count-based), preserving \ - pre-#287 behavior exactly" - .to_string(), - }); - } - - let has_recurring = recurrence.evaluation_rate.is_some() || recurrence.update_rate.is_some(); - if recurrence.one_shot_consumers > 0 && has_recurring && horizon.is_none() { - return Err(RecurrenceError::MissingHorizon); - } - - let update_rate = recurrence.update_rate.map_or(0.0, |r| r.0); - let evaluation_rate = recurrence.evaluation_rate.map_or(0.0, |r| r.0); - - let maintenance_cost_per_update = cost_model.maintenance_cost_per_update(candidate); - let summary_read_cost = cost_model.summary_read_cost(candidate); - let raw_recompute_cost = cost_model.raw_recompute_cost(candidate); - // The one-time cost of materializing the shared summary at all, before - // any read or update — see `CostModel::summary_build_cost`'s own doc. - // Without this term, a purely (or mostly) one-shot comparison modeled - // "maintained" as free to construct, so `Share` won unconditionally - // for *any* number of one-shot consumers (issue #287 review, bug 1) — - // this term is what makes materializing-and-reading actually cost more - // than a single direct recompute for a lone consumer, while still - // amortizing correctly across many. - let summary_build_cost = cost_model.summary_build_cost(candidate); - - let maintained_cost_rate = CostRate( - update_rate * maintenance_cost_per_update.0 + evaluation_rate * summary_read_cost.0, - ); - let recompute_cost_rate = CostRate(evaluation_rate * raw_recompute_cost.0); - - // A pure one-shot comparison (no recurring rate at all — `has_recurring` - // is false, so the `MissingHorizon` gate above never fired even though - // `one_shot_consumers > 0`) still needs *some* horizon to run - // `total_cost` through, or its one-shot costs would never enter the - // decision at all. Any positive horizon gives the same ordering here, - // since `maintained_cost_rate`/`recompute_cost_rate` are both exactly - // zero in this case (`rate * H` contributes nothing regardless of `H`) - // — so an implicit `Horizon(1.0)` is exact, not approximate, and this - // is never reached for the genuinely mixed case (that already required - // an explicit `horizon` above). - let effective_horizon = horizon - .or_else(|| (recurrence.one_shot_consumers > 0 && !has_recurring).then_some(Horizon(1.0))); - - let (maintained_total, recompute_total, decision) = if let Some(h) = effective_horizon { - // `summary_build_cost` is paid exactly once — whether the summary - // is ever read again by a repeating consumer or not — never scaled - // by `one_shot_consumers`. - let one_shot_maintained = - Cost(summary_read_cost.0 * recurrence.one_shot_consumers as f64) + summary_build_cost; - let one_shot_recompute = Cost(raw_recompute_cost.0 * recurrence.one_shot_consumers as f64); - let maintained = total_cost(maintained_cost_rate, h, one_shot_maintained); - let recompute = total_cost(recompute_cost_rate, h, one_shot_recompute); - let decision = if maintained.0 <= recompute.0 { - ShareDecision::Share - } else { - ShareDecision::RecomputeIndependently - }; - (Some(maintained), Some(recompute), decision) - } else { - // Pure recurring, no one-shot consumers at all: comparing the bare - // rates is exactly equivalent to comparing `rate * H` for any fixed - // `H > 0` in the limit of a long-lived, continuously-maintained - // summary, so no horizon is needed to get a correct decision — the - // one-time `summary_build_cost` is asymptotically negligible next - // to an ongoing rate term and is deliberately not charged here (it - // only enters the comparison when a caller actually needs an - // absolute total over a finite horizon, via the branch above). - let decision = if maintained_cost_rate.0 <= recompute_cost_rate.0 { - ShareDecision::Share - } else { - ShareDecision::RecomputeIndependently - }; - (None, None, decision) - }; - - Ok(RecurrenceCostExplanation { - decision, - maintained_cost_rate: Some(maintained_cost_rate), - recompute_cost_rate: Some(recompute_cost_rate), - maintained_total, - recompute_total, - horizon: effective_horizon, - update_rate: recurrence.update_rate, - evaluation_rate: recurrence.evaluation_rate, - one_shot_consumers: recurrence.one_shot_consumers, - maintenance_cost_per_update: Some(maintenance_cost_per_update), - summary_read_cost: Some(summary_read_cost), - raw_recompute_cost: Some(raw_recompute_cost), - summary_build_cost: Some(summary_build_cost), - provenance: "recurrence-aware: maintained_cost_rate vs recompute_cost_rate (cost \ - units/second), per issue #287" - .to_string(), - }) -} - #[cfg(test)] mod tests { use super::*; - use crate::cost::cost_model::DefaultCostModel; fn interval(ms: u32) -> RepetitionInterval { RepetitionInterval(ms) @@ -666,483 +83,4 @@ mod tests { let err = evaluation_rate_of(vec![interval(1000), interval(0)]).unwrap_err(); assert_eq!(err, RecurrenceError::InvalidInterval(interval(0))); } - - // ── update_rate_from_data_workload ─────────────────────────────────── - - #[test] - fn update_rate_from_data_workload_reads_ingestion_rate_evidence() { - let workload = DataWorkload { - ingestion_rate: asap_types::workload::Evidence { - value: Some(asap_types::workload::Rate(100.0)), - ..Default::default() - }, - ..Default::default() - }; - let rate = update_rate_from_data_workload(&workload).unwrap().unwrap(); - assert!((rate.0 - 100.0).abs() < 1e-9); - } - - #[test] - fn update_rate_from_data_workload_rejects_a_negative_rate() { - let workload = DataWorkload { - ingestion_rate: asap_types::workload::Evidence { - value: Some(asap_types::workload::Rate(-0.1)), - ..Default::default() - }, - ..Default::default() - }; - let err = update_rate_from_data_workload(&workload).unwrap_err(); - assert!(matches!(err, RecurrenceError::InvalidUpdateRate(_))); - } - - #[test] - fn update_rate_from_data_workload_preserves_missing_evidence() { - assert_eq!( - update_rate_from_data_workload(&DataWorkload::default()).unwrap(), - None - ); - } - - // ── RecurrenceProfile ───────────────────────────────────────────────── - - #[test] - fn empty_profile_is_empty() { - assert!(RecurrenceProfile::EMPTY.is_empty()); - assert!(RecurrenceProfile::default().is_empty()); - } - - #[test] - fn profile_with_any_field_set_is_not_empty() { - assert!(!RecurrenceProfile::EMPTY - .with_one_shot_consumers(1) - .is_empty()); - assert!(!RecurrenceProfile::EMPTY - .with_update_rate(UpdateRate(1.0)) - .unwrap() - .is_empty()); - assert!( - !RecurrenceProfile::from_repeating_intervals(vec![interval(1000)]) - .unwrap() - .is_empty() - ); - } - - #[test] - fn with_update_rate_rejects_nan_infinite_and_negative() { - for bad in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY, -1.0] { - let err = RecurrenceProfile::EMPTY - .with_update_rate(UpdateRate(bad)) - .unwrap_err(); - assert!( - matches!(err, RecurrenceError::InvalidUpdateRate(_)), - "bad={bad}, err={err:?}" - ); - } - // Zero is a legitimate (if unusual) update rate: no ingest at all. - assert!(RecurrenceProfile::EMPTY - .with_update_rate(UpdateRate(0.0)) - .is_ok()); - } - - #[test] - fn from_repeating_intervals_rejects_zero_interval() { - let err = RecurrenceProfile::from_repeating_intervals(vec![interval(0)]).unwrap_err(); - assert_eq!(err, RecurrenceError::InvalidInterval(interval(0))); - } - - // ── total_cost / unit consistency ──────────────────────────────────── - - #[test] - fn total_cost_combines_rate_and_one_shot_explicitly() { - let rate = CostRate(2.0); - let horizon = Horizon(10.0); - let one_shot = Cost(5.0); - assert_eq!(total_cost(rate, horizon, one_shot), Cost(25.0)); - } - - #[test] - fn cost_rate_and_cost_are_distinct_types() { - // Compile-time unit-consistency check: `CostRate` has no `Add` - // or `From` — the only way shown here to combine them is - // `total_cost`, which forces an explicit `Horizon`. (This test's - // real assertion is that the crate compiles at all with no such - // impls; the runtime assertion below just exercises the intended - // combination path.) - let combined = total_cost(CostRate(1.0), Horizon(1.0), Cost(1.0)); - assert_eq!(combined, Cost(2.0)); - } - - // ── decide (structural fallback) ───────────────────────────────────── - - use crate::cost::cost_model::CseCandidate; - use asap_types::ir::operator::operator_properties::{Reduction, Source}; - use asap_types::ir::properties::ResultGuarantee; - use asap_types::ir::scalar::ColumnRef; - use asap_types::ir::schema::DataType; - use asap_types::ir::schema::{ - ExactKind, ExactParams, Field, FieldDataType, GroupingStrategy, Schema, - }; - use asap_types::ir::{ASAPOp, NonASAPOp, OperatorNode}; - - use std::rc::Rc; - - fn scan() -> Rc { - OperatorNode::new_shared(asap_types::ir::Operator::NonASAP(NonASAPOp::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - })) - .unwrap() - } - - /// A summary of `family` over the kept pre-ASAP scan. - fn summary_node(family: FieldDataType) -> Rc { - let kept = Rc::new( - scan() - .as_ref() - .clone() - .with_guarantee(Some(ResultGuarantee::exact("RetainedExact"))), - ); - std::rc::Rc::new( - OperatorNode::with_schema( - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - child: kept, - family: family.clone(), - input: asap_types::ir::schema::SummaryUpdate::column(ColumnRef::Named( - "value".into(), - )), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }), - Schema::lifted(vec![Field::new("state", family, false)], None), - ) - .with_guarantee(None), - ) - } - - #[test] - fn decide_falls_back_to_structural_decision_when_profile_is_empty() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 1000, - }; - let explanation = decide( - &DefaultCostModel, - &candidate, - &RecurrenceProfile::EMPTY, - None, - ) - .unwrap(); - assert_eq!( - explanation.decision, - DefaultCostModel.cse_share_decision(&candidate) - ); - assert!(explanation.provenance.contains("structural fallback")); - } - - #[test] - fn decide_rejects_mixed_one_shot_and_repeating_without_horizon() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 2, - }; - let profile = RecurrenceProfile::from_repeating_intervals(vec![interval(1000)]) - .unwrap() - .with_one_shot_consumers(1); - let err = decide(&DefaultCostModel, &candidate, &profile, None).unwrap_err(); - assert_eq!(err, RecurrenceError::MissingHorizon); - } - - #[test] - fn decide_accepts_mixed_one_shot_and_repeating_with_an_explicit_horizon() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 2, - }; - let profile = RecurrenceProfile::from_repeating_intervals(vec![interval(1000)]) - .unwrap() - .with_one_shot_consumers(1); - let explanation = decide( - &DefaultCostModel, - &candidate, - &profile, - Some(Horizon(3600.0)), - ) - .unwrap(); - assert!(explanation.maintained_total.is_some()); - assert!(explanation.recompute_total.is_some()); - assert!(explanation.summary_build_cost.is_some()); - assert_eq!(explanation.horizon, Some(Horizon(3600.0))); - } - - // ── A deterministic cost model exercising the trait hooks directly ── - - struct DeterministicUnitCostModel; - impl CostModel for DeterministicUnitCostModel { - fn allow_uncosted_legacy_selection(&self) -> bool { - true - } - - fn rank_candidates( - &self, - _intent: &asap_types::ir::operator::agg_intent::AggIntent, - candidates: &[asap_types::ir::schema::SketchAlgorithm], - ) -> Vec { - candidates.to_vec() - } - fn maintenance_cost_per_update(&self, _candidate: &CseCandidate) -> Cost { - Cost(1.0) - } - fn summary_read_cost(&self, _candidate: &CseCandidate) -> Cost { - Cost(1.0) - } - fn raw_recompute_cost(&self, _candidate: &CseCandidate) -> Cost { - Cost(50.0) - } - } - - /// Issue #287's headline acceptance criterion: with a deterministic test - /// cost model and identical IR, a high evaluation frequency selects - /// maintained/shared state while a sufficiently infrequent workload - /// selects recomputation — driven purely by `evaluation_rate`, with a - /// fixed, nonzero `update_rate` representing continuous ingest that - /// keeps a maintained summary's floor cost independent of how often - /// it's read. - #[test] - fn high_frequency_selects_maintained_low_frequency_selects_recompute() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 1, - }; - - // A steady 10Hz ingest rate underlies both scenarios — the physical - // maintenance cost is unaffected by how often the summary is read - // (issue #287: "repetition ... does not reduce the physical - // maintenance work caused by ingest updates"). - let update_rate = UpdateRate(10.0); - - // High frequency: a consumer firing every 10ms => 100Hz. - let frequent = RecurrenceProfile::from_repeating_intervals(vec![interval(10)]) - .unwrap() - .with_update_rate(update_rate) - .unwrap(); - let frequent_explanation = - decide(&DeterministicUnitCostModel, &candidate, &frequent, None).unwrap(); - assert_eq!(frequent_explanation.decision, ShareDecision::Share); - assert!( - frequent_explanation.maintained_cost_rate.unwrap().0 - < frequent_explanation.recompute_cost_rate.unwrap().0 - ); - - // Low frequency: a consumer firing every 100s => 0.01Hz. - let infrequent = RecurrenceProfile::from_repeating_intervals(vec![interval(100_000)]) - .unwrap() - .with_update_rate(update_rate) - .unwrap(); - let infrequent_explanation = - decide(&DeterministicUnitCostModel, &candidate, &infrequent, None).unwrap(); - assert_eq!( - infrequent_explanation.decision, - ShareDecision::RecomputeIndependently - ); - assert!( - infrequent_explanation.maintained_cost_rate.unwrap().0 - > infrequent_explanation.recompute_cost_rate.unwrap().0 - ); - } - - /// Update rate feeds only `maintained_cost_rate` (via - /// `maintenance_cost_per_update`), never `recompute_cost_rate` — and - /// evaluation rate feeds both `summary_read_cost` (maintained) and - /// `raw_recompute_cost` (recompute), never bypassing either. Pins the - /// issue's "update rate affects maintained-summary cost but not read - /// frequency; evaluation rate affects summary-read and recomputation - /// cost" acceptance criterion directly against the trait hooks. - #[test] - fn update_rate_only_affects_maintained_cost_evaluation_rate_affects_both() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 1, - }; - - let base = RecurrenceProfile::from_repeating_intervals(vec![interval(1000)]).unwrap(); - let with_update = base.with_update_rate(UpdateRate(1000.0)).unwrap(); - - let base_explanation = - decide(&DeterministicUnitCostModel, &candidate, &base, None).unwrap(); - let with_update_explanation = - decide(&DeterministicUnitCostModel, &candidate, &with_update, None).unwrap(); - - // Bumping update_rate alone raises maintained_cost_rate... - assert!( - with_update_explanation.maintained_cost_rate.unwrap().0 - > base_explanation.maintained_cost_rate.unwrap().0 - ); - // ...but leaves recompute_cost_rate (a pure function of - // evaluation_rate, unchanged between the two profiles) untouched. - assert_eq!( - with_update_explanation.recompute_cost_rate, - base_explanation.recompute_cost_rate - ); - } - - /// One-shot consumers alone (no repeating consumer, no update rate) — - /// still produce a real decision with no explicit `Horizon` required, - /// by comparing the one-shot costs directly (see `decide`'s - /// `effective_horizon` fallback): with enough one-shot consumers, the - /// fixed `summary_build_cost` amortizes and sharing wins even though - /// `raw_recompute_cost` is expensive. - #[test] - fn one_shot_only_consumer_decides_without_an_explicit_horizon() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 1, - }; - let profile = RecurrenceProfile::EMPTY.with_one_shot_consumers(3); - - // raw_recompute_cost=50 >> summary_read_cost=1: sharing wins even - // for purely one-shot consumers, since materializing once (paying - // summary_build_cost=50 exactly once — DeterministicUnitCostModel's - // default delegates build cost to raw_recompute_cost) and reading it - // 3 times (3 * 1 = 3) totals 53, cheaper than recomputing - // independently 3 times (50 * 3 = 150). - let explanation = decide(&DeterministicUnitCostModel, &candidate, &profile, None).unwrap(); - assert_eq!(explanation.decision, ShareDecision::Share); - assert_eq!(explanation.maintained_total, Some(Cost(53.0))); - assert_eq!(explanation.recompute_total, Some(Cost(150.0))); - } - - /// Regression for issue #287 review bug 1: without a `summary_build_cost` - /// term, a purely one-shot comparison modeled "maintained" as free to - /// construct, so `Share` won unconditionally for *any* number of - /// one-shot consumers — including exactly one, where sharing can never - /// make sense (you always pay at least as much to build-then-read once - /// as you would to just recompute once directly). With the fix, a - /// single one-shot consumer strictly prefers `RecomputeIndependently`. - #[test] - fn one_shot_only_single_consumer_does_not_unconditionally_prefer_share() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 1, - }; - let profile = RecurrenceProfile::EMPTY.with_one_shot_consumers(1); - - let explanation = decide(&DeterministicUnitCostModel, &candidate, &profile, None).unwrap(); - // build(50) + read(1) = 51 > recompute(50) * 1 = 50. - assert_eq!(explanation.decision, ShareDecision::RecomputeIndependently); - assert_eq!(explanation.maintained_total, Some(Cost(51.0))); - assert_eq!(explanation.recompute_total, Some(Cost(50.0))); - } - - /// A batch-only workload (no `DataWorkload`, only one-shot - /// consumers) with the *default* `DefaultCostModel` must not - /// unconditionally prefer `Share` regardless of how many one-shot - /// consumers there are — issue #287 review bug 1's original repro, - /// pinned against the real default cost model rather than the test's - /// own `DeterministicUnitCostModel`. - #[test] - fn batch_only_workload_does_not_unconditionally_prefer_share_under_default_cost_model() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 1, - }; - // `DefaultCostModel`'s `raw_recompute_cost`/`summary_build_cost` - // both delegate to the same structural-size proxy - // (`cse_recompute_cost`), and `summary_read_cost` defaults to a - // nominal `1.0` — build and recompute cost the same, so reading a - // materialized copy even once more than a bare recompute can never - // pay off: for every one-shot-only consumer count, the maintained - // total (`build + read * n`) must be strictly greater than the - // recompute total (`recompute * n`), i.e. `Share` must never win. - for n in [1usize, 2, 10, 1_000] { - let profile = RecurrenceProfile::EMPTY.with_one_shot_consumers(n); - let explanation = decide(&DefaultCostModel, &candidate, &profile, None).unwrap(); - assert_eq!( - explanation.decision, - ShareDecision::RecomputeIndependently, - "n={n}, explanation={explanation:?}" - ); - } - } - - #[test] - fn decide_rejects_a_zero_or_negative_horizon() { - let sub_dag = scan(); - let bound = summary_node(FieldDataType::ExactAggregate( - ExactKind::Sum, - ExactParams::Sum, - )); - let candidate = CseCandidate { - sub_dag: &sub_dag, - bound_summary: &bound, - consumer_count: 2, - }; - let profile = RecurrenceProfile::from_repeating_intervals(vec![interval(1000)]) - .unwrap() - .with_one_shot_consumers(1); - for bad in [0.0, -1.0, f64::NAN, f64::INFINITY] { - let err = - decide(&DefaultCostModel, &candidate, &profile, Some(Horizon(bad))).unwrap_err(); - assert!( - matches!(err, RecurrenceError::InvalidHorizon(_)), - "bad={bad}, err={err:?}" - ); - } - } } diff --git a/crates/plan-selection/src/lib.rs b/crates/plan-selection/src/lib.rs index 61efc5d7..96598bda 100644 --- a/crates/plan-selection/src/lib.rs +++ b/crates/plan-selection/src/lib.rs @@ -2,8 +2,8 @@ //! that computes cost. Cargo enforces the stage order: this crate depends on //! `asap-types`, Stage 1 and Stage 2, never on the facade or the executor. //! -//! - [`cost`] — the [`CostModel`] trait, analytical and evidence-based -//! pricing, recurrence, and the physical lowering and storage I/O they price. +//! - [`cost`] — analytical pricing, evaluation rates from recurrence, and the +//! physical lowering and storage I/O profiles a deployment can price. //! //! Each Stage 2 candidate is checked against every query's accuracy target //! with the accuracy model, and Count-Min is admitted only over weights proven @@ -42,16 +42,7 @@ pub mod cost; mod test_support; pub use asap_types::deployment::DeploymentCapabilities; -pub use cost::cost_model::{ - maintenance_operation_plan_cost_rate, raw_recompute_cost_rate, read_operation_plan_cost_rate, - CostModel, CostProvenance, CostUnit, DefaultCostModel, ExactCompositionCostInputs, - ExactCompositionCostRequest, ValueOperationCapabilities, -}; -pub use cost::recurrence::{ - evaluation_rate_of, total_cost, update_rate_from_data_workload, CostRate, EvaluationRate, - Horizon, RecurrenceCostExplanation, RecurrenceError, RecurrenceProfile, RootRecurrence, - UpdateRate, -}; +pub use cost::recurrence::{evaluation_rate_of, EvaluationRate, RecurrenceError}; use asap_types::ir::NonASAPOp; use std::collections::{BTreeMap, HashMap}; @@ -110,7 +101,6 @@ const DEFAULT_LOOKBACK_MS: u64 = 60_000; /// program's assumptions do not hold. pub const MAX_ENUMERATED_CANDIDATES: usize = 64; -static DEFAULT_COST_MODEL: DefaultCostModel = DefaultCostModel; static DEFAULT_ACCURACY_MODEL: DefaultAccuracyModel = DefaultAccuracyModel; static NO_ACCURACY_EVIDENCE: NoAccuracyEvidence = NoAccuracyEvidence; static UNRESTRICTED: DeploymentCapabilities = DeploymentCapabilities::UNRESTRICTED; @@ -172,13 +162,10 @@ impl Stage3Calibration { /// Planning logic, as opposed to the scoped facts it consumes: a model can have /// a built-in default, evidence about a particular deployment cannot. With /// the deployment's capabilities, these are #509's deployment inputs. -/// -/// Stage 3 prices plans analytically, so the stage pipeline does not read -/// `cost`; only the legacy replacement search does (#580). +/// Stage 3 prices plans analytically ([`Stage3Calibration`]). #[derive(Clone, Copy)] #[non_exhaustive] pub struct PlanningModels<'a> { - pub cost: &'a dyn CostModel, pub accuracy: &'a dyn AccuracyModel, pub evidence: &'a dyn AccuracyEvidenceProvider, pub calibration: Stage3Calibration, @@ -189,12 +176,10 @@ pub struct PlanningModels<'a> { impl<'a> PlanningModels<'a> { pub fn new( - cost: &'a dyn CostModel, accuracy: &'a dyn AccuracyModel, evidence: &'a dyn AccuracyEvidenceProvider, ) -> Self { Self { - cost, accuracy, evidence, calibration: Stage3Calibration::ILLUSTRATIVE, @@ -202,12 +187,10 @@ impl<'a> PlanningModels<'a> { } } - /// The built-in models. `DefaultCostModel` does not override - /// `estimate_cost`, so this configuration ranks structurally and is not a + /// The built-in models. Stage 3's calibration is illustrative, not a /// measured deployment cost. pub fn builtin() -> PlanningModels<'static> { PlanningModels { - cost: &DEFAULT_COST_MODEL, accuracy: &DEFAULT_ACCURACY_MODEL, evidence: &NO_ACCURACY_EVIDENCE, calibration: Stage3Calibration::ILLUSTRATIVE, @@ -215,11 +198,6 @@ impl<'a> PlanningModels<'a> { } } - pub fn with_cost(mut self, cost: &'a dyn CostModel) -> Self { - self.cost = cost; - self - } - pub fn with_accuracy(mut self, accuracy: &'a dyn AccuracyModel) -> Self { self.accuracy = accuracy; self @@ -2094,9 +2072,20 @@ fn summary_shape(family: &FieldDataType) -> (u64, u64) { ), // An insert reaches about two layers; each updates `d` rows' // counters and sign-checks, then its heap. - params @ SketchParams::UnivMon { sketch_rows, .. } => ( + // Each layer holds `rows x cols` eight-byte counters plus a + // conservative 64 bytes per heap entry. + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + } => ( 2 * (2 * u64::from(*sketch_rows) + 4), - asap_logical_optimizer::pass1::replacement::sketch_state_bytes(params) + u64::from(*sketch_rows) + .checked_mul(u64::from(*sketch_cols)) + .and_then(|counters| counters.checked_mul(8)) + .and_then(|bytes| bytes.checked_add(u64::from(*heap_size).checked_mul(64)?)) + .and_then(|bytes| bytes.checked_mul(u64::from(*layers))) .unwrap_or(u64::MAX), ), _ => (1, 1_024), diff --git a/crates/planner/src/pass/mod.rs b/crates/planner/src/pass/mod.rs index 54ecf451..dec92082 100644 --- a/crates/planner/src/pass/mod.rs +++ b/crates/planner/src/pass/mod.rs @@ -2,8 +2,8 @@ //! //! An [`OptimizationPass`] is the whole optimization stage behind one //! signature: pre-ASAP IR in, post-ASAP DAG out. The trait deliberately names -//! none of this crate's two-phase vocabulary — no `CandidateLogicalASAPDAGs`, no -//! `TargetSubDAGCandidates`, no `ReplacementStrategy` — so an algorithm with no +//! none of the stage pipeline's vocabulary — no Stage 1 inventory, sharing +//! variants or physical candidates — so an algorithm with no //! candidate-generation phase at all (a greedy MQO loop, say) can implement it //! without pretending to have phases it does not have. The shipped algorithm is //! one implementation, [`StagePipeline`]. diff --git a/crates/planner/tests/e2e_plan.rs b/crates/planner/tests/e2e_plan.rs index 01e28a02..81cab76d 100644 --- a/crates/planner/tests/e2e_plan.rs +++ b/crates/planner/tests/e2e_plan.rs @@ -65,7 +65,7 @@ fn sql_workload( } /// The facade turns a prepared workload into one selected DAG per query, in -/// `QueryWorkload::entries()` order, without the caller touching `CandidateLogicalASAPDAGs`. +/// `QueryWorkload::entries()` order, without the caller touching Stage 1's inventory. #[tokio::test] async fn plans_every_query_in_entry_order() { let workload = sql_workload( diff --git a/crates/planner/tests/summary_sharing.rs b/crates/planner/tests/summary_sharing.rs index f9281ee4..71572c55 100644 --- a/crates/planner/tests/summary_sharing.rs +++ b/crates/planner/tests/summary_sharing.rs @@ -5,21 +5,16 @@ use asap_types::ir::{ASAPOp, OperatorNode}; use std::rc::Rc; use asap_frontend_sql::SqlCatalog; -use asap_logical_optimizer::accuracy::{AccuracyModel, DefaultAccuracyModel, PropagationStats}; +use asap_logical_optimizer::accuracy::{AccuracyModel, DefaultAccuracyModel}; use asap_logical_optimizer::pass1::realization::{default_size_params, DEFAULT_DELTA}; -use asap_logical_optimizer::{Replacement, ReplacementSubDAG, TargetSubDAG}; use asap_plan_selection::PlanningModels; -use asap_plan_selection::{CostModel, DefaultCostModel}; use asap_planner::pass::{PlanOutput, QueryPlan}; use asap_planner::{e2e_plan, FrontendInput, UserInput}; use asap_types::ir::operator::agg_intent::default_quantile; -use asap_types::ir::operator::AggIntent; -use asap_types::ir::properties::{ - AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, ProbabilityExpr, ResultGuarantee, -}; +use asap_types::ir::properties::{BoundExpr, ErrorMetric, ProbabilityExpr, ResultGuarantee}; use asap_types::ir::schema::SketchStatistic; use asap_types::ir::schema::{DataType, Field, Schema}; -use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchKind, SketchParams}; +use asap_types::ir::schema::{FieldDataType, SketchAlgorithm, SketchParams}; use asap_types::types::AccuracyTarget; use asap_types::workload::{ AccuracyRequirement, DataArrival, DataWorkload, DurationMs, Evidence, LatencyRequirement, @@ -29,53 +24,6 @@ use asap_types::workload::{ const NOW_MS: u64 = 1_700_000_000_000; -/// Stand-in for the workload-level amortization Stage 2 materialization will -/// price: a sketch candidate costs `preference(kind)` per sketch state, any -/// other candidate more than every sketch. Ranking is otherwise built-in. -struct PreferSketch(fn(&SketchKind) -> f64); - -impl CostModel for PreferSketch { - // Selection takes the cheapest candidate by `estimate_cost`. - fn candidate_cost_covers_complete_plan(&self) -> bool { - true - } - - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - DefaultCostModel.rank_candidates(intent, candidates) - } - - fn estimate_cost(&self, candidate: &ReplacementSubDAG, _: &TargetSubDAG<'_>) -> f64 { - let Replacement::SubDAG(root) = &candidate.replacement else { - return 1e9; - }; - let kinds: Vec<_> = OperatorNode::reachable(root) - .into_iter() - .filter_map(|node| match &node.operator { - asap_types::ir::Operator::ASAP(ASAPOp::SummaryAgg { - family: FieldDataType::Sketch(kind, _), - .. - }) => Some(kind.clone()), - _ => None, - }) - .collect(); - if kinds.is_empty() { - 1e9 - } else { - kinds.iter().map(self.0).sum() - } - } -} - -/// Prefers the largest KLL, i.e. one sized for the strictest consumer. -const PREFER_LARGE_KLL: PreferSketch = PreferSketch(|kind| match kind.params() { - SketchParams::Kll { k } => 1.0 / f64::from(*k), - _ => 1.0, -}); - fn requirements(epsilon: f64) -> QueryRequirements { QueryRequirements { accuracy: AccuracyRequirement::Explicit(AccuracyTarget::Epsilon(epsilon)), @@ -122,10 +70,6 @@ fn promql_workload(queries: &[(&str, f64)]) -> PlanningWorkload { } async fn plan_promql(queries: &[(&str, f64)]) -> PlanOutput { - plan_promql_with(queries, &DefaultCostModel).await -} - -async fn plan_promql_with(queries: &[(&str, f64)], cost: &dyn CostModel) -> PlanOutput { let workload = promql_workload(queries); let input = UserInput::new( &workload, @@ -133,7 +77,7 @@ async fn plan_promql_with(queries: &[(&str, f64)], cost: &dyn CostModel) -> Plan now_ms: NOW_MS, histograms: None, }, - PlanningModels::builtin().with_cost(cost), + PlanningModels::builtin(), ); e2e_plan(input).await.expect("workload plans") } @@ -293,35 +237,6 @@ fn kll_k_for(epsilon: f64) -> u32 { k } -/// p50 at ε=0.01 and p99 at ε=0.001 over the same input share one KLL sized -/// for the strictest consumer when the cost model prefers that candidate; each -/// reader's guarantee meets its own target. -#[tokio::test] -#[ignore = "the stage pipeline shares the KLL sized for the strictest consumer, but attaches no \ - guarantee to plan roots, and Stage 3 ignores PlanningModels.cost, so the looser query \ - alone selects the raw plan: #580"] -async fn quantiles_share_one_producer_sized_for_the_strictest_consumer() { - let p50 = ("quantile_over_time(0.5, lat[5m])", 0.01); - let p99 = ("quantile_over_time(0.99, lat[5m])", 0.001); - assert!(kll_k_for(0.001) > kll_k_for(0.01)); - - let output = plan_promql_with(&[p50, p99], &PREFER_LARGE_KLL).await; - assert!(same_states(&states(&output))); - assert_eq!(unique_deployments(&output), 1); - for (plan, (_, epsilon)) in output.plans.iter().zip([p50, p99]) { - assert_eq!(kll_k(plan), kll_k_for(0.001)); - let guarantee = plan.root.guarantee.as_ref().expect("certified"); - assert!( - guarantee.bound.evaluate().unwrap() <= epsilon, - "{guarantee:?}" - ); - } - - // Alone, the looser query keeps its own, smaller KLL. - let alone = plan_promql_with(&[p50], &PREFER_LARGE_KLL).await; - assert_eq!(kll_k(&alone.plans[0]), kll_k_for(0.01)); -} - /// The stage pipeline's summary-capability rule: p50 at ε=0.01 and p99 at /// ε=0.001 over one input read one KLL sized for ε=0.001, and Stage 3 accepts /// each query against its own target. @@ -469,16 +384,6 @@ impl AccuracyModel for UnivMonEvidence { } } - fn propagate( - &self, - op: &CompositionOperator, - inputs: &[ResultGuarantee], - local: Option<&ResultGuarantee>, - stats: &PropagationStats, - ) -> Result { - DefaultAccuracyModel.propagate(op, inputs, local, stats) - } - fn satisfies(&self, guarantee: &ResultGuarantee, target: &AccuracyTarget) -> bool { DefaultAccuracyModel.satisfies(guarantee, target) } diff --git a/crates/types/src/ir/properties/guarantee.rs b/crates/types/src/ir/properties/guarantee.rs index d9d3a3db..77712897 100644 --- a/crates/types/src/ir/properties/guarantee.rs +++ b/crates/types/src/ir/properties/guarantee.rs @@ -7,10 +7,10 @@ //! could silently consume an approximate child. This module is the //! *vocabulary* that fixes that — the typed metric, the symbolic bound and //! failure-probability expressions, the provenance trail, and the typed -//! rejection reasons. The *algebra* that composes these (the `AccuracyModel` -//! trait, its default conservative rules, and budget allocation) lives one -//! layer up in `asap_logical_optimizer::accuracy`: this crate defines the -//! shapes, the planning crate decides. +//! rejection reasons. The models that derive and check these (the +//! `AccuracyModel` trait and its default conservative rules) live one layer +//! up in `asap_logical_optimizer::accuracy`: this crate defines the shapes, +//! the planning crate decides. //! //! ## What a guarantee says //! @@ -260,8 +260,8 @@ impl ProbabilityExpr { } } -/// How a parent operator consumes its inputs' values — the shape an -/// `AccuracyModel::propagate` rule is registered against. `#[non_exhaustive]` +/// How a parent operator consumes its inputs' values — the shape a +/// composition rule is registered against. `#[non_exhaustive]` /// for the same reason [`ErrorMetric`] is. #[non_exhaustive] #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/crates/types/src/ir/schema/state_type.rs b/crates/types/src/ir/schema/state_type.rs index 0b2f4c8b..0578fd92 100644 --- a/crates/types/src/ir/schema/state_type.rs +++ b/crates/types/src/ir/schema/state_type.rs @@ -13,7 +13,8 @@ //! [`GroupingStrategy`] is a second, orthogonal axis: how many physical //! instances of a summary exist across a grouped aggregate's `by` //! subpopulations (per-subpopulation vs. one shared Hydra instance — see -//! `asap_logical_optimizer::pass1::grouping`). It rides on `ASAPOp::SummaryAgg` and on +//! `asap_logical_optimizer::pass1::logical_candidates::add_hydra_alternatives`). +//! It rides on `ASAPOp::SummaryAgg` and on //! sketch-valued edge types. use serde::{Deserialize, Serialize}; @@ -322,9 +323,8 @@ pub enum StatModelParams { // family/kind answers an intent — any family could in principle grow its own // per-subpopulation vs. shared-multi-subpopulation variant, so it is a // second, independent axis, not a member of any one family's own kind -// vocabulary. See `asap_logical_optimizer::pass1::grouping`'s module docs for where this -// axis actually plugs into the post-ASAP IR and the legality rules gating -// when `SharedMultiSubpopulation` is offered as a candidate at all. +// vocabulary. See `asap_logical_optimizer::pass1::logical_candidates::add_hydra_alternatives` +// for when Stage 1 offers `SharedMultiSubpopulation` as a candidate. /// A shared-multi-subpopulation summary family — one physical structure /// serving every subpopulation of a grouped aggregate instead of one @@ -410,8 +410,7 @@ pub enum HydraParams { /// Sizing knob for the one physical structure shared across every /// subpopulation. Correctly sizing this against an estimated /// subpopulation cardinality is a cost-model concern — out of scope - /// for the legality axis this type lives on (see - /// `asap_logical_optimizer::pass1::grouping`'s module docs) — so this is + /// for the legality axis this type lives on — so this is /// deliberately not derived from any cardinality estimate here. shared_buckets: u32, }, @@ -467,9 +466,8 @@ pub fn hydra_kind_for(algorithm: &SketchAlgorithm) -> Option { /// `HydraParams` fields. `None` when `per_subpopulation_params` doesn't /// belong to the [`SketchAlgorithm`] `kind` wraps: a caller bug, since /// [`hydra_kind_for`] and the algorithm a `SketchParams` came from must -/// agree; callers that got both from the same already-ranked -/// `Realization` (as `asap_logical_optimizer::pass1::grouping` does) cannot hit -/// this. +/// agree; callers that got both from the same `Realization` (as Stage 1's +/// `add_hydra_alternatives` does) cannot hit this. /// /// This function is generic over which inner sketch type `kind` wraps /// precisely because [`SketchParams`] already is: it destructures whichever