From a21b46e349526e9373fc309e7dcaac6e04c05855 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Thu, 29 Oct 2026 11:29:50 +0530 Subject: [PATCH] fix(query): align ordinary aggregate value handling (#789) --- CHANGELOG.md | 11 +- CLAUDE.md | 6 +- README.md | 7 + ROADMAP.md | 4 +- docs/ERROR_REFERENCE.md | 39 +- docs/TEST_COVERAGE.md | 42 ++- .../plans/2026-10-29-789-aggregate-values.md | 147 ++++++++ .../2026-10-29-789-aggregate-values-design.md | 178 +++++++++ src/query/datalog/aggregate.rs | 76 ++-- src/query/datalog/aggregate/tests.rs | 162 ++++++++- src/query/engine/aggregate_order_tests.rs | 50 ++- src/query/engine/tests.rs | 133 ++++++- tests/aggregate_value_handling_test.rs | 343 ++++++++++++++++++ tests/aggregate_window_rules_test.rs | 10 +- tests/aggregation_test.rs | 24 +- tests/error_codes_qry_test.rs | 24 +- tests/error_handling_test.rs | 49 +-- tests/ordinary_avg_test.rs | 12 +- 18 files changed, 1133 insertions(+), 184 deletions(-) create mode 100644 docs/superpowers/plans/2026-10-29-789-aggregate-values.md create mode 100644 docs/superpowers/specs/2026-10-29-789-aggregate-values-design.md create mode 100644 tests/aggregate_value_handling_test.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 73ae944049..143aed74e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,9 +11,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Ordinary average** (#777): `(avg ?v)` no longer raises PRS-027. Grouped and ungrouped queries use the same numeric population and arithmetic as window `avg`: skip nil and nonnumeric values, sum exactly, round the total once to Float, then divide by the numeric count. Integer totals are not saturated. Results are Float, or nil for an existing group without numbers; no matching rows yields no rows. The total can overflow before division, and division rounds separately. Existing window behavior is unchanged. Retained exact-sum allocations for ordinary `sum` and `avg` are now charged to the query memory limit. +- **Align ordinary aggregate value handling with windows** (#789, v3.0.0). + `sum` and `sum-distinct` skip nil and nonnumeric arguments; `min`/`max` compare + every non-nil type using the existing window total order, including mixed + Integer/Float values. Count variants retain their non-nil counting and + distinctness. Existing groups with no numeric sum arguments return Integer(0); + all-nil min/max groups remain omitted, and no-match query conventions remain + unchanged. INT-041/043/044 are deprecated for these built-ins. No v2 runtime + backport, window behavior, ordered-group eligibility, or storage change. + ### Documentation -- Retire tracker #383 in favor of the v3.0.0 milestone and its release gates. Other ordinary/window value-handling differences are tracked in #789. +- Retire tracker #383 in favor of the v3.0.0 milestone and its release gates. Ordinary/window value-handling consistency is completed by #789. ### Performance diff --git a/CLAUDE.md b/CLAUDE.md index e842cff671..f5ce704e1f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -218,17 +218,19 @@ Auto-migrates v7 → v8 on open (spec §9, with a backup meta page). v1–v6 are ## Test Coverage -**2018 tests** (2001 passing, 17 ignored; unit + integration + doc). Engine answers once checked by the executor are frozen in `tests/answers/` (#551). +**2030 tests** (2013 active, 17 ignored; unit + integration + doc). Engine answers once checked by the executor are frozen in `tests/answers/` (#551). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. The #760 deterministic storage suite adds three native unit tests in a separate `--cfg minigraf_shuttle` build, outside the default count. Run only `graph::storage::deterministic::` with that cfg; ordinary unit tests do not enter its scheduler. `deterministic-schedules.yml` runs relevant v3 PRs on Linux/macOS/Windows plus Linux MSRV, and larger nightly/manual searches with replay artifacts. The identical main workflow activates the v3 schedule after both PRs merge. See `docs/testing/deterministic-schedules.md` for scope, controls and findings. **Testing conventions** — see the Testing Conventions section below before writing any tests. -On v3, ordinary `avg` skips nil/nonnumeric values and divides the exact rounded total by the numeric count, matching window `avg` (#777). It returns Float, or nil for an existing group with no numbers; no matching bindings yields no rows. Other aggregate consistency work is #789; ordered-avg optimization remains #781. +On v3, ordinary `avg` skips nil/nonnumeric values and divides the exact rounded total by the numeric count, matching window `avg` (#777). It returns Float, or nil for an existing group with no numbers; no matching bindings yields no rows. Sum and extrema follow the value rules from #789; ordered-avg optimization remains #781. ## Current Maintainer Context +On `v3`, ordinary sum/sum-distinct skip nonnumbers and min/max compare all non-nil types by the window total order (#789). Empty-query conventions and all-nil min/max group dropping are preserved; INT-041/043/044 are deprecated for these paths. Ordered Aggregate remains count/count-distinct only (#781 owns expansion). Ordinary avg shipped in #777. + On `v3`, expression `=` / `!=` resolves literal keywords once per operator (#774), including nested expressions and substituted prepared slots. Dynamic values do not populate the cache. This is a performance-only change, with no v2.x backport; see `docs/benchmarks/774-expression-keywords.md`. **v2.0.5 released** (2026-10-11) — short-file open refused with STG-045 (#506), commit after a torn WAL entry no longer lost, WAL-007 (#513); backported rule-query `max_results` (#528), all-checkpointed WAL removal (#457) and `usize::MAX` close-time save (#476); weekly soak test (#392), real coverage gates (#396). **v2.0.4** (2026-10-08) — `BrowserDb` IndexedDB flush returns an error for an unreadable dirty page (#470); golden-file v7 corpus (#391) and SIGKILL committed-data crash test (#384). **v2.0.3** (2026-10-06) — data-integrity patch under the support policy: indexed queries with >65,535 uncheckpointed facts (#445), tx counter rewound on reopen next to an already-checkpointed WAL (#447). **v2.0.2** (2026-09-27) — last planned v2.x feature/fix release: query fixes (#297, #405, #407, #410), directory fsync (#389), point-query/attribute-scan/checkpoint perf mitigations (#323, #381, #315), CI policy checks (#395), support policy and tiers (#397, #399, #400). **v2.0.1** (2026-09-25) fixed index rebuild on open (#370). **v2.0.0** introduced kernel file locking (#317, #304), structured runtime error codes (#277), `OpenOptions` `#[non_exhaustive]`, and an MSRV of Rust 1.89. See `CHANGELOG.md` for the full rationale and release history. diff --git a/README.md b/README.md index 0f6357c16d..045c1c12c9 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,13 @@ On v3.0.0, an ascending value-only query such as `[:find ?v :where [?e :score ?v On the unreleased v3 branch, selective in-memory ancestor queries can use entity probes instead of reading the whole edge attribute (#723). This is a query-planning optimization; it adds no storage index or configuration. +On v3.0.0, ordinary `sum` and `sum-distinct` skip nonnumeric values. Ordinary +`min`/`max` compare every non-nil type using the window value order, including +mixed integers/floats (#789). Existing all-nil min/max groups are still omitted; +no matching rows still returns no rows for these aggregates. `count` and +`count-distinct` continue to count non-nil values of every type. No v2 runtime +change is included. + **Status**: See [ROADMAP.md](ROADMAP.md) for planned work and current direction. The unreleased v3 branch also avoids resolving the same keyword literal on every row of an expression comparison such as `[(= ?entity :alice)]` (#774); see the [benchmark report](docs/benchmarks/774-expression-keywords.md). diff --git a/ROADMAP.md b/ROADMAP.md index 2dd8392dcf..74bed4b7ef 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -4,7 +4,7 @@ **Philosophy**: Embedded graph memory for agents, mobile, and the browser — built on the SQLite approach: be boring, be reliable, be embeddable. -Ordinary `avg` is implemented for v3.0.0 (#777), matching window numeric filtering. Consistent handling for the other ordinary aggregates remains a v3.0.0 release requirement (#789). +Ordinary `avg` is implemented for v3.0.0 (#777), matching window numeric filtering. Ordinary sum and extrema also use window-compatible value handling (#789); their existing empty-group conventions are preserved. Completed releases and their implementation details live in the [CHANGELOG](CHANGELOG.md). This document tracks planned, exploratory, and other future work. @@ -96,6 +96,8 @@ Scope — format and storage integrity: Scope — query API: +- Align ordinary aggregate value handling with windows (#789; implemented for v3.0.0): sum/sum-distinct skip nonnumbers; min/max accept all non-nil types and mixed numbers. Preserve empty-query and all-nil min/max group-drop behavior. Ordinary avg is #777; ordered-group expansion remains #781. + - Selective in-memory ancestor joins avoid building a whole-attribute value map (#723; implemented for eligible three-pattern recursive bodies). Populated committed generations keep AVET probes; general in-memory value joins remain unchanged. - Optional `:distinct` result tuples, preserving bag output by default (#493; done on `v3`) diff --git a/docs/ERROR_REFERENCE.md b/docs/ERROR_REFERENCE.md index 0829449424..f7d2f8f94b 100644 --- a/docs/ERROR_REFERENCE.md +++ b/docs/ERROR_REFERENCE.md @@ -280,10 +280,10 @@ with no `CodedError` anywhere in its chain. | INT-038 | Header too short for a specific field | Internal | | INT-039 | Invalid magic number | Internal | | INT-040 | Function/predicate name already registered | Internal | -| INT-041 | sum: unsupported value type | Internal | +| INT-041 | sum: unsupported value type (deprecated) | Internal | | INT-042 | min/max: no non-null values in group | Internal | -| INT-043 | Cannot compare values of different types | Internal | -| INT-044 | Aggregate: unsupported value type | Internal | +| INT-043 | Cannot compare values of different types (deprecated) | Internal | +| INT-044 | Aggregate: unsupported value type (deprecated) | Internal | | INT-045 | Pending fact index out of bounds | Internal | | INT-046 | Missing CommittedFactReader for a committed FactRef | Internal | | INT-047 | into_backend: backend Arc has multiple owners | Internal | @@ -4194,46 +4194,29 @@ previously registered user-defined function. **Error text**: `sum: expected Integer, Float, or Null, got {}` -**Cause**: The built-in `sum` aggregate encountered a value in its input -group that is not an `Integer`, `Float`, or `Null`. - -**Resolution**: -- Ensure the attribute being summed only ever holds numeric or null - values. +> **Deprecated in v3.0.0 (#789).** Ordinary `sum` and `sum-distinct` skip nil and all nonnumeric values. An existing group without numbers returns Integer(0); a query with no matching rows returns no rows. This code and its message remain reserved for compatibility; these built-in paths no longer emit it. In v2, the type restriction still applies. ### INT-042 min/max: no non-null values in group **Error text**: `min/max: no non-null values in group` -**Cause**: The built-in `min`/`max` aggregate was applied to a group whose -values were all `Null`, leaving nothing to compare. - -**Resolution**: -- Filter out rows with a null value before aggregating, or handle the - all-null-group case in your query. +**Behavior**: An ordinary `min`/`max` group containing only nil values has no +extremum and is omitted. On v3 this is an internal group-drop condition, not +an error returned by the query. A window instead emits nil for each affected +source row. Queries with no matching rows do not create a min/max group. +The code and message remain reserved; #789 preserves this behavior. ### INT-043 Cannot compare values of different types **Error text**: `{}: cannot compare {} and {} values` -**Cause**: An aggregate that orders values (`min`, `max`) encountered two -values of different, non-comparable `Value` types within the same group. - -**Resolution**: -- Ensure the attribute being aggregated holds a consistent value type - across all matching facts. +> **Deprecated in v3.0.0 (#789).** Ordinary `min`/`max` accept mixed types, including Integer/Float, using the same total value order as their window forms. This code and its message remain reserved for compatibility; these built-in paths no longer emit it. In v2, the type restriction still applies. ### INT-044 Aggregate: unsupported value type **Error text**: `{}: expected Integer, Float, String, or Null, got {}` -**Cause**: An aggregate that requires an orderable scalar type (e.g. `min`, -`max`) encountered a value type it does not support (such as a `Boolean` -or `Ref`). - -**Resolution**: -- Ensure the aggregated attribute only holds Integer, Float, String, or - Null values. +> **Deprecated in v3.0.0 (#789).** Ordinary `min`/`max` accept every non-nil Value variant, including Boolean, Ref, Keyword and Set. They use the window total order. This code and its message remain reserved for compatibility; these built-in paths no longer emit it. In v2, the type restriction still applies. ### INT-045 Pending fact index out of bounds diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index f07f763a54..b473eeafb4 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,9 +4,12 @@ | | Default features | With `--features test-hooks` | |---|---|---| -| **Passing** | 2,001 | 2,035 | +| **Active** | 2,013 | 2,048 | | **Ignored** | 17 | 18 | -| **Total** | 2,018 | 2,053 | +| **Total** | 2,030 | 2,066 | + +Counts above are the current Cargo test inventory (active versus ignored), +including #777 and #789. Verification results below identify which suites ran. #777 adds 12 default-feature tests (three accumulator tests, one engine memory test and eight integration tests) and one additional `test-hooks` plan matrix. The focused 13-test run passes. Strict native/browser/WASI clippy passes, and the rebased release binary is 1,263,144 bytes against the unchanged 1,263,616-byte cap. @@ -43,9 +46,36 @@ Also outside those counts: three native unit tests in the separate #760 `minigra **Scope**: the unreleased file-format-v8 work (#287, #371): the index-key value-collision fix, the v7→v8 migration, and the drop of formats v1–v6. Otherwise the public database API, the Datalog engine, storage and recovery paths, v2.0.0's locking and structured-error-code changes, and v2.0.1's regression test for rebuilding indexes on open after several checkpoints (#370). Tests merged from the v2.x line are marked *(v2.x)*. +**#789 verification (2026-10-29)**: the change adds 12 default-feature tests +and one additional `test-hooks` test. Before the runtime edits, all seven new +API tests failed on INT-041/043 and the distinct-membership test failed on +charging a skipped type. The implementation is reconciled with merged #777 +at v3 base `0fc97bd65`: 64 aggregate unit/engine tests (one ignored) and all +15 avg/value-handling API tests pass. The full pre-rebase unit suite passed +1,167 tests, including all 16 unchanged storage fault-matrix cases. That run +and local continuation exposed six obsolete type-error assertions across +`aggregation_test.rs` and `error_handling_test.rs`; they now assert the +successful values required by #789. +On the final base, all 26 aggregation tests, eight error-handling tests and +34 focused `test-hooks` tests +pass, as does the dedicated Sum/Avg retained-heap test. Strict native/browser/WASI +clippy, formatting and WASI test compilation pass (no WASI execution). +The release binary is 1,261,736 bytes against the unchanged 1,263,616-byte cap. +The docs site builds all 13 versions and 368 pages with link validation. +The final default run (`cargo test --no-fail-fast -- --skip fault_matrix::`) +passes 1,997 tests with zero failures and 17 ignored; the 16 unchanged fault +cases passed separately as described above. This covers all 2,013 active +default tests across the two runs. The default run used the local fallback +(Rust 1.94.0) after a bench crash-test child-progress timeout under competing +I/O; the exact recorded seed passes locally with the unchanged 30-second +limit. Platform/focused checks used bench Rust 1.99.1. Full `test-hooks` +execution was not repeated; its 2,066-test inventory was verified. + ## Covered Areas -- **Ordinary average** (#777; `tests/ordinary_avg_test.rs`, `aggregate/tests.rs`, `engine/tests.rs`): grouped and ungrouped averages, every nonnumeric value kind skipped, nil versus no matching rows, Float results, exact unsaturated totals, signed zero/subnormal underflow, infinities/NaNs, input permutations, prepared execution/cursors, batches, recursive rules, transaction reads, checkpoint/reopen and alternative plans. Default and bounded windows preserve skipped rows. Sum/avg exact-sum heap state is charged once before allocation, transferred on finish, and released on errors, rewind and close. Existing neighboring aggregate errors and dropped groups are preserved. +- **Aggregate value handling** (#789; `tests/aggregate_value_handling_test.rs`, seven default tests and one with `test-hooks`): all value variants, numeric-only sum/distinct sum, mixed-type extrema, existing nil/nonnumeric groups versus empty queries, grouped and ungrouped queries, hidden `:with` groups and join multiplicity, prepared binds/cursors, recursive rules, transaction-time views and staged rollback, checkpoint/reopen, and default/bounded partition windows over identical populations. Three added accumulator tests check adversarial permutations, signed-zero/NaN bits, infinities, integer boundaries, the explicit total order and distinct membership prediction. Two added engine tests check retained numeric-member accounting, duplicates, QRY-012 boundaries, upstream failures and release on rewind/close. Rewritten error-precedence tests use real UDF finalizer failures under Limit/Offset; ordered-aggregate eligibility and frozen answers are unchanged. + +- **Ordinary average** (#777; `tests/ordinary_avg_test.rs`, `aggregate/tests.rs`, `engine/tests.rs`): grouped and ungrouped averages, every nonnumeric value kind skipped, nil versus no matching rows, Float results, exact unsaturated totals, signed zero/subnormal underflow, infinities/NaNs, input permutations, prepared execution/cursors, batches, recursive rules, transaction reads, checkpoint/reopen and alternative plans. Default and bounded windows preserve skipped rows. Sum/avg exact-sum heap state is charged once before allocation, transferred on finish, and released on errors, rewind and close. Neighboring sums follow #789 numeric filtering; empty-min group dropping is preserved. - **Expression keyword literals** (#774): a test-only conversion counter verifies that increasing input from 8 to 2,048 rows does not increase literal resolution work, across `=` / `!=`, both operand orders, nested expressions, bindings, negation, disjunction and rule bodies. Existing join-equality tests also cover changing prepared keyword slots across executions and unchanged keyword output, in memory, after checkpoint and after reopen. The envelope harness checks all eight new expression/control shapes on regular and irregular fixtures. @@ -467,11 +497,11 @@ These are not `#[test]`s; run them with `cargo kani` or `.github/workflows/kani. - **Aggregates and windows** (`tests/aggregate_window_rules_test.rs`, 8 tests; #547) - Exact float `sum` and window `avg`; integer `sum` saturating once; exact window `sum` of integers above 2^53; `min`/`max` of `-0.0` and `0.0`. - `rank` by keywords (peers 1 1 3 3); a running `sum` giving peers the value at the last peer; `row-number` breaking ties by the other `:find` variables. - - `INT-041` from a failing group. - - `src/query/datalog/aggregate.rs` (15 unit tests) checks every accumulator over every order of small inputs, and that `value_order` is a total order. + - Nonnumeric arguments are skipped by ordinary sum (#789), rather than raising INT-041. + - `src/query/datalog/aggregate/tests.rs` (27 unit tests) checks every accumulator over every order of small inputs, and that `value_order` is a total order. - `tests/window_functions_test.rs` checks `PRS-082` and a window over the groups of an aggregate query. - `tests/window_functions_test.rs` (#182) also checks `lag`/`lead` on whole and partitioned results, descending order and ties, singleton and mixed-type partitions, `nil` edges, long strings, keywords, refs and sets, grouped rows, distinct output, pagination, prepared queries, cursors and preserved user aggregates of those names. Parser tests cover malformed shapes and `tests/plan_independence_test.rs` checks both directions under reordered plans. - - `Aggregate` on the engine (#547 PR 2, `src/query/engine/tests.rs`, 3 tests): every built-in aggregate grouped by keywords, refs, booleans, floats and long strings, over `nil` values, keywords and refs, with `:with` and `or`, at six temporal settings in memory, after a checkpoint and after reopen, against their frozen answers. Also: an ungrouped aggregate reads every row and returns one batch of one row; the zero row and empty answers; groups in value order over several batches; `INT-041` from a later group; a group dropped by `min` of no value. + - `Aggregate` on the engine (#547 PR 2, `src/query/engine/tests.rs`, 3 tests): every built-in aggregate grouped by keywords, refs, booleans, floats and long strings, over `nil` values, keywords and refs, with `:with` and `or`, at six temporal settings in memory, after a checkpoint and after reopen, against their frozen answers. Also: an ungrouped aggregate reads every row and returns one batch of one row; the zero row and empty answers; groups in value order over several batches; nonnumeric sum arguments skipped in a later group; a group dropped by `min` of no value. - `tests/aggregation_test.rs` checks that a group of `0.0` and `-0.0` returns `-0.0` in either insertion order. - **Query validation** (#406, #414) - A plain `:find` variable, or a window function's argument, `:partition-by` or `:order-by` variable, that no `:where` clause binds (including `?_` wildcards) is `PRS-080` from both `execute()` and `prepare()`, naming the variable. diff --git a/docs/superpowers/plans/2026-10-29-789-aggregate-values.md b/docs/superpowers/plans/2026-10-29-789-aggregate-values.md new file mode 100644 index 0000000000..0454ceff09 --- /dev/null +++ b/docs/superpowers/plans/2026-10-29-789-aggregate-values.md @@ -0,0 +1,147 @@ +# #789 implementation plan + +Status: Claude-approved (2026-10-29); maintainer approved implementation on 2026-10-29. +Spec: ../specs/2026-10-29-789-aggregate-values-design.md. +Worktree: `.worktrees/789-aggregate-values`, branch `feat/789-aggregate-values`. +Review session: `0a6c54d6-eea8-43f7-a2b8-4de1e99fb416`; both documents approved +with no blockers. Incorporates the review's documentation and test-location notes. + +## 1. Establish the approved baseline + +Reconcile with landed #777 before delivery, preserving ordinary avg and Sum +heap accounting. Read current aggregate.rs, engine aggregate/plan/tests, +existing ordinary/window suites, frozen answers and applicable guidance. +Use the bench machine for builds and tests whenever reachable and idle; +report any fallback. Record revision, clean baseline results and toolchain. +No implementation before design approval. + +## 2. Write failing contract tests + +Add shared tests in aggregate/tests.rs and a focused integration suite +`tests/aggregate_value_handling_test.rs`. Start with ordinary sum on +10, "unknown", 20 => Integer(30); mixed Integer/Float extrema; and +sum-distinct skipping repeated nonnumbers. Confirm they fail on the old +type errors before editing production code. Add the spec's explicit expected +all-variant, empty/group, zero/NaN, distinctness and permutation cases. +Keep ordinary avg implementation/tests owned by #777. + +## 3. Make the narrow accumulator change + +Remove Sum's deferred bad-type field and Extreme's kind validation. Keep +Sum arithmetic, value_order and window behavior unchanged. Filter sum-distinct +before both retention and memory charging. Update all enum patterns and #777 +heap-accounting helpers. Preserve no-extremum `Ok(None)` and finish ordering. +Run the targeted red/green tests on the bench machine. + +## 4. Verify execution and failure behavior + +Extend engine unit tests for deterministic remaining error precedence, +memory charges, cleanup and plan independence. Replace obsolete tests that +expect INT-041/043/044 with successful explicit results; keep unrelated error +coverage intact. Audit `tests/answers` case by case before changing any golden. +Use injected upstream failures and UDF step/finalize failures to prove full +drain and find-order behavior, including empty min/max before/after a UDF. +In particular, replace the old type-error witnesses in +`src/query/engine/aggregate_order_tests.rs`, `tests/error_codes_qry_test.rs` +and `tests/aggregate_window_rules_test.rs`; preserve the precedence assertions +with real remaining failures rather than deleting their coverage. + +Exercise prepared binds, multi-batch cursors and rewind/close, rules, temporal +views, transactions, pending and checkpointed data. Compare ordinary sums and +extrema with corresponding default and bounded windows over identical input +populations, with the explicit empty-extremum exception. Keep ordered +Aggregate eligibility unchanged and document the #781 handoff in the PR. + +Run targeted engine meter tests for numeric-only sum-distinct retention, +canonical duplicates, unchanged count-distinct growth, QRY-012 thresholds and +cleanup. Recheck #777's Sum/Avg retained-heap tests after enum changes. + +## 5. Synchronize documentation + +Update the six repository status/reference documents listed in the spec using +actual final test counts. Document the stable error-code reservations and +the preserved empty/group behavior. Make a separate docs-site worktree from +the existing sibling checkout and version-scope v3 semantics. Audit the site's +other required pages and site.toml, preserving v2 and #777 changes. Run the +site's own checks and link validation; no version bump or release. + +## 6. Verify and obtain final review + +Run full `cargo test`, `cargo fmt --check`, and +`cargo clippy --all-features -- -D warnings`, plus the portable target/feature +and release-size checks used by current CI, on the bench machine. Preserve +logs and exact tested revisions; do not repeat expensive checks without a +new edit, failure or unresolved concern. Verify the final diff and test-count +documentation. Ask Claude to review complete core and docs diffs with the +approved spec, plan and verification evidence; resolve confirmed findings. +Open core PR targeting v3 and the docs-site PR only after final review. Do not +merge or release. Report remaining dependency or CI limitations accurately. + +## #777 integration check + +#777 merged as core PR #791 while verification was running. This worktree +was advanced to its actual v3 merge, `0fc97bd65`, and reconciled using the +previously reviewed and tested scratch combination. It preserves `Agg::Avg`, +its constructor/step/mean, and #777's engine accounting. Both +`exact_heap_growth` and `exact_heap_bytes` now match `Agg::Sum(s)`. +The avg integration test now expects `[nil, 0]` / `[0, nil]` for avg beside sum +on a nonnumeric population, retains the empty-min group-drop assertion, and +is named `ordinary_avg_matches_sum_results_and_preserves_dropped_groups`. +#789's UDF tests protect the remaining finish-error precedence. The obsolete +unit-test error-code helper was removed after merging the avg tests. + +The reconciled base passes 64 aggregate unit/engine tests (one ignored existing +diagnostic) and all 15 avg/value-handling API tests. Before the merge, the +scratch combination also passed the dedicated Sum/Avg retained-heap test. +The docs branch was advanced to the merged #777 site PR #86 (`3f65810`), +preserving its avg content and version guards. + +## Delivery verification + +Final-base platform checks use Rust/Cargo 1.99.1. SHA-256 comparison confirms every +changed Rust source/test file matches the verification snapshot. Native +all-feature clippy, browser clippy, WASI clippy (all with `-D warnings`), +formatting and WASI lib/new-suite compilation pass. WASI tests were compiled, +not executed; no runner was installed. The release binary is 1,261,736 bytes, +below the unchanged 1,263,616-byte cap. + +The focused `test-hooks` run passes 34 tests across `aggregate_value_handling_test`, +`ordinary_avg_test`, `plan_independence_test`, and `input_order_test`. +The dedicated `sum_and_avg_charge_exact_heap_once_and_release_it` test passes +on the final base. All 26 aggregation API tests and eight error-handling tests +pass after replacing six legacy type-error assertions found by the broader +runs with explicit results. Unrelated error-path checks remain intact. + +All 16 unchanged storage fault-matrix cases passed in the pre-rebase full unit +run (1,167 passed, five ignored). A first attempt was interrupted after severe +filesystem contention; the completed matrix used a task-owned fixture directory. +The final-base default suite excludes only this already completed matrix. +No storage source changed in the rebase. Local fallback checks while the bench +was busy used Rust 1.94.0; the final checks above use the designated bench toolchain. + +The reconciled docs site builds all 13 versions and 368 pages with link validation. +The build uses a temporary source snapshot containing the final core changelog +and error reference. Rendered v2 retains the old type restrictions, and v3 +contains both the new aggregate value rules and the merged avg documentation. + +A bench default rerun hit the crash-kill test's 30-second child-progress deadline +while other compilation and another task's full suite contended for I/O. +The same recorded seed (`1793252894809935624`) passes locally in 0.37 seconds, +with the unchanged test and timeout. The final default rerun uses the approved +local fallback (Rust 1.94.0), excluding only the completed storage fault matrix. +The deadline failure and the earlier obsolete-expectation failures are retained +in the verification logs; they are not counted as passing runs. + +The final `cargo test --offline -j 2 --no-fail-fast -- --skip fault_matrix::` +completed successfully: 1,997 passed, zero failed, 17 ignored. Together with +the 16 completed unchanged fault tests, this covers all 2,013 active default +tests. Cargo inventories verify 2,030 default tests (17 ignored) and 2,066 +`test-hooks` tests (18 ignored); only the 34 focused hooks tests were executed +for final verification. The final review uses this explicit split evidence. + +Final Claude review (`fbe4d289-e7b3-4ef6-8b7d-546d51aa975d`, 2026-10-29): +**approved, no blockers for either PR**. The reviewer accepted the split default +verification for opening the PRs, confirmed no remaining obsolete aggregate +error assertions, and checked the docs/inventory consistency. Both PRs must +pass CI before merge, including full hooks coverage, final-base storage fault +tests and the docs-site link check. No merge or release is part of this task. diff --git a/docs/superpowers/specs/2026-10-29-789-aggregate-values-design.md b/docs/superpowers/specs/2026-10-29-789-aggregate-values-design.md new file mode 100644 index 0000000000..7096500674 --- /dev/null +++ b/docs/superpowers/specs/2026-10-29-789-aggregate-values-design.md @@ -0,0 +1,178 @@ +# #789: align ordinary aggregate values with windows + +Date: 2026-10-29. Target: v3.0.0 only. +Status: Claude-approved (2026-10-29); maintainer approved implementation on 2026-10-29. +Issue: https://github.com/project-minigraf/minigraf/issues/789 +Inspected base: `c2e862fe8` on v3. + +## Scope and philosophy + +Make ordinary `sum`, `sum-distinct`, `min` and `max` accept the same argument +domain as their corresponding window functions. Audit the count variants. +This follows PHILOSOPHY.md's reliability and simplicity principles, with no +dependency, configuration, syntax, public API or storage/WAL change. Changed +query semantics ship only in the next major release; v2 runtime is untouched. +Ordinary `avg` belongs to #777. Ordered-group expansion belongs to #781. + +## Contract + +| Function | Accepted values | Existing group with only nil | Existing group with only nonnumeric non-nil values | +| --- | --- | --- | --- | +| count | Every non-nil value | Integer(0) | Number of values, including repetitions | +| count-distinct | Every distinct non-nil value | Integer(0) | Number of distinct values | +| sum | Integer and Float only | Integer(0) | Integer(0) | +| sum-distinct | Distinct Integer and Float only | Integer(0) | Integer(0) | +| min / max | Every non-nil value | Drop the ordinary group, as today | Least / greatest value by the existing window total order | + +The all-nil min/max rule is an explicit preservation of ordinary result +structure: windows emit nil for each affected source row, whereas an ordinary +group with no extremum is omitted. Do not turn that into an error or fabricate +a nil ordinary result. An all-nonnumeric group has an extremum whenever at +least one value is non-nil, including an empty set. +Ordinary avg (#777) and an unstepped registered aggregate instead return nil +for an existing all-nil group. That intentional difference remains; do not +document empty-group handling as uniform across functions. + +No matching rows remains distinct from an existing group. Preserve the exact +`engine/plan.rs` `zero_row` predicate: only a query with no plain find variable +and exclusively count/count-distinct aggregates synthesizes a zero row. This +also preserves current `:with` behavior rather than inventing an empty group +rule. Any sum, sum-distinct, min or max in an otherwise empty query means no +rows. Windows over no rows emit no rows. A constructed empty accumulator is +not evidence that a query should synthesize a row. + +Skipping an argument never filters a source row, changes grouping or `:with`, +deduplicates ordinary sum/count, or removes a bounded-window position. Keep +the existing aggregate/window syntax restrictions. Registered aggregates and +their nil/callback behavior are unchanged. + +### Numbers, ordering and distinctness + +Reuse `Sum::add` and `Sum::value` unchanged. Integer-only totals saturate once +to i64; any numeric Float makes the result Float. The exact accumulator gives +input-order-independent results. NaN or opposing infinities produce NaN; +one infinity sign produces that infinity. Numeric -0.0 alone yields -0.0; +another numeric zero or exact cancellation produces +0.0. Skipped values +neither promote the result type nor affect the zero sign. + +Reuse `Extreme::add` / `value_order` for min/max, without type validation. +Numbers compare by exact numeric value, Integer before an equal Float; float +ties use total_cmp, and NaNs follow all other numbers. The remaining classes +are strings, booleans, refs, keywords, nil, sets. Nil is skipped. Sets compare +lexicographically by elements sorted using this same order. This is the +existing window order, NOT `Value::Ord` (which still governs group equality +and distinctness). No comparator changes are part of #789. + +Keep current `Value::Ord` distinctness and `canonical`: Integer(1) and +Float(1.0) are distinct; float signed zeros collapse to +0.0; all float NaNs +collapse to one canonical NaN. Thus sum-distinct of only -0.0 is +0.0, unlike +sum of those rows. Compare sum-distinct with sum over the explicitly +canonicalized distinct numeric population, not over the original rows. + +Filter sum-distinct before insertion AND before `would_add` memory charging. +Nonnumeric values do not grow its retained set. Count-distinct retains every +non-nil type and its current accounting. Upstream decoding, row storage and +IdSpace costs still apply even when an aggregate ignores an argument. + +## Implementation and errors + +In `src/query/datalog/aggregate.rs`: + +- Change `Agg::Sum(Sum, bad_type)` to `Agg::Sum(Sum)` and ignore false from add. +- Remove Extreme's kinds bitmask and its now-unused kind/KINDS helpers; + preserve best-value selection and ordinary `Ok(None)` on no extremum. +- Separate sum-distinct from count-distinct in step and would_add, with the + identical numeric predicate for the former. Finish sums only retained + numeric members through the existing Sum accumulator. +- Retain type_name if other callers use it. Update stale comments and enum + pattern matches, including any #777 heap-accounting helpers. + +INT-041, INT-043 and INT-044 cease to be emitted by these built-in aggregate +paths. Keep their public ErrorCode variants, registry entries and stable text; +mark their reference table titles "(deprecated)" following the existing +INT-033..035 convention, and explain the v3 behavior. INT-042 already has +no emission site on the inspected v3 base: it describes the ordinary min/max +group-drop condition, not an error returned to a caller. Preserve that rule +and clarify the reference rather than claiming a newly removed error. + +Keep full-drain evaluation, group order and find-order finishing unchanged. +An earlier empty min/max can still drop a group before a later UDF finalizer; +a preceding failing UDF finalizer still wins. Step failures, decoding/storage +errors, QRY-012 budgets/checked allocation failures, INT-032 pipeline errors +and remaining UDF errors remain +observable through their existing paths. Data that formerly caused a removed +type error now produces a result or the next applicable real error. Tests +must assert this explicitly, including `:limit` / `:offset`. + +For #781: sum and min/max no longer have aggregate type errors, but this does +not prove streaming eligibility. Preserve the count/count-distinct-only +ordered path and every fallback. Sum-distinct remains growing state. Whole +pipeline order, error timing and memory proofs remain #781's responsibility. + +## Coordination with #777 + +The local #777 worktree has in-progress ordinary avg and Sum heap accounting. +Do not copy or edit it. Reconcile against its landed v3 revision before code +delivery; preserve Avg, its parser/grammar work, and exact-accumulator charges. +If #777 is still pending, keep changes independently scoped and explicitly +review the shared enum/helper conflict before integration. This spec does not +authorize silently dropping either change. Test both contracts together when +the dependency is available. + +## Evidence and acceptance + +1. Shared-accumulator tests cover all eight Value variants, mixed classes, + sets with mixed scalar elements, nil, signed zeros, multiple signed/payload NaNs, infinities, + i64 extrema, integers beyond 2^53, and cancellation. Exhaust small + adversarial permutations. Explicit expected values accompany comparisons + with Running and bounded windows; use bits for zero and NaN ordering. +2. Compare each ordinary aggregate with its corresponding full-partition + window population, including grouped/partitioned cases. Default cumulative + windows compare at the final row; bounded windows compare each selected + argument slice with an ordinary accumulator. For all-nil extrema, assert + the deliberate ordinary-drop/window-nil structural distinction. Distinct + variants have no built-in windows; assert exact distinct counts and compare + sum-distinct to the canonical numeric population. +3. Public queries cover grouped/ungrouped results, all-nil/nonnumber/empty + cases, mixed find lists, :with, repeated values, multiplicity, prepared + binds, cursors across batches, recursive/rule queries, temporal selections, + staged write transactions and pending/checkpointed/reopened snapshots. + Preserve min/max group drops even beside count/sum/avg. +4. Force legal plan alternatives with existing engine test facilities and + compare values, type/zero/NaN details and surviving errors. Cover error + order, cursor close/rewind, memory cleanup and Limit/Offset. Frozen answers + change only for the intentionally removed aggregate type restrictions. +5. Meter tests prove sum-distinct's skipped types add no retained members, + accepted distinct numbers are charged once, duplicates add no charge, + count-distinct retains its charges, and release/failed-step cleanup works. + Test QRY-012 boundaries without claiming upstream memory is free. + Sum-distinct's finishing Sum uses temporary Exact scratch, already outside + the retained-state accounting model. Do not expand that model in #789. +6. On the bench machine, verify baseline then red/green targeted tests, full + default suite, clippy's repository gate, fmt, portable feature/target checks + and release binary size against the unchanged cap. Capture exact revisions + and commands. No new performance claim or broad benchmark project. + +## Documentation and delivery + +Synchronize current README, ROADMAP, CHANGELOG (unreleased v3), CLAUDE, +docs/TEST_COVERAGE and docs/ERROR_REFERENCE. Audit current aggregate comments +and references. Preserve historical design/release records; this spec supersedes +the ordinary type restrictions in #547 without rewriting its history. + +Use the existing sibling minigraf-docs checkout for a separate PR. Scope the +aggregate table, examples, nil/empty behavior and error descriptions using +`@until v3.0.0` / `@since v3.0.0`. Preserve v2's restrictions and avg behavior; +coordinate with #777's site changes. Audit architecture, comparison and +use-cases for affected claims; update only relevant text. Inspect site.toml's +existing v3 entry; do not invent a release or bump versions. + +Claude reviews this spec and the accompanying plan before implementation; +maintainer design approval is required. Claude reviews the final implementation +before core/docs PRs are raised. No merge, release or v2 runtime backport. + +## Implementation clarifications + +The approved test matrix's reference to recursive sets was corrected to sets +with mixed scalar elements: ValueSet rejects nested sets (API-021). #789 does +not add nested-set support. All other approved semantic boundaries are unchanged. diff --git a/src/query/datalog/aggregate.rs b/src/query/datalog/aggregate.rs index c0ace7c20c..d425b80cff 100644 --- a/src/query/datalog/aggregate.rs +++ b/src/query/datalog/aggregate.rs @@ -439,29 +439,10 @@ pub(crate) fn type_name(v: &Value) -> &'static str { } } -/// Variant kinds, in `Value`'s declaration order, for `INT-043`. -const KINDS: [&str; 8] = [ - "String", "Integer", "Float", "Boolean", "Ref", "Keyword", "Null", "Set", -]; - -fn kind(v: &Value) -> u8 { - match v { - Value::String(_) => 0, - Value::Integer(_) => 1, - Value::Float(_) => 2, - Value::Boolean(_) => 3, - Value::Ref(_) => 4, - Value::Keyword(_) => 5, - Value::Null => 6, - Value::Set(_) => 7, - } -} - -/// The least or greatest value by [`value_order`], and the kinds met. +/// The least or greatest non-nil value by [`value_order`]. #[derive(Clone, Default)] pub(crate) struct Extreme { best: Option, - kinds: u8, } impl Extreme { @@ -469,7 +450,6 @@ impl Extreme { if matches!(v, Value::Null) { return; } - self.kinds |= 1 << kind(v); let better = match &self.best { None => true, Some(b) => { @@ -537,8 +517,8 @@ impl Udf {} /// [`Agg::finish`] decides the result. pub(crate) enum Agg { Count(i64), - /// The sum, and the type of the first value that is not a number. - Sum(Sum, Option<&'static str>), + /// Numbers only; other values are skipped, as in a window sum. + Sum(Sum), /// The mean of numeric values, skipping nil and nonnumbers like window avg. Avg(Sum), Min(Extreme), @@ -558,7 +538,7 @@ impl Agg { Ok(match &desc.impl_ { AggImpl::Builtin(b) => match b { Builtin::Count => Agg::Count(0), - Builtin::Sum => Agg::Sum(Sum::default(), None), + Builtin::Sum => Agg::Sum(Sum::default()), Builtin::Avg => Agg::Avg(Sum::default()), Builtin::Min => Agg::Min(Extreme::default()), Builtin::Max => Agg::Max(Extreme::default()), @@ -575,8 +555,9 @@ impl Agg { /// caller can count it first (#549). pub(crate) fn would_add(&self, v: &Value) -> bool { match self { - Agg::CountDistinct(set) | Agg::SumDistinct(set) => { - !matches!(v, Value::Null) && !set.contains(&canonical(v)) + Agg::CountDistinct(set) => !matches!(v, Value::Null) && !set.contains(&canonical(v)), + Agg::SumDistinct(set) => { + matches!(v, Value::Integer(_) | Value::Float(_)) && !set.contains(&canonical(v)) } _ => false, } @@ -585,7 +566,7 @@ impl Agg { /// Retained exact-sum allocation needed before stepping this value (#777). pub(crate) fn exact_heap_growth(&self, v: &Value) -> usize { match self { - Agg::Sum(s, _) | Agg::Avg(s) + Agg::Sum(s) | Agg::Avg(s) if s.floats.is_none() && matches!(v, Value::Float(f) if f.is_finite()) => { size_of::() @@ -597,7 +578,7 @@ impl Agg { /// Exact-sum heap bytes that move with this accumulator when finishing. pub(crate) fn exact_heap_bytes(&self) -> usize { match self { - Agg::Sum(s, _) | Agg::Avg(s) if s.floats.is_some() => size_of::(), + Agg::Sum(s) | Agg::Avg(s) if s.floats.is_some() => size_of::(), _ => 0, } } @@ -610,58 +591,45 @@ impl Agg { } match self { Agg::Count(n) => *n = n.saturating_add(1), - Agg::Sum(s, bad) => { - if !s.add(v) && bad.is_none() { - *bad = Some(type_name(v)); - } + Agg::Sum(s) => { + s.add(v); } Agg::Avg(s) => { s.add(v); } Agg::Min(e) => e.add(v, false), Agg::Max(e) => e.add(v, true), - Agg::CountDistinct(set) | Agg::SumDistinct(set) => { + Agg::CountDistinct(set) => { set.insert(canonical(v)); } + Agg::SumDistinct(set) => { + if matches!(v, Value::Integer(_) | Value::Float(_)) { + set.insert(canonical(v)); + } + } Agg::Udf(u) => u.add(v)?, } Ok(()) } /// The result: `Ok(None)` when the group is dropped (`min`/`max` with no - /// value, `INT-042`), or the error the values make (spec §3.3). - pub(crate) fn finish(self, func: &str) -> Result> { + /// value), or a registered aggregate finalizer error. + pub(crate) fn finish(self, _func: &str) -> Result> { Ok(Some(match self { Agg::Count(n) => Value::Integer(n), - Agg::Sum(_, Some(bad)) => return Err(err_coded!(ErrorCode::Int041, bad)), - Agg::Sum(s, None) => s.value(), + Agg::Sum(s) => s.value(), Agg::Avg(s) => s.mean(), Agg::Min(e) | Agg::Max(e) => { let Some(best) = e.best else { return Ok(None); }; - if e.kinds.count_ones() > 1 { - let mut names = KINDS - .iter() - .enumerate() - .filter(|(i, _)| e.kinds & (1 << i) != 0) - .map(|(_, n)| *n); - let first = names.next().unwrap_or("?"); - let second = names.next().unwrap_or("?"); - return Err(err_coded!(ErrorCode::Int043, func, first, second)); - } - if !matches!(best, Value::Integer(_) | Value::Float(_) | Value::String(_)) { - return Err(err_coded!(ErrorCode::Int044, func, type_name(&best))); - } best } Agg::CountDistinct(set) => Value::Integer(i64::try_from(set.len()).unwrap_or(i64::MAX)), Agg::SumDistinct(set) => { let mut s = Sum::default(); for v in &set { - if !s.add(v) { - return Err(err_coded!(ErrorCode::Int041, type_name(v))); - } + s.add(v); } s.value() } @@ -677,7 +645,7 @@ impl Agg { pub(crate) type Groups = BTreeMap, (Vec, Vec)>; /// Finish `groups` in key order (spec §3.3): each group's values and its -/// aggregates' results. A group whose aggregate gives `INT-042` is dropped; +/// aggregates' results. A group with no min/max extremum is dropped; /// any other error, the first in `:find` order of the first group, fails. /// `funcs` names each aggregate, for the messages. pub(crate) fn finish_groups( diff --git a/src/query/datalog/aggregate/tests.rs b/src/query/datalog/aggregate/tests.rs index 1ab1b00eb0..acf8df66e7 100644 --- a/src/query/datalog/aggregate/tests.rs +++ b/src/query/datalog/aggregate/tests.rs @@ -52,13 +52,6 @@ fn finish(func: &str, vals: &[Value]) -> Result> { agg.finish(func) } -fn code(r: Result>) -> &'static str { - match r { - Err(e) => crate::MinigrafError::from(e).code(), - Ok(_) => "", - } -} - #[test] fn value_order_is_a_total_order() { let grid = vec![ @@ -219,22 +212,22 @@ fn integer_sum_saturates_once() { } #[test] -fn sum_error_is_decided_at_finish() { +fn sum_skips_nonnumbers_in_every_order() { for p in permutations(&[i(1), Value::String("a".into()), i(2)]) { - assert!(code(finish("sum", &p)) == "INT-041"); + assert!(finish("sum", &p).unwrap() == Some(i(3))); } } #[test] -fn min_max_errors_do_not_depend_on_order() { +fn min_max_mixed_values_do_not_depend_on_order() { for p in permutations(&[kw(":a"), kw(":b"), i(1)]) { - assert!(code(finish("min", &p)) == "INT-043"); + assert!(finish("min", &p).unwrap() == Some(i(1))); } for p in permutations(&[i(1), f(2.0)]) { - assert!(code(finish("max", &p)) == "INT-043"); + assert!(finish("max", &p).unwrap() == Some(f(2.0))); } - assert!(code(finish("min", &[kw(":a")])) == "INT-044"); - assert!(code(finish("max", &[Value::Boolean(true)])) == "INT-044"); + assert!(finish("min", &[kw(":a")]).unwrap() == Some(kw(":a"))); + assert!(finish("max", &[Value::Boolean(true)]).unwrap() == Some(Value::Boolean(true))); assert!(matches!(finish("min", &[]), Ok(None))); assert!(matches!(finish("max", &[Value::Null]), Ok(None))); } @@ -287,7 +280,7 @@ fn distinct_sets_store_canonical_floats() { finish("sum-distinct", &[i(2), i(2), i(3)]).ok().flatten(), Some(i(5)) ); - assert!(code(finish("sum-distinct", &[i(2), kw(":k")])) == "INT-041"); + assert!(finish("sum-distinct", &[i(2), kw(":k")]).unwrap() == Some(i(2))); assert_eq!(finish("count-distinct", &[]).ok().flatten(), Some(i(0))); } @@ -692,3 +685,142 @@ fn row_order_over_every_value_class() { let descending: Vec = ascending.iter().rev().cloned().collect(); assert!(same(&rows, &descending), "descending"); } + +/// Compare the observable float representation too: Value equality deliberately +/// equates signed zeros and all NaNs, whereas extrema preserve their ordering. +fn same_value_bits(actual: &Value, expected: &Value) { + match (actual, expected) { + (Value::Float(a), Value::Float(b)) => assert_eq!(a.to_bits(), b.to_bits()), + _ => assert!(actual == expected, "same aggregate value and type"), + } +} + +#[test] +fn ordinary_aggregates_match_running_for_all_value_variants() { + let set = |v| Value::Set(crate::graph::types::ValueSet::new(v).unwrap()); + let ordered = [ + f(f64::NEG_INFINITY), + i(i64::MIN), + i(0), + f(-0.0), + f(0.0), + f(9_007_199_254_740_992.0), + i(9_007_199_254_740_993), + i(i64::MAX), + f(f64::INFINITY), + f(f64::from_bits(0xfff8_0000_0000_0001)), + f(f64::from_bits(0x7ff8_0000_0000_0001)), + Value::String("x".into()), + Value::Boolean(false), + Value::Boolean(true), + Value::Ref(uuid::Uuid::from_u128(1)), + kw(":x"), + set(vec![]), + set(vec![i(1)]), + set(vec![i(1), Value::String("x".into())]), + ]; + let reg = FunctionRegistry::with_builtins(); + for a in &ordered { + for b in &ordered { + for population in permutations(&[a.clone(), Value::Null, b.clone()]) { + for func in ["sum", "min", "max", "count"] { + let mut running = + Running::new(&window(&format!("({func} ?v :over (:order-by ?o))")), ®) + .unwrap(); + for v in &population { + running.step(v).unwrap(); + } + let actual = finish(func, &population).unwrap().unwrap(); + same_value_bits(&actual, &running.value().unwrap()); + } + // Independent expected extrema, not just agreement of shared code. + let a_pos = ordered.iter().position(|v| std::ptr::eq(v, a)).unwrap(); + let b_pos = ordered.iter().position(|v| std::ptr::eq(v, b)).unwrap(); + same_value_bits( + &finish("min", &population).unwrap().unwrap(), + &ordered[a_pos.min(b_pos)], + ); + same_value_bits( + &finish("max", &population).unwrap().unwrap(), + &ordered[a_pos.max(b_pos)], + ); + } + } + } +} + +#[test] +fn skipped_arguments_preserve_exact_sums_and_special_float_results() { + let nonnumbers = [ + Value::Null, + Value::String("skip".into()), + Value::Boolean(false), + kw(":k"), + Value::Ref(uuid::Uuid::nil()), + Value::Set(Default::default()), + ]; + let cases = [ + (vec![f(-0.0)], f(-0.0)), + (vec![f(-0.0), f(0.0)], f(0.0)), + (vec![i(i64::MAX), i(1), i(-1)], i(i64::MAX)), + (vec![i(i64::MIN), i(-1)], i(i64::MIN)), + ( + vec![i(9_007_199_254_740_993), f(-9_007_199_254_740_992.0)], + f(1.0), + ), + (vec![f(1e16), f(1.0), f(-1e16)], f(1.0)), + (vec![f(f64::INFINITY), i(1)], f(f64::INFINITY)), + (vec![f(f64::NEG_INFINITY), i(1)], f(f64::NEG_INFINITY)), + (vec![f(f64::MAX), f(f64::MAX)], f(f64::INFINITY)), + (vec![f(f64::INFINITY), f(f64::NEG_INFINITY)], f(f64::NAN)), + (vec![f(-f64::NAN), i(1)], f(f64::NAN)), + ]; + for (nums, expected) in cases { + for skipped in &nonnumbers { + let mut vals = nums.clone(); + vals.push(skipped.clone()); + for population in permutations(&vals) { + same_value_bits(&finish("sum", &population).unwrap().unwrap(), &expected); + } + } + } + assert!(finish("sum", &nonnumbers).unwrap() == Some(i(0))); + assert!(finish("sum-distinct", &nonnumbers).unwrap() == Some(i(0))); + assert!(finish("count", &nonnumbers).unwrap() == Some(i(5))); + assert!(finish("count-distinct", &nonnumbers).unwrap() == Some(i(5))); +} + +#[test] +fn distinct_numeric_filter_and_membership_prediction_agree() { + let reg = FunctionRegistry::with_builtins(); + let mut agg = Agg::new("sum-distinct", ®).unwrap(); + for v in [ + Value::Null, + Value::String("s".into()), + kw(":k"), + Value::Boolean(true), + Value::Ref(uuid::Uuid::nil()), + Value::Set(Default::default()), + ] { + assert!(!agg.would_add(&v), "ignored types must not be charged"); + agg.step(&v).unwrap(); + } + let Agg::SumDistinct(set) = &agg else { + panic!("distinct accumulator") + }; + assert!(set.is_empty()); + for v in [i(1), f(1.0), f(-0.0), f(-f64::NAN)] { + assert!(agg.would_add(&v)); + agg.step(&v).unwrap(); + assert!(!agg.would_add(&v)); + } + assert!(!agg.would_add(&f(0.0))); + assert!(!agg.would_add(&f(f64::from_bits(0x7ff8_0000_0000_0001)))); + assert!(finish("sum-distinct", &[i(1), i(1), f(1.0), kw(":x")]).unwrap() == Some(f(2.0))); + same_value_bits( + &finish("sum-distinct", &[f(-0.0), kw(":x")]) + .unwrap() + .unwrap(), + &f(0.0), + ); +} diff --git a/src/query/engine/aggregate_order_tests.rs b/src/query/engine/aggregate_order_tests.rs index 8a6076d45e..b414ec1024 100644 --- a/src/query/engine/aggregate_order_tests.rs +++ b/src/query/engine/aggregate_order_tests.rs @@ -100,7 +100,7 @@ fn a_group_limit_stops_after_the_requested_group_and_one_input_batch() { } #[test] -fn hidden_with_groups_multiple_keys_and_fallible_aggregates_keep_the_hash_path() { +fn hidden_with_groups_multiple_keys_and_other_aggregates_keep_the_hash_path() { let (_dir, db) = fixture(64, 4); for query in [ "[:find ?g (count ?e) :with ?v :where [?e :bucket ?g] [?e :item ?v]]", @@ -197,33 +197,49 @@ fn fallible_finishes_keep_later_group_errors_and_dropped_group_precedence() { let _text_order = crate::query::datalog::optimizer::set_reorder(false); db.execute("(transact [[:e4095 :item \"bad\"]])").unwrap(); db.checkpoint().unwrap(); - for func in ["sum", "sum-distinct", "min"] { - let q = - format!("(query [:find ?g ({func} ?v) :where [?e :bucket ?g] [?e :item ?v] :limit 1])"); - let mut cursor = db.query(&q).unwrap(); - assert!(!ordered(&cursor.stream_mut().unwrap().counters())); - let err = cursor - .next_batch(1) - .expect_err("late group must fail before any answer"); - assert_eq!( - err.code(), - if func == "min" { "INT-043" } else { "INT-041" } - ); + db.try_register_aggregate( + "checked", + || Ok(false), + |bad: &mut bool, v| { + *bad |= matches!(v, Value::String(_)); + Ok(()) + }, + |bad: &bool, _| { + if *bad { + Err(crate::UdfError::new("late value")) + } else { + Ok(Value::Integer(0)) + } + }, + ) + .unwrap(); + for func in ["sum", "sum-distinct", "min", "max"] { + for tail in [":limit 1", ":order-by [?g] :offset 1 :limit 1"] { + let q = format!( + "(query [:find ?g ({func} ?v) (checked ?v) :where [?e :bucket ?g] [?e :item ?v] {tail}])" + ); + let mut cursor = db.query(&q).unwrap(); + assert!(!ordered(&cursor.stream_mut().unwrap().counters())); + let err = cursor + .next_batch(1) + .expect_err("late finalizer must fail before any answer"); + assert_eq!(err.code(), "QRY-014"); + } } db.execute("(transact [[:drop :bucket 999] [:drop :empty nil] [:drop :bad \"bad\"]])") .unwrap(); db.checkpoint().unwrap(); let (rows, counters, _) = collect( &db, - "[:find ?g (min ?n) (sum ?v) :where [?e :bucket ?g] [?e :empty ?n] [?e :bad ?v] :limit 1]", + "[:find ?g (min ?n) (checked ?v) :where [?e :bucket ?g] [?e :empty ?n] [?e :bad ?v] :limit 1]", ); assert!( rows.is_empty(), - "an earlier empty min drops the group before its bad sum" + "an earlier empty min drops the group before its failing finalizer" ); assert!(!ordered(&counters)); - let err = db.execute("(query [:find ?g (sum ?v) (min ?n) :where [?e :bucket ?g] [?e :empty ?n] [?e :bad ?v] :limit 1])").expect_err("find order chooses the first aggregate error"); - assert_eq!(err.code(), "INT-041"); + let err = db.execute("(query [:find ?g (checked ?v) (min ?n) :where [?e :bucket ?g] [?e :empty ?n] [?e :bad ?v] :limit 1])").expect_err("find order chooses the first aggregate error"); + assert_eq!(err.code(), "QRY-014"); } #[test] diff --git a/src/query/engine/tests.rs b/src/query/engine/tests.rs index 3aa571970b..e150356989 100644 --- a/src/query/engine/tests.rs +++ b/src/query/engine/tests.rs @@ -2198,10 +2198,10 @@ fn ungrouped_aggregate_returns_one_row() { } /// Groups come out in the order of their values, over several batches; an -/// error in a later group fails the query, and `min` of no value drops its +/// nonnumber in a later sum group is skipped, and `min` of no value drops its /// group (#547 spec §3.3, §5.3). #[test] -fn grouped_aggregate_orders_groups_and_fails_in_a_later_group() { +fn grouped_aggregate_orders_groups_and_skips_nonnumbers() { let db = Minigraf::in_memory().unwrap(); let facts: String = (0..3000) .map(|i| format!("[:e{i} :k {}] [:e{i} :n {i}]", 3000 - i)) @@ -2226,10 +2226,14 @@ fn grouped_aggregate_orders_groups_and_fails_in_a_later_group() { db.execute("(transact [[:x :k 2999] [:x :n \"text\"] [:y :k 5] [:y :n nil]])") .unwrap(); let before = runs(); - let err = db + let QueryResult::QueryResults { results, .. } = db .execute("(query [:find ?k (sum ?n) :where [?e :k ?k] [?e :n ?n]])") - .unwrap_err(); - assert_eq!(crate::MinigrafError::from(err).code(), "INT-041"); + .unwrap() + else { + panic!("query results") + }; + assert_eq!(results.len(), 3000); + assert!(results[2998] == vec![crate::Value::Integer(2999), crate::Value::Integer(1)]); assert_eq!(runs(), before + 1, "query ran on the engine"); db.execute("(transact [[:z :k 7] [:z :m nil]])").unwrap(); @@ -4266,3 +4270,122 @@ fn partition_window_rejects_decreasing_boundaries_before_and_after_output() { assert_eq!(meter.used(), 0); } } + +/// Isolate Aggregate's retained members from source/IdSpace costs: Sequence +/// supplies decoded scalar cells without charging upstream buffers. +#[test] +fn numeric_only_sum_distinct_charges_members_once_and_releases_on_rewind() { + use super::{ + Operator, + cell::{Cell, Scalar}, + }; + let int = |n| Cell::Val(Scalar::Int(n)); + let run = |func: &str, cells: Vec, limit| { + let meter = super::Meter::new(limit); + let mut op = super::aggregate::Aggregate::new( + Box::new(Sequence { cells, at: 0 }), + vec![], + vec![(func.into(), 0)], + false, + false, + &meter, + ); + let result = pull_once(&mut op, &meter); + let peak = meter.peak(); + op.rewind(&[]); + assert_eq!( + meter.used(), + 0, + "rewind releases success and partial failure state" + ); + op.close(); + assert_eq!(meter.used(), 0); + (result, peak) + }; + let base = run("sum-distinct", vec![int(1)], usize::MAX).1; + let mut ignored = vec![int(1); 1000]; + ignored.extend((0..1000).map(|n| Cell::Val(Scalar::Str(format!("ignored-{n}").into())))); + ignored.extend([ + Cell::Val(Scalar::Null), + Cell::Val(Scalar::Bool(true)), + Cell::Val(Scalar::Set(Default::default())), + ]); + let (result, peak) = run("sum-distinct", ignored.clone(), base); + assert!( + result.is_ok(), + "ignored members and duplicates fit the original budget" + ); + assert_eq!(peak, base); + let count_peak = run("count-distinct", ignored, usize::MAX).1; + assert!(count_peak > base + 1000 * super::meter::HASH_ENTRY_OVERHEAD); + let (_, two) = run("sum-distinct", vec![int(1), int(2)], usize::MAX); + assert_eq!( + two - base, + super::meter::value_bytes(&crate::Value::Integer(2)) + super::meter::HASH_ENTRY_OVERHEAD + ); + let (result, _) = run("sum-distinct", vec![int(1), int(2)], base); + assert_eq!( + crate::MinigrafError::from(result.expect_err("new numeric member exceeds the budget")) + .code(), + "QRY-012" + ); + let (_, canonical) = run( + "sum-distinct", + vec![Cell::Val(Scalar::Float(-0.0))], + usize::MAX, + ); + let (_, duplicates) = run( + "sum-distinct", + vec![ + Cell::Val(Scalar::Float(-0.0)), + Cell::Val(Scalar::Float(0.0)), + ], + usize::MAX, + ); + assert_eq!(canonical, duplicates, "canonical zero charged once"); +} + +#[test] +fn ignored_aggregate_arguments_do_not_hide_upstream_failures() { + use super::{ + Ctx, Operator, + cell::{Cell, Scalar}, + }; + struct FailsAfter(Sequence); + impl Operator for FailsAfter { + fn next_batch(&mut self, ctx: &mut Ctx<'_>) -> anyhow::Result> { + match self.0.next_batch(ctx)? { + Some(batch) => Ok(Some(batch)), + None => Err(crate::error::err_coded!( + crate::error::ErrorCode::Int032, + "late source failure" + )), + } + } + fn rewind(&mut self, seed: &[Cell]) { + self.0.rewind(seed); + } + fn close(&mut self) { + self.0.close(); + } + fn counters(&self, _: &mut Vec) {} + } + for func in ["sum", "sum-distinct", "min", "max"] { + let meter = super::Meter::unlimited(); + let mut op = super::aggregate::Aggregate::new( + Box::new(FailsAfter(Sequence { + cells: vec![Cell::Val(Scalar::Str("skip".into()))], + at: 0, + })), + vec![], + vec![(func.into(), 0)], + false, + false, + &meter, + ); + let err = pull_once(&mut op, &meter).expect_err("full drain reaches source error"); + assert_eq!(crate::MinigrafError::from(err).code(), "INT-032"); + op.close(); + assert_eq!(meter.used(), 0); + } +} diff --git a/tests/aggregate_value_handling_test.rs b/tests/aggregate_value_handling_test.rs new file mode 100644 index 0000000000..ce09df2269 --- /dev/null +++ b/tests/aggregate_value_handling_test.rs @@ -0,0 +1,343 @@ +//! #789: ordinary and window aggregates share their argument domain. +use minigraf::{BindValue, Minigraf, QueryResult, Value}; + +fn rows(result: QueryResult) -> Vec> { + match result { + QueryResult::QueryResults { results, .. } => results, + _ => panic!("expected query results"), + } +} + +fn query(db: &Minigraf, query: &str) -> Vec> { + rows(db.execute(query).expect("aggregate query succeeds")) +} + +#[test] +fn sum_skips_nonnumbers_and_distinct_keeps_numeric_identity() { + let db = Minigraf::in_memory().unwrap(); + db.execute( + r#"(transact [[:a :v 10] [:b :v "unknown"] [:c :v 20] + [:d :v 10] [:e :v 10.0] [:f :v true] [:g :v :keyword] + [:h :v #uuid "00000000-0000-0000-0000-000000000001"] + [:i :v #{1 "x"}] [:j :v nil]])"#, + ) + .unwrap(); + assert!( + query( + &db, + "(query [:find (sum ?v) (sum-distinct ?v) (count ?v) (count-distinct ?v) :where [?e :v ?v]])" + ) == vec![vec![ + Value::Float(50.0), + Value::Float(40.0), + Value::Integer(9), + Value::Integer(8) + ]] + ); +} + +#[test] +fn extrema_accept_mixed_numbers_and_every_nonnil_class() { + let db = Minigraf::in_memory().unwrap(); + db.execute( + r#"(transact [[:a :v 2] [:b :v 1.5] [:c :v "x"] + [:d :v true] [:e :v :keyword] [:f :v nil] + [:g :v #uuid "00000000-0000-0000-0000-000000000001"] [:h :v #{}]])"#, + ) + .unwrap(); + let result = query(&db, "(query [:find (min ?v) (max ?v) :where [?e :v ?v]])"); + assert_eq!(result.len(), 1); + assert!(result[0][0] == Value::Float(1.5)); + assert!(matches!(&result[0][1], Value::Set(set) if set.is_empty())); +} + +#[test] +fn existing_nil_and_nonnumeric_groups_differ_from_no_matches() { + let db = Minigraf::in_memory().unwrap(); + db.execute(r#"(transact [[:a :g "nil"] [:a :v nil] [:b :g "text"] [:b :v "x"]])"#) + .unwrap(); + let prepared = db.prepare("(query [:find (sum ?v) (sum-distinct ?v) (count ?v) (count-distinct ?v) :where [?e :g $g] [?e :v ?v]])").unwrap(); + for (group, count) in [("nil", 0), ("text", 1)] { + let binds = [("g", BindValue::Val(Value::String(group.into())))]; + let expected = vec![vec![ + Value::Integer(0), + Value::Integer(0), + Value::Integer(count), + Value::Integer(count), + ]]; + assert!(rows(prepared.execute(&binds).unwrap()) == expected); + let mut cursor = prepared.query(&binds).unwrap(); + assert!(cursor.next_batch(1).unwrap().unwrap().into_rows() == expected); + assert!(cursor.next_batch(1).unwrap().is_none()); + } + for f in ["sum", "sum-distinct", "min", "max"] { + assert!( + query( + &db, + &format!("(query [:find ({f} ?v) :where [?e :missing ?v]])") + ) + .is_empty() + ); + assert!( + query( + &db, + &format!("(query [:find ?e ({f} ?v) :where [?e :missing ?v]])") + ) + .is_empty() + ); + } + for f in ["min", "max"] { + assert!( + query( + &db, + &format!("(query [:find (count ?v) ({f} ?v) :where [?e :g \"nil\"] [?e :v ?v]])") + ) + .is_empty() + ); + assert!( + query( + &db, + &format!("(query [:find ({f} ?v) :where [?e :g \"text\"] [?e :v ?v]])") + ) == vec![vec![Value::String("x".into())]] + ); + } + assert!( + query( + &db, + "(query [:find (count ?v) (count-distinct ?v) :with ?e :where [?e :missing ?v]])" + ) == vec![vec![Value::Integer(0), Value::Integer(0)]] + ); +} + +#[test] +fn default_and_bounded_windows_match_each_ordinary_population() { + let db = Minigraf::in_memory().unwrap(); + // Every variant, repeated numbers and nil occupy real frame positions. + let values = [ + "nil", + "10", + "\"unknown\"", + "20", + "true", + ":k", + "#{1 2}", + "#uuid \"00000000-0000-0000-0000-000000000001\"", + "1.5", + "10", + "nil", + ]; + let facts: String = values + .iter() + .enumerate() + .map(|(n, v)| format!("[:e{n} :v {v}] [:e{n} :ord {n}] [:e{n} :g {}]", n / 6)) + .collect(); + db.execute(&format!("(transact [{facts}])")).unwrap(); + for partitioned in [false, true] { + for bounded in [false, true] { + let partition = if partitioned { ":partition-by ?g " } else { "" }; + let frame = if bounded { " :rows 2 preceding" } else { "" }; + for f in ["sum", "min", "max", "count"] { + let result = query( + &db, + &format!( + "(query [:find ?o ({f} ?v :over ({partition}:order-by ?o{frame})) :where [?e :v ?v] [?e :ord ?o] [?e :g ?g] :order-by [?o]])" + ), + ); + assert_eq!( + result.len(), + values.len(), + "skipped arguments still have output rows" + ); + for (n, row) in result.iter().enumerate() { + let first = if partitioned { n / 6 * 6 } else { 0 }; + let start = if bounded { + n.saturating_sub(2).max(first) + } else { + first + }; + let ordinary = query( + &db, + &format!( + "(query [:find ({f} ?v) :where [?e :v ?v] [?e :ord ?o] [(>= ?o {start})] [(<= ?o {n})]])" + ), + ); + if ordinary.is_empty() { + assert!(f == "min" || f == "max"); + assert!( + row[1] == Value::Null, + "empty extrema drop groups but retain window rows" + ); + } else { + assert!( + row[1] == ordinary[0][0], + "same function and argument population" + ); + } + } + } + } + } + // Independent expectations for the bounded middle frame [10, text, 20]. + assert!( + query( + &db, + "(query [:find (sum ?v) (min ?v) (max ?v) :where [?e :v ?v] [?e :ord ?o] [(>= ?o 1)] [(<= ?o 3)]])" + ) == vec![vec![ + Value::Integer(30), + Value::Integer(10), + Value::String("unknown".into()) + ]] + ); +} + +#[test] +fn with_groups_and_join_multiplicity_remain_intact() { + let db = Minigraf::in_memory().unwrap(); + db.execute( + r#"(transact [[:a :v 10] [:a :g "x"] [:b :v 10] [:b :g "x"] + [:c :v "unknown"] [:c :g "y"] [:t1 :tag true] [:t2 :tag true]])"#, + ) + .unwrap(); + assert!( + query( + &db, + "(query [:find (sum ?v) (sum-distinct ?v) (count ?v) :where [?e :v ?v] [?t :tag true]])" + ) == vec![vec![ + Value::Integer(40), + Value::Integer(10), + Value::Integer(6) + ]] + ); + assert!( + query( + &db, + "(query [:find (sum ?v) :with ?g :where [?e :v ?v] [?e :g ?g]])" + ) == vec![vec![Value::Integer(20)], vec![Value::Integer(0)]] + ); +} + +#[test] +fn rule_and_temporal_queries_use_the_same_aggregate_contract() { + let db = Minigraf::in_memory().unwrap(); + db.execute( + r#"(transact [[:a :v 10] [:b :v "unknown"] [:c :v 20] + [:a :next :b] [:b :next :c]])"#, + ) + .unwrap(); + db.execute("(rule [(walk ?e ?v) [?e :v ?v]])").unwrap(); + db.execute("(rule [(walk ?e ?v) [?e :next ?n] (walk ?n ?v)])") + .unwrap(); + assert!( + query( + &db, + "(query [:find (sum ?v) (sum-distinct ?v) (min ?v) (max ?v) :where (walk :a ?v)])" + ) == vec![vec![ + Value::Integer(30), + Value::Integer(30), + Value::Integer(10), + Value::String("unknown".into()) + ]] + ); + db.execute("(transact [[:d :v true]])").unwrap(); + assert!( + query( + &db, + "(query [:find (sum ?v) (max ?v) :as-of 1 :where [?e :v ?v]])" + ) == vec![vec![Value::Integer(30), Value::String("unknown".into())]] + ); + assert!( + query(&db, "(query [:find (sum ?v) (max ?v) :where [?e :v ?v]])") + == vec![vec![Value::Integer(30), Value::Boolean(true)]] + ); + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:f :v 5.5]])").unwrap(); + assert!( + rows( + tx.execute("(query [:find (sum ?v) (min ?v) :where [?e :v ?v]])") + .unwrap() + ) == vec![vec![Value::Float(35.5), Value::Float(5.5)]] + ); + // Rollback does not retain the staged numeric argument. + drop(tx); + assert!( + query(&db, "(query [:find (sum ?v) :where [?e :v ?v]])") == vec![vec![Value::Integer(30)]] + ); +} + +#[cfg(not(target_arch = "wasm32"))] +#[test] +fn grouped_cursor_and_prepared_queries_survive_checkpoint_and_reopen() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("values.graph"); + let db = Minigraf::open(&path).unwrap(); + let facts: String = (0..1100) + .map(|n| { + format!( + "[:e{n} :g {n}] [:e{n} :v {}]", + if n % 2 == 0 { "10" } else { "\"unknown\"" } + ) + }) + .collect(); + db.execute(&format!("(transact [{facts}])")).unwrap(); + let q = "(query [:find ?g (sum ?v) (sum-distinct ?v) (min ?v) (max ?v) :where [?e :g ?g] [?e :v ?v] :order-by [?g]])"; + let expected: Vec<_> = (0..1100) + .map(|n| { + let v = if n % 2 == 0 { + Value::Integer(10) + } else { + Value::String("unknown".into()) + }; + let sum = Value::Integer(if n % 2 == 0 { 10 } else { 0 }); + vec![Value::Integer(n), sum.clone(), sum, v.clone(), v] + }) + .collect(); + let check = |db: &Minigraf| { + assert!(query(db, q) == expected); + let prepared = db.prepare(q).unwrap(); + assert!(rows(prepared.execute(&[]).unwrap()) == expected); + let mut cursor = prepared.query(&[]).unwrap(); + let mut actual = Vec::new(); + while let Some(batch) = cursor.next_batch(17).unwrap() { + actual.extend(batch.into_rows()); + } + assert!(actual == expected); + cursor.close(); + }; + check(&db); + db.checkpoint().unwrap(); + check(&db); + drop(db); + check(&Minigraf::open(&path).unwrap()); +} + +#[cfg(feature = "test-hooks")] +#[test] +fn value_handling_is_independent_of_join_and_negation_plans() { + let db = Minigraf::in_memory().unwrap(); + db.execute( + r#"(transact [[:a :v 10] [:b :v "x"] [:c :v 1.5] + [:a :keep true] [:b :keep true] [:c :keep true] [:z :blocked true]])"#, + ) + .unwrap(); + let expected = vec![vec![ + Value::Float(11.5), + Value::Float(1.5), + Value::String("x".into()), + Value::Integer(3), + ]]; + for mode in [ + minigraf::Subplans::Choose, + minigraf::Subplans::Hash, + minigraf::Subplans::Correlated, + ] { + let _mode = minigraf::__subplans(mode); + for reorder in [false, true] { + let _order = minigraf::__reorder(reorder); + assert!( + query( + &db, + "(query [:find (sum-distinct ?v) (min ?v) (max ?v) (count-distinct ?v) :where [?e :keep true] [?e :v ?v] (not [?e :blocked true])])" + ) == expected + ); + } + } +} diff --git a/tests/aggregate_window_rules_test.rs b/tests/aggregate_window_rules_test.rs index 8b3642a9fc..427e917c28 100644 --- a/tests/aggregate_window_rules_test.rs +++ b/tests/aggregate_window_rules_test.rs @@ -145,13 +145,11 @@ fn row_number_breaks_ties_by_the_other_find_variables() { } #[test] -fn aggregate_error_is_decided_per_group() { +fn aggregate_skips_nonnumbers_in_its_group() { let db = Minigraf::in_memory().unwrap(); db.execute(r#"(transact [[:a :v 1] [:b :v "x"] [:c :v 2]])"#) .unwrap(); - let err = db - .execute("(query [:find (sum ?v) :where [?e :v ?v]])") - .err() - .expect("sum of a string fails"); - assert_eq!(err.code(), "INT-041"); + assert!( + rows(&db, "(query [:find (sum ?v) :where [?e :v ?v]])") == vec![vec![Value::Integer(3)]] + ); } diff --git a/tests/aggregation_test.rs b/tests/aggregation_test.rs index d144a15940..84d5b10af1 100644 --- a/tests/aggregation_test.rs +++ b/tests/aggregation_test.rs @@ -177,12 +177,14 @@ fn count_and_sum_skip_nil() { } #[test] -fn sum_type_error() { +fn sum_skips_nonnumeric_values() { let db = db(); db.execute(r#"(transact [[:a :v "not-a-number"]])"#) .unwrap(); - let r = db.execute(r#"(query [:find (sum ?v) :where [?e :v ?v]])"#); - assert!(r.is_err(), "sum of string should fail"); + let r = db + .execute(r#"(query [:find (sum ?v) :where [?e :v ?v]])"#) + .unwrap(); + assert!(results(&r) == &vec![vec![Value::Integer(0)]]); } // ── min / max ───────────────────────────────────────────────────────────────── @@ -218,19 +220,23 @@ fn min_max_strings() { } #[test] -fn min_type_error_boolean() { +fn min_accepts_boolean() { let db = db(); db.execute(r#"(transact [[:a :v true]])"#).unwrap(); - let r = db.execute(r#"(query [:find (min ?v) :where [?e :v ?v]])"#); - assert!(r.is_err(), "min of boolean should fail"); + let r = db + .execute(r#"(query [:find (min ?v) :where [?e :v ?v]])"#) + .unwrap(); + assert!(results(&r) == &vec![vec![Value::Boolean(true)]]); } #[test] -fn min_mixed_int_float_error() { +fn min_compares_mixed_int_float() { let db = db(); db.execute(r#"(transact [[:a :v 1] [:b :v 2.0]])"#).unwrap(); - let r = db.execute(r#"(query [:find (min ?v) :where [?e :v ?v]])"#); - assert!(r.is_err(), "min of mixed Integer/Float should fail"); + let r = db + .execute(r#"(query [:find (min ?v) :where [?e :v ?v]])"#) + .unwrap(); + assert!(results(&r) == &vec![vec![Value::Integer(1)]]); } // ── :with ───────────────────────────────────────────────────────────────────── diff --git a/tests/error_codes_qry_test.rs b/tests/error_codes_qry_test.rs index b3b28ec7fa..f4ff8cb151 100644 --- a/tests/error_codes_qry_test.rs +++ b/tests/error_codes_qry_test.rs @@ -34,7 +34,8 @@ //! which is raised in `src/query/datalog/functions.rs`. Both were assigned //! `INT-0xx` codes (INT-020 and INT-041 respectively) by the #277 step 6 //! (API+INT) category PR, which audited and coded every remaining uncoded -//! call site crate-wide — see `docs/ERROR_REFERENCE.md`. +//! call site crate-wide — see `docs/ERROR_REFERENCE.md`. #789 retires INT-041 +//! for v3 built-ins; its regression below now asserts numeric-only summation. use minigraf::Minigraf; @@ -93,22 +94,19 @@ fn recursive_rule_derived_facts_limit_returns_int020() { } } -// ─── Runtime type-mismatch family: out of this issue's file scope ───────── - -/// `sum` over a non-numeric attribute fails at query execution time. The -/// error originates in `apply_builtin_aggregate` -/// (src/query/datalog/functions.rs), which is NOT one of this issue's five -/// in-scope files (executor.rs, evaluator.rs, optimizer.rs, matcher.rs, -/// magic_sets.rs) — so it was intentionally left uncoded by this PR. #277 -/// step 6 (API+INT category PR) later assigned it INT-041. +// #789 retires INT-041 for ordinary built-ins in v3. #[test] -fn aggregate_type_mismatch_returns_int041() { +fn aggregate_nonnumeric_values_no_longer_return_int041() { let db = Minigraf::in_memory().unwrap(); db.execute(r#"(transact [[:a :score "high"] [:b :score "low"]])"#) .unwrap(); - let result = db.execute(r#"(query [:find (sum ?s) :where [?e :score ?s]])"#); - let err = result.expect_err("sum of strings must fail at runtime"); - assert_eq!(err.code(), "INT-041"); + let minigraf::QueryResult::QueryResults { results, .. } = db + .execute("(query [:find (sum ?s) :where [?e :score ?s]])") + .unwrap() + else { + panic!("query results") + }; + assert!(results == vec![vec![minigraf::Value::Integer(0)]]); } // ─── QRY-010 / QRY-011: unknown aggregate / window function (#414) ──────── diff --git a/tests/error_handling_test.rs b/tests/error_handling_test.rs index 4c99779459..801cfe295b 100644 --- a/tests/error_handling_test.rs +++ b/tests/error_handling_test.rs @@ -1,48 +1,55 @@ -//! Integration-level error-path tests for Phase 7.5. -//! Drives the full Minigraf::execute() API with invalid programs/data and -//! asserts that errors propagate correctly to the caller. +//! Integration-level error paths and aggregate type-boundary regressions. +//! Invalid programs still propagate errors; aggregate values follow #789. -use minigraf::{Minigraf, OpenOptions}; +use minigraf::{Minigraf, OpenOptions, QueryResult, Value}; fn db() -> Minigraf { OpenOptions::new().open_memory().unwrap() } -// ── Runtime type errors in aggregation ─────────────────────────────────────── +// ── Aggregate type boundaries ──────────────────────────────────────────────── + +fn single_value(db: &Minigraf, query: &str, expected: Value) { + let QueryResult::QueryResults { results, .. } = db.execute(query).unwrap() else { + panic!("expected query results"); + }; + assert!(results == vec![vec![expected]]); +} -/// sum over a string attribute fails at query execution time. #[test] -fn sum_string_attribute_error() { +fn sum_string_attribute_returns_zero() { let db = db(); db.execute(r#"(transact [[:a :score "high"] [:b :score "low"]])"#) .unwrap(); - let r = db.execute(r#"(query [:find (sum ?s) :where [?e :score ?s]])"#); - assert!(r.is_err(), "sum of strings must fail at runtime"); + single_value( + &db, + "(query [:find (sum ?s) :where [?e :score ?s]])", + Value::Integer(0), + ); } -/// sum fails when an attribute has mixed integer and string values. #[test] -fn sum_mixed_int_string_error() { +fn sum_mixed_int_string_skips_string() { let db = db(); - // :score has both integer and string values across entities db.execute(r#"(transact [[:a :score 10] [:b :score "twenty"]])"#) .unwrap(); - let r = db.execute(r#"(query [:find (sum ?s) :where [?e :score ?s]])"#); - assert!( - r.is_err(), - "sum of mixed integer/string must fail at runtime" + single_value( + &db, + "(query [:find (sum ?s) :where [?e :score ?s]])", + Value::Integer(10), ); } -/// max over a boolean attribute fails at query execution time. -/// (min on boolean is already tested in aggregation_test.rs; this covers max.) #[test] -fn max_boolean_attribute_error() { +fn max_boolean_attribute_returns_true() { let db = db(); db.execute(r#"(transact [[:a :flag true] [:b :flag false]])"#) .unwrap(); - let r = db.execute(r#"(query [:find (max ?f) :where [?e :flag ?f]])"#); - assert!(r.is_err(), "max of booleans must fail at runtime"); + single_value( + &db, + "(query [:find (max ?f) :where [?e :flag ?f]])", + Value::Boolean(true), + ); } // ── Rule-level errors ───────────────────────────────────────────────────────── diff --git a/tests/ordinary_avg_test.rs b/tests/ordinary_avg_test.rs index 649d4e6992..cf2a9c6cb4 100644 --- a/tests/ordinary_avg_test.rs +++ b/tests/ordinary_avg_test.rs @@ -63,15 +63,15 @@ fn ordinary_avg_groups_skip_nonnumbers_and_preserve_with_multiplicity() { } #[test] -fn ordinary_avg_preserves_other_aggregate_errors_and_dropped_groups() { +fn ordinary_avg_matches_sum_results_and_preserves_dropped_groups() { let db = Minigraf::in_memory().unwrap(); db.execute(r#"(transact [[:a :n "unknown"] [:a :empty nil]])"#) .unwrap(); - for find in ["(avg ?v) (sum ?v)", "(sum ?v) (avg ?v)"] { - let err = db - .execute(&format!("(query [:find {find} :where [?e :n ?v]])")) - .unwrap_err(); - assert_eq!(err.code(), "INT-041"); + for (find, expected) in [ + ("(avg ?v) (sum ?v)", vec![Value::Null, Value::Integer(0)]), + ("(sum ?v) (avg ?v)", vec![Value::Integer(0), Value::Null]), + ] { + assert!(rows(&db, &format!("(query [:find {find} :where [?e :n ?v]])")) == vec![expected]); } assert!( rows(