From 19c95d672014075c1d017f3b344a216259741d46 Mon Sep 17 00:00:00 2001 From: Tienson Qin Date: Mon, 5 Oct 2026 06:53:22 +0000 Subject: [PATCH] Resolve lookup-refs via bounded AVET seeks instead of full-index scans Lookup_refs.entity_id materialized visible_datoms and scanned the whole list for the unique attr/value pair. Route it through the context's entid function (entid_db -> find_avet_exact), which performs a bounded AVET slice on the (attr, value) prefix plus the duplicate table. The context now carries entid instead of visible_datoms; entity_id_in_datoms keeps its list semantics for staged transaction datoms. find_avet_exact now applies db.filter_pred so the bounded path preserves the old visible_datoms filter behavior on filtered dbs (and restores upstream parity for entid on filtered dbs). Removes dead Transact_datoms.entid (no callers) and the visible_datoms context requirement. --- docs/upstream_differences.md | 18 ++++++------ impl/datascript.ml | 53 +++++++++++++++++------------------- impl/datascript.mli | 2 +- impl/lookup_refs.ml | 17 +++++++++--- impl/lookup_refs.mli | 2 +- impl/transact_datoms.ml | 3 -- 6 files changed, 48 insertions(+), 47 deletions(-) diff --git a/docs/upstream_differences.md b/docs/upstream_differences.md index d4e02e6..dedb189 100644 --- a/docs/upstream_differences.md +++ b/docs/upstream_differences.md @@ -105,14 +105,12 @@ through them. ### DataScript: Direct Lookup Refs Scan Visible Datoms -Some unique-value checks already use the `AVET` index, including transaction -conflict checks. Direct `entid` and lookup-ref resolution still go through -`visible_datoms` and then scan a list for the matching unique attribute/value. - -That preserves result behavior, but it is an eager path where upstream can use -indexed lookup. Exact parity should route direct lookup-ref resolution through a -bounded `AVET` lookup, while keeping transaction-local datom-list paths for -staged transaction state. +Resolved. Direct `entid` and lookup-ref resolution now go through `find_avet_exact`, +a bounded `AVET` seek on the attribute/value slice, instead of materializing +`visible_datoms` and scanning a list. Transaction-local datom-list paths remain +for staged transaction state (`entity_id_in_datoms` keeps its list semantics), and +`find_avet_exact` applies `filter_pred` so filtered-db results match the old +`visible_datoms` path. ### DataScript: Public Options Are Narrower Than Upstream @@ -250,9 +248,9 @@ evidence shows parity or there is no upstream surface in the checked-out source: values by default to match upstream API shape more closely. 5. Add lazy seek/range APIs for `seek_datoms`, `rseek_datoms`, and `index_range`, then route compatibility layers through those APIs. -6. Route direct lookup-ref resolution through bounded `AVET` lookup instead of +6. ~~Route direct lookup-ref resolution through bounded `AVET` lookup instead of scanning `visible_datoms`, while preserving transaction-local datom-list - behavior. + behavior.~~ Done. 7. ~~Rework entity values into lazy views so constructing an entity does not materialize all forward and reverse attributes before `entity_attr` access.~~ Done: `entity` construction is O(1) (db + eid plus a bounded diff --git a/impl/datascript.ml b/impl/datascript.ml index beb8781..09a4de1 100644 --- a/impl/datascript.ml +++ b/impl/datascript.ml @@ -92,8 +92,6 @@ let empty db = Db_impl.empty db_core_context db let init_db ?(schema = []) ?storage datoms = Db_impl.init_db db_core_context ~schema ?storage datoms -let visible_datoms = Db_impl.visible_datoms - let is_filtered = Db_impl.is_filtered let unfiltered_db db = Db_impl.unfiltered db_core_context db @@ -205,7 +203,6 @@ module Transact_datoms_impl = Transact_datoms.Make (struct let normalize_value = normalize_value let validate_entity_id = validate_entity_id let max_allocatable_entity_id = max_allocatable_entity_id - let visible_datoms = visible_datoms end) let normalize_entity_attr_value = Transact_datoms_impl.normalize_entity_attr_value @@ -251,20 +248,6 @@ let cas_expected_value_string = function | None -> "nil" | Some value -> edn_string_of_value value -let lookup_refs_context : Lookup_refs.context = - { is_unique - ; entid_in_datoms - ; visible_datoms - ; value_to_string = edn_string_of_value - } - -let unresolved_lookup_ref_message attr value = - Lookup_refs.unresolved_message lookup_refs_context attr value - -let unresolved_entity_ref_message = function - | Lookup_ref (attr, value) -> unresolved_lookup_ref_message attr value - | _ -> "lookup ref did not resolve" - let find_avet_exact db attr value = let bound = datom ~e:0 ~a:attr ~v:value () in let compare_prefix left right = @@ -276,15 +259,15 @@ let find_avet_exact db attr value = else if left == bound then -compare_prefix right left else Util.compare_datom Avet left right in - match - PSet.slice ~from_:bound ~to_:bound ~cmp db.avet_index - @ List.filter - (fun datom -> datom.a = attr && value_equal datom.v value) - (Option.value (Hashtbl.find_opt db.duplicate_avet_by_attr attr) ~default:[]) - |> List.sort (Util.compare_datom Avet) - with - | datom :: _ -> Some datom - | [] -> None + PSet.slice ~from_:bound ~to_:bound ~cmp db.avet_index + @ List.filter + (fun datom -> datom.a = attr && value_equal datom.v value) + (Option.value (Hashtbl.find_opt db.duplicate_avet_by_attr attr) ~default:[]) + |> List.sort (Util.compare_datom Avet) + |> List.find_opt (fun datom -> + match db.filter_pred with + | Some pred -> pred datom + | None -> true) let find_eavt_exact db entity_id attr value = let bound = datom ~e:entity_id ~a:attr ~v:value () in @@ -357,6 +340,22 @@ and entid_db db attr value = else None +let entid = entid_db + +let lookup_refs_context : Lookup_refs.context = + { is_unique + ; entid = entid_db + ; entid_in_datoms + ; value_to_string = edn_string_of_value + } + +let unresolved_lookup_ref_message attr value = + Lookup_refs.unresolved_message lookup_refs_context attr value + +let unresolved_entity_ref_message = function + | Lookup_ref (attr, value) -> unresolved_lookup_ref_message attr value + | _ -> "lookup ref did not resolve" + let lookup_ref_entity_id_db ?(strict_missing = false) db attr value = if not (is_unique db attr) then invalid_arg (Lookup_refs.non_unique_message lookup_refs_context attr value); @@ -368,8 +367,6 @@ let lookup_ref_entity_id_db ?(strict_missing = false) db attr value = else None -let entid = entid_db - module Transact_impl = Transact let transact_resolve_context : Transact_impl.context = diff --git a/impl/datascript.mli b/impl/datascript.mli index 11f4243..ce80137 100644 --- a/impl/datascript.mli +++ b/impl/datascript.mli @@ -198,8 +198,8 @@ end module Lookup_refs : sig type context = { is_unique : db -> attr -> bool + ; entid : db -> attr -> value -> entity_id option ; entid_in_datoms : db -> datom list -> attr -> value -> entity_id option - ; visible_datoms : db -> datom list ; value_to_string : value -> string } diff --git a/impl/lookup_refs.ml b/impl/lookup_refs.ml index 29b6c04..125feb3 100644 --- a/impl/lookup_refs.ml +++ b/impl/lookup_refs.ml @@ -2,8 +2,8 @@ open Datascript_types type context = { is_unique : db -> attr -> bool + ; entid : db -> attr -> value -> entity_id option ; entid_in_datoms : db -> datom list -> attr -> value -> entity_id option - ; visible_datoms : db -> datom list ; value_to_string : value -> string } @@ -17,10 +17,10 @@ let non_unique_message context attr value = ^ context.value_to_string value ^ "]" -let entity_id_in_datoms ?(strict_missing = false) context db datoms attr value = +let entity_id_of_resolution ?(strict_missing = false) context db attr value resolved = if not (context.is_unique db attr) then invalid_arg (non_unique_message context attr value); - match context.entid_in_datoms db datoms attr value with + match resolved with | Some entity_id -> Some entity_id | None -> if strict_missing then @@ -28,5 +28,14 @@ let entity_id_in_datoms ?(strict_missing = false) context db datoms attr value = else None +let entity_id_in_datoms ?strict_missing context db datoms attr value = + entity_id_of_resolution + ?strict_missing + context + db + attr + value + (context.entid_in_datoms db datoms attr value) + let entity_id ?strict_missing context db attr value = - entity_id_in_datoms ?strict_missing context db (context.visible_datoms db) attr value + entity_id_of_resolution ?strict_missing context db attr value (context.entid db attr value) diff --git a/impl/lookup_refs.mli b/impl/lookup_refs.mli index fc709ea..9322dd2 100644 --- a/impl/lookup_refs.mli +++ b/impl/lookup_refs.mli @@ -2,8 +2,8 @@ open Datascript_types type context = { is_unique : db -> attr -> bool + ; entid : db -> attr -> value -> entity_id option ; entid_in_datoms : db -> datom list -> attr -> value -> entity_id option - ; visible_datoms : db -> datom list ; value_to_string : value -> string } diff --git a/impl/transact_datoms.ml b/impl/transact_datoms.ml index 919fffc..16292c7 100644 --- a/impl/transact_datoms.ml +++ b/impl/transact_datoms.ml @@ -17,7 +17,6 @@ module Make (Context : sig val normalize_value : value -> value val validate_entity_id : int -> entity_id val max_allocatable_entity_id : int - val visible_datoms : db -> datom list end) = struct open Context @@ -315,6 +314,4 @@ end) = struct else None - let entid db attr value = entid_in_datoms db (visible_datoms db) attr value - end