From de3249dcfcff5c628f77da3c3d1e65f5fe8b0d7f Mon Sep 17 00:00:00 2001 From: Marcos Magueta Date: Wed, 11 Mar 2026 01:59:19 -0300 Subject: todos --- lib/algebra.ml | 6 ++++++ lib/merkle.ml | 6 ++++++ 2 files changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/algebra.ml b/lib/algebra.ml index f03aa53..a1970bd 100644 --- a/lib/algebra.ml +++ b/lib/algebra.ml @@ -24,6 +24,12 @@ module Make(Storage : Management.Physical.S) = struct match rel.Relation.generator with | Some gen -> gen | None -> + (* TODO: Avoid materializing all tuple hashes/tuples at once. + Current implementation calls tuple_hashes rel which loads all hashes + into memory, then load_tuples which loads all tuples. This defeats + lazy evaluation and scales badly (fails for billions of tuples). + Replace with paginated tuple_hashes or Merkle streaming to load + hashes/tuples on demand in chunks. *) let hashes = Ops.tuple_hashes rel in (match Ops.load_tuples storage hashes with | Error _ -> (fun _pos -> Generator.Error ("Failed to load: " ^ rel.Relation.name)) diff --git a/lib/merkle.ml b/lib/merkle.ml index e3f68d9..ffa7171 100644 --- a/lib/merkle.ml +++ b/lib/merkle.ml @@ -54,6 +54,12 @@ module HashSet : S = struct let member hash tree = StringSet.mem hash tree let keys tree = StringSet.elements tree + (* TODO: Streaming/pagination for large tuple sets. Currently materializes + entire keyset into memory, which fails for relations with billions of tuples. + Replace with paginated access (keys_paginated offset limit) or lazy generator + to avoid loading all hashes upfront. Consider replacing HashSet with proper + radix-merkle tree (plebeia) for native lazy traversal. *) + let root_hash tree = if StringSet.is_empty tree then -- cgit v1.2.3