From e120443f18c781c75d8c93033225baf9464c9ebd Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:19:57 -0500 Subject: [PATCH 01/60] feat(storage): make the lmdb feature the backend's slot and retire the duplicate enum --- Cargo.toml | 1 + crates/nostream-storage/Cargo.toml | 16 ++- crates/nostream-storage/src/backend.rs | 136 +++++++------------- crates/nostream-storage/src/error.rs | 7 + crates/nostream-storage/src/lib.rs | 27 ++-- crates/nostream-storage/src/repositories.rs | 46 ++++--- 6 files changed, 113 insertions(+), 120 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index aeab00ba8..852232b2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ tracing = "0.1" tracing-opentelemetry = "0.34" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } url = "2" +uuid = { version = "1", features = ["v4"] } [workspace.lints.rust] # `deny` rather than `forbid`: heed's LMDB `EnvOpenOptions::open` is an `unsafe fn` diff --git a/crates/nostream-storage/Cargo.toml b/crates/nostream-storage/Cargo.toml index 41536ea2d..4217ba222 100644 --- a/crates/nostream-storage/Cargo.toml +++ b/crates/nostream-storage/Cargo.toml @@ -5,19 +5,22 @@ edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true -description = "Repository traits, backend selection and the Postgres storage backend" +description = "Repository traits, backend selection and the Postgres and LMDB storage backends" [features] -# The Postgres backend and its driver. On by default because Postgres is the reference -# implementation; a build that only needs the trait seam (the LMDB follow-up) can turn it -# off and lose the sqlx link entirely. -default = ["postgres"] +# Both backends are on by default: Postgres is the reference implementation and the LMDB +# backend is what the rewrite exists to provide. A build that only needs the trait seam +# turns both off and links no driver at all. +default = ["postgres", "lmdb"] postgres = ["dep:sqlx"] +# The LMDB backend, its environment and its row encodings. `uuid` is only used to mint +# the canonical text ids the outbox and delivery-log columns generate. +lmdb = ["dep:heed", "dep:uuid"] [dependencies] chrono.workspace = true futures-core.workspace = true -heed.workspace = true +heed = { workspace = true, optional = true } nostream-config.workspace = true nostream-core.workspace = true serde.workspace = true @@ -25,6 +28,7 @@ serde_json.workspace = true sqlx = { workspace = true, optional = true } thiserror.workspace = true tokio.workspace = true +uuid = { workspace = true, optional = true } [dev-dependencies] serde_yaml.workspace = true diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 81e2d2218..980ec522f 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -1,47 +1,33 @@ -//! Backend selection and the two backends. Mirrors `src/utils/event-store.ts` (the selection -//! half) and the storage boundary in `src/@types/adapters.ts`. +//! Backend selection and the LMDB environment. Mirrors `src/utils/event-store.ts` (the +//! selection half) and the storage boundary in `src/@types/adapters.ts`. use std::path::{Path, PathBuf}; -use heed::types::Str; +use nostream_core::EventStoreBackend; -use crate::error::StorageError; -use heed::{Database, Env, EnvOpenOptions}; - -/// The storage backends the rewrite can be configured with (`src/constants/base.ts` -/// `EventStoreBackend`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Backend { - Postgres, - Lmdb, -} +#[cfg(feature = "lmdb")] +use heed::{Env, EnvOpenOptions}; -impl Backend { - /// The settings spelling of this backend. - pub const fn as_str(self) -> &'static str { - match self { - Self::Postgres => "postgres", - Self::Lmdb => "lmdb", - } - } +#[cfg(feature = "lmdb")] +use crate::error::StorageError; +#[cfg(feature = "lmdb")] +use crate::lmdb::{LMDB_MAX_DBS, LMDB_MAX_READERS}; - /// Parses a settings value. - pub fn parse(value: &str) -> Option { - match value { - "postgres" => Some(Self::Postgres), - "lmdb" => Some(Self::Lmdb), - _ => None, - } - } -} +/// The directory one backend's data lives in, under the root a caller supplies. +/// `src/@types/adapters.ts` names the LMDB store as a directory of its own beside the +/// database it replaces. +pub const LMDB_DIR: &str = "lmdb"; -impl std::fmt::Display for Backend { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.as_str()) +/// Resolves the data directory under `root` for one backend. A backend that keeps its data +/// somewhere other than a directory of its own answers `None`. +pub fn data_dir(root: &Path, backend: EventStoreBackend) -> Option { + match backend { + EventStoreBackend::Lmdb => Some(root.join(LMDB_DIR)), + EventStoreBackend::Postgres => None, } } -/// An opened LMDB environment with one named database of string keys and values. +/// An opened LMDB environment, with the named databases the backend creates. /// /// The constructor is the crate's single `unsafe` boundary. heed 0.20.5 marks /// `EnvOpenOptions::open` unsafe because an LMDB environment memory-maps a file shared with @@ -51,34 +37,36 @@ impl std::fmt::Display for Backend { /// and `Error::BadOpenOptions` when they differ (heed-0.20.5 `src/env.rs`, `OPENED_ENV`). /// heed's documented preconditions, and where each stands today: /// -/// - **No long-lived transactions** — upheld by the code: `put`/`get` open a transaction, -/// use it, and commit/drop it before returning; nothing holds one across an await. +/// - **No long-lived transactions** — upheld by the code: every repository call opens a +/// transaction, uses it, and commits or drops it inside one `spawn_blocking` closure; +/// nothing holds one across an await. /// - **No aborting the process with an active transaction** — relied on the operator for -/// SIGKILL-class exits; the Phase-2 daemon drain path must ensure a clean shutdown +/// SIGKILL-class exits; the daemon drain path must ensure a clean shutdown /// (SIGINT/SIGTERM drain through `ShutdownState` does not abort mid-transaction). -/// - **No network filesystems** — relied on the operator today: `open` accepts any path and -/// only `create_dir_all` before mapping; the Phase-2 backend factory must refuse a remote -/// location before this slice is wired into settings. +/// - **No network filesystems** — relied on the operator: `open` accepts any path and only +/// `create_dir_all` before mapping; the daemon's backend factory must refuse a remote +/// location before this environment is wired into settings. /// - **`NO_LOCK` requires self-managed concurrency; a broken lock file means /// synchronization issues and possible UB** — split responsibility. The code guarantees /// its half: no env flags are passed (`EnvOpenOptions::new()` only), so `NO_LOCK` is /// never in play and this crate never reads, writes or removes the lock file. The -/// operator must supply the rest: the data directory's lock file must not be broken -/// (stale locks left by a crashed process on a filesystem without proper flock support, -/// manual deletion, two processes on different hosts sharing a volume — LMDB cannot -/// recover from any of those, per heed's contract). +/// operator must supply the rest: the data directory's lock file must not be broken. /// - **One `map_size` per path for the life of the process** — upheld by heed + propagated: /// heed's per-process `OPENED_ENV` registry returns the same `Env` for a second open with /// equal options and `Error::BadOpenOptions` when they differ, and `open` surfaces that -/// error to the caller rather than opening a second environment. +/// error to the caller rather than opening a second environment. `Env::resize` is unsafe +/// as well, so the map is sized once and never grown. +#[cfg(feature = "lmdb")] #[derive(Debug, Clone)] pub struct LmdbEnv { env: Env, path: PathBuf, } +#[cfg(feature = "lmdb")] impl LmdbEnv { - /// Opens (creating if needed) an LMDB environment at `path` with `map_size` bytes. + /// Opens (creating if needed) an LMDB environment at `path` with `map_size` bytes, room + /// for the backend's named databases and its reader slots. pub fn open(path: impl Into, map_size: usize) -> Result { let path = path.into(); std::fs::create_dir_all(&path).map_err(|source| StorageError::CreateDirectory { @@ -87,40 +75,26 @@ impl LmdbEnv { })?; // # Safety: the preconditions on `LmdbEnv` above, each marked upheld or relied-on: - // no env flags (LMDB's locking runs), transactions never escape the call, and - // heed's per-process dedup turns a second open into the same Env or - // BadOpenOptions. The network-filesystem precondition rests on the operator - // passing a local path; see the list above. + // no env flags (LMDB's locking runs), transactions never escape the closure they are + // opened in, and heed's per-process dedup turns a second open into the same Env or + // BadOpenOptions. The network-filesystem precondition rests on the operator passing a + // local path; see the list above. #[allow(unsafe_code)] - let env = - unsafe { EnvOpenOptions::new().map_size(map_size).open(&path) }.map_err(|source| { - StorageError::OpenLmdb { - path: path.clone(), - source, - } - })?; + let env = unsafe { + EnvOpenOptions::new() + .map_size(map_size) + .max_dbs(LMDB_MAX_DBS) + .max_readers(LMDB_MAX_READERS) + .open(&path) + } + .map_err(|source| StorageError::OpenLmdb { + path: path.clone(), + source, + })?; Ok(Self { env, path }) } - /// Writes one value under one key, committing immediately. - pub fn put(&self, key: &str, value: &str) -> Result<(), StorageError> { - let mut transaction = self.env.write_txn()?; - let database: Database = self.env.create_database(&mut transaction, None)?; - database.put(&mut transaction, key, value)?; - transaction.commit()?; - Ok(()) - } - - /// Reads one value; `Ok(None)` means the key is absent. - pub fn get(&self, key: &str) -> Result, StorageError> { - let transaction = self.env.read_txn()?; - let Some(database) = self.env.open_database::(&transaction, None)? else { - return Ok(None); - }; - Ok(database.get(&transaction, key)?.map(str::to_owned)) - } - - /// The environment handle, for the Phase-2 repository implementations. + /// The environment handle, for the repository implementations. pub fn env(&self) -> &Env { &self.env } @@ -130,13 +104,3 @@ impl LmdbEnv { &self.path } } - -/// Resolves the database path under `root` for one backend. -/// -/// Phase 1 stub — job 005; will pick the per-backend data directory; see `src/@types/adapters.ts`. -pub fn data_dir(root: &Path, backend: Backend) -> Result { - let _ = (root, backend); - Err(StorageError::NotImplemented { - module: "storage::backend::data_dir", - }) -} diff --git a/crates/nostream-storage/src/error.rs b/crates/nostream-storage/src/error.rs index c2688f326..e0f1edbc5 100644 --- a/crates/nostream-storage/src/error.rs +++ b/crates/nostream-storage/src/error.rs @@ -49,10 +49,12 @@ pub enum StorageError { ValueOutOfRange { column: &'static str, value: String }, /// An LMDB environment or transaction failed. + #[cfg(feature = "lmdb")] #[error("lmdb operation failed: {0}")] Lmdb(#[from] heed::Error), /// An LMDB environment could not be opened at the requested path. + #[cfg(feature = "lmdb")] #[error("failed to open the lmdb environment at {path}: {source}")] OpenLmdb { path: std::path::PathBuf, @@ -60,6 +62,11 @@ pub enum StorageError { source: heed::Error, }, + /// An LMDB blocking task was cancelled before it returned (the runtime is shutting down). + #[cfg(feature = "lmdb")] + #[error("the lmdb task did not finish")] + LmdbTaskCancelled, + /// A value must be valid UTF-8 to be stored through the string codecs. #[error("lmdb values must be valid utf-8")] NotUtf8, diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index 707227afb..a259e083c 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -1,11 +1,11 @@ //! `nostream-storage` owns the persistence seam: the nine repository traits, the streaming -//! query surface, the Postgres backend with its embedded migrations, the backend selection -//! and the LMDB environment. +//! query surface, the Postgres backend with its embedded migrations, the LMDB backend, the +//! backend selection and the LMDB environment. //! //! It owns these TypeScript paths from the capture commit //! (`242a3001d77149e4d70151ec03060c4ef44a1547`): //! -//! - `src/repositories/` (9 files, ported to `src/postgres/`) +//! - `src/repositories/` (9 of the 10 files, ported to `src/postgres/` and `src/lmdb/`) //! - `src/database/client.ts`, `src/database/transaction.ts` (`src/postgres/mod.rs`) //! - the schema: `migrations/` (37 knex migrations, ported to `migrations/*.sql`) //! - `src/utils/event-store.ts` (the mirror of `src/utils/hidden-content-cache.ts` is the @@ -16,21 +16,30 @@ //! - `src/factories/event-repository-factory.ts` //! //! What it deliberately does **not** own: the relay handlers that call these traits, the -//! payments service, and the LMDB backend. The cache is not a cache server: no Redis client -//! is added and `src/cache/client.ts` / `src/adapters/redis-adapter.ts` are not ported — the -//! decision and what would overturn it are ADR -//! `0010-services-worker-model-and-cache.md`. `LmdbEnv`'s environment slice and -//! `backend::data_dir` remain labelled stubs; the Postgres backend is complete. +//! payments service, the daemon's backend factory and the `nostream-cli migrate pg-to-lmdb` +//! tool. The cache is not a cache server: no Redis client is added and +//! `src/cache/client.ts` / `src/adapters/redis-adapter.ts` are not ported — the decision and +//! what would overturn it are ADR `0010-services-worker-model-and-cache.md`. Both backends are +//! complete: Postgres against the 37-migration schema, and LMDB against the same column +//! contract for 9 of the capture's 10 repositories — `user_subscriptions` has a TypeScript +//! repository and no trait yet, and `events.first_seen` is stored but never indexed +//! (`docs/rewrite/decisions/0010-lmdb-backend.md`). pub mod backend; pub mod error; pub mod event_store; +#[cfg(feature = "lmdb")] +pub mod lmdb; #[cfg(feature = "postgres")] pub mod postgres; pub mod records; pub mod repositories; -pub use backend::{Backend, LmdbEnv}; +#[cfg(feature = "lmdb")] +pub use backend::LmdbEnv; +pub use backend::data_dir; pub use error::StorageError; +#[cfg(feature = "lmdb")] +pub use lmdb::LmdbBackend; #[cfg(feature = "postgres")] pub use postgres::{PostgresBackend, PostgresTransaction}; diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index 01f738f01..2750e13e9 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -21,10 +21,11 @@ use crate::error::StorageError; pub type StorageFuture<'a, T> = Pin> + Send + 'a>>; /// One row per item, in exactly the query's `ORDER BY`. The borrow of the store is held for -/// the stream's life, which is what both backends need: Postgres holds a portal and a -/// pooled connection; LMDB holds a read transaction. A channel would force a background -/// task and a buffer policy LMDB does not need, and a `Vec` would materialise a 10M-row -/// NIP-77 reconciliation. +/// the stream's life, which is what both backends need: Postgres holds a portal and a pooled +/// connection, while LMDB holds only a cursor key and opens a short read transaction per batch +/// (256 values), so a dropped stream pins no pages but a row written or deleted between two +/// batches may appear or vanish. A channel would force a background task and a buffer policy +/// LMDB does not need, and a `Vec` would materialise a 10M-row NIP-77 reconciliation. pub type QueryStream<'a, T> = Pin> + Send + 'a>>; /// The rows a retention pass deleted, expired and kept (`src/@types/repositories.ts` @@ -79,7 +80,7 @@ pub struct ReconciliationItem { } /// One user row (`src/@types/user.ts` `User`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct User { pub pubkey: Pubkey, pub is_admitted: bool, @@ -110,7 +111,8 @@ pub struct UserPatch { } /// `invoices.unit` (`src/@types/invoice.ts` `InvoiceUnit`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum InvoiceUnit { Msats, Sats, @@ -118,7 +120,8 @@ pub enum InvoiceUnit { } /// `invoices.status` (`src/@types/invoice.ts` `InvoiceStatus`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum InvoiceStatus { Pending, Completed, @@ -126,7 +129,7 @@ pub enum InvoiceStatus { } /// One payment invoice row (`src/@types/invoice.ts` `Invoice`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct Invoice { pub id: String, pub pubkey: Pubkey, @@ -144,7 +147,7 @@ pub struct Invoice { } /// One NIP-05 verification row (`src/@types/nip05.ts` `Nip05Verification`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct Nip05Verification { pub pubkey: Pubkey, pub nip05: String, @@ -168,7 +171,7 @@ pub struct CreateInviteCodeOptions { } /// One invite code (`src/@types/invite-code.ts` `InviteCode`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct InviteCode { pub code: String, pub created_by: Option, @@ -180,7 +183,8 @@ pub struct InviteCode { } /// `dvm_jobs.status` (`src/@types/dvm.ts` `DvmJobStatus`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] pub enum DvmJobStatus { Submitted, PickedUp, @@ -190,7 +194,7 @@ pub enum DvmJobStatus { } /// One DVM job row (`src/@types/dvm.ts` `DvmJob`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DvmJob { pub id: EventId, pub requester_pubkey: Pubkey, @@ -233,7 +237,8 @@ pub struct DvmJobUpdate { } /// `reports.report_type` (`src/@types/report.ts` `ReportType`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum ReportType { Nudity, Malware, @@ -258,7 +263,7 @@ pub struct NewReport { } /// One report row (`src/@types/report.ts` `Report`). -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct Report { pub id: i32, pub event_id: EventId, @@ -280,7 +285,8 @@ pub struct ActionableTarget { /// `notification_delivery_log.target_type` (`src/@types/operator-notifications.ts` /// `OperatorNotificationChannelType`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum NotificationTargetType { Http, Discord, @@ -290,7 +296,8 @@ pub enum NotificationTargetType { /// `notification_delivery_log.status` (`src/@types/operator-notifications.ts` /// `NotificationDeliveryStatus`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum NotificationDeliveryStatus { Success, Failed, @@ -310,7 +317,7 @@ pub struct NewDeliveryLogEntry { /// One stored delivery attempt (`src/@types/operator-notifications.ts` /// `NotificationDeliveryLogEntry`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DeliveryLogEntry { pub id: String, pub outbox_id: Option, @@ -332,7 +339,8 @@ pub struct DeliveryLogFilters { /// `notification_outbox.status` (`src/@types/notification-outbox.ts` /// `NotificationOutboxStatus`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum OutboxStatus { Pending, Processing, @@ -344,7 +352,7 @@ pub enum OutboxStatus { pub type OutboxPayload = serde_json::Value; /// One outbox message (`src/@types/notification-outbox.ts`). -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct OutboxMessage { pub id: String, pub event_type: String, From 544630bcb244574f1cc81d40873a3c02026455a5 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:19:57 -0500 Subject: [PATCH 02/60] feat(storage): implement the nine repositories over lmdb --- crates/nostream-storage/src/lmdb/encoding.rs | 527 +++++++ crates/nostream-storage/src/lmdb/events.rs | 1362 +++++++++++++++++ crates/nostream-storage/src/lmdb/invoices.rs | 193 +++ crates/nostream-storage/src/lmdb/metadata.rs | 658 ++++++++ crates/nostream-storage/src/lmdb/mod.rs | 946 ++++++++++++ .../src/lmdb/notifications.rs | 488 ++++++ crates/nostream-storage/src/lmdb/rows.rs | 189 +++ crates/nostream-storage/src/lmdb/users.rs | 148 ++ 8 files changed, 4511 insertions(+) create mode 100644 crates/nostream-storage/src/lmdb/encoding.rs create mode 100644 crates/nostream-storage/src/lmdb/events.rs create mode 100644 crates/nostream-storage/src/lmdb/invoices.rs create mode 100644 crates/nostream-storage/src/lmdb/metadata.rs create mode 100644 crates/nostream-storage/src/lmdb/mod.rs create mode 100644 crates/nostream-storage/src/lmdb/notifications.rs create mode 100644 crates/nostream-storage/src/lmdb/rows.rs create mode 100644 crates/nostream-storage/src/lmdb/users.rs diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs new file mode 100644 index 000000000..478e56302 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -0,0 +1,527 @@ +//! Key and value codecs for the LMDB store: the format byte, the fixed event header, the +//! big-endian key layout, timestamp truncation and the hex/tag ranges the event queries seek +//! with. `docs/rewrite/decisions/0010-lmdb-backend.md` records why each shape was chosen. + +use std::ops::Bound; + +use chrono::{DateTime, Utc}; +use nostream_core::{EventId, Pubkey}; + +use crate::error::StorageError; + +/// The first byte of every value in every database (ADR 0003 revisit trigger 6): a row +/// written by a later field set cannot be decoded as this one. +pub(crate) const ROW_FORMAT_VERSION: u8 = 1; + +/// The fixed event header before the JSON body: flags, `created_at`, `kind`, `expires_at`, +/// `pubkey` and one reserved byte, so a live test reads at most 55 bytes of a value. +pub(crate) const EVENT_HEADER_LEN: usize = 54; + +/// The header's "this row is soft-deleted" bit. +const FLAG_DELETED: u8 = 1; + +/// The header's "no expiry" sentinel: `expires_at` is a fixed `i64` slot. +const NO_EXPIRY: i64 = i64::MIN; + +/// The number of bytes a `created_at ‖ event_id` key appends after the timestamp. +const EVENT_ID_LEN: usize = 32; + +/// An invalid stored value, naming the database it was read from. +pub(crate) fn invalid(table: &'static str, message: String) -> StorageError { + StorageError::InvalidRow { + table, + column: "value", + value: message, + } +} + +/// The instant a row carries, truncated to the microsecond precision every PostgreSQL +/// `timestamp` column has, so a value written by either backend reads back the same. +pub(crate) fn micros(value: DateTime) -> DateTime { + DateTime::::from_timestamp_micros(value.timestamp_micros()).unwrap_or(value) +} + +/// An `i64` ordered by its bytes: the sign bit is flipped so a negative value sorts before +/// zero rather than after every positive one. +pub(crate) fn i64_key(value: i64) -> [u8; 8] { + ((value as u64) ^ (1 << 63)).to_be_bytes() +} + +/// The inverse of [`i64_key`]. +pub(crate) fn i64_from_key(bytes: &[u8]) -> Result { + let raw = u64::from_be_bytes(fixed::<8>("events", bytes)?); + Ok((raw ^ (1 << 63)) as i64) +} + +/// An `i32` ordered by its bytes, sign-flipped for the same reason as [`i64_key`]. +pub(crate) fn i32_key(value: i32) -> [u8; 4] { + ((value as u32) ^ (1 << 31)).to_be_bytes() +} + +/// The inverse of [`i32_key`]. +pub(crate) fn i32_from_key(bytes: &[u8]) -> Result { + let raw = u32::from_be_bytes(fixed::<4>("events", bytes)?); + Ok((raw ^ (1 << 31)) as i32) +} + +fn fixed(table: &'static str, bytes: &[u8]) -> Result<[u8; N], StorageError> { + bytes + .try_into() + .map_err(|_| invalid(table, format!("{N} bytes expected, {} found", bytes.len()))) +} + +/// One event value's fixed header, decoded without touching the JSON body. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct EventHeader { + pub deleted: bool, + pub created_at: i64, + pub kind: i32, + pub expires_at: Option, + pub pubkey: Pubkey, +} + +/// Encodes the fixed header of an event value. +pub(crate) fn header_bytes(header: &EventHeader) -> [u8; EVENT_HEADER_LEN] { + let mut bytes = [0u8; EVENT_HEADER_LEN]; + if header.deleted { + bytes[0] = FLAG_DELETED; + } + bytes[1..9].copy_from_slice(&header.created_at.to_be_bytes()); + bytes[9..13].copy_from_slice(&header.kind.to_be_bytes()); + bytes[13..21].copy_from_slice(&header.expires_at.unwrap_or(NO_EXPIRY).to_be_bytes()); + bytes[21..53].copy_from_slice(header.pubkey.as_bytes()); + // bytes[53] is reserved and written as zero. + bytes +} + +/// Checks the format byte and returns the value's body (the version byte and header gone). +pub(crate) fn body<'a>(table: &'static str, value: &'a [u8]) -> Result<&'a [u8], StorageError> { + check_version(table, value)?; + let start = 1 + EVENT_HEADER_LEN; + value + .get(start..) + .filter(|body| !body.is_empty()) + .ok_or_else(|| { + invalid( + table, + format!("the header is truncated at {} bytes", value.len()), + ) + }) +} + +/// The 32-byte event id an index key or a database value carries. +pub(crate) fn event_id_from(bytes: &[u8]) -> Result { + Ok(EventId::from_byte_array(fixed::<32>("events", bytes)?)) +} + +/// The 32-byte pubkey an index key carries. +pub(crate) fn pubkey_from(bytes: &[u8]) -> Result { + Ok(Pubkey::from_byte_array(fixed::<32>("events", bytes)?)) +} + +/// One key range as the borrowed pair heed's `range` accepts. An empty bound is +/// [`Bound::Unbounded`]: LMDB answers `MDB_BAD_VALSIZE` for a zero-length seek key, and the +/// created-at path uses the empty bound to mean "from the first key". +pub(crate) fn bounds<'a>(lower: &'a [u8], upper: &'a [u8]) -> (Bound<&'a [u8]>, Bound<&'a [u8]>) { + let start = if lower.is_empty() { + Bound::Unbounded + } else { + Bound::Included(lower) + }; + let end = if upper.is_empty() { + Bound::Unbounded + } else { + Bound::Included(upper) + }; + (start, end) +} + +/// One key range that resumes after `after`, exclusive, when a previous batch stopped there. +pub(crate) fn resume_bounds<'a>( + after: Option<&'a [u8]>, + lower: &'a [u8], + upper: &'a [u8], +) -> (Bound<&'a [u8]>, Bound<&'a [u8]>) { + let (start, end) = bounds(lower, upper); + let start = match after { + Some(after) => Bound::Excluded(after), + None => start, + }; + (start, end) +} + +/// Decodes an event value's fixed header, without its JSON body. +pub(crate) fn event_header(table: &'static str, value: &[u8]) -> Result { + check_version(table, value)?; + let end = 1 + EVENT_HEADER_LEN; + let header = value.get(1..end).ok_or_else(|| { + invalid( + table, + format!("the header is truncated at {} bytes", value.len()), + ) + })?; + let expires_at = i64::from_be_bytes(fixed::<8>(table, &header[13..21])?); + Ok(EventHeader { + deleted: header[0] & FLAG_DELETED != 0, + created_at: i64::from_be_bytes(fixed::<8>(table, &header[1..9])?), + kind: i32::from_be_bytes(fixed::<4>(table, &header[9..13])?), + expires_at: (expires_at != NO_EXPIRY).then_some(expires_at), + pubkey: Pubkey::from_byte_array(fixed::<32>(table, &header[21..53])?), + }) +} + +/// Rejects a value written by a different row format, and one too short to carry a version. +pub(crate) fn check_version(table: &'static str, value: &[u8]) -> Result<(), StorageError> { + match value.split_first() { + Some((version, _)) if *version == ROW_FORMAT_VERSION => Ok(()), + Some((version, _)) => Err(invalid( + table, + format!("unsupported format version {version}"), + )), + None => Err(invalid(table, "the value is empty".to_owned())), + } +} + +/// `created_at ‖ event_id`. +pub(crate) fn created_at_key(created_at: i64, event_id: &EventId) -> Vec { + let mut key = Vec::with_capacity(8 + EVENT_ID_LEN); + key.extend_from_slice(&i64_key(created_at)); + key.extend_from_slice(event_id.as_bytes()); + key +} + +/// `expires_at ‖ event_id` and `deleted_at ‖ event_id`, which have the same shape as a +/// `created_at` key. +pub(crate) fn time_key(value: i64, event_id: &EventId) -> Vec { + created_at_key(value, event_id) +} + +/// `pubkey ‖ kind ‖ created_at ‖ event_id`. +pub(crate) fn author_key( + pubkey: &Pubkey, + kind: i32, + created_at: i64, + event_id: &EventId, +) -> Vec { + let mut key = Vec::with_capacity(32 + 4 + 8 + EVENT_ID_LEN); + key.extend_from_slice(pubkey.as_bytes()); + key.extend_from_slice(&i32_key(kind)); + key.extend_from_slice(&i64_key(created_at)); + key.extend_from_slice(event_id.as_bytes()); + key +} + +/// `pubkey ‖ kind ‖ dedup_len ‖ canonical JSON`, where `dedup_len` is the JSON's byte length. +pub(crate) fn replaceable_key( + pubkey: &Pubkey, + kind: i32, + deduplication: &[serde_json::Value], +) -> Result, StorageError> { + let json = serde_json::to_vec(deduplication).map_err(|error| StorageError::InvalidRow { + table: "events", + column: "event_deduplication", + value: error.to_string(), + })?; + let length = u16::try_from(json.len()).map_err(|_| StorageError::InvalidRow { + table: "events", + column: "event_deduplication", + value: "the deduplication value is longer than 65535 bytes".to_owned(), + })?; + let mut key = Vec::with_capacity(32 + 4 + 2 + json.len()); + key.extend_from_slice(pubkey.as_bytes()); + key.extend_from_slice(&i32_key(kind)); + key.extend_from_slice(&length.to_be_bytes()); + key.extend_from_slice(&json); + Ok(key) +} + +/// `tag_name ‖ 0x00 ‖ tag_value ‖ 0x00 ‖ created_at ‖ event_id`. +/// +/// A name or value containing `0x00` is unrepresentable in the capture's `text` columns and +/// would make the separator ambiguous, so it is rejected rather than encoded. +pub(crate) fn tag_key( + name: &str, + value: &str, + created_at: i64, + event_id: &EventId, +) -> Result, StorageError> { + if name.contains('\0') { + return Err(StorageError::InvalidRow { + table: "event_tags", + column: "tag_name", + value: name.to_owned(), + }); + } + if value.contains('\0') { + return Err(StorageError::InvalidRow { + table: "event_tags", + column: "tag_value", + value: value.to_owned(), + }); + } + let mut key = Vec::with_capacity(name.len() + value.len() + 2 + 8 + EVENT_ID_LEN); + key.extend_from_slice(name.as_bytes()); + key.push(0); + key.extend_from_slice(value.as_bytes()); + key.push(0); + key.extend_from_slice(&i64_key(created_at)); + key.extend_from_slice(event_id.as_bytes()); + Ok(key) +} + +/// The tag rows the capture's trigger writes: one per element whose name is exactly one +/// character and whose value is present and non-empty, and never beyond the second element. +pub(crate) fn tag_entries(record: &crate::repositories::EventRecord) -> Vec<(&str, &str)> { + let mut entries = Vec::new(); + for tag in &record.event.tags { + let (Some(name), Some(value)) = (tag.first(), tag.get(1)) else { + continue; + }; + if name.chars().count() == 1 && !value.is_empty() { + entries.push((name.as_str(), value.as_str())); + } + } + entries +} + +/// Whether a kind is replaceable, exactly as the capture's conflict-target predicate says: +/// 0, 3, 41 and the two parameterized ranges. +pub(crate) fn replaceable_kind(kind: i32) -> bool { + matches!(kind, 0 | 3 | 41) + || (10_000..20_000).contains(&kind) + || (30_000..40_000).contains(&kind) +} + +/// Whether the lowercase hex rendering of `bytes` starts with `prefix`, case-sensitively, +/// exactly as `String.prototype.startsWith` compares the capture's ids and pubkeys. +pub(crate) fn hex_matches(bytes: &[u8], prefix: &str) -> bool { + if prefix.len() > bytes.len() * 2 { + return false; + } + prefix.bytes().enumerate().all(|(index, character)| { + let byte = bytes[index / 2]; + let nibble = if index % 2 == 0 { + byte >> 4 + } else { + byte & 0x0f + }; + let expected = match character { + b'0'..=b'9' => character - b'0', + b'a'..=b'f' => character - b'a' + 10, + _ => return false, + }; + nibble == expected + }) +} + +/// Hex to bytes, mapping an invalid digit to zero exactly as the SQL composer does. +pub(crate) fn hex_bytes(value: &str) -> Vec { + let mut bytes = Vec::with_capacity(value.len() / 2); + let digits = value.as_bytes(); + let mut index = 0; + while index + 1 < digits.len() { + bytes.push((hex_digit(digits[index]) << 4) | hex_digit(digits[index + 1])); + index += 2; + } + bytes +} + +fn hex_digit(digit: u8) -> u8 { + match digit { + b'0'..=b'9' => digit - b'0', + b'a'..=b'f' => digit - b'a' + 10, + b'A'..=b'F' => digit - b'A' + 10, + _ => 0, + } +} + +/// The inclusive key ranges one hex criterion family covers, over a 32-byte id or pubkey. +/// +/// A 64-character criterion is one equality range; an even prefix compares its bytes; an odd +/// one compares the bytes of the nibble range `0` to `f`, as `sql.rs` composes it. `suffix` is +/// the number of bytes the index key carries after the 32-byte field. +pub(crate) fn hex_ranges(criteria: &[String], suffix: usize) -> Vec<(Vec, Vec)> { + let mut ranges = Vec::with_capacity(criteria.len()); + for criterion in criteria { + let (lower, mut upper) = match criterion.len() % 2 { + 1 => ( + hex_bytes(&format!("{criterion}0")), + hex_bytes(&format!("{criterion}f")), + ), + _ => { + let prefix = hex_bytes(criterion); + (prefix.clone(), prefix) + } + }; + upper.extend(std::iter::repeat_n(0xff, suffix)); + ranges.push((lower, upper)); + } + ranges +} + +/// The inclusive key ranges one tag criterion covers: an exact value is the range from the +/// value's own separator, and a `#g` prefix is the range over the value's first bytes. +pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(Vec, Vec)> { + let is_geohash = name == "g"; + let mut ranges = Vec::with_capacity(criteria.len()); + for criterion in criteria { + let mut head = Vec::with_capacity(name.len() + 1 + criterion.len()); + head.extend_from_slice(name.as_bytes()); + head.push(0); + match criterion.strip_suffix('*').filter(|_| is_geohash) { + Some(prefix) => { + head.extend_from_slice(prefix.as_bytes()); + let mut upper = head.clone(); + upper.extend(std::iter::repeat_n(0xff, 64)); + ranges.push((head, upper)); + } + None => { + head.extend_from_slice(criterion.as_bytes()); + head.push(0); + let mut upper = head.clone(); + upper.extend(std::iter::repeat_n(0xff, 8 + EVENT_ID_LEN)); + ranges.push((head, upper)); + } + } + } + ranges +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + fn id(byte: u8) -> EventId { + EventId::from_byte_array([byte; 32]) + } + + #[test] + fn a_timestamp_key_sorts_numerically() { + let low = created_at_key(-5, &id(1)); + let zero = created_at_key(0, &id(1)); + let high = created_at_key(1_700_000_000, &id(1)); + assert!(low < zero); + assert!(zero < high); + assert_eq!(i64_from_key(&high[..8]).unwrap(), 1_700_000_000); + assert_eq!(i64_from_key(&low[..8]).unwrap(), -5); + } + + #[test] + fn a_kind_key_sorts_numerically() { + assert!(i32_key(-1) < i32_key(0)); + assert!(i32_key(0) < i32_key(1)); + assert_eq!(i32_from_key(&i32_key(30_023)).unwrap(), 30_023); + } + + #[test] + fn an_even_prefix_is_one_range_and_an_odd_one_is_one_nibble_range() { + let ranges = hex_ranges(&["abcd".to_owned(), "abcde".to_owned()], 40); + assert_eq!(ranges.len(), 2); + assert_eq!(ranges[0].0, vec![0xab, 0xcd]); + assert_eq!(ranges[0].1.len(), 2 + 40); + assert_eq!(ranges[1].0, vec![0xab, 0xcd, 0xe0]); + assert_eq!(ranges[1].1, { + let mut high = vec![0xab, 0xcd, 0xef]; + high.extend(std::iter::repeat_n(0xff, 40)); + high + }); + } + + #[test] + fn a_geohash_criterion_is_a_prefix_range_and_another_tag_is_exact() { + let geohash = tag_ranges("g", &["u4pru*".to_owned()]); + assert_eq!(geohash[0].0, b"g\0u4pru".to_vec()); + let exact = tag_ranges("e", &["u4pru*".to_owned()]); + assert_eq!(exact[0].0, b"e\0u4pru*\0".to_vec()); + } + + #[test] + fn the_header_round_trips_without_its_body() { + let header = EventHeader { + deleted: true, + created_at: 1_700_000_000, + kind: 30_023, + expires_at: Some(2_000_000_000), + pubkey: Pubkey::from_byte_array([7; 32]), + }; + let mut value = vec![ROW_FORMAT_VERSION]; + value.extend_from_slice(&header_bytes(&header)); + assert_eq!(event_header("events", &value).unwrap(), header); + } + + #[test] + fn an_unsupported_format_version_names_the_version() { + let error = check_version("events", &[2, 0]).expect_err("version 2"); + match error { + StorageError::InvalidRow { value, .. } => { + assert_eq!(value, "unsupported format version 2"); + } + other => panic!("expected InvalidRow, got {other:?}"), + } + } + + #[test] + fn a_truncated_header_is_an_invalid_row() { + let error = event_header("events", &[ROW_FORMAT_VERSION, 0]).expect_err("truncated"); + assert!(matches!(error, StorageError::InvalidRow { .. })); + } + + #[test] + fn a_tag_name_or_value_carrying_a_separator_is_rejected() { + let error = tag_key("e", "a\0b", 0, &id(1)).expect_err("a separator in the value"); + assert!(matches!( + error, + StorageError::InvalidRow { + column: "tag_value", + .. + } + )); + let error = tag_key("e\0", "ab", 0, &id(1)).expect_err("a separator in the name"); + assert!(matches!( + error, + StorageError::InvalidRow { + column: "tag_name", + .. + } + )); + } + + #[test] + fn only_the_trigger_s_tag_shape_is_indexed() { + use nostream_core::Event; + + let event: Event = serde_json::from_value(serde_json::json!({ + "id": "0d684e8ec2431de586aa3cafbee2f6d308d19b28805e53deabcac3220e9136a5", + "pubkey": "f43c1f9bff677b8f27b602725ea0ad51af221344f69a6b352a74991a4479bac3", + "created_at": 1660407625i64, + "kind": 1, + "tags": [["e", "x", "y"], ["", "x"], ["e", ""], ["ab", "c"], ["p", "q"]], + "content": "", + "sig": "ab".repeat(64) + })) + .expect("a wire event"); + let record = crate::repositories::EventRecord::from_event(&event); + assert_eq!(tag_entries(&record), vec![("e", "x"), ("p", "q")]); + } + + #[test] + fn the_replaceable_kind_set_matches_the_conflict_target() { + for kind in [0, 3, 41, 10_000, 19_999, 30_000, 39_999] { + assert!(replaceable_kind(kind), "{kind} is replaceable"); + } + for kind in [1, 2, 4, 9_999, 20_000, 29_999, 40_000] { + assert!(!replaceable_kind(kind), "{kind} is not replaceable"); + } + } + + #[test] + fn hex_prefixes_compare_nibbles_case_sensitively() { + assert!(hex_matches(&[0xab, 0xcd], "abcd")); + assert!(hex_matches(&[0xab, 0xcd], "abc")); + assert!(!hex_matches(&[0xab, 0xcd], "ABC")); + assert!(!hex_matches(&[0xab, 0xcd], "abce")); + assert!(!hex_matches(&[0xab, 0xcd], "abcdef")); + } +} diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs new file mode 100644 index 000000000..6326331c4 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -0,0 +1,1362 @@ +//! `EventRepository` against LMDB: the seven event databases, the write paths the capture's +//! `INSERT`/`ON CONFLICT` statements define, and the filter evaluation `sql.rs` composes. +//! +//! A read never holds a transaction across an await: a query materialises its page of +//! `(created_at, event_id)` keys inside one blocking closure, and the stream that returns the +//! rows re-seeks its cursor per batch (see [`ValuesStream`] and [`ReconciliationStream`]). + +use std::collections::{BTreeMap, HashSet}; +use std::future::Future; +use std::ops::Bound; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use chrono::{DateTime, Utc}; +use futures_core::Stream; +use heed::types::Bytes; +use heed::{Database, Env, RoTxn, RwTxn}; +use nostream_core::{EventId, Filter, Kind, KindSelector, Pubkey}; +use tokio::task::JoinHandle; + +use super::encoding::{self, EventHeader}; +use super::rows; +use super::{DEFAULT_FILTER_LIMIT, Databases, Gates, RETENTION_BATCH_SIZE, STREAM_BATCH_ITEMS}; +use crate::error::StorageError; +use crate::repositories::{ + EventPurgeCounts, EventRecord, EventRetentionOptions, ReconciliationItem, +}; + +/// The kind of the capture's `REQUEST_TO_VANISH`, which every retention whitelist keeps and +/// every soft delete of one pubkey's ids spares. +const REQUEST_TO_VANISH: i32 = 62; + +// --------------------------------------------------------------------------------------- +// Write paths. +// --------------------------------------------------------------------------------------- + +/// `create`/`create_many`: an id that is already stored writes nothing and counts nothing. +pub(crate) fn create( + txn: &mut RwTxn<'_>, + databases: &Databases, + records: &[EventRecord], + now: DateTime, +) -> Result { + let mut affected = 0; + for record in records { + if databases + .events + .get(txn, record.event.id.as_bytes())? + .is_some() + { + continue; + } + let stored = EventRecord { + row_id: None, + event: record.event.clone(), + first_seen: Some(now), + deleted_at: None, + expires_at: record.expires_at, + // `INSERT_COLUMNS` omits the deduplication column, so the capture stores NULL. + event_deduplication: None, + remote_address: record.remote_address, + }; + write_event(txn, databases, &stored)?; + affected += 1; + } + Ok(affected) +} + +/// `upsert`: a replaceable event merges through its pointer, and a stored id is a duplicate. +pub(crate) fn upsert( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, + now: DateTime, +) -> Result { + Ok(u64::from(upsert_one(txn, databases, record, now)?)) +} + +/// `upsert_many`: the same merge, row by row, in one write transaction. +pub(crate) fn upsert_many( + txn: &mut RwTxn<'_>, + databases: &Databases, + records: &[EventRecord], + now: DateTime, +) -> Result { + let mut affected = 0; + for record in records { + if upsert_one(txn, databases, record, now)? { + affected += 1; + } + } + Ok(affected) +} + +fn upsert_one( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, + now: DateTime, +) -> Result { + if databases + .events + .get(txn, record.event.id.as_bytes())? + .is_some() + { + return Err(StorageError::DuplicateEvent); + } + + let kind = kind_of(record)?; + let deduplication = record.deduplication(); + let mut first_seen = None; + + if encoding::replaceable_kind(kind) { + let pointer = encoding::replaceable_key(&record.event.pubkey, kind, &deduplication)?; + if let Some(stored) = databases.replaceable.get(txn, &pointer)? { + let stored_id = encoding::event_id_from(stored)?; + let Some(value) = databases.events.get(txn, stored_id.as_bytes())? else { + // A pointer with no row: treat the incoming event as a plain insert. + return write_upserted(txn, databases, record, now, kind, &deduplication, None); + }; + let stored_header = rows::event_header(value)?; + let incoming_wins = stored_header.created_at < record.event.created_at + || (stored_header.created_at == record.event.created_at + && stored_id > record.event.id); + if !incoming_wins { + return Ok(false); + } + let stored_record = rows::decode_event(value)?; + first_seen = stored_record.first_seen; + hard_delete(txn, databases, &stored_record)?; + databases.replaceable.delete(txn, &pointer)?; + } + } + + write_upserted( + txn, + databases, + record, + now, + kind, + &deduplication, + first_seen, + ) +} + +fn write_upserted( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, + now: DateTime, + kind: i32, + deduplication: &[serde_json::Value], + first_seen: Option>, +) -> Result { + let stored = EventRecord { + row_id: None, + event: record.event.clone(), + // `DO UPDATE` never writes `first_seen`, so a merged row keeps the one it had. + first_seen: Some(first_seen.unwrap_or(now)), + // The capture's merge list clears `deleted_at`. + deleted_at: None, + expires_at: record.expires_at, + event_deduplication: Some(deduplication.to_vec()), + remote_address: record.remote_address, + }; + write_event(txn, databases, &stored)?; + if encoding::replaceable_kind(kind) { + databases.replaceable.put( + txn, + &encoding::replaceable_key(&record.event.pubkey, kind, deduplication)?, + record.event.id.as_bytes(), + )?; + } + Ok(true) +} + +/// Writes an event and every index entry it needs. +fn write_event( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, +) -> Result<(), StorageError> { + let kind = kind_of(record)?; + let id = record.event.id; + let created_at = record.event.created_at; + + put_value(txn, databases, record)?; + databases + .events_by_created_at + .put(txn, &encoding::created_at_key(created_at, &id), &[])?; + databases.events_by_author.put( + txn, + &encoding::author_key(&record.event.pubkey, kind, created_at, &id), + &[], + )?; + for (name, value) in encoding::tag_entries(record) { + databases + .tags + .put(txn, &encoding::tag_key(name, value, created_at, &id)?, &[])?; + } + if let Some(expires_at) = record.expires_at { + databases + .events_expiring + .put(txn, &encoding::time_key(expires_at, &id), &[])?; + } + Ok(()) +} + +/// Writes only the `events` value, which is what a soft delete changes. +fn put_value( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, +) -> Result<(), StorageError> { + let value = rows::encode_event(record)?; + databases + .events + .put(txn, record.event.id.as_bytes(), &value)?; + Ok(()) +} + +/// Removes an event from every database, which is what a replacement and a retention pass do. +fn hard_delete( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, +) -> Result<(), StorageError> { + let kind = kind_of(record)?; + let id = record.event.id; + let created_at = record.event.created_at; + + databases.events.delete(txn, id.as_bytes())?; + databases + .events_by_created_at + .delete(txn, &encoding::created_at_key(created_at, &id))?; + databases.events_by_author.delete( + txn, + &encoding::author_key(&record.event.pubkey, kind, created_at, &id), + )?; + if let Some(deleted_at) = record.deleted_at { + databases + .events_deleted + .delete(txn, &encoding::time_key(deleted_at.timestamp(), &id))?; + } + if let Some(expires_at) = record.expires_at { + databases + .events_expiring + .delete(txn, &encoding::time_key(expires_at, &id))?; + } + for (name, value) in encoding::tag_entries(record) { + databases + .tags + .delete(txn, &encoding::tag_key(name, value, created_at, &id)?)?; + } + if let Some(deduplication) = &record.event_deduplication + && encoding::replaceable_kind(kind) + { + let pointer = encoding::replaceable_key(&record.event.pubkey, kind, deduplication)?; + if let Some(stored) = databases.replaceable.get(txn, &pointer)? + && stored == &id.as_bytes()[..] + { + databases.replaceable.delete(txn, &pointer)?; + } + } + Ok(()) +} + +/// Marks one event soft-deleted: the header bit, the JSON field and the deleted index. Every +/// read index is left alone, which is what keeps the row reachable by `find_by_filters`. +fn soft_delete( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, + now: DateTime, +) -> Result<(), StorageError> { + let mut record = record.clone(); + record.deleted_at = Some(now); + put_value(txn, databases, &record)?; + databases.events_deleted.put( + txn, + &encoding::time_key(now.timestamp(), &record.event.id), + &[], + )?; + Ok(()) +} + +/// `delete_by_pubkey_and_ids`: a soft delete that never touches kind 62 or a deleted row. +pub(crate) fn delete_by_pubkey_and_ids( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + ids: &[EventId], + now: DateTime, +) -> Result { + let mut affected = 0; + for id in ids { + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + continue; + }; + let record = rows::decode_event(value)?; + if record.event.pubkey != *pubkey + || kind_of(&record)? == REQUEST_TO_VANISH + || record.deleted_at.is_some() + { + continue; + } + soft_delete(txn, databases, &record, now)?; + affected += 1; + } + Ok(affected) +} + +/// `delete_by_pubkey_except_kinds`: every other live row of one author, soft-deleted. +pub(crate) fn delete_by_pubkey_except_kinds( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + excluded_kinds: &[Kind], + now: DateTime, +) -> Result { + let excluded: Vec = excluded_kinds + .iter() + .map(|kind| i32::try_from(kind.0).unwrap_or(i32::MAX)) + .collect(); + let mut keys = Vec::new(); + for entry in databases + .events_by_author + .prefix_iter(txn, pubkey.as_bytes())? + { + let (key, _) = entry?; + keys.push(key.to_vec()); + } + + let mut affected = 0; + for key in keys { + let header = author_key_header(&key)?; + if excluded.contains(&header.kind) { + continue; + } + let id = header.event_id; + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + continue; + }; + let record = rows::decode_event(value)?; + if record.deleted_at.is_some() { + continue; + } + soft_delete(txn, databases, &record, now)?; + affected += 1; + } + Ok(affected) +} + +/// `has_active_request_to_vanish`: a live kind-62 row exists for the pubkey. +pub(crate) fn has_active_request_to_vanish( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result { + let mut prefix = pubkey.as_bytes().to_vec(); + prefix.extend_from_slice(&encoding::i32_key(REQUEST_TO_VANISH)); + for entry in databases.events_by_author.prefix_iter(txn, &prefix)? { + let (key, _) = entry?; + let id = header_id(key, 44, "events_by_author")?; + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + continue; + }; + if !rows::event_header(value)?.deleted { + return Ok(true); + } + } + Ok(false) +} + +/// `delete_expired_and_retained`: one hard-delete batch of at most +/// [`RETENTION_BATCH_SIZE`] rows, gathered in the capture's index order and classified +/// deleted, then expired, then over-retention. +pub(crate) fn delete_expired_and_retained( + txn: &mut RwTxn<'_>, + databases: &Databases, + options: &EventRetentionOptions, + now: i64, +) -> Result { + let Some(max_days) = options.max_days.filter(|days| *days > 0) else { + return Ok(EventPurgeCounts::default()); + }; + let retention_limit = now.saturating_sub(max_days.saturating_mul(86_400)); + let kinds = retention_kinds(options.kind_whitelist.as_deref()); + let pubkeys: Vec<[u8; 32]> = options + .pubkey_whitelist + .iter() + .flatten() + .map(|pubkey| encoding::hex_bytes(&pubkey.0)) + .filter_map(|bytes| bytes.try_into().ok()) + .collect(); + + let mut candidates = Vec::new(); + let mut seen = HashSet::new(); + for entry in databases.events_deleted.iter(txn)? { + let (key, _) = entry?; + let id = header_id(key, 8, "events_deleted")?; + gather( + txn, + databases, + id, + now, + retention_limit, + &kinds, + &pubkeys, + &mut candidates, + &mut seen, + )?; + if candidates.len() >= RETENTION_BATCH_SIZE { + break; + } + } + if candidates.len() < RETENTION_BATCH_SIZE { + for entry in databases.events_expiring.iter(txn)? { + let (key, _) = entry?; + let id = header_id(key, 8, "events_expiring")?; + gather( + txn, + databases, + id, + now, + retention_limit, + &kinds, + &pubkeys, + &mut candidates, + &mut seen, + )?; + if candidates.len() >= RETENTION_BATCH_SIZE { + break; + } + } + } + if candidates.len() < RETENTION_BATCH_SIZE { + let lower = encoding::i64_key(i64::MIN).to_vec(); + let mut upper = encoding::i64_key(retention_limit.saturating_sub(1)).to_vec(); + upper.extend([0xff; 32]); + let range = encoding::bounds(&lower, &upper); + for entry in databases.events_by_created_at.range(txn, &range)? { + let (key, _) = entry?; + let id = header_id(key, 8, "events_by_created_at")?; + if candidates.len() >= RETENTION_BATCH_SIZE { + break; + } + gather( + txn, + databases, + id, + now, + retention_limit, + &kinds, + &pubkeys, + &mut candidates, + &mut seen, + )?; + } + } + + let mut counts = EventPurgeCounts::default(); + for id in candidates { + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + continue; + }; + let header = rows::event_header(value)?; + if header.deleted { + counts.deleted += 1; + } else if header.expires_at.is_some_and(|expires_at| expires_at < now) { + counts.expired += 1; + } else if header.created_at < retention_limit { + counts.retained += 1; + } else { + continue; + } + let record = rows::decode_event(value)?; + hard_delete(txn, databases, &record)?; + } + Ok(counts) +} + +/// Adds one retention candidate when it qualifies and is not protected, deduplicating. +#[allow(clippy::too_many_arguments)] +fn gather( + txn: &RoTxn<'_>, + databases: &Databases, + id: EventId, + now: i64, + retention_limit: i64, + kinds: &[KindSelector], + pubkeys: &[[u8; 32]], + candidates: &mut Vec, + seen: &mut HashSet<[u8; 32]>, +) -> Result<(), StorageError> { + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + return Ok(()); + }; + let header = rows::event_header(value)?; + if whitelisted(header.kind, kinds) || pubkeys.contains(header.pubkey.as_bytes()) { + return Ok(()); + } + let qualifies = header.deleted + || header.expires_at.is_some_and(|expires_at| expires_at < now) + || header.created_at < retention_limit; + if qualifies && seen.insert(*id.as_bytes()) { + candidates.push(id); + } + Ok(()) +} + +/// The capture's deduplicated whitelist, which always keeps kind 62. +fn retention_kinds(configured: Option<&[KindSelector]>) -> Vec { + let mut whitelist: Vec = Vec::new(); + for selector in configured.unwrap_or_default() { + if !whitelist.contains(selector) { + whitelist.push(*selector); + } + } + let vanish = KindSelector::Single(Kind(REQUEST_TO_VANISH as u64)); + if !whitelist.contains(&vanish) { + whitelist.push(vanish); + } + whitelist +} + +fn whitelisted(kind: i32, selectors: &[KindSelector]) -> bool { + selectors.iter().any(|selector| match selector { + KindSelector::Single(selector) => i32::try_from(selector.0).unwrap_or(i32::MAX) == kind, + KindSelector::Range([start, end]) => { + let start = i32::try_from(start.0).unwrap_or(i32::MAX); + let end = i32::try_from(end.0).unwrap_or(i32::MAX); + kind >= start && kind <= end + } + }) +} + +fn kind_of(record: &EventRecord) -> Result { + i32::try_from(record.event.kind.0).map_err(|_| StorageError::ValueOutOfRange { + column: "event_kind", + value: record.event.kind.0.to_string(), + }) +} + +// --------------------------------------------------------------------------------------- +// Read paths. +// --------------------------------------------------------------------------------------- + +/// One database's inclusive key range. +type KeyRange = (Vec, Vec); + +/// The index one filter is answered from, how its keys are read back, and the ranges to walk. +type IndexRanges = (Database, AccessPath, Vec); + +/// One reconciliation batch still running on the blocking pool, with the cursor it returns. +type PendingBatch = JoinHandle, Position), StorageError>>; + +/// The index one filter is answered from, and how its keys are read back. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AccessPath { + Ids, + Authors, + Tags, + CreatedAt, +} + +/// The page of keys one filter matches, in the capture's order. +pub(crate) fn page( + txn: &RoTxn<'_>, + databases: &Databases, + filters: &[Filter], + gates: &Gates, + now: i64, +) -> Result, StorageError> { + if filters.is_empty() { + return Err(StorageError::InvalidArgument { + message: "Filters cannot be empty", + }); + } + + let mut union = Vec::new(); + let mut seen = HashSet::new(); + for filter in filters { + let limit = Some( + usize::try_from(filter.limit.unwrap_or(DEFAULT_FILTER_LIMIT)).unwrap_or(usize::MAX), + ); + for candidate in branch(txn, databases, filter, gates, now, false, limit)? { + if seen.insert(candidate) { + union.push(candidate); + } + } + } + sort_page(&mut union, filters[0].limit.is_some()); + Ok(union) +} + +/// The count of the live ids the filters match, deduplicated across filters. +pub(crate) fn count( + txn: &RoTxn<'_>, + databases: &Databases, + filters: &[Filter], + gates: &Gates, + now: i64, +) -> Result { + if filters.is_empty() { + return Err(StorageError::InvalidArgument { + message: "Filters cannot be empty", + }); + } + + let mut ids = HashSet::new(); + for filter in filters { + let limit = filter + .limit + .map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); + for (_, id) in branch(txn, databases, filter, gates, now, true, limit)? { + ids.insert(id); + } + } + Ok(ids.len() as u64) +} + +/// One filter's page: up to `limit` matching keys, ordered as the capture's branch is. +fn branch( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + gates: &Gates, + now: i64, + live: bool, + limit: Option, +) -> Result, StorageError> { + if gates.nip50 + && filter + .search + .as_deref() + .is_some_and(|search| !search.trim().is_empty()) + { + return Err(StorageError::NotImplemented { + module: "storage::lmdb::nip50_search", + }); + } + if filter.ids.as_ref().is_some_and(|ids| ids.is_empty()) + || filter + .authors + .as_ref() + .is_some_and(|authors| authors.is_empty()) + || filter + .tags + .iter() + .any(|(key, criteria)| generic_tag_name(key).is_some() && criteria.is_empty()) + { + return Ok(Vec::new()); + } + + let matcher = Matcher { + filter, + gates, + now, + live, + }; + let (database, path, ranges) = choose_ranges(filter, databases); + let descending = filter.limit.is_some(); + let mut matches: Vec<(i64, EventId)> = Vec::new(); + let mut seen = HashSet::new(); + let mut boundary: Option = None; + + for (low, high) in &ranges { + let range = encoding::bounds(low, high); + if path == AccessPath::CreatedAt && descending { + for entry in database.rev_range(txn, &range)? { + let (key, _) = entry?; + let Some((created_at, id)) = key_candidate(path, key)? else { + continue; + }; + if boundary.is_some_and(|bound| created_at < bound) { + break; + } + if matcher.matches(txn, databases, &id)?.is_some() { + push(&mut matches, &mut seen, (created_at, id)); + if limit.is_some_and(|limit| matches.len() >= limit) { + boundary = Some(created_at); + } + } + } + } else { + for entry in database.range(txn, &range)? { + let (key, _) = entry?; + let Some(found) = candidate(txn, databases, path, key)? else { + continue; + }; + if boundary.is_some_and(|bound| found.0 > bound) { + break; + } + if matcher.matches(txn, databases, &found.1)?.is_some() { + push(&mut matches, &mut seen, found); + if limit.is_some_and(|limit| matches.len() >= limit) { + boundary = Some(found.0); + } + } + } + } + } + + // A limit on any other index is applied after the page is ordered, exactly as the + // capture's `ORDER BY … LIMIT` is. + sort_page(&mut matches, descending); + if let Some(limit) = limit { + matches.truncate(limit); + } + Ok(matches) +} + +fn push(matches: &mut Vec<(i64, EventId)>, seen: &mut HashSet, candidate: (i64, EventId)) { + if seen.insert(candidate.1) { + matches.push(candidate); + } +} + +fn sort_page(page: &mut [(i64, EventId)], descending: bool) { + if descending { + page.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(&right.1))); + } else { + page.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); + } +} + +/// The index a filter is answered from, and the ranges within it. +fn choose_ranges( + filter: &Filter, + databases: &Databases, +) -> IndexRanges { + if let Some(ids) = &filter.ids + && !ids.is_empty() + { + return ( + databases.events, + AccessPath::Ids, + encoding::hex_ranges(ids, 0), + ); + } + if let Some(authors) = &filter.authors + && !authors.is_empty() + { + return ( + databases.events_by_author, + AccessPath::Authors, + encoding::hex_ranges(authors, 4 + 8 + 32), + ); + } + if let Some((name, criteria)) = tag_path(filter) { + return ( + databases.tags, + AccessPath::Tags, + encoding::tag_ranges(name, criteria), + ); + } + ( + databases.events_by_created_at, + AccessPath::CreatedAt, + created_at_ranges(filter), + ) +} + +/// The first tag criterion the capture treats as a query: `#` plus one ASCII letter with a +/// non-empty list. +fn tag_path(filter: &Filter) -> Option<(&str, &[String])> { + filter.tags.iter().find_map(|(key, criteria)| { + generic_tag_name(key) + .filter(|_| !criteria.is_empty()) + .map(|name| (name, criteria.as_slice())) + }) +} + +fn created_at_ranges(filter: &Filter) -> Vec<(Vec, Vec)> { + let lower = match filter.since { + Some(since) => encoding::i64_key(since).to_vec(), + None => Vec::new(), + }; + let upper = match filter.until { + Some(until) => { + let mut upper = encoding::i64_key(until).to_vec(); + upper.extend([0xff; 32]); + upper + } + None => vec![0xff; 64], + }; + vec![(lower, upper)] +} + +/// The `(created_at, event_id)` one index key carries, when the key carries it. +fn key_candidate(path: AccessPath, key: &[u8]) -> Result, StorageError> { + match path { + AccessPath::Ids => Ok(None), + AccessPath::Authors => { + if key.len() != 76 { + return Err(encoding::invalid( + "events_by_author", + format!("a key of {} bytes", key.len()), + )); + } + Ok(Some(( + encoding::i64_from_key(&key[36..44])?, + encoding::event_id_from(&key[44..76])?, + ))) + } + AccessPath::Tags => { + if key.len() < 42 { + return Err(encoding::invalid( + "tags", + format!("a key of {} bytes", key.len()), + )); + } + let start = key.len() - 40; + Ok(Some(( + encoding::i64_from_key(&key[start..start + 8])?, + encoding::event_id_from(&key[start + 8..])?, + ))) + } + AccessPath::CreatedAt => { + if key.len() != 40 { + return Err(encoding::invalid( + "events_by_created_at", + format!("a key of {} bytes", key.len()), + )); + } + Ok(Some(( + encoding::i64_from_key(&key[..8])?, + encoding::event_id_from(&key[8..])?, + ))) + } + } +} + +/// A candidate whose `created_at` is only in the value, read from the header. +fn candidate( + txn: &RoTxn<'_>, + databases: &Databases, + path: AccessPath, + key: &[u8], +) -> Result, StorageError> { + if let Some(candidate) = key_candidate(path, key)? { + return Ok(Some(candidate)); + } + let id = encoding::event_id_from(key)?; + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + return Ok(None); + }; + Ok(Some((rows::event_header(value)?.created_at, id))) +} + +/// One filter's criteria, evaluated against a stored event. +struct Matcher<'a> { + filter: &'a Filter, + gates: &'a Gates, + now: i64, + live: bool, +} + +impl Matcher<'_> { + fn matches( + &self, + txn: &RoTxn<'_>, + databases: &Databases, + id: &EventId, + ) -> Result, StorageError> { + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + return Ok(None); + }; + let header = rows::event_header(value)?; + + if self.live + && (header.deleted + || header + .expires_at + .is_some_and(|expires_at| expires_at <= self.now)) + { + return Ok(None); + } + if let Some(ids) = &self.filter.ids + && !ids + .iter() + .any(|criterion| encoding::hex_matches(id.as_bytes(), criterion)) + { + return Ok(None); + } + if let Some(authors) = &self.filter.authors + && !authors + .iter() + .any(|criterion| encoding::hex_matches(header.pubkey.as_bytes(), criterion)) + { + return Ok(None); + } + if let Some(kinds) = &self.filter.kinds + && !kinds + .iter() + .any(|kind| i32::try_from(kind.0).unwrap_or(i32::MAX) == header.kind) + { + return Ok(None); + } + if let Some(since) = self.filter.since + && header.created_at < since + { + return Ok(None); + } + if let Some(until) = self.filter.until + && header.created_at > until + { + return Ok(None); + } + if self.gates.nip56 && self.reported(txn, databases, id, &header.pubkey)? { + return Ok(None); + } + if self + .filter + .tags + .keys() + .any(|key| generic_tag_name(key).is_some()) + && !tag_criteria_match(&self.filter.tags, &rows::decode_event(value)?) + { + return Ok(None); + } + Ok(Some(header)) + } + + /// The capture's `NOT EXISTS (… reports …)`: one point lookup per target column. + fn reported( + &self, + txn: &RoTxn<'_>, + databases: &Databases, + id: &EventId, + pubkey: &Pubkey, + ) -> Result { + if has_prefix(databases.reports_actionable_event, txn, id.as_bytes(), 4)? { + return Ok(true); + } + has_prefix( + databases.reports_actionable_pubkey, + txn, + pubkey.as_bytes(), + 4, + ) + } +} + +fn has_prefix( + database: Database, + txn: &RoTxn<'_>, + prefix: &[u8], + suffix: usize, +) -> Result { + let mut upper = prefix.to_vec(); + upper.extend(std::iter::repeat_n(0xff, suffix)); + let range = (Bound::Included(prefix), Bound::Included(upper.as_slice())); + Ok(database.range(txn, &range)?.next().transpose()?.is_some()) +} + +/// The capture's `isGenericTagQuery`: `#` plus one ASCII letter. +pub(crate) fn generic_tag_name(key: &str) -> Option<&str> { + let mut characters = key.chars(); + if characters.next() != Some('#') { + return None; + } + let name = characters.next()?; + if characters.next().is_some() || !name.is_ascii_alphabetic() { + return None; + } + Some(&key[1..2]) +} + +/// Every tag criterion of a filter, under the capture's rules: a value is exact, and only +/// `#g` honours a trailing `*` as a prefix. +fn tag_criteria_match(tags: &BTreeMap>, record: &EventRecord) -> bool { + for (key, criteria) in tags { + let Some(name) = generic_tag_name(key) else { + continue; + }; + if criteria.is_empty() { + return false; + } + let is_geohash = key == "#g"; + let matched = record.event.tags.iter().any(|tag| { + if tag.first().map(String::as_str) != Some(name) { + return false; + } + let Some(value) = tag.get(1) else { + return false; + }; + criteria.iter().any(|criterion| { + match criterion.strip_suffix('*').filter(|_| is_geohash) { + Some(prefix) => value.starts_with(prefix), + None => value == criterion, + } + }) + }); + if !matched { + return false; + } + } + true +} + +// --------------------------------------------------------------------------------------- +// Streams. +// --------------------------------------------------------------------------------------- + +/// One batch of keys a stream has read, and where the next batch resumes. +struct Position { + ranges: Vec, + index: usize, + after: Option>, +} + +impl Position { + fn new(filter: &Filter) -> Self { + Self { + ranges: created_at_ranges(filter), + index: 0, + after: None, + } + } +} + +/// Reads one page of values in batches of [`STREAM_BATCH_ITEMS`], one read transaction per +/// poll, so a stream that is dropped pins no pages. +pub(crate) struct ValuesStream { + env: Env, + databases: Databases, + keys: std::vec::IntoIter, + pending: Option, StorageError>>>, + buffer: std::vec::IntoIter, + finished: bool, +} + +impl ValuesStream { + pub(crate) fn new(env: Env, databases: Databases, page: &[(i64, EventId)]) -> Self { + Self { + env, + databases, + keys: page + .iter() + .map(|(_, id)| *id) + .collect::>() + .into_iter(), + pending: None, + buffer: Vec::new().into_iter(), + finished: false, + } + } +} + +impl Stream for ValuesStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + if let Some(record) = this.buffer.next() { + return Poll::Ready(Some(Ok(record))); + } + if let Some(handle) = this.pending.as_mut() { + match Pin::new(handle).poll(context) { + Poll::Pending => return Poll::Pending, + Poll::Ready(joined) => { + this.pending = None; + match joined { + Ok(Ok(records)) => this.buffer = records.into_iter(), + Ok(Err(error)) => { + this.finished = true; + return Poll::Ready(Some(Err(error))); + } + Err(_) => { + this.finished = true; + return Poll::Ready(Some(Err(StorageError::LmdbTaskCancelled))); + } + } + } + } + continue; + } + if this.finished { + return Poll::Ready(None); + } + let batch: Vec = this.keys.by_ref().take(STREAM_BATCH_ITEMS).collect(); + if batch.is_empty() { + this.finished = true; + return Poll::Ready(None); + } + let env = this.env.clone(); + let database = this.databases.events; + this.pending = Some(tokio::task::spawn_blocking(move || { + let transaction = env.read_txn()?; + let mut records = Vec::with_capacity(batch.len()); + for id in batch { + let Some(value) = database.get(&transaction, id.as_bytes())? else { + continue; + }; + records.push(rows::decode_event(value)?); + } + Ok(records) + })); + } + } +} + +/// Yields the live `(created_at, event_id)` pairs a reconciliation compares, oldest first, +/// in batches of [`STREAM_BATCH_ITEMS`] keys with one read transaction per poll. +pub(crate) struct ReconciliationStream { + env: Env, + databases: Databases, + filter: Filter, + gates: Gates, + now: i64, + position: Position, + pending: Option, + buffer: std::vec::IntoIter, + finished: bool, +} + +impl ReconciliationStream { + pub(crate) fn new( + env: Env, + databases: Databases, + filter: &Filter, + gates: Gates, + now: i64, + ) -> Self { + Self { + env, + databases, + filter: filter.clone(), + gates, + now, + position: Position::new(filter), + pending: None, + buffer: Vec::new().into_iter(), + finished: false, + } + } +} + +impl Stream for ReconciliationStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + if let Some(item) = this.buffer.next() { + return Poll::Ready(Some(Ok(item))); + } + if let Some(handle) = this.pending.as_mut() { + match Pin::new(handle).poll(context) { + Poll::Pending => return Poll::Pending, + Poll::Ready(joined) => { + this.pending = None; + match joined { + Ok(Ok((items, position))) => { + this.position = position; + this.buffer = items.into_iter(); + } + Ok(Err(error)) => { + this.finished = true; + return Poll::Ready(Some(Err(error))); + } + Err(_) => { + this.finished = true; + return Poll::Ready(Some(Err(StorageError::LmdbTaskCancelled))); + } + } + } + } + continue; + } + if this.finished { + return Poll::Ready(None); + } + let env = this.env.clone(); + let databases = this.databases; + let filter = this.filter.clone(); + let gates = this.gates; + let now = this.now; + let mut position = std::mem::replace(&mut this.position, Position::new(&this.filter)); + let exhausted = position.index >= position.ranges.len(); + if exhausted { + this.finished = true; + return Poll::Ready(None); + } + this.pending = Some(tokio::task::spawn_blocking(move || { + let transaction = env.read_txn()?; + let items = scan_batch( + &transaction, + &databases, + &filter, + &gates, + now, + &mut position, + )?; + Ok((items, position)) + })); + } + } +} + +/// One batch of a reconciliation scan: up to [`STREAM_BATCH_ITEMS`] live keys, resuming +/// after the last key the previous batch read. +fn scan_batch( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + gates: &Gates, + now: i64, + position: &mut Position, +) -> Result, StorageError> { + let matcher = Matcher { + filter, + gates, + now, + live: true, + }; + let mut items = Vec::new(); + while position.index < position.ranges.len() { + let lower = position.ranges[position.index].0.clone(); + let upper = position.ranges[position.index].1.clone(); + let after = position.after.clone(); + let range = encoding::resume_bounds(after.as_deref(), &lower, &upper); + let mut reached_cap = false; + for entry in databases.events_by_created_at.range(txn, &range)? { + let (key, _) = entry?; + position.after = Some(key.to_vec()); + if items.len() >= STREAM_BATCH_ITEMS { + reached_cap = true; + break; + } + let (created_at, event_id) = + key_candidate(AccessPath::CreatedAt, key)?.ok_or_else(|| { + encoding::invalid( + "events_by_created_at", + "a key without a timestamp".to_owned(), + ) + })?; + if matcher.matches(txn, databases, &event_id)?.is_some() { + items.push(ReconciliationItem { + created_at, + event_id, + }); + } + } + if reached_cap { + break; + } + position.index += 1; + position.after = None; + } + Ok(items) +} + +/// The `event_id` one `events_by_author` or `events_deleted` key ends with. +fn header_id(key: &[u8], start: usize, table: &'static str) -> Result { + match key.get(start..) { + Some(bytes) => encoding::event_id_from(bytes), + None => Err(encoding::invalid( + table, + format!("a key of {} bytes", key.len()), + )), + } +} + +/// The kind and event id an `events_by_author` key carries. +struct AuthorKeyHeader { + kind: i32, + event_id: EventId, +} + +fn author_key_header(key: &[u8]) -> Result { + if key.len() != 76 { + return Err(encoding::invalid( + "events_by_author", + format!("a key of {} bytes", key.len()), + )); + } + Ok(AuthorKeyHeader { + kind: encoding::i32_from_key(&key[32..36])?, + event_id: encoding::event_id_from(&key[44..76])?, + }) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + fn id(byte: u8) -> EventId { + EventId::from_byte_array([byte; 32]) + } + + #[test] + fn a_page_sorts_newest_first_with_the_smaller_id_first_on_a_tie() { + let mut page = vec![(10, id(2)), (10, id(1)), (11, id(9))]; + sort_page(&mut page, true); + assert_eq!(page, vec![(11, id(9)), (10, id(1)), (10, id(2))]); + + sort_page(&mut page, false); + assert_eq!(page, vec![(10, id(1)), (10, id(2)), (11, id(9))]); + } + + #[test] + fn only_a_hash_and_one_letter_is_a_tag_query() { + assert_eq!(generic_tag_name("#e"), Some("e")); + assert_eq!(generic_tag_name("#1"), None); + assert_eq!(generic_tag_name("#ee"), None); + assert_eq!(generic_tag_name("e"), None); + assert_eq!(generic_tag_name("#"), None); + } + + #[test] + fn the_retention_whitelist_always_keeps_kind_62_and_deduplicates() { + let whitelist = retention_kinds(Some(&[ + KindSelector::Single(Kind(1)), + KindSelector::Single(Kind(1)), + ])); + assert_eq!( + whitelist, + vec![ + KindSelector::Single(Kind(1)), + KindSelector::Single(Kind(62)), + ] + ); + assert!(whitelisted(62, &whitelist)); + assert!(whitelisted(1, &[KindSelector::Range([Kind(1), Kind(9)])])); + assert!(!whitelisted(10, &[KindSelector::Range([Kind(1), Kind(9)])])); + } + + #[test] + fn a_geohash_wildcard_is_a_prefix_and_another_tag_treats_it_literally() { + use nostream_core::Event; + + let event: Event = serde_json::from_value(serde_json::json!({ + "id": "0d684e8ec2431de586aa3cafbee2f6d308d19b28805e53deabcac3220e9136a5", + "pubkey": "f43c1f9bff677b8f27b602725ea0ad51af221344f69a6b352a74991a4479bac3", + "created_at": 1660407625i64, + "kind": 1, + "tags": [["g", "u4pruyd"], ["e", "other"]], + "content": "", + "sig": "ab".repeat(64) + })) + .expect("a wire event"); + let record = EventRecord::from_event(&event); + + let mut tags = BTreeMap::new(); + tags.insert("#g".to_owned(), vec!["u4pru*".to_owned()]); + assert!(tag_criteria_match(&tags, &record)); + + let mut tags = BTreeMap::new(); + tags.insert("#e".to_owned(), vec!["u4pru*".to_owned()]); + assert!(!tag_criteria_match(&tags, &record)); + + let mut tags = BTreeMap::new(); + tags.insert("#em".to_owned(), vec!["nothing".to_owned()]); + assert!(tag_criteria_match(&tags, &record)); + } +} diff --git a/crates/nostream-storage/src/lmdb/invoices.rs b/crates/nostream-storage/src/lmdb/invoices.rs new file mode 100644 index 000000000..09e3ef6c0 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/invoices.rs @@ -0,0 +1,193 @@ +//! `InvoiceRepository` against LMDB. Mirrors `src/repositories/invoice-repository.ts`. +//! +//! Two databases: `invoices` by id, and `invoices_pending` (`created_at ‖ id`) holding only +//! the pending rows, which is the partial index `find_pending_invoices` pages through. + +use chrono::{DateTime, Utc}; +use heed::{RoTxn, RwTxn}; + +use super::Databases; +use super::encoding; +use super::rows; +use super::users; +use crate::error::StorageError; +use crate::repositories::{Invoice, InvoiceStatus}; + +/// `findById`: one row by primary key. +pub(crate) fn find_by_id( + txn: &RoTxn<'_>, + databases: &Databases, + id: &str, +) -> Result, StorageError> { + let Some(value) = databases.invoices.get(txn, id.as_bytes())? else { + return Ok(None); + }; + rows::decode("invoices", value).map(Some) +} + +/// `created_at ‖ id`, the key of the pending index. +fn pending_key(created_at: DateTime, id: &str) -> Vec { + let mut key = encoding::i64_key(created_at.timestamp_micros()).to_vec(); + key.extend_from_slice(id.as_bytes()); + key +} + +/// Writes a row and moves its pending entry, when the status or the row changed. +fn write( + txn: &mut RwTxn<'_>, + databases: &Databases, + invoice: &Invoice, + previous: Option<&Invoice>, +) -> Result<(), StorageError> { + let value = rows::encode("invoices", invoice)?; + databases.invoices.put(txn, invoice.id.as_bytes(), &value)?; + if let Some(previous) = previous + && previous.status == InvoiceStatus::Pending + { + databases + .invoices_pending + .delete(txn, &pending_key(previous.created_at, &previous.id))?; + } + if invoice.status == InvoiceStatus::Pending { + databases + .invoices_pending + .put(txn, &pending_key(invoice.created_at, &invoice.id), &[])?; + } + Ok(()) +} + +/// `upsert`: inserts the row, or updates only `status` and `updated_at` — the capture's +/// merge list, which never carries `amount_paid` or `confirmed_at`. +pub(crate) fn upsert( + txn: &mut RwTxn<'_>, + databases: &Databases, + invoice: &Invoice, + now: DateTime, +) -> Result { + match find_by_id(txn, databases, &invoice.id)? { + Some(mut stored) => { + let previous = stored.clone(); + stored.status = invoice.status; + stored.updated_at = encoding::micros(now); + write(txn, databases, &stored, Some(&previous))?; + } + None => { + let mut stored = invoice.clone(); + // The insert's column list omits both, so the database fills them with NULL. + stored.amount_paid = None; + stored.confirmed_at = None; + stored.updated_at = encoding::micros(now); + write(txn, databases, &stored, None)?; + } + } + Ok(1) +} + +/// `updateStatus`: updates one row and returns it. +pub(crate) fn update_status( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &str, + status: InvoiceStatus, + now: DateTime, +) -> Result, StorageError> { + let Some(mut stored) = find_by_id(txn, databases, id)? else { + return Ok(None); + }; + let previous = stored.clone(); + stored.status = status; + stored.updated_at = encoding::micros(now); + write(txn, databases, &stored, Some(&previous))?; + Ok(Some(stored)) +} + +/// The balance credit one unit is worth (`scale_balance_addition_with_unit`). +fn unit_scale(unit: crate::repositories::InvoiceUnit) -> i64 { + use crate::repositories::InvoiceUnit; + match unit { + InvoiceUnit::Msats => 1, + InvoiceUnit::Sats => 1_000, + InvoiceUnit::Btc => 100_000_000 * 1_000, + } +} + +/// `confirmInvoice`: the schema function's body, with `int8` overflow answered as +/// [`StorageError::ValueOutOfRange`]. +pub(crate) fn confirm_invoice( + txn: &mut RwTxn<'_>, + databases: &Databases, + invoice_id: &str, + amount_received: i64, + confirmed_at: DateTime, + now: DateTime, +) -> Result<(), StorageError> { + let Some(stored) = find_by_id(txn, databases, invoice_id)? else { + // The capture's `SELECT … INTO` leaves `payee` NULL and both updates match nothing. + return Ok(()); + }; + if stored.confirmed_at.is_some() { + return Ok(()); + } + + let credit = amount_received + .checked_mul(unit_scale(stored.unit)) + .ok_or_else(|| StorageError::ValueOutOfRange { + column: "users.balance", + value: amount_received.to_string(), + })?; + let payee = users::load(txn, databases, &stored.pubkey)?; + let balance = match &payee { + Some(payee) => Some(payee.balance.checked_add(credit).ok_or_else(|| { + StorageError::ValueOutOfRange { + column: "users.balance", + value: payee.balance.to_string(), + } + })?), + None => None, + }; + + let mut confirmed = stored.clone(); + confirmed.confirmed_at = Some(encoding::micros(confirmed_at)); + confirmed.amount_paid = Some(amount_received); + confirmed.updated_at = encoding::micros(now); + write(txn, databases, &confirmed, Some(&stored))?; + + if let (Some(mut payee), Some(balance)) = (payee, balance) { + payee.balance = balance; + users::store(txn, databases, &payee)?; + } + Ok(()) +} + +/// `findPendingInvoices`: the pending rows, oldest first, paged. +pub(crate) fn find_pending_invoices( + txn: &RoTxn<'_>, + databases: &Databases, + offset: u64, + limit: u64, +) -> Result, StorageError> { + let mut invoices = Vec::new(); + let mut skipped = 0; + for entry in databases.invoices_pending.iter(txn)? { + let (key, _) = entry?; + if skipped < offset { + skipped += 1; + continue; + } + if invoices.len() as u64 >= limit { + break; + } + let id = key.get(8..).ok_or_else(|| { + encoding::invalid("invoices_pending", format!("a key of {} bytes", key.len())) + })?; + let id = std::str::from_utf8(id).map_err(|_| StorageError::InvalidRow { + table: "invoices_pending", + column: "id", + value: "not utf-8".to_owned(), + })?; + if let Some(value) = databases.invoices.get(txn, id.as_bytes())? { + invoices.push(rows::decode("invoices", value)?); + } + } + Ok(invoices) +} diff --git a/crates/nostream-storage/src/lmdb/metadata.rs b/crates/nostream-storage/src/lmdb/metadata.rs new file mode 100644 index 000000000..b395cb2fc --- /dev/null +++ b/crates/nostream-storage/src/lmdb/metadata.rs @@ -0,0 +1,658 @@ +//! `Nip05VerificationRepository`, `InviteCodeRepository`, `DvmJobRepository` and +//! `ReportRepository` against LMDB. Mirrors the four `src/repositories/*` files. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Duration, Utc}; +use heed::{RoTxn, RwTxn}; +use nostream_core::{EventId, Kind, Pubkey}; + +use super::Databases; +use super::encoding; +use super::rows; +use crate::error::StorageError; +use crate::repositories::{ + ActionableTarget, CreateInviteCodeOptions, DvmJob, DvmJobStatus, DvmJobUpdate, FieldUpdate, + InviteCode, NewReport, Nip05Verification, Report, +}; + +/// The `meta` key holding the `reports.id` sequence (`reports_id_seq`). +const REPORT_SEQUENCE: &[u8] = b"reports.next_id"; + +/// A text component of a composite key, terminated so the next component reads unambiguously. +fn text_part(value: &str) -> Vec { + let mut bytes = value.as_bytes().to_vec(); + bytes.push(0); + bytes +} + +fn text_from<'a>(bytes: &'a [u8], table: &'static str) -> Result<&'a str, StorageError> { + std::str::from_utf8(bytes).map_err(|_| StorageError::InvalidRow { + table, + column: "key", + value: "not utf-8".to_owned(), + }) +} + +// --------------------------------------------------------------------------------------- +// NIP-05 verifications. +// --------------------------------------------------------------------------------------- + +/// `created_at`-style keys for the tail indexes order by microseconds. +fn stamp_key(value: DateTime) -> [u8; 8] { + encoding::i64_key(value.timestamp_micros()) +} + +fn nip05_key(pubkey: &Pubkey) -> Vec { + pubkey.as_bytes().to_vec() +} + +/// `last_checked_at ‖ pubkey`, the key of the pending-check index. +fn nip05_pending_key(verification: &Nip05Verification) -> Vec { + let mut key = stamp_key(verification.last_checked_at).to_vec(); + key.extend_from_slice(verification.pubkey.as_bytes()); + key +} + +pub(crate) fn nip05_find_by_pubkey( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result, StorageError> { + let Some(value) = databases.nip05.get(txn, &nip05_key(pubkey))? else { + return Ok(None); + }; + rows::decode("nip05_verifications", value).map(Some) +} + +/// `Nip05VerificationRepository::upsert`: every column but `pubkey` and `created_at` moves. +pub(crate) fn nip05_upsert( + txn: &mut RwTxn<'_>, + databases: &Databases, + verification: &Nip05Verification, + now: DateTime, +) -> Result { + let previous = nip05_find_by_pubkey(txn, databases, &verification.pubkey)?; + let mut stored = verification.clone(); + stored.created_at = previous + .as_ref() + .map_or_else(|| encoding::micros(now), |row| row.created_at); + stored.updated_at = encoding::micros(now); + + let value = rows::encode("nip05_verifications", &stored)?; + databases + .nip05 + .put(txn, &nip05_key(&stored.pubkey), &value)?; + if let Some(previous) = &previous { + databases + .nip05_by_last_checked_at + .delete(txn, &nip05_pending_key(previous))?; + } + databases + .nip05_by_last_checked_at + .put(txn, &nip05_pending_key(&stored), &[])?; + Ok(1) +} + +/// `Nip05VerificationRepository::find_pending_verifications`: rows last checked before the +/// cutoff with fewer than `max_failures`, oldest check first. +pub(crate) fn nip05_find_pending( + txn: &RoTxn<'_>, + databases: &Databases, + update_frequency_ms: i64, + max_failures: i32, + limit: u64, + now: DateTime, +) -> Result, StorageError> { + let cutoff = encoding::micros(now - Duration::milliseconds(update_frequency_ms)); + let mut verifications = Vec::new(); + for entry in databases.nip05_by_last_checked_at.iter(txn)? { + let (key, _) = entry?; + let last_checked_at = encoding::i64_from_key(&key[..8])?; + if last_checked_at >= cutoff.timestamp_micros() { + break; + } + let pubkey = encoding::pubkey_from(&key[8..])?; + let Some(verification) = nip05_find_by_pubkey(txn, databases, &pubkey)? else { + continue; + }; + if verification.failure_count >= max_failures { + continue; + } + verifications.push(verification); + if verifications.len() as u64 >= limit { + break; + } + } + Ok(verifications) +} + +/// `Nip05VerificationRepository::delete_by_pubkey`. +pub(crate) fn nip05_delete_by_pubkey( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result { + let Some(verification) = nip05_find_by_pubkey(txn, databases, pubkey)? else { + return Ok(0); + }; + databases.nip05.delete(txn, &nip05_key(pubkey))?; + databases + .nip05_by_last_checked_at + .delete(txn, &nip05_pending_key(&verification))?; + Ok(1) +} + +// --------------------------------------------------------------------------------------- +// Invite codes. +// --------------------------------------------------------------------------------------- + +fn invite_created_key(invite: &InviteCode) -> Vec { + let mut key = stamp_key(invite.created_at).to_vec(); + key.extend_from_slice(invite.code.as_bytes()); + key +} + +fn invite_expires_key(invite: &InviteCode, expires_at: DateTime) -> Vec { + let mut key = stamp_key(expires_at).to_vec(); + key.extend_from_slice(invite.code.as_bytes()); + key +} + +pub(crate) fn invite_find_by_code( + txn: &RoTxn<'_>, + databases: &Databases, + code: &str, +) -> Result, StorageError> { + let Some(value) = databases.invite_codes.get(txn, code.as_bytes())? else { + return Ok(None); + }; + rows::decode("invite_codes", value).map(Some) +} + +fn invite_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + invite: &InviteCode, +) -> Result<(), StorageError> { + let value = rows::encode("invite_codes", invite)?; + databases + .invite_codes + .put(txn, invite.code.as_bytes(), &value)?; + databases + .invite_codes_by_created_at + .put(txn, &invite_created_key(invite), &[])?; + if let Some(expires_at) = invite.expires_at { + databases.invite_codes_by_expires_at.put( + txn, + &invite_expires_key(invite, expires_at), + &[], + )?; + } + Ok(()) +} + +/// `InviteCodeRepository::create`: the row the capture returns, minted with its options. +pub(crate) fn invite_create( + txn: &mut RwTxn<'_>, + databases: &Databases, + code: &str, + options: &CreateInviteCodeOptions, + now: DateTime, +) -> Result { + let remaining_uses = match &options.remaining_uses { + FieldUpdate::Absent => Some(1), + FieldUpdate::Clear => None, + FieldUpdate::Set(uses) => { + if *uses < 0 { + return Err(StorageError::InvalidArgument { + message: "invite_codes.remaining_uses must not be negative", + }); + } + Some(*uses) + } + }; + if invite_find_by_code(txn, databases, code)?.is_some() { + return Err(StorageError::InvalidArgument { + message: "invite code already exists", + }); + } + + let invite = InviteCode { + code: code.to_owned(), + created_by: options.created_by, + claimed_by: None, + expires_at: options.expires_at.map(encoding::micros), + remaining_uses, + created_at: encoding::micros(now), + updated_at: encoding::micros(now), + }; + invite_write(txn, databases, &invite)?; + Ok(invite) +} + +/// `InviteCodeRepository::claim_code`: one conditional update, so exactly one caller wins. +pub(crate) fn invite_claim_code( + txn: &mut RwTxn<'_>, + databases: &Databases, + code: &str, + pubkey: &Pubkey, + now: DateTime, +) -> Result { + let Some(mut invite) = invite_find_by_code(txn, databases, code)? else { + return Ok(false); + }; + let now = encoding::micros(now); + if invite + .expires_at + .is_some_and(|expires_at| expires_at <= now) + || invite.remaining_uses.is_some_and(|uses| uses <= 0) + { + return Ok(false); + } + + invite.remaining_uses = invite.remaining_uses.map(|uses| uses - 1); + invite.claimed_by = invite.claimed_by.or(Some(*pubkey)); + invite.updated_at = now; + let value = rows::encode("invite_codes", &invite)?; + databases.invite_codes.put(txn, code.as_bytes(), &value)?; + Ok(true) +} + +/// `InviteCodeRepository::find_active_codes`: unexpired and claimable, newest first. +pub(crate) fn invite_find_active_codes( + txn: &RoTxn<'_>, + databases: &Databases, + limit: u64, + now: DateTime, +) -> Result, StorageError> { + let now = encoding::micros(now); + let mut invites = Vec::new(); + for entry in databases.invite_codes_by_created_at.rev_iter(txn)? { + let (key, _) = entry?; + let code = text_from(&key[8..], "invite_codes_by_created_at")?; + let Some(invite) = invite_find_by_code(txn, databases, code)? else { + continue; + }; + if invite + .expires_at + .is_some_and(|expires_at| expires_at <= now) + || invite.remaining_uses.is_some_and(|uses| uses <= 0) + { + continue; + } + invites.push(invite); + if invites.len() as u64 >= limit { + break; + } + } + Ok(invites) +} + +/// `InviteCodeRepository::delete_expired_codes`. +pub(crate) fn invite_delete_expired( + txn: &mut RwTxn<'_>, + databases: &Databases, + now: DateTime, +) -> Result { + let now = encoding::micros(now); + let mut expired = Vec::new(); + for entry in databases.invite_codes_by_expires_at.iter(txn)? { + let (key, _) = entry?; + let expires_at = encoding::i64_from_key(&key[..8])?; + if expires_at > now.timestamp_micros() { + break; + } + expired.push(text_from(&key[8..], "invite_codes_by_expires_at")?.to_owned()); + } + + let mut affected = 0; + for code in expired { + let Some(invite) = invite_find_by_code(txn, databases, &code)? else { + continue; + }; + databases.invite_codes.delete(txn, code.as_bytes())?; + databases + .invite_codes_by_created_at + .delete(txn, &invite_created_key(&invite))?; + if let Some(expires_at) = invite.expires_at { + databases + .invite_codes_by_expires_at + .delete(txn, &invite_expires_key(&invite, expires_at))?; + } + affected += 1; + } + Ok(affected) +} + +// --------------------------------------------------------------------------------------- +// DVM jobs. +// --------------------------------------------------------------------------------------- + +fn dvm_status_key(job: &DvmJob, created_at: DateTime) -> Vec { + let mut key = text_part(job.status.as_str()); + key.extend_from_slice(&stamp_key(created_at)); + key.extend_from_slice(job.id.as_bytes()); + key +} + +pub(crate) fn dvm_find_by_id( + txn: &RoTxn<'_>, + databases: &Databases, + id: &EventId, +) -> Result, StorageError> { + let Some(value) = databases.dvm_jobs.get(txn, id.as_bytes())? else { + return Ok(None); + }; + rows::decode("dvm_jobs", value).map(Some) +} + +fn dvm_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + job: &DvmJob, + previous: Option<&DvmJob>, +) -> Result<(), StorageError> { + let value = rows::encode("dvm_jobs", job)?; + databases.dvm_jobs.put(txn, job.id.as_bytes(), &value)?; + if let Some(previous) = previous { + databases + .dvm_jobs_by_status_created_at + .delete(txn, &dvm_status_key(previous, previous.created_at))?; + } + databases + .dvm_jobs_by_status_created_at + .put(txn, &dvm_status_key(job, job.created_at), &[])?; + Ok(()) +} + +/// `DvmJobRepository::create`: a submitted job with every optional column empty. +pub(crate) fn dvm_create( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &EventId, + requester: &Pubkey, + kind: Kind, + now: DateTime, +) -> Result { + let now = encoding::micros(now); + let job = DvmJob { + id: *id, + requester_pubkey: *requester, + kind, + worker_index: None, + status: DvmJobStatus::Submitted, + result_event_id: None, + error: None, + picked_up_at: None, + completed_at: None, + created_at: now, + updated_at: now, + }; + dvm_write(txn, databases, &job, None)?; + Ok(job) +} + +/// `DvmJobRepository::assign_worker`: a conditional update from `submitted`. +pub(crate) fn dvm_assign_worker( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &EventId, + worker_index: i32, + now: DateTime, +) -> Result { + let Some(mut job) = dvm_find_by_id(txn, databases, id)? else { + return Ok(false); + }; + if job.status != DvmJobStatus::Submitted { + return Ok(false); + } + let previous = job.clone(); + let now = encoding::micros(now); + job.worker_index = Some(worker_index); + job.status = DvmJobStatus::PickedUp; + job.picked_up_at = Some(now); + job.updated_at = now; + dvm_write(txn, databases, &job, Some(&previous))?; + Ok(true) +} + +/// `DvmJobRepository::update_status`: the `FieldUpdate` fields are applied by key presence. +pub(crate) fn dvm_update_status( + txn: &mut RwTxn<'_>, + databases: &Databases, + update: &DvmJobUpdate, + now: DateTime, +) -> Result, StorageError> { + let Some(mut job) = dvm_find_by_id(txn, databases, &update.id)? else { + return Ok(None); + }; + let previous = job.clone(); + let now = encoding::micros(now); + job.status = update.status; + job.updated_at = now; + if update.status.is_terminal() { + job.completed_at = Some(now); + } + match &update.result_event_id { + FieldUpdate::Absent => {} + FieldUpdate::Clear => job.result_event_id = None, + FieldUpdate::Set(id) => job.result_event_id = Some(*id), + } + match &update.error { + FieldUpdate::Absent => {} + FieldUpdate::Clear => job.error = None, + FieldUpdate::Set(error) => job.error = Some(error.clone()), + } + dvm_write(txn, databases, &job, Some(&previous))?; + Ok(Some(job)) +} + +/// `DvmJobRepository::find_pending_jobs`: submitted and picked-up jobs, oldest first. +pub(crate) fn dvm_find_pending_jobs( + txn: &RoTxn<'_>, + databases: &Databases, + limit: u64, + kinds: Option<&[Kind]>, +) -> Result, StorageError> { + let mut jobs = Vec::new(); + for status in [DvmJobStatus::Submitted, DvmJobStatus::PickedUp] { + let prefix = text_part(status.as_str()); + for entry in databases + .dvm_jobs_by_status_created_at + .prefix_iter(txn, &prefix)? + { + let (key, _) = entry?; + let id = encoding::event_id_from(&key[key.len() - 32..])?; + let Some(job) = dvm_find_by_id(txn, databases, &id)? else { + continue; + }; + if kinds.is_some_and(|kinds| !kinds.is_empty() && !kinds.contains(&job.kind)) { + continue; + } + jobs.push(job); + } + } + jobs.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.id.cmp(&right.id)) + }); + jobs.truncate(usize::try_from(limit).unwrap_or(usize::MAX)); + Ok(jobs) +} + +// --------------------------------------------------------------------------------------- +// Reports. +// --------------------------------------------------------------------------------------- + +fn report_key(id: i32) -> [u8; 4] { + encoding::i32_key(id) +} + +fn report_event_key(report: &Report, event_id: &EventId) -> Vec { + let mut key = event_id.as_bytes().to_vec(); + key.extend_from_slice(&report_key(report.id)); + key +} + +fn report_created_key(report: &Report) -> Vec { + let mut key = stamp_key(report.created_at).to_vec(); + key.extend_from_slice(&report_key(report.id)); + key +} + +fn report_pubkey_key(report: &Report, pubkey: &Pubkey) -> Vec { + let mut key = pubkey.as_bytes().to_vec(); + key.extend_from_slice(&report_key(report.id)); + key +} + +fn report_load( + txn: &RoTxn<'_>, + databases: &Databases, + id: i32, +) -> Result, StorageError> { + let Some(value) = databases.reports.get(txn, &report_key(id))? else { + return Ok(None); + }; + rows::decode("reports", value).map(Some) +} + +/// Allocates the next `reports.id` from the `meta` sequence, which starts at 1. +fn next_report_id(txn: &mut RwTxn<'_>, databases: &Databases) -> Result { + let current: u64 = match databases.meta.get(txn, REPORT_SEQUENCE)? { + Some(value) => rows::decode("meta", value)?, + None => 0, + }; + let next = current + 1; + let value = rows::encode("meta", &next)?; + databases.meta.put(txn, REPORT_SEQUENCE, &value)?; + i32::try_from(next).map_err(|_| StorageError::ValueOutOfRange { + column: "reports.id", + value: next.to_string(), + }) +} + +fn report_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + report: &Report, +) -> Result<(), StorageError> { + let value = rows::encode("reports", report)?; + databases.reports.put(txn, &report_key(report.id), &value)?; + databases + .reports_by_event_id + .put(txn, &report_event_key(report, &report.event_id), &[])?; + if report.actionable { + databases + .reports_actionable_by_created_at + .put(txn, &report_created_key(report), &[])?; + if let Some(event_id) = &report.reported_event_id { + databases.reports_actionable_event.put( + txn, + &report_event_key(report, event_id), + &[], + )?; + } + if let Some(pubkey) = &report.reported_pubkey { + databases.reports_actionable_pubkey.put( + txn, + &report_pubkey_key(report, pubkey), + &[], + )?; + } + } + Ok(()) +} + +/// `ReportRepository::create`: inserts the row and returns it with its generated id. +pub(crate) fn report_create( + txn: &mut RwTxn<'_>, + databases: &Databases, + new: &NewReport, + now: DateTime, +) -> Result { + let report = Report { + id: next_report_id(txn, databases)?, + event_id: new.event_id, + reporter_pubkey: new.reporter_pubkey, + reported_pubkey: new.reported_pubkey, + reported_event_id: new.reported_event_id, + report_type: new.report_type, + weight: new.weight, + actionable: new.actionable, + created_at: encoding::micros(now), + }; + report_write(txn, databases, &report)?; + Ok(report) +} + +/// `ReportRepository::find_by_event_id`. +pub(crate) fn report_find_by_event_id( + txn: &RoTxn<'_>, + databases: &Databases, + event_id: &EventId, +) -> Result, StorageError> { + let mut reports = Vec::new(); + for entry in databases + .reports_by_event_id + .prefix_iter(txn, event_id.as_bytes())? + { + let (key, _) = entry?; + let id = encoding::i32_from_key(&key[32..])?; + if let Some(report) = report_load(txn, databases, id)? { + reports.push(report); + } + } + Ok(reports) +} + +/// `ReportRepository::find_actionable`: actionable rows, newest first. +pub(crate) fn report_find_actionable( + txn: &RoTxn<'_>, + databases: &Databases, + limit: u64, +) -> Result, StorageError> { + let mut reports = Vec::new(); + for entry in databases.reports_actionable_by_created_at.rev_iter(txn)? { + let (key, _) = entry?; + let id = encoding::i32_from_key(&key[8..])?; + if let Some(report) = report_load(txn, databases, id)? { + reports.push(report); + } + if reports.len() as u64 >= limit { + break; + } + } + Ok(reports) +} + +/// One actionable target, as the deduplicating walk keys it. +type ActionableKey = (Option<[u8; 32]>, Option<[u8; 32]>); + +/// `ReportRepository::find_actionable_targets`: every distinct target, deduplicated the way +/// the capture's `DISTINCT` does. +pub(crate) fn report_find_actionable_targets( + txn: &RoTxn<'_>, + databases: &Databases, +) -> Result, StorageError> { + let mut targets: BTreeSet = BTreeSet::new(); + for entry in databases.reports_actionable_by_created_at.iter(txn)? { + let (key, _) = entry?; + let id = encoding::i32_from_key(&key[8..])?; + if let Some(report) = report_load(txn, databases, id)? { + targets.insert(( + report.reported_pubkey.map(|pubkey| *pubkey.as_bytes()), + report.reported_event_id.map(|id| *id.as_bytes()), + )); + } + } + Ok(targets + .into_iter() + .map(|(pubkey, event_id)| ActionableTarget { + reported_pubkey: pubkey.map(Pubkey::from_byte_array), + reported_event_id: event_id.map(EventId::from_byte_array), + }) + .collect()) +} diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs new file mode 100644 index 000000000..d9e22195b --- /dev/null +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -0,0 +1,946 @@ +//! The LMDB backend: one environment, the 29 named databases, and the nine repository traits. +//! Mirrors `src/repositories/` through the seam the Postgres backend also implements, and +//! `docs/rewrite/decisions/0010-lmdb-backend.md` records why each choice below was made. +//! +//! Every heed call runs inside `tokio::task::spawn_blocking`, because `Env::write_txn` blocks +//! its thread on the single writer's mutex and a read transaction must not be held across an +//! await. A repository method is one whole write transaction, so it is atomic across every +//! database it touches. + +mod encoding; +mod events; +mod invoices; +mod metadata; +mod notifications; +mod rows; +mod users; + +use std::path::PathBuf; +use std::sync::Arc; + +use chrono::Utc; +use heed::types::Bytes; +use heed::{Database, Env, RoTxn, RwTxn}; +use nostream_config::Settings; +use nostream_core::{EventId, EventStoreBackend, Filter, Kind, Pubkey}; + +use crate::backend::LmdbEnv; +use crate::error::StorageError; +use crate::repositories::{ + ActionableTarget, CreateInviteCodeOptions, DeliveryLogEntry, DeliveryLogFilters, DvmJob, + DvmJobRepository, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, + EventRetentionOptions, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, + InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, + NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, OutboxPayload, + QueryStream, ReconciliationItem, Report, ReportRepository, StorageFuture, User, UserPatch, + UserRepository, +}; + +/// The fixed map size every environment is opened with: `Env::resize` is `unsafe` and this +/// crate has one audited `unsafe` call, so the map is preallocated at 64 GiB and never grown. +pub const LMDB_MAP_SIZE: usize = 64 << 30; + +/// The number of named databases the environment reserves, comfortably above the 29 it opens. +pub const LMDB_MAX_DBS: u32 = 64; + +/// The number of concurrent read transactions the environment reserves. +pub const LMDB_MAX_READERS: u32 = 512; + +/// The capture's default page size (`DEFAULT_FILTER_LIMIT`). +pub(crate) const DEFAULT_FILTER_LIMIT: u64 = 500; + +/// The retention batch size (`RETENTION_BATCH_SIZE`). +pub(crate) const RETENTION_BATCH_SIZE: usize = 1_000; + +/// How many rows one stream poll copies before it drops its read transaction. +pub(crate) const STREAM_BATCH_ITEMS: usize = 256; + +/// The settings gates the capture reads per query. The LMDB backend records only whether +/// NIP-50 is on: it has no search implementation (ADR 0010, Out of scope). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Gates { + pub(crate) nip50: bool, + pub(crate) nip56: bool, +} + +/// The named databases of one environment. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Databases { + pub(crate) events: Database, + pub(crate) events_by_created_at: Database, + pub(crate) events_by_author: Database, + pub(crate) events_deleted: Database, + pub(crate) events_expiring: Database, + pub(crate) replaceable: Database, + pub(crate) tags: Database, + pub(crate) users: Database, + pub(crate) invoices: Database, + pub(crate) invoices_pending: Database, + pub(crate) nip05: Database, + pub(crate) nip05_by_last_checked_at: Database, + pub(crate) invite_codes: Database, + pub(crate) invite_codes_by_created_at: Database, + pub(crate) invite_codes_by_expires_at: Database, + pub(crate) dvm_jobs: Database, + pub(crate) dvm_jobs_by_status_created_at: Database, + pub(crate) reports: Database, + pub(crate) reports_by_event_id: Database, + pub(crate) reports_actionable_by_created_at: Database, + pub(crate) reports_actionable_event: Database, + pub(crate) reports_actionable_pubkey: Database, + pub(crate) outbox: Database, + pub(crate) outbox_by_status_created_at: Database, + pub(crate) outbox_by_status_updated_at: Database, + pub(crate) notification_delivery_log: Database, + pub(crate) delivery_log_by_created_at: Database, + pub(crate) delivery_log_by_outbox: Database, + pub(crate) meta: Database, +} + +impl Databases { + /// Creates (or opens) every database inside the caller's write transaction. + fn create(env: &Env, txn: &mut RwTxn<'_>) -> Result { + Ok(Self { + events: env.create_database(txn, Some("events"))?, + events_by_created_at: env.create_database(txn, Some("events_by_created_at"))?, + events_by_author: env.create_database(txn, Some("events_by_author"))?, + events_deleted: env.create_database(txn, Some("events_deleted"))?, + events_expiring: env.create_database(txn, Some("events_expiring"))?, + replaceable: env.create_database(txn, Some("replaceable"))?, + tags: env.create_database(txn, Some("tags"))?, + users: env.create_database(txn, Some("users"))?, + invoices: env.create_database(txn, Some("invoices"))?, + invoices_pending: env.create_database(txn, Some("invoices_pending"))?, + nip05: env.create_database(txn, Some("nip05_verifications"))?, + nip05_by_last_checked_at: env.create_database(txn, Some("nip05_by_last_checked_at"))?, + invite_codes: env.create_database(txn, Some("invite_codes"))?, + invite_codes_by_created_at: env + .create_database(txn, Some("invite_codes_by_created_at"))?, + invite_codes_by_expires_at: env + .create_database(txn, Some("invite_codes_by_expires_at"))?, + dvm_jobs: env.create_database(txn, Some("dvm_jobs"))?, + dvm_jobs_by_status_created_at: env + .create_database(txn, Some("dvm_jobs_by_status_created_at"))?, + reports: env.create_database(txn, Some("reports"))?, + reports_by_event_id: env.create_database(txn, Some("reports_by_event_id"))?, + reports_actionable_by_created_at: env + .create_database(txn, Some("reports_actionable_by_created_at"))?, + reports_actionable_event: env.create_database(txn, Some("reports_actionable_event"))?, + reports_actionable_pubkey: env + .create_database(txn, Some("reports_actionable_pubkey"))?, + outbox: env.create_database(txn, Some("notification_outbox"))?, + outbox_by_status_created_at: env + .create_database(txn, Some("outbox_by_status_created_at"))?, + outbox_by_status_updated_at: env + .create_database(txn, Some("outbox_by_status_updated_at"))?, + notification_delivery_log: env + .create_database(txn, Some("notification_delivery_log"))?, + delivery_log_by_created_at: env + .create_database(txn, Some("delivery_log_by_created_at"))?, + delivery_log_by_outbox: env.create_database(txn, Some("delivery_log_by_outbox"))?, + meta: env.create_database(txn, Some("meta"))?, + }) + } +} + +/// An embedded LMDB event store with its metadata tail, built from the root a caller supplies. +#[derive(Debug, Clone)] +pub struct LmdbBackend { + env: Env, + databases: Databases, + settings: Option>, +} + +impl LmdbBackend { + /// Opens (creating if needed) the store under `/lmdb` and creates every database. + pub async fn open(root: impl Into) -> Result { + let root = root.into(); + let path = crate::backend::data_dir(&root, EventStoreBackend::Lmdb).ok_or( + StorageError::InvalidArgument { + message: "the lmdb backend has no data directory", + }, + )?; + let env = tokio::task::spawn_blocking(move || LmdbEnv::open(path, LMDB_MAP_SIZE)) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)??; + let handle = env.env().clone(); + let databases = tokio::task::spawn_blocking(move || { + let mut transaction = handle.write_txn()?; + let databases = Databases::create(&handle, &mut transaction)?; + transaction.commit()?; + Ok::(databases) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)??; + + Ok(Self { + env: env.env().clone(), + databases, + settings: None, + }) + } + + /// Supplies the `nip50`/`nip56` gates. Absent settings mean both features are off. + pub fn with_settings(mut self, settings: Arc) -> Self { + self.settings = Some(settings); + self + } + + /// The environment handle, for an operator's tooling. + pub fn env(&self) -> &Env { + &self.env + } + + pub(crate) fn gates(&self) -> Gates { + let Some(settings) = &self.settings else { + return Gates::default(); + }; + Gates { + nip50: settings + .nip50 + .as_ref() + .is_some_and(|nip50| nip50.enabled == Some(true)), + nip56: settings + .nip56 + .as_ref() + .is_some_and(|nip56| nip56.enabled && nip56.hide_actionable_reports), + } + } + + /// Runs one write transaction on the blocking pool; an error rolls it back. + pub(crate) async fn write(&self, work: F) -> Result + where + T: Send + 'static, + F: FnOnce(&mut RwTxn<'_>, &Databases) -> Result + Send + 'static, + { + let env = self.env.clone(); + let databases = self.databases; + tokio::task::spawn_blocking(move || { + let mut transaction = env.write_txn()?; + let value = work(&mut transaction, &databases)?; + transaction.commit()?; + Ok(value) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)? + } + + /// Runs one read transaction on the blocking pool. + pub(crate) async fn read(&self, work: F) -> Result + where + T: Send + 'static, + F: FnOnce(&RoTxn<'_>, &Databases) -> Result + Send + 'static, + { + let env = self.env.clone(); + let databases = self.databases; + tokio::task::spawn_blocking(move || { + let transaction = env.read_txn()?; + work(&transaction, &databases) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)? + } +} + +impl EventRepository for LmdbBackend { + fn create<'a>(&'a self, record: &'a EventRecord) -> StorageFuture<'a, u64> { + Box::pin(async move { + let record = record.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + events::create(txn, databases, std::slice::from_ref(&record), now) + }) + .await + }) + } + + fn create_many<'a>(&'a self, records: &'a [EventRecord]) -> StorageFuture<'a, u64> { + Box::pin(async move { + let records = records.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| events::create(txn, databases, &records, now)) + .await + }) + } + + fn upsert<'a>(&'a self, record: &'a EventRecord) -> StorageFuture<'a, u64> { + Box::pin(async move { + let record = record.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| events::upsert(txn, databases, &record, now)) + .await + }) + } + + fn upsert_many<'a>(&'a self, records: &'a [EventRecord]) -> StorageFuture<'a, u64> { + Box::pin(async move { + let records = records.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| events::upsert_many(txn, databases, &records, now)) + .await + }) + } + + fn find_by_filters<'a>( + &'a self, + filters: &'a [Filter], + ) -> StorageFuture<'a, QueryStream<'a, EventRecord>> { + Box::pin(async move { + let filters = filters.to_vec(); + let gates = self.gates(); + let now = Utc::now().timestamp(); + let page = self + .read(move |txn, databases| events::page(txn, databases, &filters, &gates, now)) + .await?; + let stream: QueryStream<'_, EventRecord> = Box::pin(events::ValuesStream::new( + self.env.clone(), + self.databases, + &page, + )); + Ok(stream) + }) + } + + fn count_by_filters<'a>(&'a self, filters: &'a [Filter]) -> StorageFuture<'a, u64> { + Box::pin(async move { + let filters = filters.to_vec(); + let gates = self.gates(); + let now = Utc::now().timestamp(); + self.read(move |txn, databases| events::count(txn, databases, &filters, &gates, now)) + .await + }) + } + + fn find_reconciliation_items<'a>( + &'a self, + filter: &'a Filter, + ) -> StorageFuture<'a, QueryStream<'a, ReconciliationItem>> { + Box::pin(async move { + let filter = filter.clone(); + let gates = self.gates(); + let now = Utc::now().timestamp(); + let stream: QueryStream<'_, ReconciliationItem> = + Box::pin(events::ReconciliationStream::new( + self.env.clone(), + self.databases, + &filter, + gates, + now, + )); + Ok(stream) + }) + } + + fn delete_by_pubkey_and_ids<'a>( + &'a self, + pubkey: &'a Pubkey, + ids: &'a [EventId], + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let ids = ids.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + events::delete_by_pubkey_and_ids(txn, databases, &pubkey, &ids, now) + }) + .await + }) + } + + fn delete_by_pubkey_except_kinds<'a>( + &'a self, + pubkey: &'a Pubkey, + excluded_kinds: &'a [Kind], + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let kinds = excluded_kinds.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + events::delete_by_pubkey_except_kinds(txn, databases, &pubkey, &kinds, now) + }) + .await + }) + } + + fn has_active_request_to_vanish<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, bool> { + Box::pin(async move { + let pubkey = *pubkey; + self.read(move |txn, databases| { + events::has_active_request_to_vanish(txn, databases, &pubkey) + }) + .await + }) + } + + fn delete_expired_and_retained<'a>( + &'a self, + options: &'a EventRetentionOptions, + ) -> StorageFuture<'a, EventPurgeCounts> { + Box::pin(async move { + let options = options.clone(); + let now = Utc::now().timestamp(); + self.write(move |txn, databases| { + events::delete_expired_and_retained(txn, databases, &options, now) + }) + .await + }) + } +} + +impl InvoiceRepository for LmdbBackend { + fn find_by_id<'a>(&'a self, id: &'a str) -> StorageFuture<'a, Option> { + Box::pin(async move { + let id = id.to_owned(); + self.read(move |txn, databases| invoices::find_by_id(txn, databases, &id)) + .await + }) + } + + fn upsert<'a>(&'a self, invoice: &'a Invoice) -> StorageFuture<'a, u64> { + Box::pin(async move { + let invoice = invoice.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| invoices::upsert(txn, databases, &invoice, now)) + .await + }) + } + + fn update_status<'a>( + &'a self, + id: &'a str, + status: InvoiceStatus, + ) -> StorageFuture<'a, Option> { + Box::pin(async move { + let id = id.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + invoices::update_status(txn, databases, &id, status, now) + }) + .await + }) + } + + fn confirm_invoice<'a>( + &'a self, + invoice_id: &'a str, + amount_received: i64, + confirmed_at: chrono::DateTime, + ) -> StorageFuture<'a, ()> { + Box::pin(async move { + let invoice_id = invoice_id.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + invoices::confirm_invoice( + txn, + databases, + &invoice_id, + amount_received, + confirmed_at, + now, + ) + }) + .await + }) + } + + fn find_pending_invoices<'a>( + &'a self, + offset: u64, + limit: u64, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + self.read(move |txn, databases| { + invoices::find_pending_invoices(txn, databases, offset, limit) + }) + .await + }) + } +} + +impl UserRepository for LmdbBackend { + fn find_by_pubkey<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, Option> { + Box::pin(async move { + let pubkey = *pubkey; + self.read(move |txn, databases| users::find_by_pubkey(txn, databases, &pubkey)) + .await + }) + } + + fn upsert<'a>(&'a self, user: &'a UserPatch) -> StorageFuture<'a, u64> { + Box::pin(async move { + let user = user.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| users::upsert(txn, databases, &user, now)) + .await + }) + } + + fn get_balance_by_pubkey<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, i64> { + Box::pin(async move { + let pubkey = *pubkey; + self.read(move |txn, databases| users::get_balance_by_pubkey(txn, databases, &pubkey)) + .await + }) + } + + fn is_vanished<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, bool> { + Box::pin(async move { + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| users::is_vanished(txn, databases, &pubkey, now)) + .await + }) + } + + fn set_vanished<'a>(&'a self, pubkey: &'a Pubkey, vanished: bool) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + users::upsert_vanish_state(txn, databases, &pubkey, vanished, now) + }) + .await + }) + } + + fn admit_user<'a>( + &'a self, + pubkey: &'a Pubkey, + admitted_at: chrono::DateTime, + ) -> StorageFuture<'a, ()> { + Box::pin(async move { + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + users::admit_user(txn, databases, &pubkey, admitted_at, now) + }) + .await + }) + } + + fn revoke_admission<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| users::revoke_admission(txn, databases, &pubkey, now)) + .await + }) + } +} + +impl Nip05VerificationRepository for LmdbBackend { + fn find_by_pubkey<'a>( + &'a self, + pubkey: &'a Pubkey, + ) -> StorageFuture<'a, Option> { + Box::pin(async move { + let pubkey = *pubkey; + self.read(move |txn, databases| metadata::nip05_find_by_pubkey(txn, databases, &pubkey)) + .await + }) + } + + fn upsert<'a>(&'a self, verification: &'a Nip05Verification) -> StorageFuture<'a, u64> { + Box::pin(async move { + let verification = verification.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::nip05_upsert(txn, databases, &verification, now) + }) + .await + }) + } + + fn find_pending_verifications<'a>( + &'a self, + update_frequency_ms: i64, + max_failures: i32, + limit: u64, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let now = encoding::micros(Utc::now()); + self.read(move |txn, databases| { + metadata::nip05_find_pending( + txn, + databases, + update_frequency_ms, + max_failures, + limit, + now, + ) + }) + .await + }) + } + + fn delete_by_pubkey<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + self.write(move |txn, databases| { + metadata::nip05_delete_by_pubkey(txn, databases, &pubkey) + }) + .await + }) + } +} + +impl InviteCodeRepository for LmdbBackend { + fn create<'a>( + &'a self, + code: &'a str, + options: &'a CreateInviteCodeOptions, + ) -> StorageFuture<'a, InviteCode> { + Box::pin(async move { + let code = code.to_owned(); + let options = options.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::invite_create(txn, databases, &code, &options, now) + }) + .await + }) + } + + fn find_by_code<'a>(&'a self, code: &'a str) -> StorageFuture<'a, Option> { + Box::pin(async move { + let code = code.to_owned(); + self.read(move |txn, databases| metadata::invite_find_by_code(txn, databases, &code)) + .await + }) + } + + fn claim_code<'a>(&'a self, code: &'a str, pubkey: &'a Pubkey) -> StorageFuture<'a, bool> { + Box::pin(async move { + let code = code.to_owned(); + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::invite_claim_code(txn, databases, &code, &pubkey, now) + }) + .await + }) + } + + fn find_active_codes<'a>(&'a self, limit: u64) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let now = encoding::micros(Utc::now()); + self.read(move |txn, databases| { + metadata::invite_find_active_codes(txn, databases, limit, now) + }) + .await + }) + } + + fn delete_expired_codes<'a>(&'a self) -> StorageFuture<'a, u64> { + Box::pin(async move { + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| metadata::invite_delete_expired(txn, databases, now)) + .await + }) + } +} + +impl DvmJobRepository for LmdbBackend { + fn create<'a>( + &'a self, + id: &'a EventId, + requester: &'a Pubkey, + kind: Kind, + ) -> StorageFuture<'a, DvmJob> { + Box::pin(async move { + let id = *id; + let requester = *requester; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::dvm_create(txn, databases, &id, &requester, kind, now) + }) + .await + }) + } + + fn find_by_id<'a>(&'a self, id: &'a EventId) -> StorageFuture<'a, Option> { + Box::pin(async move { + let id = *id; + self.read(move |txn, databases| metadata::dvm_find_by_id(txn, databases, &id)) + .await + }) + } + + fn assign_worker<'a>(&'a self, id: &'a EventId, worker_index: i32) -> StorageFuture<'a, bool> { + Box::pin(async move { + let id = *id; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::dvm_assign_worker(txn, databases, &id, worker_index, now) + }) + .await + }) + } + + fn update_status<'a>(&'a self, update: &'a DvmJobUpdate) -> StorageFuture<'a, Option> { + Box::pin(async move { + let update = update.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::dvm_update_status(txn, databases, &update, now) + }) + .await + }) + } + + fn find_pending_jobs<'a>( + &'a self, + limit: u64, + kinds: Option<&'a [Kind]>, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let kinds = kinds.map(<[Kind]>::to_vec); + self.read(move |txn, databases| { + metadata::dvm_find_pending_jobs(txn, databases, limit, kinds.as_deref()) + }) + .await + }) + } +} + +impl ReportRepository for LmdbBackend { + fn create<'a>(&'a self, report: &'a NewReport) -> StorageFuture<'a, Report> { + Box::pin(async move { + let report = report.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| metadata::report_create(txn, databases, &report, now)) + .await + }) + } + + fn create_many<'a>(&'a self, reports: &'a [NewReport]) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let reports = reports.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + let mut created = Vec::with_capacity(reports.len()); + for report in &reports { + created.push(metadata::report_create(txn, databases, report, now)?); + } + Ok(created) + }) + .await + }) + } + + fn find_by_event_id<'a>(&'a self, event_id: &'a EventId) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let event_id = *event_id; + self.read(move |txn, databases| { + metadata::report_find_by_event_id(txn, databases, &event_id) + }) + .await + }) + } + + fn find_actionable<'a>(&'a self, limit: u64) -> StorageFuture<'a, Vec> { + Box::pin(async move { + self.read(move |txn, databases| metadata::report_find_actionable(txn, databases, limit)) + .await + }) + } + + fn find_actionable_targets<'a>(&'a self) -> StorageFuture<'a, Vec> { + Box::pin(async move { self.read(metadata::report_find_actionable_targets).await }) + } +} + +impl NotificationDeliveryLogRepository for LmdbBackend { + fn append<'a>(&'a self, entry: &'a NewDeliveryLogEntry) -> StorageFuture<'a, ()> { + Box::pin(async move { + let entry = entry.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| notifications::append(txn, databases, &entry, now)) + .await + }) + } + + fn find_recent<'a>( + &'a self, + limit: u64, + filters: &'a DeliveryLogFilters, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let filters = filters.clone(); + self.read(move |txn, databases| { + notifications::find_recent(txn, databases, limit, &filters) + }) + .await + }) + } + + fn find_successful_target_ids<'a>( + &'a self, + outbox_id: &'a str, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let outbox_id = outbox_id.to_owned(); + self.read(move |txn, databases| { + notifications::find_successful_target_ids(txn, databases, &outbox_id) + }) + .await + }) + } + + fn delete_older_than<'a>(&'a self, cutoff: chrono::DateTime) -> StorageFuture<'a, u64> { + Box::pin(async move { + self.write(move |txn, databases| { + notifications::delete_older_than(txn, databases, cutoff) + }) + .await + }) + } +} + +impl NotificationOutboxRepository for LmdbBackend { + fn enqueue<'a>( + &'a self, + event_type: &'a str, + payload: &'a OutboxPayload, + ) -> StorageFuture<'a, OutboxMessage> { + Box::pin(async move { + let event_type = event_type.to_owned(); + let payload = payload.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + notifications::enqueue(txn, databases, &event_type, &payload, now) + }) + .await + }) + } + + fn claim_batch<'a>(&'a self, limit: u64) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| notifications::claim_batch(txn, databases, limit, now)) + .await + }) + } + + fn mark_delivered<'a>(&'a self, id: &'a str) -> StorageFuture<'a, ()> { + Box::pin(async move { + let id = id.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + notifications::mark_delivered(txn, databases, &id, now) + }) + .await + }) + } + + fn mark_failed<'a>( + &'a self, + id: &'a str, + error: &'a str, + attempt_count: i32, + max_attempts: i32, + base_delay_ms: i64, + ) -> StorageFuture<'a, ()> { + Box::pin(async move { + let id = id.to_owned(); + let error = error.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + notifications::mark_failed( + txn, + databases, + &id, + &error, + attempt_count, + max_attempts, + base_delay_ms, + now, + ) + }) + .await + }) + } + + fn delete_terminal_older_than<'a>( + &'a self, + cutoff: chrono::DateTime, + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + self.write(move |txn, databases| { + notifications::delete_terminal_older_than(txn, databases, cutoff) + }) + .await + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + #[tokio::test] + async fn the_backend_opens_and_creates_every_database() { + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let transaction = backend.env.read_txn().expect("a read transaction"); + for name in [ + "events", + "events_by_created_at", + "events_by_author", + "events_deleted", + "events_expiring", + "replaceable", + "tags", + "users", + "meta", + "delivery_log_by_outbox", + ] { + let database = backend + .env + .open_database::(&transaction, Some(name)) + .expect("the database opens"); + assert!(database.is_some(), "{name} exists"); + } + } + + #[tokio::test] + async fn absent_settings_leave_both_gates_off() { + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let gates = backend.gates(); + assert!(!gates.nip50); + assert!(!gates.nip56); + } + + #[tokio::test] + async fn a_named_database_past_the_maximum_is_refused() { + // The environment reserves LMDB_MAX_DBS handles; the 29 the backend opens leave the + // rest free, and the 65th creation has to fail rather than silently succeed. + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let mut transaction = backend.env.write_txn().expect("a write transaction"); + let mut created = 0; + for index in 0..LMDB_MAX_DBS { + let name = format!("probe-{index}"); + if backend + .env + .create_database::(&mut transaction, Some(&name)) + .is_err() + { + break; + } + created += 1; + } + assert!(created < LMDB_MAX_DBS, "the handle limit is enforced"); + transaction.abort(); + } +} diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs new file mode 100644 index 000000000..5126d1a9b --- /dev/null +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -0,0 +1,488 @@ +//! `NotificationDeliveryLogRepository` and `NotificationOutboxRepository` against LMDB. +//! Mirrors `src/repositories/notification-delivery-log-repository.ts` and +//! `src/repositories/notification-outbox-repository.ts`. +//! +//! The outbox's `FOR UPDATE SKIP LOCKED` claim needs no port: LMDB has one writer, so a claim +//! cannot interleave with another claim, and it runs inside one write transaction. + +use std::collections::HashSet; + +use chrono::{DateTime, Duration, Utc}; +use heed::{RoTxn, RwTxn}; + +use super::Databases; +use super::encoding; +use super::rows; +use crate::error::StorageError; +use crate::repositories::{ + DeliveryLogEntry, DeliveryLogFilters, NOTIFICATION_OUTBOX_PROCESSING_LEASE_MS, + NewDeliveryLogEntry, OutboxMessage, OutboxPayload, OutboxStatus, +}; + +/// A text component of a composite key, terminated so the next component reads unambiguously. +fn text_part(value: &str) -> Vec { + let mut bytes = value.as_bytes().to_vec(); + bytes.push(0); + bytes +} + +fn text_from<'a>(bytes: &'a [u8], table: &'static str) -> Result<&'a str, StorageError> { + std::str::from_utf8(bytes).map_err(|_| StorageError::InvalidRow { + table, + column: "key", + value: "not utf-8".to_owned(), + }) +} + +fn stamp_key(value: DateTime) -> [u8; 8] { + encoding::i64_key(value.timestamp_micros()) +} + +fn new_id() -> String { + uuid::Uuid::new_v4().to_string() +} + +// --------------------------------------------------------------------------------------- +// The delivery log. +// --------------------------------------------------------------------------------------- + +fn delivery_created_key(entry: &DeliveryLogEntry) -> Vec { + let mut key = stamp_key(entry.created_at).to_vec(); + key.extend_from_slice(entry.id.as_bytes()); + key +} + +fn delivery_outbox_key(outbox_id: &str, entry: &DeliveryLogEntry) -> Vec { + let mut key = text_part(outbox_id); + key.extend_from_slice(&text_part(entry.status.as_str())); + key.extend_from_slice(entry.target_id.as_bytes()); + key +} + +fn delivery_load( + txn: &RoTxn<'_>, + databases: &Databases, + id: &str, +) -> Result, StorageError> { + let Some(value) = databases + .notification_delivery_log + .get(txn, id.as_bytes())? + else { + return Ok(None); + }; + rows::decode("notification_delivery_log", value).map(Some) +} + +/// `append`: one delivery attempt, timestamped now. The column's foreign key is enforced +/// here, because the store has no server to enforce it. +pub(crate) fn append( + txn: &mut RwTxn<'_>, + databases: &Databases, + entry: &NewDeliveryLogEntry, + now: DateTime, +) -> Result<(), StorageError> { + if let Some(outbox_id) = &entry.outbox_id + && databases.outbox.get(txn, outbox_id.as_bytes())?.is_none() + { + return Err(StorageError::InvalidArgument { + message: "notification_delivery_log.outbox_id must name an outbox row", + }); + } + + let entry = DeliveryLogEntry { + id: new_id(), + outbox_id: entry.outbox_id.clone(), + event_type: entry.event_type.clone(), + target_id: entry.target_id.clone(), + target_type: entry.target_type, + status: entry.status, + attempt_number: entry.attempt_number, + error_snippet: entry.error_snippet.clone(), + created_at: encoding::micros(now), + }; + let value = rows::encode("notification_delivery_log", &entry)?; + databases + .notification_delivery_log + .put(txn, entry.id.as_bytes(), &value)?; + databases + .delivery_log_by_created_at + .put(txn, &delivery_created_key(&entry), &[])?; + if let Some(outbox_id) = &entry.outbox_id { + databases + .delivery_log_by_outbox + .put(txn, &delivery_outbox_key(outbox_id, &entry), &[])?; + } + Ok(()) +} + +/// `find_recent`: newest first, optionally filtered by status and event type. +pub(crate) fn find_recent( + txn: &RoTxn<'_>, + databases: &Databases, + limit: u64, + filters: &DeliveryLogFilters, +) -> Result, StorageError> { + let mut entries = Vec::new(); + for entry in databases.delivery_log_by_created_at.rev_iter(txn)? { + let (key, _) = entry?; + let id = text_from(&key[8..], "delivery_log_by_created_at")?; + let Some(entry) = delivery_load(txn, databases, id)? else { + continue; + }; + if filters.status.is_some_and(|status| status != entry.status) + || filters + .event_type + .as_ref() + .is_some_and(|event_type| *event_type != entry.event_type) + { + continue; + } + entries.push(entry); + if entries.len() as u64 >= limit { + break; + } + } + Ok(entries) +} + +/// `findSuccessfulTargetIds`: the distinct targets that succeeded for one message. +pub(crate) fn find_successful_target_ids( + txn: &RoTxn<'_>, + databases: &Databases, + outbox_id: &str, +) -> Result, StorageError> { + let mut prefix = text_part(outbox_id); + prefix.extend_from_slice(&text_part( + crate::repositories::NotificationDeliveryStatus::Success.as_str(), + )); + + let mut seen = HashSet::new(); + let mut targets = Vec::new(); + for entry in databases.delivery_log_by_outbox.prefix_iter(txn, &prefix)? { + let (key, _) = entry?; + let target_id = text_from(&key[prefix.len()..], "delivery_log_by_outbox")?; + if seen.insert(target_id.to_owned()) { + targets.push(target_id.to_owned()); + } + } + Ok(targets) +} + +/// `deleteOlderThan`: prunes the log by creation time. +pub(crate) fn delete_older_than( + txn: &mut RwTxn<'_>, + databases: &Databases, + cutoff: DateTime, +) -> Result { + let cutoff = encoding::micros(cutoff).timestamp_micros(); + let mut expired = Vec::new(); + for entry in databases.delivery_log_by_created_at.iter(txn)? { + let (key, _) = entry?; + if encoding::i64_from_key(&key[..8])? >= cutoff { + break; + } + expired.push(text_from(&key[8..], "delivery_log_by_created_at")?.to_owned()); + } + + let mut affected = 0; + for id in expired { + let Some(entry) = delivery_load(txn, databases, &id)? else { + continue; + }; + databases + .notification_delivery_log + .delete(txn, id.as_bytes())?; + databases + .delivery_log_by_created_at + .delete(txn, &delivery_created_key(&entry))?; + if let Some(outbox_id) = &entry.outbox_id { + databases + .delivery_log_by_outbox + .delete(txn, &delivery_outbox_key(outbox_id, &entry))?; + } + affected += 1; + } + Ok(affected) +} + +// --------------------------------------------------------------------------------------- +// The outbox. +// --------------------------------------------------------------------------------------- + +fn outbox_created_key(message: &OutboxMessage) -> Vec { + let mut key = text_part(message.status.as_str()); + key.extend_from_slice(&stamp_key(message.created_at)); + key.extend_from_slice(message.id.as_bytes()); + key +} + +fn outbox_updated_key(message: &OutboxMessage) -> Vec { + let mut key = text_part(message.status.as_str()); + key.extend_from_slice(&stamp_key(message.updated_at)); + key.extend_from_slice(message.id.as_bytes()); + key +} + +fn outbox_load( + txn: &RoTxn<'_>, + databases: &Databases, + id: &str, +) -> Result, StorageError> { + let Some(value) = databases.outbox.get(txn, id.as_bytes())? else { + return Ok(None); + }; + rows::decode("notification_outbox", value).map(Some) +} + +/// Writes a message and its two status indexes, removing the entries of the row it replaces. +fn outbox_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + message: &OutboxMessage, + previous: Option<&OutboxMessage>, +) -> Result<(), StorageError> { + let value = rows::encode("notification_outbox", message)?; + databases.outbox.put(txn, message.id.as_bytes(), &value)?; + if let Some(previous) = previous { + databases + .outbox_by_status_created_at + .delete(txn, &outbox_created_key(previous))?; + databases + .outbox_by_status_updated_at + .delete(txn, &outbox_updated_key(previous))?; + } + databases + .outbox_by_status_created_at + .put(txn, &outbox_created_key(message), &[])?; + databases + .outbox_by_status_updated_at + .put(txn, &outbox_updated_key(message), &[])?; + Ok(()) +} + +/// `enqueue`: inserts a pending message and returns the stored row. +pub(crate) fn enqueue( + txn: &mut RwTxn<'_>, + databases: &Databases, + event_type: &str, + payload: &OutboxPayload, + now: DateTime, +) -> Result { + let now = encoding::micros(now); + let message = OutboxMessage { + id: new_id(), + event_type: event_type.to_owned(), + payload: payload.clone(), + status: OutboxStatus::Pending, + attempt_count: 0, + available_at: now, + last_error: None, + delivered_at: None, + created_at: now, + updated_at: now, + }; + outbox_write(txn, databases, &message, None)?; + Ok(message) +} + +/// `claimBatch`: pending rows that are due and processing rows whose lease expired, oldest +/// first, marked processing in the same write transaction. +pub(crate) fn claim_batch( + txn: &mut RwTxn<'_>, + databases: &Databases, + limit: u64, + now: DateTime, +) -> Result, StorageError> { + let now = encoding::micros(now); + let stale = now - Duration::milliseconds(NOTIFICATION_OUTBOX_PROCESSING_LEASE_MS); + + let mut candidates = Vec::new(); + for status in [OutboxStatus::Pending, OutboxStatus::Processing] { + let prefix = text_part(status.as_str()); + for entry in databases + .outbox_by_status_created_at + .prefix_iter(txn, &prefix)? + { + let (key, _) = entry?; + let id = text_from(&key[prefix.len() + 8..], "outbox_by_status_created_at")?; + let Some(message) = outbox_load(txn, databases, id)? else { + continue; + }; + let due = match status { + OutboxStatus::Pending => message.available_at <= now, + OutboxStatus::Processing => message.updated_at < stale, + OutboxStatus::Delivered | OutboxStatus::Dead => false, + }; + if due { + candidates.push(message); + } + } + } + candidates.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.id.cmp(&right.id)) + }); + candidates.truncate(usize::try_from(limit).unwrap_or(usize::MAX)); + + let mut claimed = Vec::with_capacity(candidates.len()); + for message in candidates { + let previous = message.clone(); + let mut message = message; + message.status = OutboxStatus::Processing; + message.updated_at = now; + outbox_write(txn, databases, &message, Some(&previous))?; + claimed.push(message); + } + Ok(claimed) +} + +/// `markDelivered`: delivered, timestamped, error cleared. +pub(crate) fn mark_delivered( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &str, + now: DateTime, +) -> Result<(), StorageError> { + let Some(mut message) = outbox_load(txn, databases, id)? else { + return Ok(()); + }; + let previous = message.clone(); + let now = encoding::micros(now); + message.status = OutboxStatus::Delivered; + message.delivered_at = Some(now); + message.last_error = None; + message.updated_at = now; + outbox_write(txn, databases, &message, Some(&previous)) +} + +/// `markFailed`: a retry with capped exponential backoff, or the dead transition. +#[allow(clippy::too_many_arguments)] +pub(crate) fn mark_failed( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &str, + error: &str, + attempt_count: i32, + max_attempts: i32, + base_delay_ms: i64, + now: DateTime, +) -> Result<(), StorageError> { + let Some(mut message) = outbox_load(txn, databases, id)? else { + return Ok(()); + }; + let previous = message.clone(); + let now = encoding::micros(now); + let dead = attempt_count >= max_attempts; + message.status = if dead { + OutboxStatus::Dead + } else { + OutboxStatus::Pending + }; + message.attempt_count = attempt_count; + message.available_at = if dead { + now + } else { + now + Duration::milliseconds(backoff_ms(base_delay_ms, attempt_count)) + }; + message.last_error = Some(truncate_bytes(error, 2_000)); + message.updated_at = now; + outbox_write(txn, databases, &message, Some(&previous)) +} + +/// `deleteTerminalOlderThan`: prunes delivered and dead messages, and nulls the outbox id of +/// every delivery-log entry that named one. +pub(crate) fn delete_terminal_older_than( + txn: &mut RwTxn<'_>, + databases: &Databases, + cutoff: DateTime, +) -> Result { + let cutoff = encoding::micros(cutoff).timestamp_micros(); + let mut terminal = Vec::new(); + for status in [OutboxStatus::Delivered, OutboxStatus::Dead] { + let prefix = text_part(status.as_str()); + for entry in databases + .outbox_by_status_updated_at + .prefix_iter(txn, &prefix)? + { + let (key, _) = entry?; + if encoding::i64_from_key(&key[prefix.len()..prefix.len() + 8])? >= cutoff { + break; + } + terminal.push( + text_from(&key[prefix.len() + 8..], "outbox_by_status_updated_at")?.to_owned(), + ); + } + } + + let mut affected = 0; + for id in terminal { + let Some(message) = outbox_load(txn, databases, &id)? else { + continue; + }; + databases.outbox.delete(txn, id.as_bytes())?; + databases + .outbox_by_status_created_at + .delete(txn, &outbox_created_key(&message))?; + databases + .outbox_by_status_updated_at + .delete(txn, &outbox_updated_key(&message))?; + null_delivery_outbox(txn, databases, &id)?; + affected += 1; + } + Ok(affected) +} + +/// Sets `outbox_id` to `NULL` on every delivery-log row that named this outbox row. +fn null_delivery_outbox( + txn: &mut RwTxn<'_>, + databases: &Databases, + outbox_id: &str, +) -> Result<(), StorageError> { + let mut named = Vec::new(); + for entry in databases.notification_delivery_log.iter(txn)? { + let (_, value) = entry?; + let row: DeliveryLogEntry = rows::decode("notification_delivery_log", value)?; + if row.outbox_id.as_deref() == Some(outbox_id) { + named.push(row); + } + } + + for mut row in named { + databases + .delivery_log_by_outbox + .delete(txn, &delivery_outbox_key(outbox_id, &row))?; + row.outbox_id = None; + let value = rows::encode("notification_delivery_log", &row)?; + databases + .notification_delivery_log + .put(txn, row.id.as_bytes(), &value)?; + } + Ok(()) +} + +/// `min(60_000, max(0, base_delay_ms) * 2^max(0, attempt_count - 1))`, the same rule as +/// `postgres::notifications::backoff_ms`; kept local because that module is behind its own +/// feature and neither backend may depend on the other's. +fn backoff_ms(base_delay_ms: i64, attempt_count: i32) -> i64 { + let base = i128::from(base_delay_ms.max(0)); + let exponent = u32::try_from(attempt_count.saturating_sub(1).max(0)).unwrap_or(u32::MAX); + let factor = if exponent >= 62 { + i128::MAX + } else { + 1i128 << exponent + }; + i64::try_from(base.saturating_mul(factor).min(60_000)).unwrap_or(60_000) +} + +/// Truncates at a character boundary, keeping at most `limit` bytes. +fn truncate_bytes(value: &str, limit: usize) -> String { + if value.len() <= limit { + return value.to_owned(); + } + let mut end = limit; + while end > 0 && !value.is_char_boundary(end) { + end -= 1; + } + value[..end].to_owned() +} diff --git a/crates/nostream-storage/src/lmdb/rows.rs b/crates/nostream-storage/src/lmdb/rows.rs new file mode 100644 index 000000000..013f56c11 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/rows.rs @@ -0,0 +1,189 @@ +//! Per-row encode and decode for the LMDB store: one format byte, the event's fixed header, +//! then `serde_json` of the row the seam already declares. + +use chrono::{DateTime, Utc}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use super::encoding::{self, EVENT_HEADER_LEN, EventHeader, ROW_FORMAT_VERSION}; +use crate::error::StorageError; +use crate::repositories::{EventRecord, User}; + +/// One stored `users` row: the seam's [`User`] plus `tos_accepted_at`, which +/// `fromDBUser` omits from the read shape but a write still carries. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub(crate) struct StoredUser { + pub pubkey: nostream_core::Pubkey, + pub is_admitted: bool, + pub is_vanished: bool, + pub balance: i64, + pub tos_accepted_at: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl StoredUser { + /// A new row with the capture's column defaults. + pub(crate) fn new(pubkey: nostream_core::Pubkey, now: DateTime) -> Self { + Self { + pubkey, + is_admitted: false, + is_vanished: false, + balance: 0, + tos_accepted_at: None, + created_at: now, + updated_at: now, + } + } + + /// The read shape the seam returns (`fromDBUser` drops `tos_accepted_at`). + pub(crate) fn user(self) -> User { + User { + pubkey: self.pubkey, + is_admitted: self.is_admitted, + is_vanished: self.is_vanished, + balance: self.balance, + created_at: self.created_at, + updated_at: self.updated_at, + } + } +} + +/// Encodes one value: the format byte, then the JSON document. +pub(crate) fn encode(table: &'static str, row: &T) -> Result, StorageError> +where + T: Serialize, +{ + let mut value = Vec::from([ROW_FORMAT_VERSION]); + value.extend( + serde_json::to_vec(row).map_err(|error| StorageError::InvalidRow { + table, + column: "value", + value: error.to_string(), + })?, + ); + Ok(value) +} + +/// Decodes one value written by [`encode`]. +pub(crate) fn decode(table: &'static str, value: &[u8]) -> Result +where + T: DeserializeOwned, +{ + encoding::check_version(table, value)?; + serde_json::from_slice(&value[1..]).map_err(|error| StorageError::InvalidRow { + table, + column: "value", + value: error.to_string(), + }) +} + +/// Encodes one event value: version byte, fixed header, JSON of the whole record. +pub(crate) fn encode_event(record: &EventRecord) -> Result, StorageError> { + let kind = i32::try_from(record.event.kind.0).map_err(|_| StorageError::ValueOutOfRange { + column: "event_kind", + value: record.event.kind.0.to_string(), + })?; + let header = EventHeader { + deleted: record.deleted_at.is_some(), + created_at: record.event.created_at, + kind, + expires_at: record.expires_at, + pubkey: record.event.pubkey, + }; + let mut value = Vec::with_capacity(1 + EVENT_HEADER_LEN + 256); + value.push(ROW_FORMAT_VERSION); + value.extend_from_slice(&encoding::header_bytes(&header)); + value.extend( + serde_json::to_vec(record).map_err(|error| StorageError::InvalidRow { + table: "events", + column: "value", + value: error.to_string(), + })?, + ); + Ok(value) +} + +/// Decodes one event value. `row_id` is backend-defined and this backend writes none. +pub(crate) fn decode_event(value: &[u8]) -> Result { + let body = encoding::body("events", value)?; + let mut record: EventRecord = + serde_json::from_slice(body).map_err(|error| StorageError::InvalidRow { + table: "events", + column: "value", + value: error.to_string(), + })?; + record.row_id = None; + Ok(record) +} + +/// Decodes one event value's fixed header, without its JSON body. +pub(crate) fn event_header(value: &[u8]) -> Result { + encoding::event_header("events", value) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use nostream_core::Event; + + fn event() -> Event { + serde_json::from_value(serde_json::json!({ + "id": "0d684e8ec2431de586aa3cafbee2f6d308d19b28805e53deabcac3220e9136a5", + "pubkey": "f43c1f9bff677b8f27b602725ea0ad51af221344f69a6b352a74991a4479bac3", + "created_at": 1660407625i64, + "kind": 3, + "tags": [["p", "b34417513f66497d7b0e1a8406b6689ac32afb184027717e57d281ea19186315"]], + "content": "hello", + "sig": "ab".repeat(64) + })) + .expect("a wire event") + } + + #[test] + fn the_header_carries_the_live_state_without_decoding_the_body() { + let mut record = EventRecord::from_event(&event()).with_expires_at(2_000_000_000); + record.deleted_at = Some(DateTime::UNIX_EPOCH); + let value = encode_event(&record).expect("an encoded event"); + let header = event_header(&value).expect("a header"); + assert!(header.deleted); + assert_eq!(header.expires_at, Some(2_000_000_000)); + assert_eq!(header.created_at, 1_660_407_625); + assert_eq!(header.kind, 3); + assert_eq!(header.pubkey, record.event.pubkey); + } + + #[test] + fn a_value_without_a_body_is_an_invalid_row() { + let mut value = Vec::from([ROW_FORMAT_VERSION]); + value.extend_from_slice(&[0u8; EVENT_HEADER_LEN]); + let error = decode_event(&value).expect_err("no body"); + assert!(matches!(error, StorageError::InvalidRow { .. })); + } + + #[test] + fn a_tail_row_round_trips_behind_the_version_byte() { + let user = StoredUser::new(event().pubkey, DateTime::UNIX_EPOCH); + let value = encode("users", &user).expect("an encoded user"); + assert_eq!(value[0], ROW_FORMAT_VERSION); + let decoded: StoredUser = decode("users", &value).expect("a decoded user"); + assert_eq!(decoded.pubkey, user.pubkey); + assert!(!decoded.is_admitted); + } + + #[test] + fn a_bumped_version_is_refused_by_every_row_type() { + let user = StoredUser::new(event().pubkey, DateTime::UNIX_EPOCH); + let mut value = encode("users", &user).expect("an encoded user"); + value[0] = 2; + let error = decode::("users", &value).expect_err("version 2"); + match error { + StorageError::InvalidRow { value, .. } => { + assert_eq!(value, "unsupported format version 2"); + } + other => panic!("expected InvalidRow, got {other:?}"), + } + } +} diff --git a/crates/nostream-storage/src/lmdb/users.rs b/crates/nostream-storage/src/lmdb/users.rs new file mode 100644 index 000000000..1bd5f19e6 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/users.rs @@ -0,0 +1,148 @@ +//! `UserRepository` against LMDB. Mirrors `src/repositories/user-repository.ts`. +//! +//! A user row is one value in the `users` database, keyed by the 32-byte pubkey. Every write +//! is one whole write transaction, so `admit_user` and the balance credit are atomic against +//! every other database. + +use chrono::{DateTime, Utc}; +use heed::{RoTxn, RwTxn}; +use nostream_core::Pubkey; + +use super::Databases; +use super::encoding; +use super::events; +use super::rows::{self, StoredUser}; +use crate::error::StorageError; +use crate::repositories::{FieldUpdate, User, UserPatch}; + +/// The stored row of one pubkey. +pub(crate) fn load( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result, StorageError> { + let Some(value) = databases.users.get(txn, pubkey.as_bytes())? else { + return Ok(None); + }; + rows::decode("users", value).map(Some) +} + +/// Writes one user row, preserving nothing the caller did not supply. +pub(crate) fn store( + txn: &mut RwTxn<'_>, + databases: &Databases, + stored: &StoredUser, +) -> Result<(), StorageError> { + let value = rows::encode("users", stored)?; + databases.users.put(txn, stored.pubkey.as_bytes(), &value)?; + Ok(()) +} + +/// `findByPubkey`: the full row as the capture's `fromDBUser` maps it. +pub(crate) fn find_by_pubkey( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result, StorageError> { + Ok(load(txn, databases, pubkey)?.map(StoredUser::user)) +} + +/// `upsert`: admission, the vanished flag and `updated_at`, plus `tos_accepted_at` only when +/// the patch carries it. `balance` and `created_at` are never touched. +pub(crate) fn upsert( + txn: &mut RwTxn<'_>, + databases: &Databases, + patch: &UserPatch, + now: DateTime, +) -> Result { + let mut stored = match load(txn, databases, &patch.pubkey)? { + Some(stored) => stored, + None => StoredUser::new(patch.pubkey, encoding::micros(now)), + }; + stored.is_admitted = patch.is_admitted; + stored.is_vanished = patch.is_vanished; + stored.updated_at = encoding::micros(now); + match patch.tos_accepted_at { + FieldUpdate::Absent => {} + FieldUpdate::Clear => stored.tos_accepted_at = None, + FieldUpdate::Set(value) => stored.tos_accepted_at = Some(encoding::micros(value)), + } + store(txn, databases, &stored)?; + Ok(1) +} + +/// `getBalanceByPubkey`: the stored balance, `0` when no row exists. +pub(crate) fn get_balance_by_pubkey( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result { + Ok(load(txn, databases, pubkey)?.map_or(0, |stored| stored.balance)) +} + +/// `upsertVanishState`: inserts a minimal row or updates only the vanished flag. +pub(crate) fn upsert_vanish_state( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + vanished: bool, + now: DateTime, +) -> Result { + let mut stored = match load(txn, databases, pubkey)? { + Some(stored) => stored, + None => StoredUser::new(*pubkey, encoding::micros(now)), + }; + stored.is_vanished = vanished; + stored.updated_at = encoding::micros(now); + store(txn, databases, &stored)?; + Ok(1) +} + +/// The capture's `isVanished`: the stored flag, or one hydrated from a live vanish request. +pub(crate) fn is_vanished( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + now: DateTime, +) -> Result { + if let Some(stored) = load(txn, databases, pubkey)? { + return Ok(stored.is_vanished); + } + let vanished = events::has_active_request_to_vanish(txn, databases, pubkey)?; + upsert_vanish_state(txn, databases, pubkey, vanished, now)?; + Ok(vanished) +} + +/// `admitUser`: the schema function's upsert, which a write transaction makes atomic. +pub(crate) fn admit_user( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + admitted_at: DateTime, + now: DateTime, +) -> Result<(), StorageError> { + let mut stored = match load(txn, databases, pubkey)? { + Some(stored) => stored, + None => StoredUser::new(*pubkey, encoding::micros(now)), + }; + stored.is_admitted = true; + stored.tos_accepted_at = Some(encoding::micros(admitted_at)); + stored.updated_at = encoding::micros(now); + store(txn, databases, &stored) +} + +/// `revokeAdmission`: a targeted update so unrelated columns are never clobbered. +pub(crate) fn revoke_admission( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + now: DateTime, +) -> Result { + let Some(mut stored) = load(txn, databases, pubkey)? else { + return Ok(0); + }; + stored.is_admitted = false; + stored.updated_at = encoding::micros(now); + store(txn, databases, &stored)?; + Ok(1) +} From 391ee8d9b068ba44121f327e203221090029f414 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:19:57 -0500 Subject: [PATCH 03/60] test(storage): replay the parity fixture through a shared nine-trait executor and add the lmdb arm --- crates/nostream-storage/tests/common/mod.rs | 890 +++++++++++++++++- crates/nostream-storage/tests/parity.rs | 942 +------------------ crates/nostream-storage/tests/parity_lmdb.rs | 516 ++++++++++ crates/nostream-storage/tests/storage.rs | 74 +- 4 files changed, 1508 insertions(+), 914 deletions(-) create mode 100644 crates/nostream-storage/tests/parity_lmdb.rs diff --git a/crates/nostream-storage/tests/common/mod.rs b/crates/nostream-storage/tests/common/mod.rs index 1c0034750..ffba90522 100644 --- a/crates/nostream-storage/tests/common/mod.rs +++ b/crates/nostream-storage/tests/common/mod.rs @@ -1,17 +1,83 @@ //! Helpers shared by the storage integration tests. A directory module, so cargo does not //! build it as a target of its own. +//! +//! The executor below is the replay both parity targets share: it takes the nine repository +//! traits as `&dyn` references, so the same script drives Postgres and LMDB. It cannot reach +//! a serializable transaction or raw SQL, which is why the Postgres target keeps +//! `user.tos_accepted_at` and the `transaction` groups. #![allow(dead_code)] +use std::collections::HashMap; +use std::future::poll_fn; + +use chrono::{DateTime, Utc}; use nostream_config::Settings; -use serde_json::Value; +use serde_json::{Value, json}; + +use nostream_core::{Filter, Kind}; +use nostream_storage::repositories::{ + ActionableTarget, CreateInviteCodeOptions, DeliveryLogEntry, DeliveryLogFilters, DvmJob, + DvmJobRepository, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, + EventRetentionOptions, FieldUpdate, InviteCode, InviteCodeRepository, Invoice, + InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, + Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, + OutboxMessage, QueryStream, ReconciliationItem, Report, ReportRepository, User, UserPatch, + UserRepository, +}; + +/// The nine repositories one replay calls, as the traits rather than a concrete backend. +#[derive(Clone, Copy)] +pub struct Repos<'a> { + pub events: &'a dyn EventRepository, + pub invoices: &'a dyn InvoiceRepository, + pub users: &'a dyn UserRepository, + pub nip05: &'a dyn Nip05VerificationRepository, + pub invites: &'a dyn InviteCodeRepository, + pub dvm: &'a dyn DvmJobRepository, + pub reports: &'a dyn ReportRepository, + pub delivery_log: &'a dyn NotificationDeliveryLogRepository, + pub outbox: &'a dyn NotificationOutboxRepository, +} + +impl<'a> Repos<'a> { + /// The nine traits one backend implements. + pub fn all(backend: &'a T) -> Self + where + T: EventRepository + + InvoiceRepository + + UserRepository + + Nip05VerificationRepository + + InviteCodeRepository + + DvmJobRepository + + ReportRepository + + NotificationDeliveryLogRepository + + NotificationOutboxRepository, + { + Self { + events: backend, + invoices: backend, + users: backend, + nip05: backend, + invites: backend, + dvm: backend, + reports: backend, + delivery_log: backend, + outbox: backend, + } + } +} + +/// The capture's recording, compiled into the test binary. +pub fn fixture() -> Value { + serde_json::from_str(include_str!("../fixtures/repository-parity.json")) + .expect("tests/fixtures/repository-parity.json is valid JSON") +} /// The shipped defaults with a fixture's overrides applied — the settings both the capture /// and the replay load, so the gates under test are the fixture's. `PARITY_NIP50_LANGUAGE`, -/// when exported, replaces the fixture's language for every target that uses this helper -/// (both database targets), which is how the plan guard sees a deployment's language. -/// `PARITY_NIP50_LANGUAGE` overrides the fixture's `nip50.language`, so the NIP-50 plan -/// guard can be run against a deployed configuration (ADR 0003, revisit trigger 4). +/// when exported, replaces the fixture's language, which is how the plan guard sees a +/// deployment's language (ADR 0003, revisit trigger 4). pub fn settings_from_fixture(fixture: &Value) -> Settings { let raw = std::fs::read_to_string(concat!( env!("CARGO_MANIFEST_DIR"), @@ -37,3 +103,817 @@ pub fn settings_from_fixture(fixture: &Value) -> Settings { } serde_yaml::from_value(document).expect("the settings fit the typed schema") } + +// --------------------------------------------------------------------------------------- +// Encoders mirroring the generator. +// --------------------------------------------------------------------------------------- + +pub fn iso(value: &DateTime) -> Value { + Value::String(value.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) +} + +pub fn optional_iso(value: &Option>) -> Value { + match value { + Some(value) => iso(value), + None => Value::Null, + } +} + +pub fn parse_iso(value: &Value) -> DateTime { + DateTime::parse_from_rfc3339(value.as_str().expect("an ISO timestamp")) + .expect("a valid ISO timestamp") + .with_timezone(&Utc) +} + +pub fn text(value: &Value, key: &str) -> String { + value[key] + .as_str() + .unwrap_or_else(|| panic!("{key} is a string")) + .to_owned() +} + +pub fn integer(value: &Value, key: &str) -> i64 { + value[key] + .as_i64() + .unwrap_or_else(|| panic!("{key} is an integer")) +} + +pub fn optional_text(value: &Value, key: &str) -> Option { + value.get(key).and_then(Value::as_str).map(str::to_owned) +} + +pub fn text_of(value: &Value) -> String { + value.as_str().expect("a string").to_owned() +} + +pub fn event_row(record: &EventRecord) -> Value { + json!([ + record.event.id.to_string(), + record.event.created_at, + record.expires_at, + record.deleted_at.is_some(), + record.event_deduplication, + ]) +} + +pub fn event_record(value: &Value) -> EventRecord { + let event: nostream_core::Event = + serde_json::from_value(value["event"].clone()).expect("a wire event"); + let mut record = EventRecord::from_event(&event); + if let Some(address) = optional_text(value, "remote_address") { + record = record.with_remote_address(address.parse().expect("an IP address")); + } + if let Some(expires_at) = value.get("expires_at").and_then(Value::as_i64) { + record = record.with_expires_at(expires_at); + } + if let Some(deduplication) = value.get("deduplication") { + record = record.with_deduplication( + serde_json::from_value(deduplication.clone()).expect("deduplication values"), + ); + } + record +} + +pub fn invoice_view(invoice: &Invoice) -> Value { + json!({ + "id": invoice.id, + "pubkey": invoice.pubkey.to_string(), + "bolt11": invoice.bolt11, + "amount_requested": invoice.amount_requested, + "amount_paid": invoice.amount_paid, + "unit": invoice.unit.as_str(), + "status": invoice.status.as_str(), + "description": invoice.description, + "confirmed_at": optional_iso(&invoice.confirmed_at), + "expires_at": optional_iso(&invoice.expires_at), + "verify_url": invoice.verify_url, + }) +} + +pub fn optional_invoice_view(invoice: &Option) -> Value { + match invoice { + Some(invoice) => invoice_view(invoice), + None => Value::Null, + } +} + +pub fn user_view(user: &User) -> Value { + json!({ + "pubkey": user.pubkey.to_string(), + "is_admitted": user.is_admitted, + "is_vanished": user.is_vanished, + "balance": user.balance, + }) +} + +pub fn optional_user_view(user: &Option) -> Value { + match user { + Some(user) => user_view(user), + None => Value::Null, + } +} + +pub fn nip05_view(verification: &Nip05Verification) -> Value { + json!({ + "pubkey": verification.pubkey.to_string(), + "nip05": verification.nip05, + "domain": verification.domain, + "is_verified": verification.is_verified, + "last_verified_at": optional_iso(&verification.last_verified_at), + "last_checked_at": iso(&verification.last_checked_at), + "failure_count": verification.failure_count, + }) +} + +pub fn invite_view(invite: &InviteCode) -> Value { + json!({ + "code": invite.code, + "created_by": invite.created_by.map(|pubkey| pubkey.to_string()), + "claimed_by": invite.claimed_by.map(|pubkey| pubkey.to_string()), + "expires_at": optional_iso(&invite.expires_at), + "remaining_uses": invite.remaining_uses, + }) +} + +pub fn dvm_view(job: &DvmJob) -> Value { + json!({ + "id": job.id.to_string(), + "requester_pubkey": job.requester_pubkey.to_string(), + "kind": job.kind.0, + "worker_index": job.worker_index, + "status": job.status.as_str(), + "result_event_id": job.result_event_id.map(|id| id.to_string()), + "error": job.error, + }) +} + +pub fn optional_dvm_view(job: &Option) -> Value { + match job { + Some(job) => dvm_view(job), + None => Value::Null, + } +} + +pub fn report_view(report: &Report) -> Value { + json!({ + "id": report.id, + "event_id": report.event_id.to_string(), + "reporter_pubkey": report.reporter_pubkey.to_string(), + "reported_pubkey": report.reported_pubkey.map(|pubkey| pubkey.to_string()), + "reported_event_id": report.reported_event_id.map(|id| id.to_string()), + "report_type": report.report_type.as_str(), + "weight": report.weight, + "actionable": report.actionable, + }) +} + +pub fn delivery_view(entry: &DeliveryLogEntry) -> Value { + json!({ + "outbox_id": entry.outbox_id, + "event_type": entry.event_type, + "target_id": entry.target_id, + "target_type": entry.target_type.as_str(), + "status": entry.status.as_str(), + "attempt_number": entry.attempt_number, + "error_snippet": entry.error_snippet, + }) +} + +pub fn outbox_view(message: &OutboxMessage) -> Value { + json!({ + "event_type": message.event_type, + "payload": message.payload, + "status": message.status.as_str(), + "attempt_count": message.attempt_count, + "last_error": message.last_error, + }) +} + +pub fn sort_by_key(items: &mut [T], key: impl Fn(&T) -> String) { + items.sort_by_key(|item| key(item)); +} + +pub async fn collect( + mut stream: QueryStream<'_, T>, +) -> Result, nostream_storage::StorageError> { + let mut items = Vec::new(); + while let Some(item) = poll_fn(|context| stream.as_mut().poll_next(context)).await { + items.push(item?); + } + Ok(items) +} + +fn field_update(value: &Value) -> FieldUpdate { + match value["kind"].as_str().expect("a FieldUpdate kind") { + "absent" => FieldUpdate::Absent, + "clear" => FieldUpdate::Clear, + _ => FieldUpdate::Set(value["value"].clone()), + } +} + +fn parse_event_id(value: &Value) -> nostream_core::EventId { + value + .as_str() + .expect("an event id") + .parse() + .expect("a valid event id") +} + +fn parse_pubkey(value: &Value) -> nostream_core::Pubkey { + value + .as_str() + .expect("a pubkey") + .parse() + .expect("a valid pubkey") +} + +fn resolve_ref( + refs: &HashMap, + request: &Value, +) -> Result { + let event_type = request + .get("enqueue_event_type") + .or_else(|| request.get("outbox_id_from")) + .and_then(Value::as_str) + .expect("an enqueue reference"); + refs.get(event_type) + .cloned() + .ok_or(nostream_storage::StorageError::InvalidArgument { + message: "no enqueued row to address", + }) +} + +fn new_report(value: &Value) -> NewReport { + NewReport { + event_id: parse_event_id(&value["event_id"]), + reporter_pubkey: parse_pubkey(&value["reporter_pubkey"]), + reported_pubkey: optional_text(value, "reported_pubkey") + .map(|pubkey| pubkey.parse().expect("a pubkey")), + reported_event_id: optional_text(value, "reported_event_id") + .map(|id| id.parse().expect("an event id")), + report_type: value["report_type"] + .as_str() + .expect("a report type") + .parse() + .expect("a report type"), + weight: value["weight"].as_f64().expect("a weight") as f32, + actionable: value["actionable"].as_bool().expect("actionable"), + } +} + +/// One script entry, through the nine traits. +/// +/// The two arms this executor cannot run are `user.tos_accepted_at` (raw SQL, no seam form) +/// and `transaction` (no transaction type on the LMDB seam); the Postgres target handles the +/// former, and each target's replay loop handles the latter. +pub async fn run_op( + repos: Repos<'_>, + op: &Value, + refs: &mut HashMap, +) -> Result { + let name = op["op"].as_str().expect("an op name"); + let request = &op["request"]; + + match name { + "events.create" => { + let record = event_record(&request["record"]); + Ok(json!({ "count": repos.events.create(&record).await? })) + } + "events.create_many" => { + let records: Vec = request["records"] + .as_array() + .expect("records") + .iter() + .map(event_record) + .collect(); + Ok(json!({ "count": repos.events.create_many(&records).await? })) + } + "events.upsert" => { + let record = event_record(&request["record"]); + Ok(json!({ "count": repos.events.upsert(&record).await? })) + } + "events.upsert_many" => { + let records: Vec = request["records"] + .as_array() + .expect("records") + .iter() + .map(event_record) + .collect(); + Ok(json!({ "count": repos.events.upsert_many(&records).await? })) + } + "events.find_by_filters" => { + let filters: Vec = request["filters"] + .as_array() + .expect("filters") + .iter() + .map(|filter| serde_json::from_value(filter.clone()).expect("a filter")) + .collect(); + if filters.is_empty() { + let failed = repos.events.find_by_filters(&[]).await.is_err(); + return Ok(json!({ "error": failed })); + } + let rows = collect(repos.events.find_by_filters(&filters).await?).await?; + Ok(json!({ "rows": rows.iter().map(event_row).collect::>() })) + } + "events.count_by_filters" => { + let filters: Vec = request["filters"] + .as_array() + .expect("filters") + .iter() + .map(|filter| serde_json::from_value(filter.clone()).expect("a filter")) + .collect(); + Ok(json!({ "count": repos.events.count_by_filters(&filters).await? })) + } + "events.find_reconciliation_items" => { + let filter: Filter = + serde_json::from_value(request["filter"].clone()).expect("a filter"); + let items = collect(repos.events.find_reconciliation_items(&filter).await?).await?; + Ok(json!({ + "items": items.iter().map(|ReconciliationItem { created_at, event_id }| { + json!([created_at, event_id.to_string()]) + }).collect::>() + })) + } + "events.delete_by_pubkey_and_ids" => { + let pubkey = parse_pubkey(&request["pubkey"]); + let ids: Vec = request["ids"] + .as_array() + .expect("ids") + .iter() + .map(parse_event_id) + .collect(); + Ok(json!({ "count": repos.events.delete_by_pubkey_and_ids(&pubkey, &ids).await? })) + } + "events.delete_by_pubkey_except_kinds" => { + let pubkey = parse_pubkey(&request["pubkey"]); + let kinds: Vec = request["kinds"] + .as_array() + .expect("kinds") + .iter() + .map(|kind| Kind(kind.as_u64().expect("a kind"))) + .collect(); + Ok( + json!({ "count": repos.events.delete_by_pubkey_except_kinds(&pubkey, &kinds).await? }), + ) + } + "events.has_active_request_to_vanish" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "value": repos.events.has_active_request_to_vanish(&pubkey).await? })) + } + "events.delete_expired_and_retained" => { + let options = EventRetentionOptions { + max_days: request["options"].get("max_days").and_then(Value::as_i64), + kind_whitelist: request["options"] + .get("kind_whitelist") + .and_then(Value::as_array) + .map(|selectors| { + selectors + .iter() + .map(|selector| match selector { + Value::Array(range) => nostream_core::KindSelector::Range([ + Kind(range[0].as_u64().expect("a range start")), + Kind(range[1].as_u64().expect("a range end")), + ]), + other => nostream_core::KindSelector::Single(Kind( + other.as_u64().expect("a kind"), + )), + }) + .collect() + }), + pubkey_whitelist: request["options"] + .get("pubkey_whitelist") + .and_then(Value::as_array) + .map(|pubkeys| { + pubkeys + .iter() + .map(|pubkey| nostream_core::SettingsPubkey(text_of(pubkey))) + .collect() + }), + }; + let EventPurgeCounts { + deleted, + expired, + retained, + } = repos.events.delete_expired_and_retained(&options).await?; + Ok(json!({ "deleted": deleted, "expired": expired, "retained": retained })) + } + "user.find_by_pubkey" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "user": optional_user_view(&repos.users.find_by_pubkey(&pubkey).await?) })) + } + "user.upsert" => { + let patch = &request["patch"]; + let tos_accepted_at = match field_update(&patch["tos_accepted_at"]) { + FieldUpdate::Absent => FieldUpdate::Absent, + FieldUpdate::Clear => FieldUpdate::Clear, + FieldUpdate::Set(value) => FieldUpdate::Set(parse_iso(&value)), + }; + let user = UserPatch { + pubkey: parse_pubkey(&patch["pubkey"]), + is_admitted: FieldUpdate::Set(patch["is_admitted"].as_bool().expect("is_admitted")), + is_vanished: FieldUpdate::Set(patch["is_vanished"].as_bool().expect("is_vanished")), + tos_accepted_at, + }; + Ok(json!({ "count": repos.users.upsert(&user).await? })) + } + "user.get_balance_by_pubkey" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "balance": repos.users.get_balance_by_pubkey(&pubkey).await? })) + } + "user.is_vanished" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "value": repos.users.is_vanished(&pubkey).await? })) + } + "user.set_vanished" => { + let pubkey = parse_pubkey(&request["pubkey"]); + let vanished = request["vanished"].as_bool().expect("vanished"); + Ok(json!({ "count": repos.users.set_vanished(&pubkey, vanished).await? })) + } + "user.admit_user" => { + let pubkey = parse_pubkey(&request["pubkey"]); + repos + .users + .admit_user(&pubkey, parse_iso(&request["admitted_at"])) + .await?; + Ok(json!({ "ok": true })) + } + "user.tos_accepted_at" => Err(nostream_storage::StorageError::NotImplemented { + module: "tests::common::user_tos_accepted_at", + }), + "user.revoke_admission" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "count": repos.users.revoke_admission(&pubkey).await? })) + } + "invoice.find_by_id" => { + let id = text(request, "id"); + Ok(json!({ "invoice": optional_invoice_view(&repos.invoices.find_by_id(&id).await?) })) + } + "invoice.upsert" => { + let value = &request["invoice"]; + let invoice = Invoice { + id: text(value, "id"), + pubkey: parse_pubkey(&value["pubkey"]), + bolt11: text(value, "bolt11"), + amount_requested: integer(value, "amount_requested"), + amount_paid: value.get("amount_paid").and_then(Value::as_i64), + unit: value["unit"] + .as_str() + .expect("a unit") + .parse() + .expect("a unit"), + status: value["status"] + .as_str() + .expect("a status") + .parse() + .expect("a status"), + description: text(value, "description"), + confirmed_at: value + .get("confirmed_at") + .filter(|value| !value.is_null()) + .map(parse_iso), + expires_at: value + .get("expires_at") + .filter(|value| !value.is_null()) + .map(parse_iso), + updated_at: parse_iso(&value["created_at"]), + created_at: parse_iso(&value["created_at"]), + verify_url: optional_text(value, "verify_url"), + }; + Ok(json!({ "count": repos.invoices.upsert(&invoice).await? })) + } + "invoice.update_status" => { + let id = text(request, "id"); + let status: InvoiceStatus = request["status"] + .as_str() + .expect("a status") + .parse() + .expect("a status"); + Ok(json!({ + "invoice": optional_invoice_view(&repos.invoices.update_status(&id, status).await?) + })) + } + "invoice.confirm_invoice" => { + let id = text(request, "invoice_id"); + repos + .invoices + .confirm_invoice( + &id, + integer(request, "amount_received"), + parse_iso(&request["confirmed_at"]), + ) + .await?; + Ok(json!({ "ok": true })) + } + "invoice.find_pending_invoices" => { + let offset = integer(request, "offset") as u64; + let limit = integer(request, "limit") as u64; + let invoices = repos.invoices.find_pending_invoices(offset, limit).await?; + Ok(json!({ + "invoices": invoices.iter().map(invoice_view).collect::>() + })) + } + "nip05.find_by_pubkey" => { + let pubkey = parse_pubkey(&request["pubkey"]); + let verification = repos.nip05.find_by_pubkey(&pubkey).await?; + Ok(json!({ + "verification": verification.as_ref().map(nip05_view).unwrap_or(Value::Null) + })) + } + "nip05.upsert" => { + let value = &request["verification"]; + let verification = Nip05Verification { + pubkey: parse_pubkey(&value["pubkey"]), + nip05: text(value, "nip05"), + domain: text(value, "domain"), + is_verified: value["is_verified"].as_bool().expect("is_verified"), + last_verified_at: value + .get("last_verified_at") + .filter(|value| !value.is_null()) + .map(parse_iso), + last_checked_at: parse_iso(&value["last_checked_at"]), + failure_count: integer(value, "failure_count") as i32, + created_at: Utc::now(), + updated_at: Utc::now(), + }; + Ok(json!({ "count": repos.nip05.upsert(&verification).await? })) + } + "nip05.find_pending_verifications" => { + let verifications = repos + .nip05 + .find_pending_verifications( + integer(request, "update_frequency_ms"), + integer(request, "max_failures") as i32, + integer(request, "limit") as u64, + ) + .await?; + Ok(json!({ + "verifications": verifications.iter().map(nip05_view).collect::>() + })) + } + "nip05.delete_by_pubkey" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "count": repos.nip05.delete_by_pubkey(&pubkey).await? })) + } + "invite.create" => { + let options = CreateInviteCodeOptions { + expires_at: request["options"] + .get("expires_at") + .filter(|value| !value.is_null()) + .map(parse_iso), + remaining_uses: match field_update( + request["options"] + .get("remaining_uses") + .unwrap_or(&json!({"kind": "absent"})), + ) { + FieldUpdate::Absent => FieldUpdate::Absent, + FieldUpdate::Clear => FieldUpdate::Clear, + FieldUpdate::Set(value) => { + FieldUpdate::Set(value.as_i64().expect("remaining uses") as i32) + } + }, + created_by: optional_text(&request["options"], "created_by") + .map(|pubkey| pubkey.parse().expect("a pubkey")), + }; + let code = text(request, "code"); + let invite = repos.invites.create(&code, &options).await?; + Ok(json!({ "invite": invite_view(&invite) })) + } + "invite.find_by_code" => { + let code = text(request, "code"); + let invite = repos.invites.find_by_code(&code).await?; + Ok(json!({ "invite": invite.as_ref().map(invite_view).unwrap_or(Value::Null) })) + } + "invite.claim_code" => { + let code = text(request, "code"); + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "value": repos.invites.claim_code(&code, &pubkey).await? })) + } + "invite.find_active_codes" => { + let mut invites = repos + .invites + .find_active_codes(integer(request, "limit") as u64) + .await?; + sort_by_key(&mut invites, |invite| invite.code.clone()); + Ok(json!({ "invites": invites.iter().map(invite_view).collect::>() })) + } + "invite.delete_expired_codes" => { + Ok(json!({ "count": repos.invites.delete_expired_codes().await? })) + } + "dvm.create" => { + let id = parse_event_id(&request["id"]); + let requester = parse_pubkey(&request["requester"]); + let kind = Kind(request["kind"].as_u64().expect("a kind")); + let job = repos.dvm.create(&id, &requester, kind).await?; + Ok(json!({ "job": dvm_view(&job) })) + } + "dvm.find_by_id" => { + let id = parse_event_id(&request["id"]); + Ok(json!({ "job": optional_dvm_view(&repos.dvm.find_by_id(&id).await?) })) + } + "dvm.assign_worker" => { + let id = parse_event_id(&request["id"]); + let worker_index = integer(request, "worker_index") as i32; + Ok(json!({ "value": repos.dvm.assign_worker(&id, worker_index).await? })) + } + "dvm.update_status" => { + let update = &request["update"]; + let dvm_update = DvmJobUpdate { + id: parse_event_id(&update["id"]), + status: update["status"] + .as_str() + .expect("a status") + .parse() + .expect("a status"), + result_event_id: match field_update(&update["result_event_id"]) { + FieldUpdate::Absent => FieldUpdate::Absent, + FieldUpdate::Clear => FieldUpdate::Clear, + FieldUpdate::Set(value) => FieldUpdate::Set(parse_event_id(&value)), + }, + error: match field_update(&update["error"]) { + FieldUpdate::Absent => FieldUpdate::Absent, + FieldUpdate::Clear => FieldUpdate::Clear, + FieldUpdate::Set(value) => { + FieldUpdate::Set(value.as_str().expect("an error").to_owned()) + } + }, + }; + Ok(json!({ "job": optional_dvm_view(&repos.dvm.update_status(&dvm_update).await?) })) + } + "dvm.find_pending_jobs" => { + let kinds: Option> = + request.get("kinds").and_then(Value::as_array).map(|kinds| { + kinds + .iter() + .map(|kind| Kind(kind.as_u64().expect("a kind"))) + .collect() + }); + let mut jobs = repos + .dvm + .find_pending_jobs(integer(request, "limit") as u64, kinds.as_deref()) + .await?; + sort_by_key(&mut jobs, |job| job.id.to_string()); + Ok(json!({ "jobs": jobs.iter().map(dvm_view).collect::>() })) + } + "report.create" => { + let report = new_report(&request["report"]); + let created = repos.reports.create(&report).await?; + Ok(json!({ "report": report_view(&created) })) + } + "report.create_many" => { + let reports: Vec = request["reports"] + .as_array() + .expect("reports") + .iter() + .map(new_report) + .collect(); + let mut created = repos.reports.create_many(&reports).await?; + sort_by_key(&mut created, |report| report.id.to_string()); + Ok(json!({ "reports": created.iter().map(report_view).collect::>() })) + } + "report.find_by_event_id" => { + let event_id = parse_event_id(&request["event_id"]); + let mut reports = repos.reports.find_by_event_id(&event_id).await?; + sort_by_key(&mut reports, |report| report.id.to_string()); + Ok(json!({ "reports": reports.iter().map(report_view).collect::>() })) + } + "report.find_actionable" => { + let mut reports = repos + .reports + .find_actionable(integer(request, "limit") as u64) + .await?; + sort_by_key(&mut reports, |report| report.id.to_string()); + Ok(json!({ "reports": reports.iter().map(report_view).collect::>() })) + } + "report.find_actionable_targets" => { + let mut targets: Vec = + repos.reports.find_actionable_targets().await?; + sort_by_key(&mut targets, |target| { + format!( + "{}/{}", + target + .reported_pubkey + .map(|pubkey| pubkey.to_string()) + .unwrap_or_else(|| "null".to_owned()), + target + .reported_event_id + .map(|id| id.to_string()) + .unwrap_or_else(|| "null".to_owned()) + ) + }); + Ok(json!({ + "targets": targets.iter().map(|target| json!({ + "reported_pubkey": target.reported_pubkey.map(|pubkey| pubkey.to_string()), + "reported_event_id": target + .reported_event_id + .map(|id| id.to_string()), + })).collect::>() + })) + } + "delivery_log.append" => { + let entry = &request["entry"]; + let outbox_id = match entry.get("outbox_id_from") { + Some(_) => Some(resolve_ref(refs, entry)?), + None => optional_text(entry, "outbox_id"), + }; + let new_entry = NewDeliveryLogEntry { + outbox_id, + event_type: text(entry, "event_type"), + target_id: text(entry, "target_id"), + target_type: entry["target_type"] + .as_str() + .expect("a target type") + .parse() + .expect("a target type"), + status: entry["status"] + .as_str() + .expect("a status") + .parse() + .expect("a status"), + attempt_number: integer(entry, "attempt_number") as i32, + error_snippet: optional_text(entry, "error_snippet"), + }; + repos.delivery_log.append(&new_entry).await?; + Ok(json!({ "ok": true })) + } + "delivery_log.find_recent" => { + let filters = DeliveryLogFilters { + status: request["filters"] + .get("status") + .and_then(Value::as_str) + .map(|status| status.parse().expect("a status")), + event_type: request["filters"] + .get("event_type") + .and_then(Value::as_str) + .map(str::to_owned), + }; + let mut entries = repos + .delivery_log + .find_recent(integer(request, "limit") as u64, &filters) + .await?; + sort_by_key(&mut entries, |entry| { + format!("{}/{:02}", entry.target_id, entry.attempt_number) + }); + Ok(json!({ "entries": entries.iter().map(delivery_view).collect::>() })) + } + "delivery_log.find_successful_target_ids" => { + let outbox_id = match request.get("outbox_id_from") { + Some(_) => resolve_ref(refs, request)?, + None => text(request, "outbox_id"), + }; + let mut ids = repos + .delivery_log + .find_successful_target_ids(&outbox_id) + .await?; + ids.sort(); + Ok(json!({ "target_ids": ids })) + } + "delivery_log.delete_older_than" => { + let cutoff = parse_iso(&request["cutoff"]); + Ok(json!({ "count": repos.delivery_log.delete_older_than(cutoff).await? })) + } + "outbox.enqueue" => { + let event_type = text(request, "event_type"); + let message = repos + .outbox + .enqueue(&event_type, &request["payload"]) + .await?; + refs.insert(message.event_type.clone(), message.id.clone()); + Ok(json!({ "message": outbox_view(&message) })) + } + "outbox.claim_batch" => { + let mut messages = repos + .outbox + .claim_batch(integer(request, "limit") as u64) + .await?; + sort_by_key(&mut messages, |message| message.event_type.clone()); + Ok(json!({ "messages": messages.iter().map(outbox_view).collect::>() })) + } + "outbox.mark_delivered" => { + let id = resolve_ref(refs, request)?; + repos.outbox.mark_delivered(&id).await?; + Ok(json!({ "ok": true })) + } + "outbox.mark_failed" => { + let id = resolve_ref(refs, request)?; + repos + .outbox + .mark_failed( + &id, + &text(request, "error"), + integer(request, "attempt_count") as i32, + integer(request, "max_attempts") as i32, + integer(request, "base_delay_ms"), + ) + .await?; + Ok(json!({ "ok": true })) + } + "outbox.delete_terminal_older_than" => { + let cutoff = parse_iso(&request["cutoff"]); + Ok(json!({ "count": repos.outbox.delete_terminal_older_than(cutoff).await? })) + } + "transaction" => Err(nostream_storage::StorageError::NotImplemented { + module: "tests::common::transaction", + }), + other => panic!("unknown op {other}"), + } +} diff --git a/crates/nostream-storage/tests/parity.rs b/crates/nostream-storage/tests/parity.rs index 1d2d517b5..805364a6c 100644 --- a/crates/nostream-storage/tests/parity.rs +++ b/crates/nostream-storage/tests/parity.rs @@ -4,6 +4,10 @@ //! Run with `--ignored` and `PARITY_ADMIN_URL` set; `scripts/storage-parity.sh` does both. //! The test creates the database if absent and runs the migrator, so this target and //! `migrations.rs` have no ordering dependency. +//! +//! The 51 method arms live in `common`, shared with the LMDB arm. What stays here is what +//! only this backend can reach: raw SQL on `users.tos_accepted_at` and the serializable +//! transaction groups. #![cfg(feature = "postgres")] // This integration-test target may unwrap: a failed assertion here is a failed test, @@ -12,35 +16,20 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use std::collections::HashMap; -use std::future::poll_fn; use std::sync::Arc; -use chrono::{DateTime, Utc}; use serde_json::{Value, json}; use sqlx::Executor; use sqlx::postgres::PgPool; -use nostream_core::{Filter, Kind}; -use nostream_storage::postgres::{PostgresBackend, PostgresTransaction}; -mod common; +use common::{Repos, fixture, settings_from_fixture}; +use nostream_storage::postgres::PostgresBackend; +use nostream_storage::repositories::EventRepository; -use nostream_storage::repositories::{ - ActionableTarget, CreateInviteCodeOptions, DeliveryLogEntry, DeliveryLogFilters, DvmJob, - DvmJobRepository, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, - EventRetentionOptions, FieldUpdate, InviteCode, InviteCodeRepository, Invoice, - InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, - Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, - OutboxMessage, QueryStream, ReconciliationItem, Report, ReportRepository, User, UserPatch, - UserRepository, -}; +mod common; const DATABASE: &str = "nostream_parity_rust"; -fn fixture() -> Value { - serde_json::from_str(include_str!("fixtures/repository-parity.json")) - .expect("tests/fixtures/repository-parity.json is valid JSON") -} - fn admin_url() -> String { std::env::var("PARITY_ADMIN_URL").expect( "PARITY_ADMIN_URL must name an administrative database, e.g. \ @@ -63,246 +52,40 @@ fn with_database(url: &str, database: &str) -> String { } } -fn iso(value: &DateTime) -> Value { - Value::String(value.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) -} - -fn optional_iso(value: &Option>) -> Value { - match value { - Some(value) => iso(value), - None => Value::Null, - } -} - -fn parse_iso(value: &Value) -> DateTime { - DateTime::parse_from_rfc3339(value.as_str().expect("an ISO timestamp")) - .expect("a valid ISO timestamp") - .with_timezone(&Utc) -} - -fn text(value: &Value, key: &str) -> String { - value[key] - .as_str() - .unwrap_or_else(|| panic!("{key} is a string")) - .to_owned() -} - -fn integer(value: &Value, key: &str) -> i64 { - value[key] - .as_i64() - .unwrap_or_else(|| panic!("{key} is an integer")) -} - -fn optional_text(value: &Value, key: &str) -> Option { - value.get(key).and_then(Value::as_str).map(str::to_owned) -} - -// --------------------------------------------------------------------------------------- -// Encoders mirroring the generator. -// --------------------------------------------------------------------------------------- - -fn event_row(record: &EventRecord) -> Value { - json!([ - record.event.id.to_string(), - record.event.created_at, - record.expires_at, - record.deleted_at.is_some(), - record.event_deduplication, - ]) -} - -fn event_record(value: &Value) -> EventRecord { - let event: nostream_core::Event = - serde_json::from_value(value["event"].clone()).expect("a wire event"); - let mut record = EventRecord::from_event(&event); - if let Some(address) = optional_text(value, "remote_address") { - record = record.with_remote_address(address.parse().expect("an IP address")); - } - if let Some(expires_at) = value.get("expires_at").and_then(Value::as_i64) { - record = record.with_expires_at(expires_at); - } - if let Some(deduplication) = value.get("deduplication") { - record = record.with_deduplication( - serde_json::from_value(deduplication.clone()).expect("deduplication values"), - ); - } - record -} - -fn invoice_view(invoice: &Invoice) -> Value { - json!({ - "id": invoice.id, - "pubkey": invoice.pubkey.to_string(), - "bolt11": invoice.bolt11, - "amount_requested": invoice.amount_requested, - "amount_paid": invoice.amount_paid, - "unit": invoice.unit.as_str(), - "status": invoice.status.as_str(), - "description": invoice.description, - "confirmed_at": optional_iso(&invoice.confirmed_at), - "expires_at": optional_iso(&invoice.expires_at), - "verify_url": invoice.verify_url, - }) -} - -fn optional_invoice_view(invoice: &Option) -> Value { - match invoice { - Some(invoice) => invoice_view(invoice), - None => Value::Null, - } -} - -fn user_view(user: &User) -> Value { - json!({ - "pubkey": user.pubkey.to_string(), - "is_admitted": user.is_admitted, - "is_vanished": user.is_vanished, - "balance": user.balance, - }) -} - -fn optional_user_view(user: &Option) -> Value { - match user { - Some(user) => user_view(user), - None => Value::Null, +/// One script entry: the two probes this backend owns, then the shared executor. +async fn run_op( + backend: &PostgresBackend, + repos: Repos<'_>, + op: &Value, + refs: &mut HashMap, +) -> Result { + if op["op"].as_str() == Some("user.tos_accepted_at") { + return user_tos_accepted_at(backend, &op["request"]).await; } + common::run_op(repos, op, refs).await } -fn nip05_view(verification: &Nip05Verification) -> Value { - json!({ - "pubkey": verification.pubkey.to_string(), - "nip05": verification.nip05, - "domain": verification.domain, - "is_verified": verification.is_verified, - "last_verified_at": optional_iso(&verification.last_verified_at), - "last_checked_at": iso(&verification.last_checked_at), - "failure_count": verification.failure_count, - }) -} - -fn invite_view(invite: &InviteCode) -> Value { - json!({ - "code": invite.code, - "created_by": invite.created_by.map(|pubkey| pubkey.to_string()), - "claimed_by": invite.claimed_by.map(|pubkey| pubkey.to_string()), - "expires_at": optional_iso(&invite.expires_at), - "remaining_uses": invite.remaining_uses, - }) -} - -fn dvm_view(job: &DvmJob) -> Value { - json!({ - "id": job.id.to_string(), - "requester_pubkey": job.requester_pubkey.to_string(), - "kind": job.kind.0, - "worker_index": job.worker_index, - "status": job.status.as_str(), - "result_event_id": job.result_event_id.map(|id| id.to_string()), - "error": job.error, - }) -} - -fn optional_dvm_view(job: &Option) -> Value { - match job { - Some(job) => dvm_view(job), +/// `users.tos_accepted_at` has no seam form (`fromDBUser` omits the column), so this probe is +/// raw SQL against the master pool and stays in the Postgres arm. +async fn user_tos_accepted_at( + backend: &PostgresBackend, + request: &Value, +) -> Result { + let pubkey = parse_pubkey(&request["pubkey"]); + let row = sqlx::query("SELECT tos_accepted_at FROM users WHERE pubkey = $1") + .bind(pubkey.as_bytes().to_vec()) + .fetch_optional(backend.master()) + .await?; + let value = match row { None => Value::Null, - } -} - -fn report_view(report: &Report) -> Value { - json!({ - "id": report.id, - "event_id": report.event_id.to_string(), - "reporter_pubkey": report.reporter_pubkey.to_string(), - "reported_pubkey": report.reported_pubkey.map(|pubkey| pubkey.to_string()), - "reported_event_id": report.reported_event_id.map(|id| id.to_string()), - "report_type": report.report_type.as_str(), - "weight": report.weight, - "actionable": report.actionable, - }) -} - -fn delivery_view(entry: &DeliveryLogEntry) -> Value { - json!({ - "outbox_id": entry.outbox_id, - "event_type": entry.event_type, - "target_id": entry.target_id, - "target_type": entry.target_type.as_str(), - "status": entry.status.as_str(), - "attempt_number": entry.attempt_number, - "error_snippet": entry.error_snippet, - }) -} - -fn outbox_view(message: &OutboxMessage) -> Value { - json!({ - "event_type": message.event_type, - "payload": message.payload, - "status": message.status.as_str(), - "attempt_count": message.attempt_count, - "last_error": message.last_error, - }) -} - -fn sort_by_key(items: &mut [T], key: impl Fn(&T) -> String) { - items.sort_by_key(|item| key(item)); -} - -async fn collect( - mut stream: QueryStream<'_, T>, -) -> Result, nostream_storage::StorageError> { - let mut items = Vec::new(); - while let Some(item) = poll_fn(|context| stream.as_mut().poll_next(context)).await { - items.push(item?); - } - Ok(items) -} - -/// The three repositories a serializable transaction carries; every other op uses the -/// backend's pools. -#[derive(Clone, Copy)] -enum Client<'a> { - Backend(&'a PostgresBackend), - Transaction(&'a PostgresTransaction), -} - -impl<'a> Client<'a> { - fn users(&self) -> &'a dyn UserRepository { - match self { - Self::Backend(backend) => *backend, - Self::Transaction(transaction) => *transaction, + Some(row) => { + let value: Option = sqlx::Row::try_get(&row, "tos_accepted_at")?; + value + .map(|value| common::iso(&value.and_utc())) + .unwrap_or(Value::Null) } - } - - fn invoices(&self) -> &'a dyn InvoiceRepository { - match self { - Self::Backend(backend) => *backend, - Self::Transaction(transaction) => *transaction, - } - } - - fn outbox(&self) -> &'a dyn NotificationOutboxRepository { - match self { - Self::Backend(backend) => *backend, - Self::Transaction(transaction) => *transaction, - } - } -} - -fn field_update(value: &Value) -> FieldUpdate { - match value["kind"].as_str().expect("a FieldUpdate kind") { - "absent" => FieldUpdate::Absent, - "clear" => FieldUpdate::Clear, - _ => FieldUpdate::Set(value["value"].clone()), - } -} - -fn parse_event_id(value: &Value) -> nostream_core::EventId { - value - .as_str() - .expect("an event id") - .parse() - .expect("a valid event id") + }; + Ok(json!({ "value": value })) } fn parse_pubkey(value: &Value) -> nostream_core::Pubkey { @@ -313,639 +96,8 @@ fn parse_pubkey(value: &Value) -> nostream_core::Pubkey { .expect("a valid pubkey") } -async fn run_op( - backend: &PostgresBackend, - client: Client<'_>, - op: &Value, - refs: &mut HashMap, -) -> Result { - let name = op["op"].as_str().expect("an op name"); - let request = &op["request"]; - - match name { - "events.create" => { - let record = event_record(&request["record"]); - Ok(json!({ "count": EventRepository::create(backend, &record).await? })) - } - "events.create_many" => { - let records: Vec = request["records"] - .as_array() - .expect("records") - .iter() - .map(event_record) - .collect(); - Ok(json!({ "count": EventRepository::create_many(backend, &records).await? })) - } - "events.upsert" => { - let record = event_record(&request["record"]); - Ok(json!({ "count": EventRepository::upsert(backend, &record).await? })) - } - "events.upsert_many" => { - let records: Vec = request["records"] - .as_array() - .expect("records") - .iter() - .map(event_record) - .collect(); - Ok(json!({ "count": EventRepository::upsert_many(backend, &records).await? })) - } - "events.find_by_filters" => { - let filters: Vec = request["filters"] - .as_array() - .expect("filters") - .iter() - .map(|filter| serde_json::from_value(filter.clone()).expect("a filter")) - .collect(); - if filters.is_empty() { - let failed = EventRepository::find_by_filters(backend, &[]) - .await - .is_err(); - return Ok(json!({ "error": failed })); - } - let rows = collect(EventRepository::find_by_filters(backend, &filters).await?).await?; - Ok(json!({ "rows": rows.iter().map(event_row).collect::>() })) - } - "events.count_by_filters" => { - let filters: Vec = request["filters"] - .as_array() - .expect("filters") - .iter() - .map(|filter| serde_json::from_value(filter.clone()).expect("a filter")) - .collect(); - Ok(json!({ "count": EventRepository::count_by_filters(backend, &filters).await? })) - } - "events.find_reconciliation_items" => { - let filter: Filter = - serde_json::from_value(request["filter"].clone()).expect("a filter"); - let items = - collect(EventRepository::find_reconciliation_items(backend, &filter).await?) - .await?; - Ok(json!({ - "items": items.iter().map(|ReconciliationItem { created_at, event_id }| { - json!([created_at, event_id.to_string()]) - }).collect::>() - })) - } - "events.delete_by_pubkey_and_ids" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let ids: Vec = request["ids"] - .as_array() - .expect("ids") - .iter() - .map(parse_event_id) - .collect(); - Ok( - json!({ "count": EventRepository::delete_by_pubkey_and_ids(backend, &pubkey, &ids).await? }), - ) - } - "events.delete_by_pubkey_except_kinds" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let kinds: Vec = request["kinds"] - .as_array() - .expect("kinds") - .iter() - .map(|kind| Kind(kind.as_u64().expect("a kind"))) - .collect(); - Ok( - json!({ "count": EventRepository::delete_by_pubkey_except_kinds(backend, &pubkey, &kinds).await? }), - ) - } - "events.has_active_request_to_vanish" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok( - json!({ "value": EventRepository::has_active_request_to_vanish(backend, &pubkey).await? }), - ) - } - "events.delete_expired_and_retained" => { - let options = EventRetentionOptions { - max_days: request["options"].get("max_days").and_then(Value::as_i64), - kind_whitelist: request["options"] - .get("kind_whitelist") - .and_then(Value::as_array) - .map(|selectors| { - selectors - .iter() - .map(|selector| match selector { - Value::Array(range) => nostream_core::KindSelector::Range([ - Kind(range[0].as_u64().expect("a range start")), - Kind(range[1].as_u64().expect("a range end")), - ]), - other => nostream_core::KindSelector::Single(Kind( - other.as_u64().expect("a kind"), - )), - }) - .collect() - }), - pubkey_whitelist: request["options"] - .get("pubkey_whitelist") - .and_then(Value::as_array) - .map(|pubkeys| { - pubkeys - .iter() - .map(|pubkey| nostream_core::SettingsPubkey(text_of(pubkey))) - .collect() - }), - }; - let EventPurgeCounts { - deleted, - expired, - retained, - } = EventRepository::delete_expired_and_retained(backend, &options).await?; - Ok(json!({ "deleted": deleted, "expired": expired, "retained": retained })) - } - "user.find_by_pubkey" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok( - json!({ "user": optional_user_view(&client.users().find_by_pubkey(&pubkey).await?) }), - ) - } - "user.upsert" => { - let patch = &request["patch"]; - let tos_accepted_at = match field_update(&patch["tos_accepted_at"]) { - FieldUpdate::Absent => FieldUpdate::Absent, - FieldUpdate::Clear => FieldUpdate::Clear, - FieldUpdate::Set(value) => FieldUpdate::Set(parse_iso(&value)), - }; - let user = UserPatch { - pubkey: parse_pubkey(&patch["pubkey"]), - is_admitted: FieldUpdate::Set(patch["is_admitted"].as_bool().expect("is_admitted")), - is_vanished: FieldUpdate::Set(patch["is_vanished"].as_bool().expect("is_vanished")), - tos_accepted_at, - }; - Ok(json!({ "count": client.users().upsert(&user).await? })) - } - "user.get_balance_by_pubkey" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok(json!({ "balance": client.users().get_balance_by_pubkey(&pubkey).await? })) - } - "user.is_vanished" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok(json!({ "value": client.users().is_vanished(&pubkey).await? })) - } - "user.set_vanished" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let vanished = request["vanished"].as_bool().expect("vanished"); - Ok(json!({ "count": client.users().set_vanished(&pubkey, vanished).await? })) - } - "user.admit_user" => { - let pubkey = parse_pubkey(&request["pubkey"]); - client - .users() - .admit_user(&pubkey, parse_iso(&request["admitted_at"])) - .await?; - Ok(json!({ "ok": true })) - } - "user.tos_accepted_at" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let row = sqlx::query("SELECT tos_accepted_at FROM users WHERE pubkey = $1") - .bind(pubkey.as_bytes().to_vec()) - .fetch_optional(backend.master()) - .await?; - let value = match row { - None => Value::Null, - Some(row) => { - let value: Option = - sqlx::Row::try_get(&row, "tos_accepted_at")?; - value - .map(|value| iso(&value.and_utc())) - .unwrap_or(Value::Null) - } - }; - Ok(json!({ "value": value })) - } - "user.revoke_admission" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok(json!({ "count": client.users().revoke_admission(&pubkey).await? })) - } - "invoice.find_by_id" => { - let id = text(request, "id"); - Ok( - json!({ "invoice": optional_invoice_view(&client.invoices().find_by_id(&id).await?) }), - ) - } - "invoice.upsert" => { - let value = &request["invoice"]; - let invoice = Invoice { - id: text(value, "id"), - pubkey: parse_pubkey(&value["pubkey"]), - bolt11: text(value, "bolt11"), - amount_requested: integer(value, "amount_requested"), - amount_paid: value.get("amount_paid").and_then(Value::as_i64), - unit: value["unit"] - .as_str() - .expect("a unit") - .parse() - .expect("a unit"), - status: value["status"] - .as_str() - .expect("a status") - .parse() - .expect("a status"), - description: text(value, "description"), - confirmed_at: value - .get("confirmed_at") - .filter(|value| !value.is_null()) - .map(parse_iso), - expires_at: value - .get("expires_at") - .filter(|value| !value.is_null()) - .map(parse_iso), - updated_at: parse_iso(&value["created_at"]), - created_at: parse_iso(&value["created_at"]), - verify_url: optional_text(value, "verify_url"), - }; - Ok(json!({ "count": client.invoices().upsert(&invoice).await? })) - } - "invoice.update_status" => { - let id = text(request, "id"); - let status: InvoiceStatus = request["status"] - .as_str() - .expect("a status") - .parse() - .expect("a status"); - Ok(json!({ - "invoice": optional_invoice_view( - &client.invoices().update_status(&id, status).await? - ) - })) - } - "invoice.confirm_invoice" => { - let id = text(request, "invoice_id"); - client - .invoices() - .confirm_invoice( - &id, - integer(request, "amount_received"), - parse_iso(&request["confirmed_at"]), - ) - .await?; - Ok(json!({ "ok": true })) - } - "invoice.find_pending_invoices" => { - let offset = integer(request, "offset") as u64; - let limit = integer(request, "limit") as u64; - let invoices = client - .invoices() - .find_pending_invoices(offset, limit) - .await?; - Ok(json!({ - "invoices": invoices.iter().map(invoice_view).collect::>() - })) - } - "nip05.find_by_pubkey" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let verification = - Nip05VerificationRepository::find_by_pubkey(backend, &pubkey).await?; - Ok(json!({ - "verification": verification.as_ref().map(nip05_view).unwrap_or(Value::Null) - })) - } - "nip05.upsert" => { - let value = &request["verification"]; - let verification = Nip05Verification { - pubkey: parse_pubkey(&value["pubkey"]), - nip05: text(value, "nip05"), - domain: text(value, "domain"), - is_verified: value["is_verified"].as_bool().expect("is_verified"), - last_verified_at: value - .get("last_verified_at") - .filter(|value| !value.is_null()) - .map(parse_iso), - last_checked_at: parse_iso(&value["last_checked_at"]), - failure_count: integer(value, "failure_count") as i32, - created_at: Utc::now(), - updated_at: Utc::now(), - }; - Ok( - json!({ "count": Nip05VerificationRepository::upsert(backend, &verification).await? }), - ) - } - "nip05.find_pending_verifications" => { - let verifications = backend - .find_pending_verifications( - integer(request, "update_frequency_ms"), - integer(request, "max_failures") as i32, - integer(request, "limit") as u64, - ) - .await?; - Ok(json!({ - "verifications": verifications.iter().map(nip05_view).collect::>() - })) - } - "nip05.delete_by_pubkey" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok( - json!({ "count": Nip05VerificationRepository::delete_by_pubkey(backend, &pubkey).await? }), - ) - } - "invite.create" => { - let options = CreateInviteCodeOptions { - expires_at: request["options"] - .get("expires_at") - .filter(|value| !value.is_null()) - .map(parse_iso), - remaining_uses: match field_update( - request["options"] - .get("remaining_uses") - .unwrap_or(&json!({"kind": "absent"})), - ) { - FieldUpdate::Absent => FieldUpdate::Absent, - FieldUpdate::Clear => FieldUpdate::Clear, - FieldUpdate::Set(value) => { - FieldUpdate::Set(value.as_i64().expect("remaining uses") as i32) - } - }, - created_by: optional_text(&request["options"], "created_by") - .map(|pubkey| pubkey.parse().expect("a pubkey")), - }; - let code = text(request, "code"); - let invite = InviteCodeRepository::create(backend, &code, &options).await?; - Ok(json!({ "invite": invite_view(&invite) })) - } - "invite.find_by_code" => { - let code = text(request, "code"); - let invite = InviteCodeRepository::find_by_code(backend, &code).await?; - Ok(json!({ "invite": invite.as_ref().map(invite_view).unwrap_or(Value::Null) })) - } - "invite.claim_code" => { - let code = text(request, "code"); - let pubkey = parse_pubkey(&request["pubkey"]); - Ok(json!({ "value": InviteCodeRepository::claim_code(backend, &code, &pubkey).await? })) - } - "invite.find_active_codes" => { - let mut invites = - InviteCodeRepository::find_active_codes(backend, integer(request, "limit") as u64) - .await?; - sort_by_key(&mut invites, |invite| invite.code.clone()); - Ok(json!({ "invites": invites.iter().map(invite_view).collect::>() })) - } - "invite.delete_expired_codes" => { - Ok(json!({ "count": InviteCodeRepository::delete_expired_codes(backend).await? })) - } - "dvm.create" => { - let id = parse_event_id(&request["id"]); - let requester = parse_pubkey(&request["requester"]); - let kind = Kind(request["kind"].as_u64().expect("a kind")); - let job = DvmJobRepository::create(backend, &id, &requester, kind).await?; - Ok(json!({ "job": dvm_view(&job) })) - } - "dvm.find_by_id" => { - let id = parse_event_id(&request["id"]); - Ok( - json!({ "job": optional_dvm_view(&DvmJobRepository::find_by_id(backend, &id).await?) }), - ) - } - "dvm.assign_worker" => { - let id = parse_event_id(&request["id"]); - let worker_index = integer(request, "worker_index") as i32; - Ok( - json!({ "value": DvmJobRepository::assign_worker(backend, &id, worker_index).await? }), - ) - } - "dvm.update_status" => { - let update = &request["update"]; - let dvm_update = DvmJobUpdate { - id: parse_event_id(&update["id"]), - status: update["status"] - .as_str() - .expect("a status") - .parse() - .expect("a status"), - result_event_id: match field_update(&update["result_event_id"]) { - FieldUpdate::Absent => FieldUpdate::Absent, - FieldUpdate::Clear => FieldUpdate::Clear, - FieldUpdate::Set(value) => FieldUpdate::Set(parse_event_id(&value)), - }, - error: match field_update(&update["error"]) { - FieldUpdate::Absent => FieldUpdate::Absent, - FieldUpdate::Clear => FieldUpdate::Clear, - FieldUpdate::Set(value) => { - FieldUpdate::Set(value.as_str().expect("an error").to_owned()) - } - }, - }; - Ok( - json!({ "job": optional_dvm_view(&DvmJobRepository::update_status(backend, &dvm_update).await?) }), - ) - } - "dvm.find_pending_jobs" => { - let kinds: Option> = - request.get("kinds").and_then(Value::as_array).map(|kinds| { - kinds - .iter() - .map(|kind| Kind(kind.as_u64().expect("a kind"))) - .collect() - }); - let mut jobs = DvmJobRepository::find_pending_jobs( - backend, - integer(request, "limit") as u64, - kinds.as_deref(), - ) - .await?; - sort_by_key(&mut jobs, |job| job.id.to_string()); - Ok(json!({ "jobs": jobs.iter().map(dvm_view).collect::>() })) - } - "report.create" => { - let report = new_report(&request["report"]); - let created = ReportRepository::create(backend, &report).await?; - Ok(json!({ "report": report_view(&created) })) - } - "report.create_many" => { - let reports: Vec = request["reports"] - .as_array() - .expect("reports") - .iter() - .map(new_report) - .collect(); - let mut created = ReportRepository::create_many(backend, &reports).await?; - sort_by_key(&mut created, |report| report.id.to_string()); - Ok(json!({ "reports": created.iter().map(report_view).collect::>() })) - } - "report.find_by_event_id" => { - let event_id = parse_event_id(&request["event_id"]); - let mut reports = ReportRepository::find_by_event_id(backend, &event_id).await?; - sort_by_key(&mut reports, |report| report.id.to_string()); - Ok(json!({ "reports": reports.iter().map(report_view).collect::>() })) - } - "report.find_actionable" => { - let mut reports = - ReportRepository::find_actionable(backend, integer(request, "limit") as u64) - .await?; - sort_by_key(&mut reports, |report| report.id.to_string()); - Ok(json!({ "reports": reports.iter().map(report_view).collect::>() })) - } - "report.find_actionable_targets" => { - let mut targets: Vec = - ReportRepository::find_actionable_targets(backend).await?; - sort_by_key(&mut targets, |target| { - format!( - "{}/{}", - target - .reported_pubkey - .map(|pubkey| pubkey.to_string()) - .unwrap_or_else(|| "null".to_owned()), - target - .reported_event_id - .map(|id| id.to_string()) - .unwrap_or_else(|| "null".to_owned()) - ) - }); - Ok(json!({ - "targets": targets.iter().map(|target| json!({ - "reported_pubkey": target.reported_pubkey.map(|pubkey| pubkey.to_string()), - "reported_event_id": target - .reported_event_id - .map(|id| id.to_string()), - })).collect::>() - })) - } - "delivery_log.append" => { - let entry = &request["entry"]; - let outbox_id = match entry.get("outbox_id_from") { - Some(_) => Some(resolve_ref(refs, entry)?), - None => optional_text(entry, "outbox_id"), - }; - let new_entry = NewDeliveryLogEntry { - outbox_id, - event_type: text(entry, "event_type"), - target_id: text(entry, "target_id"), - target_type: entry["target_type"] - .as_str() - .expect("a target type") - .parse() - .expect("a target type"), - status: entry["status"] - .as_str() - .expect("a status") - .parse() - .expect("a status"), - attempt_number: integer(entry, "attempt_number") as i32, - error_snippet: optional_text(entry, "error_snippet"), - }; - NotificationDeliveryLogRepository::append(backend, &new_entry).await?; - Ok(json!({ "ok": true })) - } - "delivery_log.find_recent" => { - let filters = DeliveryLogFilters { - status: request["filters"] - .get("status") - .and_then(Value::as_str) - .map(|status| status.parse().expect("a status")), - event_type: request["filters"] - .get("event_type") - .and_then(Value::as_str) - .map(str::to_owned), - }; - let mut entries = NotificationDeliveryLogRepository::find_recent( - backend, - integer(request, "limit") as u64, - &filters, - ) - .await?; - sort_by_key(&mut entries, |entry| { - format!("{}/{:02}", entry.target_id, entry.attempt_number) - }); - Ok(json!({ "entries": entries.iter().map(delivery_view).collect::>() })) - } - "delivery_log.find_successful_target_ids" => { - let outbox_id = match request.get("outbox_id_from") { - Some(_) => resolve_ref(refs, request)?, - None => text(request, "outbox_id"), - }; - let mut ids = - NotificationDeliveryLogRepository::find_successful_target_ids(backend, &outbox_id) - .await?; - ids.sort(); - Ok(json!({ "target_ids": ids })) - } - "delivery_log.delete_older_than" => { - let cutoff = parse_iso(&request["cutoff"]); - Ok( - json!({ "count": NotificationDeliveryLogRepository::delete_older_than(backend, cutoff).await? }), - ) - } - "outbox.enqueue" => { - let event_type = text(request, "event_type"); - let message = client - .outbox() - .enqueue(&event_type, &request["payload"]) - .await?; - refs.insert(message.event_type.clone(), message.id.clone()); - Ok(json!({ "message": outbox_view(&message) })) - } - "outbox.claim_batch" => { - let mut messages = client - .outbox() - .claim_batch(integer(request, "limit") as u64) - .await?; - sort_by_key(&mut messages, |message| message.event_type.clone()); - Ok(json!({ "messages": messages.iter().map(outbox_view).collect::>() })) - } - "outbox.mark_delivered" => { - let id = resolve_ref(refs, request)?; - client.outbox().mark_delivered(&id).await?; - Ok(json!({ "ok": true })) - } - "outbox.mark_failed" => { - let id = resolve_ref(refs, request)?; - client - .outbox() - .mark_failed( - &id, - &text(request, "error"), - integer(request, "attempt_count") as i32, - integer(request, "max_attempts") as i32, - integer(request, "base_delay_ms"), - ) - .await?; - Ok(json!({ "ok": true })) - } - "outbox.delete_terminal_older_than" => { - let cutoff = parse_iso(&request["cutoff"]); - Ok(json!({ "count": client.outbox().delete_terminal_older_than(cutoff).await? })) - } - other => panic!("unknown op {other}"), - } -} - -fn resolve_ref( - refs: &HashMap, - request: &Value, -) -> Result { - let event_type = request - .get("enqueue_event_type") - .or_else(|| request.get("outbox_id_from")) - .and_then(Value::as_str) - .expect("an enqueue reference"); - refs.get(event_type) - .cloned() - .ok_or(nostream_storage::StorageError::InvalidArgument { - message: "no enqueued row to address", - }) -} - -fn text_of(value: &Value) -> String { - value.as_str().expect("a string").to_owned() -} - -fn new_report(value: &Value) -> NewReport { - NewReport { - event_id: parse_event_id(&value["event_id"]), - reporter_pubkey: parse_pubkey(&value["reporter_pubkey"]), - reported_pubkey: optional_text(value, "reported_pubkey") - .map(|pubkey| pubkey.parse().expect("a pubkey")), - reported_event_id: optional_text(value, "reported_event_id") - .map(|id| id.parse().expect("an event id")), - report_type: value["report_type"] - .as_str() - .expect("a report type") - .parse() - .expect("a report type"), - weight: value["weight"].as_f64().expect("a weight") as f32, - actionable: value["actionable"].as_bool().expect("actionable"), - } -} - +/// The capture's four serializable transaction groups. Each nested call runs against the +/// transaction for the three traits it carries and against the backend for the rest. async fn run_transaction( backend: &PostgresBackend, op: &Value, @@ -961,7 +113,13 @@ async fn run_transaction( // A fresh map per nested op loses it, and `mark_delivered` then has no row // to address. for (index, nested) in ops.iter().enumerate() { - match run_op(backend, Client::Transaction(&transaction), nested, refs).await { + let repos = Repos { + invoices: &transaction, + users: &transaction, + outbox: &transaction, + ..Repos::all(backend) + }; + match run_op(backend, repos, nested, refs).await { Ok(result) => results.push(result), Err(_) => { failed_at = Some(index); @@ -1035,14 +193,14 @@ async fn replay(admin_url: &str, database: &str) -> Vec { let backend = PostgresBackend::connect(&with_database(admin_url, database)) .await .expect("the parity database is reachable") - .with_settings(Arc::new(common::settings_from_fixture(&fixture))); + .with_settings(Arc::new(settings_from_fixture(&fixture))); backend.migrate().await.expect("the schema migrates"); - let corpus: Vec = fixture["corpus"] + let corpus: Vec = fixture["corpus"] .as_array() .expect("corpus") .iter() - .map(event_record) + .map(common::event_record) .collect(); EventRepository::create_many(&backend, &corpus) .await @@ -1060,7 +218,7 @@ async fn replay(admin_url: &str, database: &str) -> Vec { let actual = if name == "transaction" { run_transaction(&backend, op, &mut refs).await } else { - run_op(&backend, Client::Backend(&backend), op, &mut refs) + run_op(&backend, Repos::all(&backend), op, &mut refs) .await .unwrap_or_else(|error| panic!("op {index} ({name}) failed: {error}")) }; diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs new file mode 100644 index 000000000..bb4899eb8 --- /dev/null +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -0,0 +1,516 @@ +//! Behavioural parity for the embedded backend: replays `tests/fixtures/repository-parity.json` +//! through `LmdbBackend` and compares every result with the TypeScript implementation's +//! recording, then covers the cases the replay cannot reach. +//! +//! No database and no network: the store is opened on a `tempfile::TempDir`. Every entry the +//! seam can run is replayed, and the list of the ones it cannot is asserted entry by entry, so +//! an exclusion can only grow by editing `EXCLUDED`, which the diff shows. + +#![cfg(feature = "lmdb")] +// This integration-test target may unwrap: a failed assertion here is a failed test, +// which is the point. The allowance names this file rather than exempting test targets +// workspace-wide, so a new test file still has to ask. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::HashMap; +use std::sync::Arc; + +use chrono::{DateTime, Duration, Utc}; +use serde_json::{Value, json}; + +use common::{Repos, fixture, settings_from_fixture}; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; +use nostream_storage::LmdbBackend; +use nostream_storage::StorageError; +use nostream_storage::repositories::{ + CreateInviteCodeOptions, DeliveryLogFilters, EventRecord, EventRepository, FieldUpdate, + InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, InvoiceUnit, + NewDeliveryLogEntry, NotificationDeliveryLogRepository, NotificationDeliveryStatus, + NotificationOutboxRepository, NotificationTargetType, UserRepository, +}; + +mod common; + +/// The entries this arm cannot replay, fixed by index: 21 is `search` (NIP-50 is out of scope), +/// 48 and 50 are `user.tos_accepted_at` (raw SQL, no seam form), and 112 and 115 are the two +/// `transaction` groups the capture rolled back — a seam with no rollback cannot reproduce them. +/// The two groups it committed (111, 117) are replayed: their nested ops in order leave exactly +/// the state the commit left, which is why entries after 111 observe its rows. +const EXCLUDED: [usize; 5] = [21, 48, 50, 112, 115]; + +/// The entries that are replayed; `EXCLUDED.len() + REPLAYED` is the fixture's entry count. +const REPLAYED: usize = 113; + +/// The exclusion list as the marker line prints it. +const EXCLUDED_MARKER: &str = "21,48,50,112,115"; + +/// Replays a transaction group the capture committed. The group's comparison is the aggregate +/// `{failed: false, results: [...]}`, and running its nested ops in order through the seam +/// produces exactly that shape and leaves the state the commit left. A group the capture rolled +/// back needs rollback, which this seam has no form for, and is listed in `EXCLUDED` instead. +async fn run_committed_transaction( + backend: &LmdbBackend, + op: &Value, + refs: &mut HashMap, +) -> Value { + let mut results = Vec::new(); + for nested in op["request"]["ops"].as_array().expect("transaction ops") { + let name = nested["op"].as_str().expect("an op name"); + let result = common::run_op(Repos::all(backend), nested, refs) + .await + .unwrap_or_else(|error| panic!("a committed transaction's {name} failed: {error}")); + results.push(result); + } + json!({ "failed": false, "results": results }) +} + +/// One store on a temporary directory, with the fixture's settings gates. +async fn open_store() -> (tempfile::TempDir, LmdbBackend) { + let fixture = fixture(); + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens") + .with_settings(Arc::new(settings_from_fixture(&fixture))); + (directory, backend) +} + +/// A signed event of the given kind and creation time, by the key `secret`. +fn event(secret: u8, kind: u64, created_at: i64) -> EventRecord { + let mut event = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(kind), + tags: Vec::new(), + content: format!("{secret}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut event).expect("the event signs"); + EventRecord::from_event(&event) +} + +fn one_entry(outbox_id: Option) -> NewDeliveryLogEntry { + NewDeliveryLogEntry { + outbox_id, + event_type: "note.created".to_owned(), + target_id: "https://example.test/hook".to_owned(), + target_type: NotificationTargetType::Http, + status: NotificationDeliveryStatus::Success, + attempt_number: 1, + error_snippet: None, + } +} + +/// `8-4-4-4-12`, all lowercase hex, version 4, variant 1. +fn assert_uuid_v4(value: &str) { + let groups: Vec<&str> = value.split('-').collect(); + let lengths: Vec = groups.iter().map(|group| group.len()).collect(); + assert_eq!( + lengths, + vec![8, 4, 4, 4, 12], + "{value} is hyphenated 8-4-4-4-12" + ); + assert!( + groups.iter().all(|group| group + .chars() + .all(|character| character.is_ascii_digit() || matches!(character, 'a'..='f'))), + "{value} is lowercase hex" + ); + assert_eq!(groups[2].as_bytes()[0], b'4', "{value} is version 4"); + assert!( + matches!(groups[3].as_bytes()[0], b'8' | b'9' | b'a' | b'b'), + "{value} carries the RFC 4122 variant" + ); +} + +#[tokio::test] +async fn the_typescript_script_replays_with_an_empty_diff() { + let fixture = fixture(); + let script = fixture["script"].as_array().expect("script"); + assert_eq!( + script.len(), + REPLAYED + EXCLUDED.len(), + "the fixture's entry count is the one the exclusion list was written against" + ); + for (index, name) in [ + (21, "events.find_by_filters"), + (48, "user.tos_accepted_at"), + (50, "user.tos_accepted_at"), + (112, "transaction"), + (115, "transaction"), + ] { + assert_eq!( + script[index]["op"].as_str(), + Some(name), + "entry {index} is not what the exclusion list says it is" + ); + } + // Each entry is excluded exactly when the seam has no form for it: a transaction group + // the capture rolled back, the two raw-SQL probes, and the one filter carrying `search`. + for (index, op) in script.iter().enumerate() { + let name = op["op"].as_str().expect("an op name"); + let excluded_kind = match name { + "transaction" => op["result"]["compare"]["failed"] != Value::Bool(false), + "user.tos_accepted_at" => true, + _ => op["request"]["filters"] + .as_array() + .is_some_and(|filters| filters.iter().any(|filter| filter.get("search").is_some())), + }; + assert_eq!( + EXCLUDED.contains(&index), + excluded_kind, + "entry {index} ({name}) is excluded exactly when the seam cannot run it" + ); + } + + let (_directory, backend) = open_store().await; + let corpus: Vec = fixture["corpus"] + .as_array() + .expect("corpus") + .iter() + .map(common::event_record) + .collect(); + backend + .create_many(&corpus) + .await + .expect("the corpus inserts"); + + let mut refs = HashMap::new(); + let mut replayed = 0; + for (index, op) in script.iter().enumerate() { + if EXCLUDED.contains(&index) { + continue; + } + let name = op["op"].as_str().expect("an op name"); + let actual = if name == "transaction" { + run_committed_transaction(&backend, op, &mut refs).await + } else { + common::run_op(Repos::all(&backend), op, &mut refs) + .await + .unwrap_or_else(|error| panic!("op {index} ({name}) failed: {error}")) + }; + assert_eq!( + actual, op["result"]["compare"], + "op {index} ({name}) differs from the TypeScript capture" + ); + replayed += 1; + } + assert_eq!(replayed, REPLAYED); + println!("LMDB_PARITY_COMPLETE replayed={replayed} excluded={EXCLUDED_MARKER}"); +} + +#[tokio::test] +async fn a_soft_deleted_event_is_readable_by_filter_and_excluded_from_count() { + let (_directory, backend) = open_store().await; + let record = event(1, 1, 1_700_000_000); + EventRepository::create(&backend, &record) + .await + .expect("the event stores"); + + let filter = Filter { + ids: Some(vec![record.event.id.to_string()]), + ..Filter::default() + }; + let live = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(live.len(), 1); + assert!(live[0].deleted_at.is_none()); + + assert_eq!( + backend + .delete_by_pubkey_and_ids(&record.event.pubkey, &[record.event.id]) + .await + .unwrap(), + 1 + ); + + let after = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + after.len(), + 1, + "find_by_filters does not apply the live predicate (V14)" + ); + assert!( + after[0].deleted_at.is_some(), + "the row carries its deletion time" + ); + assert_eq!( + backend + .count_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + 0, + "count_by_filters applies the live predicate (V13)" + ); + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + assert!(items.is_empty(), "reconciliation lists live rows only"); +} + +#[tokio::test] +async fn a_filter_result_larger_than_one_batch_arrives_whole_and_in_order() { + let (_directory, backend) = open_store().await; + // 500 is above STREAM_BATCH_ITEMS (256), so the stream copies at least two batches. + let records: Vec = (0..500) + .map(|index| event((index % 200 + 1) as u8, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }; + let seen = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(seen.len(), 500, "the whole page arrives"); + assert!( + seen.windows(2) + .all(|pair| pair[0].event.created_at < pair[1].event.created_at), + "a filter without a limit orders by created_at ascending (V6)" + ); +} + +#[tokio::test] +async fn admit_user_creates_an_admitted_row() { + let (_directory, backend) = open_store().await; + let pubkey = Pubkey::from_byte_array([7; 32]); + assert!(backend.find_by_pubkey(&pubkey).await.unwrap().is_none()); + + let admitted_at = Utc::now(); + backend.admit_user(&pubkey, admitted_at).await.unwrap(); + + let user = backend + .find_by_pubkey(&pubkey) + .await + .unwrap() + .expect("the call creates the row"); + assert!(user.is_admitted); + assert_eq!(user.balance, 0); +} + +#[tokio::test] +async fn confirm_invoice_scales_by_unit_and_rejects_an_overflow() { + let (_directory, backend) = open_store().await; + let now = Utc::now(); + + // sats: the credit is `amount_received × 1000` (V18). + let sats_payee = Pubkey::from_byte_array([9; 32]); + backend.admit_user(&sats_payee, now).await.unwrap(); + let sats_invoice = Invoice { + id: "invoice-sats".to_owned(), + pubkey: sats_payee, + bolt11: "lnbc1".to_owned(), + amount_requested: 1_000, + amount_paid: None, + unit: InvoiceUnit::Sats, + status: InvoiceStatus::Pending, + description: "test".to_owned(), + confirmed_at: None, + expires_at: None, + updated_at: now, + created_at: now, + verify_url: None, + }; + InvoiceRepository::upsert(&backend, &sats_invoice) + .await + .unwrap(); + backend + .confirm_invoice("invoice-sats", 1_000, now) + .await + .unwrap(); + + let confirmed = backend.find_by_id("invoice-sats").await.unwrap().unwrap(); + assert_eq!(confirmed.amount_paid, Some(1_000)); + assert!(confirmed.confirmed_at.is_some()); + assert_eq!( + backend.get_balance_by_pubkey(&sats_payee).await.unwrap(), + 1_000_000 + ); + + // btc: `amount_received × 100_000_000 × 1000`. + let btc_payee = Pubkey::from_byte_array([10; 32]); + backend.admit_user(&btc_payee, now).await.unwrap(); + let btc_invoice = Invoice { + id: "invoice-btc".to_owned(), + pubkey: btc_payee, + unit: InvoiceUnit::Btc, + ..sats_invoice.clone() + }; + InvoiceRepository::upsert(&backend, &btc_invoice) + .await + .unwrap(); + backend + .confirm_invoice("invoice-btc", 1, now) + .await + .unwrap(); + assert_eq!( + backend.get_balance_by_pubkey(&btc_payee).await.unwrap(), + 100_000_000_000 + ); + + // An `int8` overflow is an error, not a wrap, and nothing is written (V19). + let overflowing = Invoice { + id: "invoice-overflow".to_owned(), + ..btc_invoice.clone() + }; + InvoiceRepository::upsert(&backend, &overflowing) + .await + .unwrap(); + let before = backend.get_balance_by_pubkey(&btc_payee).await.unwrap(); + let error = backend + .confirm_invoice("invoice-overflow", i64::MAX, now) + .await + .unwrap_err(); + assert!( + matches!( + error, + StorageError::ValueOutOfRange { + column: "users.balance", + .. + } + ), + "an amount that cannot be scaled is ValueOutOfRange, got {error:?}" + ); + assert_eq!( + backend.get_balance_by_pubkey(&btc_payee).await.unwrap(), + before, + "the failed confirmation credits nothing" + ); + assert!( + backend + .find_by_id("invoice-overflow") + .await + .unwrap() + .unwrap() + .confirmed_at + .is_none(), + "the failed confirmation leaves the invoice unconfirmed" + ); +} + +#[tokio::test] +async fn a_delivery_log_entry_must_name_an_existing_outbox_row() { + let (_directory, backend) = open_store().await; + let unknown = one_entry(Some("00000000-0000-4000-8000-000000000000".to_owned())); + match backend.append(&unknown).await { + Err(StorageError::InvalidArgument { .. }) => {} + other => panic!("an unknown outbox id is rejected, got {other:?}"), + } + assert!( + backend + .find_recent(10, &DeliveryLogFilters::default()) + .await + .unwrap() + .is_empty(), + "the rejected entry is not stored" + ); + + let message = backend + .enqueue("note.created", &json!({ "id": "note-1" })) + .await + .unwrap(); + backend.append(&one_entry(Some(message.id))).await.unwrap(); + assert_eq!( + backend + .find_recent(10, &DeliveryLogFilters::default()) + .await + .unwrap() + .len(), + 1 + ); +} + +#[tokio::test] +async fn deleting_an_outbox_row_nulls_its_delivery_log_entries() { + let (_directory, backend) = open_store().await; + let message = backend + .enqueue("note.created", &json!({ "id": "note-2" })) + .await + .unwrap(); + backend + .append(&one_entry(Some(message.id.clone()))) + .await + .unwrap(); + backend.mark_delivered(&message.id).await.unwrap(); + + let cutoff: DateTime = Utc::now() + Duration::seconds(60); + assert_eq!( + backend.delete_terminal_older_than(cutoff).await.unwrap(), + 1, + "the delivered row is deleted" + ); + + let entries = backend + .find_recent(10, &DeliveryLogFilters::default()) + .await + .unwrap(); + assert_eq!(entries.len(), 1, "the delivery log keeps its entry"); + assert!( + entries[0].outbox_id.is_none(), + "the deleted outbox row leaves no dangling id (V40)" + ); +} + +#[tokio::test] +async fn an_invite_code_rejects_a_negative_remaining_uses() { + let (_directory, backend) = open_store().await; + let negative = CreateInviteCodeOptions { + remaining_uses: FieldUpdate::Set(-5), + ..CreateInviteCodeOptions::default() + }; + match InviteCodeRepository::create(&backend, "negative", &negative).await { + Err(StorageError::InvalidArgument { .. }) => {} + other => panic!("a negative remaining_uses is rejected, got {other:?}"), + } + + let zero = CreateInviteCodeOptions { + remaining_uses: FieldUpdate::Set(0), + ..CreateInviteCodeOptions::default() + }; + let invite = InviteCodeRepository::create(&backend, "zero", &zero) + .await + .unwrap(); + assert_eq!(invite.remaining_uses, Some(0)); +} + +#[tokio::test] +async fn two_enqueues_return_distinct_uuids() { + let (_directory, backend) = open_store().await; + let first = backend + .enqueue("note.created", &json!({ "id": "a" })) + .await + .unwrap(); + let second = backend + .enqueue("note.created", &json!({ "id": "b" })) + .await + .unwrap(); + + assert_ne!(first.id, second.id); + assert_uuid_v4(&first.id); + assert_uuid_v4(&second.id); +} diff --git a/crates/nostream-storage/tests/storage.rs b/crates/nostream-storage/tests/storage.rs index f0e790900..5f425bbe3 100644 --- a/crates/nostream-storage/tests/storage.rs +++ b/crates/nostream-storage/tests/storage.rs @@ -1,31 +1,40 @@ -//! The LMDB environment slice, the trait surface, and the backend's dyn compatibility. +//! The backend seam: the enum, the data directory, and the dyn-compatible trait surface. // This integration-test target may unwrap: a failed assertion here is a failed test, // which is the point. The allowance names this file rather than exempting test targets // workspace-wide, so a new test file still has to ask. #![allow(clippy::unwrap_used, clippy::expect_used)] -use std::path::Path; +use std::path::{Path, PathBuf}; -use nostream_storage::{Backend, LmdbEnv}; +use nostream_core::EventStoreBackend; +use nostream_storage::data_dir; -#[tokio::test] -async fn the_lmdb_environment_round_trips_a_value() { - let directory = tempfile::tempdir().expect("tempdir"); - let env = LmdbEnv::open(directory.path(), 1024 * 1024).expect("the environment opens"); - - env.put("greeting", "hello").expect("put"); - assert_eq!(env.get("greeting").expect("get"), Some("hello".to_owned())); - assert_eq!(env.get("absent").expect("get"), None); - assert_eq!(env.path(), Path::new(directory.path())); +#[test] +fn the_backend_parses_the_settings_spelling() { + assert_eq!( + EventStoreBackend::parse("postgres"), + Some(EventStoreBackend::Postgres) + ); + assert_eq!( + EventStoreBackend::parse("lmdb"), + Some(EventStoreBackend::Lmdb) + ); + assert_eq!(EventStoreBackend::parse("mongodb"), None); + assert_eq!(EventStoreBackend::parse("LMDB"), None); + assert_eq!(EventStoreBackend::Lmdb.to_string(), "lmdb"); } #[test] -fn the_backend_parses_the_settings_spelling() { - assert_eq!(Backend::parse("postgres"), Some(Backend::Postgres)); - assert_eq!(Backend::parse("lmdb"), Some(Backend::Lmdb)); - assert_eq!(Backend::parse("mongodb"), None); - assert_eq!(Backend::Lmdb.to_string(), "lmdb"); +fn the_lmdb_backend_keeps_a_data_directory_of_its_own() { + assert_eq!( + data_dir(Path::new("/data"), EventStoreBackend::Lmdb), + Some(PathBuf::from("/data/lmdb")) + ); + assert_eq!( + data_dir(Path::new("/data"), EventStoreBackend::Postgres), + None + ); } #[tokio::test] @@ -411,3 +420,34 @@ async fn the_postgres_backend_coerces_to_all_nine_trait_objects() { let _: &dyn NotificationDeliveryLogRepository = &backend; let _: &dyn NotificationOutboxRepository = &backend; } + +/// The embedded backend coerces into the same nine forms, from a store it opens itself. +#[cfg(feature = "lmdb")] +#[tokio::test] +async fn the_lmdb_backend_coerces_to_all_nine_trait_objects() { + use nostream_storage::LmdbBackend; + use nostream_storage::repositories::{ + DvmJobRepository, EventRepository, InviteCodeRepository, InvoiceRepository, + Nip05VerificationRepository, NotificationDeliveryLogRepository, + NotificationOutboxRepository, ReportRepository, UserRepository, + }; + + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + assert!( + data_dir(directory.path(), EventStoreBackend::Lmdb) + .expect("a data directory") + .is_dir() + ); + let _: &dyn EventRepository = &backend; + let _: &dyn InvoiceRepository = &backend; + let _: &dyn UserRepository = &backend; + let _: &dyn Nip05VerificationRepository = &backend; + let _: &dyn InviteCodeRepository = &backend; + let _: &dyn DvmJobRepository = &backend; + let _: &dyn ReportRepository = &backend; + let _: &dyn NotificationDeliveryLogRepository = &backend; + let _: &dyn NotificationOutboxRepository = &backend; +} From b53c646429cb77e6afe246488036d9b78db24c52 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:19:57 -0500 Subject: [PATCH 04/60] docs(storage): record the lmdb backend and add its oracle step --- .changeset/storage-lmdb-backend.md | 2 + CLAUDE.md | 5 +- docs/rewrite/decisions/0010-lmdb-backend.md | 194 ++++++++++++++++++++ scripts/storage-parity.sh | 4 + 4 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 .changeset/storage-lmdb-backend.md create mode 100644 docs/rewrite/decisions/0010-lmdb-backend.md diff --git a/.changeset/storage-lmdb-backend.md b/.changeset/storage-lmdb-backend.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/storage-lmdb-backend.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/CLAUDE.md b/CLAUDE.md index 0e885d22c..034dd0b06 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,10 @@ cargo test -p nostream-storage --test migrations --test parity -- --ignored --te `PARITY_ADMIN_URL` names the administrative database (CI sets `postgres://postgres:postgres@localhost:5432/postgres`); the targets create, migrate and drop their own scratch databases. `scripts/storage-parity.sh` runs both plus the -TypeScript Cucumber suite against the sqlx-migrated schema — the complete local oracle. +TypeScript Cucumber suite against the sqlx-migrated schema — the complete local oracle. The +LMDB arm of that oracle needs no database: `cargo test -p nostream-storage --test parity_lmdb +-- --nocapture` replays the same fixture on a temporary store and runs inside the fifth +validation command above. - **`--workspace`, never a single crate.** A crate-local `cargo test -p` proves nothing about a workspace lint that another member trips. diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md new file mode 100644 index 000000000..37f455423 --- /dev/null +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -0,0 +1,194 @@ +# ADR 0010: The LMDB event store behind the existing seam (job 024) + +- **Status:** accepted (job 024, Phase 2) +- **Deciders:** the rewrite team, on the evidence in job 024's recon and spec +- **Date:** 2026-11-06 + +## Context + +`nostream-storage` shipped one backend. `src/backend.rs` selected between `postgres` and +`lmdb` and then answered `Phase 1 stub` from `data_dir`, so the second backend the rewrite +exists to provide had a name, a settings value and no code. This job adds it: the event +store, the metadata tail, and the nine repository traits implemented over heed 0.20.5, +with the same integration test the Postgres backend passes replayed against it. + +`spec.md` records ten decisions (D1–D10) with their reasons and their measurements. This +document keeps the parts a later reader needs to judge the code: where the state lives, how +a value is shaped, what the seam does not offer, and what the oracle therefore measures. + +## The backend is a feature of the crate, not a crate + +`lmdb = ["dep:heed", "dep:uuid"]`, default on beside `postgres` (D1). heed was already a +hard dependency, the seams' own ADR says the `--no-default-features` build is what this +follow-up uses (ADR 0003, "Backend coupling"), and a second crate would repeat five +manifest fields plus `[lints] workspace = true` to isolate code that shares every type in +`nostream-storage`. The cost is that `--no-default-features` now links no driver at all — +which is also the point: `cargo check -p nostream-storage --no-default-features --all-targets` +must compile, and it does. + +`nostream_storage::Backend` is deleted (D8) and everything uses +`nostream_core::EventStoreBackend`. Two enums with identical variants and spellings were a +drift waiting to happen, and neither had a consumer: `data_dir` now returns +`Option` — `Some(root/lmdb)` for LMDB, `None` for Postgres — and the LMDB backend +opens `/lmdb` itself. + +## Metadata lives in LMDB databases of its own + +The eight tail tables are 22 named databases in the same environment (D2): one primary +database per table keyed by its primary key, plus one per query path (`invoices_pending`, +`nip05_by_last_checked_at`, `invite_codes_by_created_at`/`_by_expires_at`, +`dvm_jobs_by_status_created_at`, four `reports_*`, three `outbox_*`/`delivery_log_*`), and a +`meta` database for the report sequence. + +**The reason is one write transaction spanning events and tail.** NIP-56's hidden-content +predicate is a `NOT EXISTS` over `reports` inside the events query +(`crates/nostream-storage/src/postgres/sql.rs:291-297`). One environment makes that two +point lookups — `reports_actionable_event` by event id and `reports_actionable_pubkey` by +author — inside the same read transaction that already holds the event row, instead of a +second store read per candidate. An embedded SQL store would have needed a second +dependency and could still tear against the LMDB file. + +The price is recorded with the decision: 29 named databases maintained by hand, `DISTINCT` +and `OFFSET` become cursor walks (`BTreeSet` for `find_actionable_targets`, a skip count for +`find_pending_invoices`), and the columns' `CHECK` and foreign-key constraints move into +Rust (§ Constraints, below). + +Rows are versioned: one format byte (`ROW_FORMAT_VERSION = 1`), then `serde_json`, the byte +per database rather than per table. ADR 0003 trigger 6 asked for a version marker before a +second backend persisted `EventRecord`, and this is it. + +## An event value is a 54-byte header, then JSON + +`events` holds `1 + 54` bytes of header, then `serde_json` of `EventRecord` (D5): + +|bytes|field| +|---|---| +|1|format version (`1`)| +|8|`created_at`, big-endian i64| +|4|`kind`, big-endian i32| +|8|`expires_at`, big-endian i64, `i64::MIN` for NULL| +|32|author pubkey| +|1|reserved, zero| + +The live predicate and the NIP-56 lookup read that header without decoding the body, so +`count_by_filters` and reconciliation copy 55 bytes per candidate instead of an event. +The reserved byte keeps the header at the 54 bytes R7, V35 and AC8 pin, and it is where a +future flag goes without a version bump. + +`row_id` stays `None` (D5): it is the Postgres row's `uuid` and nothing reads it through +this backend. Text components of a composite key are NUL-terminated except the final one +(`text_part`), so `tag_key` is `name NUL value NUL created_at ‖ event_id`, and a tag name or +value carrying `0x00` is refused as `InvalidRow` rather than splitting the key. + +Soft delete is a flag **and** the row's value, never an index removal (D4): `find_by_filters` +does not apply the live predicate (`sql.rs:119-140` against `:262-267`), so the row must stay +in every read index while `count_by_filters` and reconciliation exclude it. The +`events_deleted` database is the retention scan's path, not the query's. + +## The seam has no transaction type (D3) + +`RwTxn` borrows the `Env` (heed 0.20.5 `src/txn.rs:119-122`) and a `spawn_blocking` closure +must be `'static`, so a seam-level transaction would need a leaked `&'static Env` plus a +mutex, and would then hold LMDB's **single** writer across every `await` between `begin()` +and `commit()`. That is a worse failure mode than the one it fixes: one stuck caller stops +every write in the process. + +What replaces it: a repository method is one whole write transaction, so every write the +seam performs is atomic across every database it touches. What it leaves **unmeasured** is +atomicity *across repository calls* — a caller that needs two writes to succeed or fail +together has no form for it here. `PostgresTransaction` and `begin()` are untouched. + +This resolves nothing about ADR 0003 trigger 2 (a second backend needs a transaction +abstraction) except the shape a future one must take: a held LMDB transaction cannot satisfy +it, so the abstraction has to be closure-shaped — `with_write_txn(|tx| …)` — with the +single writer never held across an `await`. The trigger stays open. + +## Blocking and streams (D7) + +Every heed call — `write_txn`, `read_txn`, `create_database`, iteration and `commit` — runs +inside `tokio::task::spawn_blocking`, because `Env::write_txn` blocks the calling thread on +the writer's mutex and a read transaction must not be held across an `await`. `LmdbBackend` +therefore has two helpers, `write` and `read`, that own their transaction and commit or drop +it inside the closure; no `RwTxn` or `RoTxn` leaves one. + +A stream is a key page, not a held transaction. `find_by_filters` collects its union page +inside one blocking closure; the returned `QueryStream` yields values in batches of +`STREAM_BATCH_ITEMS = 256`, each poll opening a read transaction, seeking to the saved +cursor key, copying up to 256 values and dropping it. `find_reconciliation_items` runs the +same loop over keys and never reads a value. **A stream is therefore not a snapshot** (D7): +a row written or deleted between two batches may appear or vanish, where Postgres' portal +holds one snapshot. No comparable behaviour exists to compare, so the divergence is recorded +for the benchmark job, and ADR 0003 trigger 1 ("a long-lived LMDB read transaction hurts +writes") is answered by construction: no transaction outlives a poll, and a dropped stream +pins no pages. + +`Env::resize` is `unsafe` and this crate has one audited `unsafe` call, so the map is +preallocated at `LMDB_MAP_SIZE = 64 GiB` and never grown (D6). The same single +`#[allow(unsafe_code)]` open sets `LMDB_MAX_DBS = 64` (the 29 named databases, comfortably +inside) and `LMDB_MAX_READERS = 512`. + +## Constraints move into Rust + +The store has no server to enforce the columns' `CHECK` and foreign-key constraints, so the +two a caller can violate are enforced on write (R17): a negative +`CreateInviteCodeOptions::remaining_uses` is `InvalidArgument`, and a delivery-log entry +naming an unknown outbox row is `InvalidArgument` while deleting an outbox row walks +`delivery_log_by_outbox` and sets `outbox_id` to NULL. The `status`, `unit`, `report_type` +and `target_type` spellings need no code: they are typed enums parsed by `column_enum!`, so +an invalid spelling cannot reach the writer. + +Every `DateTime` is truncated to microseconds before encoding, because every column it +lands in is a PostgreSQL `timestamp`; a call takes one `Utc::now()` and uses it for every row +it writes, standing in for the statements' `CURRENT_TIMESTAMP`. + +`admit_user` and `confirm_invoice` are Rust in one write transaction each, following the +schema functions they replace. `confirm_invoice` scales by `sats × 1000`, `msats × 1`, +`btc × 100_000_000 × 1000` with checked arithmetic, and answers `ValueOutOfRange` where the +`int8` would overflow. `reports.id` is allocated from `meta` starting at 1; every other +generated id is a UUID v4 in canonical text form, which is why `uuid` (v4) is the job's only +new dependency. + +## Oracle: two arms, one executor + +The fixture is the capture's recording and both backends are compared against the same +`result.compare` (D10). The 51 method arms moved out of `tests/parity.rs` into +`tests/common/`, where the executor takes a `Repos<'a>` of nine `&dyn` repository +references, so one script drives both backends; what stays Postgres-only is what only +Postgres can reach: raw SQL on `users.tos_accepted_at` and the serializable `transaction` +groups. + +`tests/parity_lmdb.rs` opens a store on a `tempfile::TempDir`, inserts the fixture's +483-event corpus and replays **113 of 118** entries. The five it does not replay: + +|index|op|why the seam cannot run it| +|---|---|---| +|21|`events.find_by_filters` with `search`|NIP-50 is out of scope; the backend answers `NotImplemented`| +|48, 50|`user.tos_accepted_at`|raw SQL against `backend.master()`, no seam form| +|112, 115|`transaction`|the capture **rolled these back**; reproducing a rollback needs a transaction type this job deliberately does not add| + +The other two `transaction` groups, 111 and 117, **committed** in the capture and are +replayed: running their nested ops in order through the seam leaves exactly the state the +commit left, and their comparison — the aggregate `{failed: false, results: [...]}` — is what +those calls return. That is also the evidence for the count rather than a preference: entry +116, `outbox.claim_batch`, expects the outbox row that group 111 committed, so excluding 111 +cannot pass. What the transaction decision leaves unmeasured is rollback: the arm proves the +committed groups' effects, not that two calls can fail together. + +The target asserts every exclusion against the fixture — an entry is excluded exactly when +its op is one of the five above — so the list cannot grow without the diff showing it. + +## Consequences + +- One environment, one write transaction, one error taxonomy; no second dependency, and no + tear between two files. +- 29 named databases and their key encodings are maintained by hand; `DISTINCT`/`OFFSET` + became cursor walks; two `CHECK`/foreign-key constraints became Rust code. +- Postgres remains the reference: every query path is written against `sql.rs`'s predicates + and orderings, and the shared executor is what proves it. +- Retention's batch composition can differ from Postgres' planner-chosen order when more than + 1000 rows qualify; both return 1000 rows classified by the same precedence (D13 in + `spec.md`, Assumption 10). The corpus is 483 events, so no current entry reaches it. +- Two JSON values PostgreSQL's `jsonb` considers equal but that serialize differently + (`1` against `1.0`) would be distinct deduplication pointers here; the repository builds + the value in one shape and the relay passes a single string, so today's callers cannot + reach it (Assumption 11). diff --git a/scripts/storage-parity.sh b/scripts/storage-parity.sh index f8021cbbb..3e8291e42 100755 --- a/scripts/storage-parity.sh +++ b/scripts/storage-parity.sh @@ -285,6 +285,10 @@ echo "== behavioural parity" cargo test -p nostream-storage --test parity -- --ignored --test-threads=1 \ || fail "the parity target failed" +echo "== lmdb behavioural parity (no database)" +./scripts/rust.sh cargo test -p nostream-storage --test parity_lmdb -- --nocapture \ + || fail "the lmdb parity target failed" + echo "== tls parity (the same replay over sslmode=require)" # The test skips itself when the variable is absent, so the run has to prove it executed: # a renamed or quietly skipped TLS leg is the failure mode this check closes. From 509a8fcc98ff2eff37dc4fb9cab6b7a91e5466df Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:20:00 -0500 Subject: [PATCH 05/60] build(storage): lock the lmdb feature's heed and uuid edges --- Cargo.lock | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 18987eb41..541c090a1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1353,6 +1353,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "uuid", ] [[package]] @@ -2796,6 +2797,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" From c66fa9f0f0077ea0bf9ed53f25d42137a29d7505 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:21:16 -0500 Subject: [PATCH 06/60] docs(storage): mark each lmdb ceiling where a reader meets it --- crates/nostream-storage/src/lmdb/encoding.rs | 3 +++ crates/nostream-storage/src/lmdb/events.rs | 8 ++++---- crates/nostream-storage/src/lmdb/mod.rs | 6 ++++++ crates/nostream-storage/tests/parity_lmdb.rs | 3 +++ 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index 478e56302..bb2aa01e9 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -212,6 +212,9 @@ pub(crate) fn author_key( } /// `pubkey ‖ kind ‖ dedup_len ‖ canonical JSON`, where `dedup_len` is the JSON's byte length. +// shortcut: the deduplication index uses `serde_json` text as its key, so `1` and `1.0` differ +// where `jsonb` would not; upgrade when a caller can pass a hand-built JSON number, which +// today's two callers cannot (spec Assumption 11). pub(crate) fn replaceable_key( pubkey: &Pubkey, kind: i32, diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 6326331c4..aca55a444 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -375,6 +375,9 @@ pub(crate) fn has_active_request_to_vanish( /// `delete_expired_and_retained`: one hard-delete batch of at most /// [`RETENTION_BATCH_SIZE`] rows, gathered in the capture's index order and classified /// deleted, then expired, then over-retention. +// shortcut: retention takes its 1000 candidates in a fixed order (deleted, expiring, +// created_at) where Postgres' planner chooses; upgrade when a deployment exceeds 1000 +// qualifying rows per pass and the mix, not the total, starts to matter (spec Assumption 10). pub(crate) fn delete_expired_and_retained( txn: &mut RwTxn<'_>, databases: &Databases, @@ -726,10 +729,7 @@ fn sort_page(page: &mut [(i64, EventId)], descending: bool) { } /// The index a filter is answered from, and the ranges within it. -fn choose_ranges( - filter: &Filter, - databases: &Databases, -) -> IndexRanges { +fn choose_ranges(filter: &Filter, databases: &Databases) -> IndexRanges { if let Some(ids) = &filter.ids && !ids.is_empty() { diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index d9e22195b..4816f8576 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -38,9 +38,15 @@ use crate::repositories::{ /// The fixed map size every environment is opened with: `Env::resize` is `unsafe` and this /// crate has one audited `unsafe` call, so the map is preallocated at 64 GiB and never grown. +/// +/// shortcut: the map is fixed at 64 GiB and never grown; upgrade when a deployment's data can +/// approach it, which needs `Env::resize` (a second `unsafe` call) or a refusal at open. pub const LMDB_MAP_SIZE: usize = 64 << 30; /// The number of named databases the environment reserves, comfortably above the 29 it opens. +/// +/// shortcut: 29 databases open under a reserve of 64; upgrade when a tenth repository or a new +/// query path crosses that reserve, which moves this value and every store's reopen path. pub const LMDB_MAX_DBS: u32 = 64; /// The number of concurrent read transactions the environment reserves. diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index bb4899eb8..1c5af83f3 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -36,6 +36,9 @@ mod common; /// `transaction` groups the capture rolled back — a seam with no rollback cannot reproduce them. /// The two groups it committed (111, 117) are replayed: their nested ops in order leave exactly /// the state the commit left, which is why entries after 111 observe its rows. +// shortcut: five fixture entries are excluded by index and the count is a constant; upgrade +// when the capture gains an entry, which fails the count assertion and forces this list to be +// re-derived rather than quietly skipped. const EXCLUDED: [usize; 5] = [21, 48, 50, 112, 115]; /// The entries that are replayed; `EXCLUDED.len() + REPLAYED` is the fixture's entry count. From ebaa97170e112c74069601a193a588b94e7310e9 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:21:32 -0500 Subject: [PATCH 07/60] docs(storage): mark the stream's snapshot ceiling at its type --- crates/nostream-storage/src/lmdb/events.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index aca55a444..f113a23e5 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -1023,6 +1023,10 @@ impl Position { /// Reads one page of values in batches of [`STREAM_BATCH_ITEMS`], one read transaction per /// poll, so a stream that is dropped pins no pages. +/// +/// shortcut: a stream is not a snapshot — a row written or deleted between two batches may +/// appear or vanish; upgrade when a caller needs a stable view, per ADR 0003 trigger 1 and the +/// benchmark job, which would buy the snapshot back with a held transaction. pub(crate) struct ValuesStream { env: Env, databases: Databases, From 47e79d7ecab3e223cd153c90e44d6e674cd594e6 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:46:02 -0500 Subject: [PATCH 08/60] fix(storage): prune only the created-at scan and reconcile every batch --- crates/nostream-storage/src/lmdb/events.rs | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index f113a23e5..f5b011b12 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -692,12 +692,18 @@ fn branch( let Some(found) = candidate(txn, databases, path, key)? else { continue; }; - if boundary.is_some_and(|bound| found.0 > bound) { + // Only the created-at index iterates in page order. `ids`, `authors` and + // `tags` iterate in key order, so an early stop there would drop the newest + // rows of a limited page; those paths scan the whole (time-bounded) range and + // the page is selected below. + if path == AccessPath::CreatedAt && boundary.is_some_and(|bound| found.0 > bound) { break; } if matcher.matches(txn, databases, &found.1)?.is_some() { push(&mut matches, &mut seen, found); - if limit.is_some_and(|limit| matches.len() >= limit) { + if path == AccessPath::CreatedAt + && limit.is_some_and(|limit| matches.len() >= limit) + { boundary = Some(found.0); } } @@ -986,6 +992,10 @@ fn tag_criteria_match(tags: &BTreeMap>, record: &EventRecord let Some(value) = tag.get(1) else { return false; }; + // `event_tags` holds no empty value, so the capture cannot match one either. + if value.is_empty() { + return false; + } criteria.iter().any(|criterion| { match criterion.strip_suffix('*').filter(|_| is_geohash) { Some(prefix) => value.starts_with(prefix), @@ -1230,11 +1240,13 @@ fn scan_batch( let mut reached_cap = false; for entry in databases.events_by_created_at.range(txn, &range)? { let (key, _) = entry?; - position.after = Some(key.to_vec()); if items.len() >= STREAM_BATCH_ITEMS { reached_cap = true; break; } + // The cursor moves only onto a key this batch evaluated, because the next batch + // resumes exclusively after it. + position.after = Some(key.to_vec()); let (created_at, event_id) = key_candidate(AccessPath::CreatedAt, key)?.ok_or_else(|| { encoding::invalid( From 871ca6c12d819727e421da971e8c81c5532b0ee3 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:46:02 -0500 Subject: [PATCH 09/60] fix(storage): key the delivery-log outbox index by entry id --- .../src/lmdb/notifications.rs | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs index 5126d1a9b..6bd79d197 100644 --- a/crates/nostream-storage/src/lmdb/notifications.rs +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -52,13 +52,28 @@ fn delivery_created_key(entry: &DeliveryLogEntry) -> Vec { key } +/// `outbox_id ‖ status ‖ target_id ‖ id`, so one entry is addressable and two attempts at +/// the same target do not share a key. fn delivery_outbox_key(outbox_id: &str, entry: &DeliveryLogEntry) -> Vec { let mut key = text_part(outbox_id); key.extend_from_slice(&text_part(entry.status.as_str())); - key.extend_from_slice(entry.target_id.as_bytes()); + key.extend_from_slice(&text_part(&entry.target_id)); + key.extend_from_slice(entry.id.as_bytes()); key } +/// The entry id one `delivery_log_by_outbox` key ends with. +fn delivery_outbox_id(key: &[u8]) -> Result<&str, StorageError> { + let Some(start) = key.iter().rposition(|byte| *byte == 0) else { + return Err(StorageError::InvalidRow { + table: "delivery_log_by_outbox", + column: "key", + value: "a key without an entry id".to_owned(), + }); + }; + text_from(&key[start + 1..], "delivery_log_by_outbox") +} + fn delivery_load( txn: &RoTxn<'_>, databases: &Databases, @@ -160,7 +175,15 @@ pub(crate) fn find_successful_target_ids( let mut targets = Vec::new(); for entry in databases.delivery_log_by_outbox.prefix_iter(txn, &prefix)? { let (key, _) = entry?; - let target_id = text_from(&key[prefix.len()..], "delivery_log_by_outbox")?; + let rest = &key[prefix.len()..]; + let Some(end) = rest.iter().position(|byte| *byte == 0) else { + return Err(StorageError::InvalidRow { + table: "delivery_log_by_outbox", + column: "key", + value: "a key without a target terminator".to_owned(), + }); + }; + let target_id = text_from(&rest[..end], "delivery_log_by_outbox")?; if seen.insert(target_id.to_owned()) { targets.push(target_id.to_owned()); } @@ -439,19 +462,18 @@ fn null_delivery_outbox( databases: &Databases, outbox_id: &str, ) -> Result<(), StorageError> { + let prefix = text_part(outbox_id); let mut named = Vec::new(); - for entry in databases.notification_delivery_log.iter(txn)? { - let (_, value) = entry?; - let row: DeliveryLogEntry = rows::decode("notification_delivery_log", value)?; - if row.outbox_id.as_deref() == Some(outbox_id) { - named.push(row); - } + for entry in databases.delivery_log_by_outbox.prefix_iter(txn, &prefix)? { + let (key, _) = entry?; + named.push((key.to_vec(), delivery_outbox_id(key)?.to_owned())); } - for mut row in named { - databases - .delivery_log_by_outbox - .delete(txn, &delivery_outbox_key(outbox_id, &row))?; + for (index_key, id) in named { + let Some(mut row) = delivery_load(txn, databases, &id)? else { + continue; + }; + databases.delivery_log_by_outbox.delete(txn, &index_key)?; row.outbox_id = None; let value = rows::encode("notification_delivery_log", &row)?; databases From 6f461fe3fca71d32f12ef7d380590d1e829aeeb1 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:46:02 -0500 Subject: [PATCH 10/60] fix(storage): compare hex criteria as the range builder seeks them --- crates/nostream-storage/src/lmdb/encoding.rs | 35 ++++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index bb2aa01e9..1f691fc05 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -295,8 +295,9 @@ pub(crate) fn replaceable_kind(kind: i32) -> bool { || (30_000..40_000).contains(&kind) } -/// Whether the lowercase hex rendering of `bytes` starts with `prefix`, case-sensitively, -/// exactly as `String.prototype.startsWith` compares the capture's ids and pubkeys. +/// Whether `bytes` starts with `prefix`, compared one nibble at a time with the same digit +/// mapping [`hex_bytes`] gives the range that selected the row, so a criterion means the same +/// thing to the matcher as to the index it was sought in. pub(crate) fn hex_matches(bytes: &[u8], prefix: &str) -> bool { if prefix.len() > bytes.len() * 2 { return false; @@ -308,12 +309,7 @@ pub(crate) fn hex_matches(bytes: &[u8], prefix: &str) -> bool { } else { byte & 0x0f }; - let expected = match character { - b'0'..=b'9' => character - b'0', - b'a'..=b'f' => character - b'a' + 10, - _ => return false, - }; - nibble == expected + nibble == hex_digit(character) }) } @@ -440,6 +436,18 @@ mod tests { assert_eq!(exact[0].0, b"e\0u4pru*\0".to_vec()); } + #[test] + fn a_nanosecond_instant_is_truncated_to_microseconds() { + let instant = DateTime::::from_timestamp_nanos(1_700_000_000_123_456_789); + let truncated = micros(instant); + assert_eq!(truncated.timestamp_micros(), 1_700_000_000_123_456); + assert_eq!( + truncated.timestamp_subsec_nanos(), + 123_456_000, + "the sub-microsecond digits are dropped, as a PostgreSQL timestamp drops them (V38)" + ); + } + #[test] fn the_header_round_trips_without_its_body() { let header = EventHeader { @@ -520,10 +528,17 @@ mod tests { } #[test] - fn hex_prefixes_compare_nibbles_case_sensitively() { + fn a_hex_prefix_compares_the_nibbles_the_range_builder_sought() { assert!(hex_matches(&[0xab, 0xcd], "abcd")); assert!(hex_matches(&[0xab, 0xcd], "abc")); - assert!(!hex_matches(&[0xab, 0xcd], "ABC")); + assert!( + hex_matches(&[0xab, 0xcd], "ABC"), + "an uppercase digit names the same nibble the range was built from" + ); + assert!( + hex_matches(&[0x0a], "z"), + "an invalid digit maps to zero, as the capture's decode does" + ); assert!(!hex_matches(&[0xab, 0xcd], "abce")); assert!(!hex_matches(&[0xab, 0xcd], "abcdef")); } From a4a503faca356a067df392ed0374288774f43d70 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:46:02 -0500 Subject: [PATCH 11/60] refactor(storage): drop the unreachable lmdb data-directory error --- crates/nostream-storage/src/lmdb/mod.rs | 41 +++++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 4816f8576..031fe3361 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -22,7 +22,7 @@ use chrono::Utc; use heed::types::Bytes; use heed::{Database, Env, RoTxn, RwTxn}; use nostream_config::Settings; -use nostream_core::{EventId, EventStoreBackend, Filter, Kind, Pubkey}; +use nostream_core::{EventId, Filter, Kind, Pubkey}; use crate::backend::LmdbEnv; use crate::error::StorageError; @@ -161,11 +161,8 @@ impl LmdbBackend { /// Opens (creating if needed) the store under `/lmdb` and creates every database. pub async fn open(root: impl Into) -> Result { let root = root.into(); - let path = crate::backend::data_dir(&root, EventStoreBackend::Lmdb).ok_or( - StorageError::InvalidArgument { - message: "the lmdb backend has no data directory", - }, - )?; + // The same rule `data_dir` answers, from the same constant. + let path = root.join(crate::backend::LMDB_DIR); let env = tokio::task::spawn_blocking(move || LmdbEnv::open(path, LMDB_MAP_SIZE)) .await .map_err(|_| StorageError::LmdbTaskCancelled)??; @@ -949,4 +946,36 @@ mod tests { assert!(created < LMDB_MAX_DBS, "the handle limit is enforced"); transaction.abort(); } + + #[tokio::test] + async fn a_store_filled_past_its_map_size_reports_the_driver_error() { + // A small map is the only way to reach the boundary the fixed LMDB_MAP_SIZE stands + // in for: the environment never grows, so a full one is the driver's error (V27). + let directory = tempfile::tempdir().expect("tempdir"); + let environment = + LmdbEnv::open(directory.path().join("map"), 1 << 20).expect("a 1 MiB environment"); + let env = environment.env().clone(); + let mut transaction = env.write_txn().expect("a write transaction"); + let database = env + .create_database::(&mut transaction, Some("fill")) + .expect("a database"); + + let value = vec![0u8; 64 << 10]; + let mut written = 0u32; + loop { + match database.put(&mut transaction, &written.to_be_bytes(), &value) { + Ok(()) => written += 1, + Err(error) => { + let error = StorageError::from(error); + assert!( + matches!(error, StorageError::Lmdb(_)), + "a full map is the driver's error, got {error:?}" + ); + break; + } + } + } + assert!(written > 0, "the map held rows before it filled"); + transaction.abort(); + } } From 05ef86236c890780818fe08f99b1cb4d2e329034 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:46:02 -0500 Subject: [PATCH 12/60] test(storage): claim the limited page, batch, tag and outbox-key rules --- crates/nostream-storage/tests/parity_lmdb.rs | 245 +++++++++++++++++++ 1 file changed, 245 insertions(+) diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index 1c5af83f3..d8df787bb 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -416,6 +416,51 @@ async fn confirm_invoice_scales_by_unit_and_rejects_an_overflow() { ); } +#[tokio::test] +async fn a_nanosecond_instant_reads_back_at_microsecond_precision() { + let (_directory, backend) = open_store().await; + let now = Utc::now(); + let payee = Pubkey::from_byte_array([11; 32]); + backend.admit_user(&payee, now).await.unwrap(); + let invoice = Invoice { + id: "invoice-micros".to_owned(), + pubkey: payee, + bolt11: "lnbc1".to_owned(), + amount_requested: 1, + amount_paid: None, + unit: InvoiceUnit::Sats, + status: InvoiceStatus::Pending, + description: "test".to_owned(), + confirmed_at: None, + expires_at: None, + updated_at: now, + created_at: now, + verify_url: None, + }; + InvoiceRepository::upsert(&backend, &invoice).await.unwrap(); + + // Three digits past the microsecond, which no `timestamp` column can hold (V38). + let instant = DateTime::::from_timestamp_nanos(1_700_000_000_123_456_789); + backend + .confirm_invoice("invoice-micros", 1, instant) + .await + .unwrap(); + + let confirmed_at = backend + .find_by_id("invoice-micros") + .await + .unwrap() + .unwrap() + .confirmed_at + .expect("the confirmation is stamped"); + assert_eq!( + confirmed_at.timestamp_micros(), + 1_700_000_000_123_456, + "a nanosecond instant reads back at microsecond precision (V38)" + ); + assert_eq!(confirmed_at.timestamp_subsec_nanos(), 123_456_000); +} + #[tokio::test] async fn a_delivery_log_entry_must_name_an_existing_outbox_row() { let (_directory, backend) = open_store().await; @@ -517,3 +562,203 @@ async fn two_enqueues_return_distinct_uuids() { assert_uuid_v4(&first.id); assert_uuid_v4(&second.id); } + +/// Five events of one author, created at 100, 200, … 500 seconds. +async fn five_events_of_one_author(backend: &LmdbBackend) -> Vec { + let records: Vec = (1..=5).map(|step| event(5, 1, step * 100)).collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + records +} + +/// The `created_at` values a page returned, newest first. +fn created_ats(records: &[EventRecord]) -> Vec { + records + .iter() + .map(|record| record.event.created_at) + .collect() +} + +/// A signed event of one author, by the key `secret`, carrying `tags`. +fn tagged_event(secret: u8, created_at: i64, tags: Vec>) -> EventRecord { + let mut wire = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(1), + tags, + content: format!("{secret}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut wire).expect("the event signs"); + EventRecord::from_event(&wire) +} + +#[tokio::test] +async fn a_limited_ids_filter_returns_the_newest_events() { + let (_directory, backend) = open_store().await; + let records = five_events_of_one_author(&backend).await; + let filter = Filter { + ids: Some( + records + .iter() + .map(|record| record.event.id.to_string()) + .collect(), + ), + limit: Some(3), + ..Filter::default() + }; + + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + created_ats(&page), + vec![500, 400, 300], + "a limit orders the page newest first whatever index answered it (V6)" + ); + assert_eq!( + backend + .count_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + 3, + "the count applies the same limit and order (V6)" + ); +} + +#[tokio::test] +async fn a_limited_author_filter_returns_the_newest_events() { + let (_directory, backend) = open_store().await; + let records = five_events_of_one_author(&backend).await; + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + limit: Some(3), + ..Filter::default() + }; + + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + created_ats(&page), + vec![500, 400, 300], + "a limit orders the page newest first whatever index answered it (V6)" + ); +} + +#[tokio::test] +async fn a_reconciliation_lists_every_live_row_across_batches() { + let (_directory, backend) = open_store().await; + let records: Vec = (0..500) + .map(|index| event((index % 200 + 1) as u8, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(1), + ..Filter::default() + }; + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + assert_eq!( + items.len(), + 500, + "every live row arrives, not one per batch fewer (R20)" + ); + assert!( + items + .windows(2) + .all(|pair| (pair[0].created_at, pair[0].event_id) + < (pair[1].created_at, pair[1].event_id)), + "reconciliation is oldest first and ignores the filter's limit" + ); +} + +#[tokio::test] +async fn an_empty_tag_value_criterion_matches_nothing() { + let (_directory, backend) = open_store().await; + let record = tagged_event(3, 1_700_000_000, vec![vec!["e".to_owned(), String::new()]]); + EventRepository::create(&backend, &record) + .await + .expect("the event stores"); + + let mut filter = Filter { + ids: Some(vec![record.event.id.to_string()]), + limit: Some(10), + ..Filter::default() + }; + let reachable = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(reachable.len(), 1, "the row is reachable by its id"); + + filter.tags.insert("#e".to_owned(), vec![String::new()]); + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert!( + page.is_empty(), + "event_tags holds no empty value, so no row can match one (V42)" + ); +} + +#[tokio::test] +async fn a_delivery_log_entry_keeps_its_index_entry_when_a_sibling_is_pruned() { + let (_directory, backend) = open_store().await; + let message = backend + .enqueue("note.created", &json!({ "id": "note-3" })) + .await + .unwrap(); + let entry = one_entry(Some(message.id.clone())); + backend.append(&entry).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + backend.append(&entry).await.unwrap(); + + let entries = backend + .find_recent(10, &DeliveryLogFilters::default()) + .await + .unwrap(); + assert_eq!(entries.len(), 2, "two attempts, one target"); + let cutoff = entries[1].created_at + Duration::milliseconds(1); + assert_eq!( + backend.delete_older_than(cutoff).await.unwrap(), + 1, + "only the older attempt is pruned" + ); + assert_eq!( + backend + .find_successful_target_ids(&message.id) + .await + .unwrap(), + vec![entry.target_id.clone()], + "the surviving attempt keeps its index entry (V40)" + ); +} From 6d774ff4e6d77a6e0ac88fed73d5dfa0a4f17c68 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:46:02 -0500 Subject: [PATCH 13/60] docs(storage): correct the reconciliation note in ADR 0010 --- docs/rewrite/decisions/0010-lmdb-backend.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index 37f455423..70392fd53 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -114,8 +114,9 @@ it inside the closure; no `RwTxn` or `RoTxn` leaves one. A stream is a key page, not a held transaction. `find_by_filters` collects its union page inside one blocking closure; the returned `QueryStream` yields values in batches of `STREAM_BATCH_ITEMS = 256`, each poll opening a read transaction, seeking to the saved -cursor key, copying up to 256 values and dropping it. `find_reconciliation_items` runs the -same loop over keys and never reads a value. **A stream is therefore not a snapshot** (D7): +cursor key, copying up to 256 values and dropping it. `find_reconciliation_items` resumes +the same cursor over `events_by_created_at`, reading a candidate's header and decoded record +only to apply the filter's live predicate. **A stream is therefore not a snapshot** (D7): a row written or deleted between two batches may appear or vanish, where Postgres' portal holds one snapshot. No comparable behaviour exists to compare, so the divergence is recorded for the benchmark job, and ADR 0003 trigger 1 ("a long-lived LMDB read transaction hurts From c06b5d276287383034fd5c9a5f2d7752d2907b94 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:52:10 -0500 Subject: [PATCH 14/60] test(storage): claim the retention batch ceiling and its classification --- crates/nostream-storage/tests/parity_lmdb.rs | 120 ++++++++++++++++++- 1 file changed, 116 insertions(+), 4 deletions(-) diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index d8df787bb..5347b4926 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -23,10 +23,11 @@ use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; use nostream_storage::LmdbBackend; use nostream_storage::StorageError; use nostream_storage::repositories::{ - CreateInviteCodeOptions, DeliveryLogFilters, EventRecord, EventRepository, FieldUpdate, - InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, InvoiceUnit, - NewDeliveryLogEntry, NotificationDeliveryLogRepository, NotificationDeliveryStatus, - NotificationOutboxRepository, NotificationTargetType, UserRepository, + CreateInviteCodeOptions, DeliveryLogFilters, EventRecord, EventRepository, + EventRetentionOptions, FieldUpdate, InviteCodeRepository, Invoice, InvoiceRepository, + InvoiceStatus, InvoiceUnit, NewDeliveryLogEntry, NotificationDeliveryLogRepository, + NotificationDeliveryStatus, NotificationOutboxRepository, NotificationTargetType, + UserRepository, }; mod common; @@ -762,3 +763,114 @@ async fn a_delivery_log_entry_keeps_its_index_entry_when_a_sibling_is_pruned() { "the surviving attempt keeps its index entry (V40)" ); } + +#[tokio::test] +async fn a_retention_pass_classifies_a_row_once_and_spares_the_whitelist() { + let (_directory, backend) = open_store().await; + let now = Utc::now(); + let long_ago = 1_700_000_000; + let expired_at = now.timestamp() - 1; + + // Deleted and expired at once: the capture counts it as deleted, and its precedence is + // what makes the total one row rather than two (R13, V8). + let deleted_and_expired = event(21, 1, long_ago).with_expires_at(expired_at); + let expired_only = event(22, 1, long_ago).with_expires_at(expired_at); + let retained_only = event(23, 1, long_ago); + // Every whitelist keeps kind 62, so this row is never a candidate. + let vanish = event(24, 62, long_ago); + backend + .create_many(&[ + deleted_and_expired.clone(), + expired_only, + retained_only, + vanish.clone(), + ]) + .await + .expect("the events store"); + assert_eq!( + backend + .delete_by_pubkey_and_ids( + &deleted_and_expired.event.pubkey, + &[deleted_and_expired.event.id] + ) + .await + .unwrap(), + 1 + ); + + let skipped = backend + .delete_expired_and_retained(&EventRetentionOptions { + max_days: Some(0), + kind_whitelist: None, + pubkey_whitelist: None, + }) + .await + .unwrap(); + assert_eq!( + (skipped.deleted, skipped.expired, skipped.retained), + (0, 0, 0), + "max_days 0 touches nothing (V8)" + ); + + let counts = backend + .delete_expired_and_retained(&EventRetentionOptions { + max_days: Some(1), + kind_whitelist: None, + pubkey_whitelist: None, + }) + .await + .unwrap(); + assert_eq!( + (counts.deleted, counts.expired, counts.retained), + (1, 1, 1), + "each qualifying row is classified once, in the capture's precedence" + ); + + let filter = Filter { + ids: Some(vec![vanish.event.id.to_string()]), + ..Filter::default() + }; + let survivors = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + survivors.len(), + 1, + "a kind-62 row survives an empty configured whitelist (V8)" + ); +} + +#[tokio::test] +async fn a_retention_pass_takes_at_most_one_batch_of_candidates() { + let (_directory, backend) = open_store().await; + let records: Vec = (0..1_001) + .map(|index| event((index % 200 + 1) as u8, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let options = EventRetentionOptions { + max_days: Some(1), + kind_whitelist: None, + pubkey_whitelist: None, + }; + let first = backend.delete_expired_and_retained(&options).await.unwrap(); + assert_eq!( + (first.deleted, first.expired, first.retained), + (0, 0, 1_000), + "one pass takes RETENTION_BATCH_SIZE candidates (V7)" + ); + let second = backend.delete_expired_and_retained(&options).await.unwrap(); + assert_eq!( + (second.deleted, second.expired, second.retained), + (0, 0, 1), + "the next pass takes the row the batch ceiling left behind" + ); +} From 75b7ea101316cbe7c2cd7a2971ad40080e431f68 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 00:55:00 -0500 Subject: [PATCH 15/60] test(storage): claim the empty criterion family --- crates/nostream-storage/tests/parity_lmdb.rs | 47 +++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index 5347b4926..9ced37d05 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -12,7 +12,7 @@ // workspace-wide, so a new test file still has to ask. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use chrono::{DateTime, Duration, Utc}; @@ -731,6 +731,51 @@ async fn an_empty_tag_value_criterion_matches_nothing() { ); } +#[tokio::test] +async fn an_empty_criterion_family_matches_nothing() { + let (_directory, backend) = open_store().await; + let record = event(6, 1, 1_700_000_000); + EventRepository::create(&backend, &record) + .await + .expect("the event stores"); + + for filter in [ + Filter { + ids: Some(Vec::new()), + ..Filter::default() + }, + Filter { + authors: Some(Vec::new()), + ..Filter::default() + }, + Filter { + tags: BTreeMap::from([("#e".to_owned(), Vec::new())]), + ..Filter::default() + }, + ] { + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert!( + page.is_empty(), + "a present-but-empty family matches nothing, not the next path (V45): {filter:?}" + ); + assert_eq!( + backend + .count_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + 0, + "the count of an empty family is zero (V45): {filter:?}" + ); + } +} + #[tokio::test] async fn a_delivery_log_entry_keeps_its_index_entry_when_a_sibling_is_pruned() { let (_directory, backend) = open_store().await; From 0b2d4b152e71f836c42b47bc4b5183d862742fd9 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:01:38 -0500 Subject: [PATCH 16/60] refactor(storage): give the lmdb key codecs one home --- crates/nostream-storage/src/lmdb/encoding.rs | 141 ++++++++++++++++-- crates/nostream-storage/src/lmdb/metadata.rs | 65 +++++--- .../src/lmdb/notifications.rs | 20 +-- 3 files changed, 177 insertions(+), 49 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index 1f691fc05..f9eeaaee8 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -26,6 +26,19 @@ const NO_EXPIRY: i64 = i64::MIN; /// The number of bytes a `created_at ‖ event_id` key appends after the timestamp. const EVENT_ID_LEN: usize = 32; +/// LMDB's maximum key length (`MDB_MAXKEYSIZE`, the default this environment is opened with). +/// +/// It bounds every key this module builds: a tag value longer than the room a `tags` key has +/// left cannot be stored, so an event carrying one is refused at the write. That is the one +/// place this backend rejects a row Postgres would keep, and ADR 0010 records it as a +/// divergence. +/// +/// shortcut: the tag index carries the value verbatim in the key, so a value of 470 bytes or +/// more cannot be indexed and its event is refused; upgrade when an event like that has to be +/// stored, which needs the value hashed into the key plus a second lookup, or a larger +/// `max_key_size` at the open. +const MAX_KEY_LEN: usize = 511; + /// An invalid stored value, naming the database it was read from. pub(crate) fn invalid(table: &'static str, message: String) -> StorageError { StorageError::InvalidRow { @@ -182,6 +195,32 @@ pub(crate) fn check_version(table: &'static str, value: &[u8]) -> Result<(), Sto } } +/// A text component of a composite key, terminated so the next component reads unambiguously. +/// +/// The terminator is what every key *reader* in this module splits on, so the rule lives here +/// with the writers; a component that carries `0x00` cannot be read back, and the codec that +/// puts a tag into a key ([`tag_key`]) refuses one for that reason. +pub(crate) fn text_part(value: &str) -> Vec { + let mut bytes = value.as_bytes().to_vec(); + bytes.push(0); + bytes +} + +/// The text a composite key's component carries, refusing bytes a `text` column could not hold. +pub(crate) fn text_from<'a>(bytes: &'a [u8], table: &'static str) -> Result<&'a str, StorageError> { + std::str::from_utf8(bytes).map_err(|_| StorageError::InvalidRow { + table, + column: "key", + value: "not utf-8".to_owned(), + }) +} + +/// A `created_at`-style timestamp component of a tail index key, in the microseconds every +/// PostgreSQL `timestamp` column carries. +pub(crate) fn stamp_key(value: DateTime) -> [u8; 8] { + i64_key(value.timestamp_micros()) +} + /// `created_at ‖ event_id`. pub(crate) fn created_at_key(created_at: i64, event_id: &EventId) -> Vec { let mut key = Vec::with_capacity(8 + EVENT_ID_LEN); @@ -272,6 +311,24 @@ pub(crate) fn tag_key( Ok(key) } +/// The tag value a [`tag_key`] carries: the bytes between its two `0x00` separators. +/// +/// A query that walked the `tags` index has already been told the value by the key, so it does +/// not have to decode the event's body to check the criterion that selected the row. +pub(crate) fn tag_value(key: &[u8]) -> Result<&str, StorageError> { + let Some(name_end) = key.iter().position(|byte| *byte == 0) else { + return Err(invalid("tags", "a key without a name separator".to_owned())); + }; + let rest = &key[name_end + 1..]; + let Some(value_end) = rest.iter().position(|byte| *byte == 0) else { + return Err(invalid( + "tags", + "a key without a value separator".to_owned(), + )); + }; + text_from(&rest[..value_end], "tags") +} + /// The tag rows the capture's trigger writes: one per element whose name is exactly one /// character and whose value is present and non-empty, and never beyond the second element. pub(crate) fn tag_entries(record: &crate::repositories::EventRecord) -> Vec<(&str, &str)> { @@ -355,11 +412,35 @@ pub(crate) fn hex_ranges(criteria: &[String], suffix: usize) -> Vec<(Vec, Ve upper.extend(std::iter::repeat_n(0xff, suffix)); ranges.push((lower, upper)); } - ranges + disjoint(ranges) } -/// The inclusive key ranges one tag criterion covers: an exact value is the range from the -/// value's own separator, and a `#g` prefix is the range over the value's first bytes. +/// The ranges no key is selected by more than once. +/// +/// Two hex criteria can select the same keys — `ab` and `abcd` nest, and a digit the [range +/// builder](hex_ranges) maps to zero does the same — and a walk that reads one key twice would +/// count an event twice. Dropping a range another already covers keeps every range a set of +/// keys it alone selects. +fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec)> { + ranges.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1))); + let mut kept: Vec<(Vec, Vec)> = Vec::with_capacity(ranges.len()); + for range in ranges { + let covered = kept + .last() + .is_some_and(|last| last.0 <= range.0 && range.1 <= last.1); + if !covered { + kept.push(range); + } + } + kept +} + +/// The inclusive key ranges one tag criterion covers, one range per criterion and in the +/// criteria's order: an exact value is the range from the value's own separator, and a `#g` +/// prefix is the range over the value's first bytes. +/// +/// A criterion whose own bounds do not fit [`MAX_KEY_LEN`] selects nothing: no stored key can +/// be that long, so seeking it would fail the read rather than answer it. pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(Vec, Vec)> { let is_geohash = name == "g"; let mut ranges = Vec::with_capacity(criteria.len()); @@ -367,20 +448,23 @@ pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(Vec, Vec { head.extend_from_slice(prefix.as_bytes()); let mut upper = head.clone(); upper.extend(std::iter::repeat_n(0xff, 64)); - ranges.push((head, upper)); + (head, upper) } None => { head.extend_from_slice(criterion.as_bytes()); head.push(0); let mut upper = head.clone(); upper.extend(std::iter::repeat_n(0xff, 8 + EVENT_ID_LEN)); - ranges.push((head, upper)); + (head, upper) } + }; + if lower.len() <= MAX_KEY_LEN && upper.len() <= MAX_KEY_LEN { + ranges.push((lower, upper)); } } ranges @@ -416,18 +500,55 @@ mod tests { #[test] fn an_even_prefix_is_one_range_and_an_odd_one_is_one_nibble_range() { - let ranges = hex_ranges(&["abcd".to_owned(), "abcde".to_owned()], 40); - assert_eq!(ranges.len(), 2); + let ranges = hex_ranges(&["abcd".to_owned(), "abef".to_owned()], 40); + assert_eq!(ranges.len(), 2, "two disjoint criteria seek two ranges"); assert_eq!(ranges[0].0, vec![0xab, 0xcd]); assert_eq!(ranges[0].1.len(), 2 + 40); - assert_eq!(ranges[1].0, vec![0xab, 0xcd, 0xe0]); - assert_eq!(ranges[1].1, { + assert_eq!(ranges[1].0, vec![0xab, 0xef]); + + let odd = hex_ranges(&["abcde".to_owned()], 40); + assert_eq!(odd[0].0, vec![0xab, 0xcd, 0xe0]); + assert_eq!(odd[0].1, { let mut high = vec![0xab, 0xcd, 0xef]; high.extend(std::iter::repeat_n(0xff, 40)); high }); } + #[test] + fn a_criterion_another_range_covers_seeks_no_range_of_its_own() { + // `abcd` selects every key `abcde` does, so a walk that read both would visit those + // keys twice and count them twice. + let nested = hex_ranges(&["abcd".to_owned(), "abcde".to_owned()], 40); + assert_eq!(nested.len(), 1); + assert_eq!(nested[0].0, vec![0xab, 0xcd]); + + // Two criteria that decode to the same bytes are one range too. + let duplicate = hex_ranges(&["zz".to_owned(), "z0".to_owned()], 0); + assert_eq!(duplicate.len(), 1, "both criteria map to the zero digit"); + } + + #[test] + fn a_tag_key_names_the_value_that_selected_it() { + let key = tag_key("t", "perfprobe", 1_700_000_000, &id(9)).unwrap(); + assert_eq!(tag_value(&key).unwrap(), "perfprobe"); + // A value shorter than the key's tail is still read up to its own separator. + let key = tag_key("t", "a", 0, &id(1)).unwrap(); + assert_eq!(tag_value(&key).unwrap(), "a"); + assert!(tag_value(b"t").is_err()); + assert!(tag_value(b"t\0a").is_err()); + } + + #[test] + fn a_criterion_too_long_for_a_key_seeks_nothing() { + // A key carrying it could not be written, so no stored row can match it; seeking it + // would fail the read with LMDB's key-length error instead of answering it. + let long = "x".repeat(MAX_KEY_LEN); + assert!(tag_ranges("t", std::slice::from_ref(&long)).is_empty()); + let short = "x".repeat(64); + assert_eq!(tag_ranges("t", std::slice::from_ref(&short)).len(), 1); + } + #[test] fn a_geohash_criterion_is_a_prefix_range_and_another_tag_is_exact() { let geohash = tag_ranges("g", &["u4pru*".to_owned()]); diff --git a/crates/nostream-storage/src/lmdb/metadata.rs b/crates/nostream-storage/src/lmdb/metadata.rs index b395cb2fc..f777ffa6a 100644 --- a/crates/nostream-storage/src/lmdb/metadata.rs +++ b/crates/nostream-storage/src/lmdb/metadata.rs @@ -9,6 +9,7 @@ use nostream_core::{EventId, Kind, Pubkey}; use super::Databases; use super::encoding; +use super::encoding::{stamp_key, text_from, text_part}; use super::rows; use crate::error::StorageError; use crate::repositories::{ @@ -19,30 +20,14 @@ use crate::repositories::{ /// The `meta` key holding the `reports.id` sequence (`reports_id_seq`). const REPORT_SEQUENCE: &[u8] = b"reports.next_id"; -/// A text component of a composite key, terminated so the next component reads unambiguously. -fn text_part(value: &str) -> Vec { - let mut bytes = value.as_bytes().to_vec(); - bytes.push(0); - bytes -} - -fn text_from<'a>(bytes: &'a [u8], table: &'static str) -> Result<&'a str, StorageError> { - std::str::from_utf8(bytes).map_err(|_| StorageError::InvalidRow { - table, - column: "key", - value: "not utf-8".to_owned(), - }) -} +/// The width of `reports.id` in every `reports_*` index key, pinned so a widening is a compile +/// error here rather than a prefix lookup that silently seeks a narrower range. +const REPORT_ID_LEN: usize = 4; // --------------------------------------------------------------------------------------- // NIP-05 verifications. // --------------------------------------------------------------------------------------- -/// `created_at`-style keys for the tail indexes order by microseconds. -fn stamp_key(value: DateTime) -> [u8; 8] { - encoding::i64_key(value.timestamp_micros()) -} - fn nip05_key(pubkey: &Pubkey) -> Vec { pubkey.as_bytes().to_vec() } @@ -486,7 +471,7 @@ pub(crate) fn dvm_find_pending_jobs( // Reports. // --------------------------------------------------------------------------------------- -fn report_key(id: i32) -> [u8; 4] { +fn report_key(id: i32) -> [u8; REPORT_ID_LEN] { encoding::i32_key(id) } @@ -656,3 +641,43 @@ pub(crate) fn report_find_actionable_targets( }) .collect()) } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use crate::repositories::ReportType; + + fn report() -> Report { + Report { + id: 7, + event_id: EventId::from_byte_array([1; 32]), + reporter_pubkey: Pubkey::from_byte_array([2; 32]), + reported_pubkey: Some(Pubkey::from_byte_array([3; 32])), + reported_event_id: Some(EventId::from_byte_array([4; 32])), + report_type: ReportType::Spam, + weight: 0.0, + actionable: true, + created_at: Utc::now(), + } + } + + #[test] + fn every_report_index_key_ends_with_the_id_width_the_codec_writes() { + // The NIP-56 lookups seek `reports_actionable_*` by the target's prefix and the walks + // read the id back from the tail, so the id's width is the one `report_key` writes: a + // widening has to change this assertion and the reads together, never one of them. + let report = report(); + assert_eq!(REPORT_ID_LEN, report_key(report.id).len()); + assert_eq!( + report_event_key(&report, &report.event_id).len(), + 32 + REPORT_ID_LEN + ); + assert_eq!( + report_pubkey_key(&report, &report.reporter_pubkey).len(), + 32 + REPORT_ID_LEN + ); + assert_eq!(report_created_key(&report).len(), 8 + REPORT_ID_LEN); + } +} diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs index 6bd79d197..f3ac46f12 100644 --- a/crates/nostream-storage/src/lmdb/notifications.rs +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -12,6 +12,7 @@ use heed::{RoTxn, RwTxn}; use super::Databases; use super::encoding; +use super::encoding::{stamp_key, text_from, text_part}; use super::rows; use crate::error::StorageError; use crate::repositories::{ @@ -19,25 +20,6 @@ use crate::repositories::{ NewDeliveryLogEntry, OutboxMessage, OutboxPayload, OutboxStatus, }; -/// A text component of a composite key, terminated so the next component reads unambiguously. -fn text_part(value: &str) -> Vec { - let mut bytes = value.as_bytes().to_vec(); - bytes.push(0); - bytes -} - -fn text_from<'a>(bytes: &'a [u8], table: &'static str) -> Result<&'a str, StorageError> { - std::str::from_utf8(bytes).map_err(|_| StorageError::InvalidRow { - table, - column: "key", - value: "not utf-8".to_owned(), - }) -} - -fn stamp_key(value: DateTime) -> [u8; 8] { - encoding::i64_key(value.timestamp_micros()) -} - fn new_id() -> String { uuid::Uuid::new_v4().to_string() } From 9656faa14bfc2f3cb28fa8480105d67da114a1a9 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:01:41 -0500 Subject: [PATCH 17/60] perf(storage): bound a read to its page and report the map's fill --- Cargo.lock | 1 + crates/nostream-storage/Cargo.toml | 3 +- crates/nostream-storage/src/lib.rs | 2 +- crates/nostream-storage/src/lmdb/events.rs | 1161 +++++++++++++++++--- crates/nostream-storage/src/lmdb/mod.rs | 72 +- 5 files changed, 1061 insertions(+), 178 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 541c090a1..ef6843886 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1353,6 +1353,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tracing", "uuid", ] diff --git a/crates/nostream-storage/Cargo.toml b/crates/nostream-storage/Cargo.toml index 4217ba222..516868329 100644 --- a/crates/nostream-storage/Cargo.toml +++ b/crates/nostream-storage/Cargo.toml @@ -15,7 +15,7 @@ default = ["postgres", "lmdb"] postgres = ["dep:sqlx"] # The LMDB backend, its environment and its row encodings. `uuid` is only used to mint # the canonical text ids the outbox and delivery-log columns generate. -lmdb = ["dep:heed", "dep:uuid"] +lmdb = ["dep:heed", "dep:uuid", "dep:tracing"] [dependencies] chrono.workspace = true @@ -28,6 +28,7 @@ serde_json.workspace = true sqlx = { workspace = true, optional = true } thiserror.workspace = true tokio.workspace = true +tracing = { workspace = true, optional = true } uuid = { workspace = true, optional = true } [dev-dependencies] diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index a259e083c..d8cc9aadb 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -40,6 +40,6 @@ pub use backend::LmdbEnv; pub use backend::data_dir; pub use error::StorageError; #[cfg(feature = "lmdb")] -pub use lmdb::LmdbBackend; +pub use lmdb::{LmdbBackend, MapUsage}; #[cfg(feature = "postgres")] pub use postgres::{PostgresBackend, PostgresTransaction}; diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index f5b011b12..eb8be99db 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -20,7 +20,10 @@ use tokio::task::JoinHandle; use super::encoding::{self, EventHeader}; use super::rows; -use super::{DEFAULT_FILTER_LIMIT, Databases, Gates, RETENTION_BATCH_SIZE, STREAM_BATCH_ITEMS}; +use super::{ + DEFAULT_FILTER_LIMIT, Databases, Gates, MAX_PAGE_KEYS, RETENTION_BATCH_SIZE, + STREAM_BATCH_ITEMS, STREAM_BATCH_KEYS, +}; use crate::error::StorageError; use crate::repositories::{ EventPurgeCounts, EventRecord, EventRetentionOptions, ReconciliationItem, @@ -549,11 +552,69 @@ fn kind_of(record: &EventRecord) -> Result { // Read paths. // --------------------------------------------------------------------------------------- +/// Counts a key a read evaluated. Only the tests read the count, so a release build drops it. +fn key_evaluated() { + #[cfg(test)] + probes::key(); +} + +/// Counts an event body a read decoded. Only the tests read the count. +fn body_decoded() { + #[cfg(test)] + probes::decode(); +} + +/// The work counters the cost tests assert on: a page's cost is its keys evaluated and its +/// bodies decoded, and neither is visible in its result. +#[cfg(test)] +mod probes { + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Serialises the tests that count work, so one cannot read another's numbers. + pub(super) static COUNTING: Mutex<()> = Mutex::new(()); + + static KEYS: AtomicUsize = AtomicUsize::new(0); + static DECODES: AtomicUsize = AtomicUsize::new(0); + + pub(super) fn reset() { + KEYS.store(0, Ordering::Relaxed); + DECODES.store(0, Ordering::Relaxed); + } + + pub(super) fn key() { + KEYS.fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn decode() { + DECODES.fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn keys() -> usize { + KEYS.load(Ordering::Relaxed) + } + + pub(super) fn decodes() -> usize { + DECODES.load(Ordering::Relaxed) + } +} + /// One database's inclusive key range. type KeyRange = (Vec, Vec); -/// The index one filter is answered from, how its keys are read back, and the ranges to walk. -type IndexRanges = (Database, AccessPath, Vec); +/// The index one filter is answered from, the ranges within it, and what a key proves. +struct Plan<'a> { + database: Database, + path: AccessPath, + ranges: Vec, + /// Whether a range's own key order is the page's order, so a range stops once the page is + /// full instead of being walked whole. + ordered: bool, + /// The tag family the ranges were built from, with its criteria in the criteria's order — + /// `tag_ranges` builds one range per criterion. A key in these ranges has already proved + /// that family, so the row's body is decoded only for the filter's other tag families. + tag: Option<(&'a str, &'a [String])>, +} /// One reconciliation batch still running on the blocking pool, with the cursor it returns. type PendingBatch = JoinHandle, Position), StorageError>>; @@ -584,10 +645,15 @@ pub(crate) fn page( let mut union = Vec::new(); let mut seen = HashSet::new(); for filter in filters { - let limit = Some( - usize::try_from(filter.limit.unwrap_or(DEFAULT_FILTER_LIMIT)).unwrap_or(usize::MAX), - ); - for candidate in branch(txn, databases, filter, gates, now, false, limit)? { + for candidate in branch( + txn, + databases, + filter, + gates, + now, + false, + page_limit(filter), + )? { if seen.insert(candidate) { union.push(candidate); } @@ -598,6 +664,11 @@ pub(crate) fn page( } /// The count of the live ids the filters match, deduplicated across filters. +/// +/// A count has no page, so it holds no candidate set: each filter counts the rows it visits and +/// skips one the count already holds — a row of an earlier filter's page when that filter had a +/// limit (a page is at most [`MAX_PAGE_KEYS`] ids), or one its predicate matches when it had +/// none — which is the capture's `UNION`. pub(crate) fn count( txn: &RoTxn<'_>, databases: &Databases, @@ -611,28 +682,160 @@ pub(crate) fn count( }); } - let mut ids = HashSet::new(); + let mut counted = Counted::default(); + let mut total = 0; for filter in filters { - let limit = filter - .limit - .map(|limit| usize::try_from(limit).unwrap_or(usize::MAX)); - for (_, id) in branch(txn, databases, filter, gates, now, true, limit)? { - ids.insert(id); + total += match filter.limit { + Some(limit) => { + let page = branch(txn, databases, filter, gates, now, true, count_limit(limit))?; + let mut fresh = 0; + for (_, id) in &page { + if !already_counted(txn, databases, &counted, gates, now, id)? { + fresh += 1; + } + } + counted.pages.push(page.iter().map(|(_, id)| *id).collect()); + fresh + } + None => { + let fresh = count_matches(txn, databases, filter, &counted, gates, now)?; + counted.predicates.push(filter); + fresh + } + }; + } + Ok(total) +} + +/// What a count has already counted: bounded by [`MAX_PAGE_KEYS`] per limited filter. +#[derive(Default)] +struct Counted<'a> { + /// The ids of every earlier filter that had a limit: its whole page. + pages: Vec>, + /// Every earlier filter without a limit, whose page is every row its predicate matches. + predicates: Vec<&'a Filter>, +} + +/// Whether the count already holds `id`, so counting it again would count the row twice. +fn already_counted( + txn: &RoTxn<'_>, + databases: &Databases, + counted: &Counted<'_>, + gates: &Gates, + now: i64, + id: &EventId, +) -> Result { + if counted.pages.iter().any(|page| page.contains(id)) { + return Ok(true); + } + for filter in &counted.predicates { + let matcher = Matcher { + filter, + gates, + now, + live: true, + proven: None, + }; + if matcher.matches(txn, databases, id)?.is_some() { + return Ok(true); } } - Ok(ids.len() as u64) + Ok(false) } -/// One filter's page: up to `limit` matching keys, ordered as the capture's branch is. -fn branch( +/// Whether a criterion of the filter's own tag family, other than the one this range was built +/// from, also selects the row. One event can carry two values the family asks for, and the +/// criterion whose range comes first answers for it, so the row is counted once. +/// +/// This is the one place a count reads an event's tags instead of its key: the key names a +/// single value, and which values the event also carries is in the body. +fn earlier_criterion_selects( + txn: &RoTxn<'_>, + databases: &Databases, + plan: &Plan<'_>, + index: usize, + id: &EventId, +) -> Result { + let Some((name, criteria)) = plan.tag else { + return Ok(false); + }; + if index == 0 { + return Ok(false); + } + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + return Ok(false); + }; + let record = rows::decode_event(value)?; + Ok(criteria + .iter() + .take(index) + .any(|earlier| record_has_tag(&record, name, std::slice::from_ref(earlier)))) +} + +/// Counts the live rows one unlimited filter matches that the count does not already hold. +/// +/// Every range is walked whole — a count has no page to stop at — and a row two of them select +/// is counted once, at the first criterion's range. +fn count_matches( txn: &RoTxn<'_>, databases: &Databases, filter: &Filter, + counted: &Counted<'_>, gates: &Gates, now: i64, - live: bool, - limit: Option, -) -> Result, StorageError> { +) -> Result { + if !admits(filter, gates)? { + return Ok(0); + } + let plan = plan(filter, databases); + let matcher = Matcher { + filter, + gates, + now, + live: true, + proven: plan.tag.map(|(name, _)| name), + }; + let mut total = 0; + for (index, (low, high)) in plan.ranges.iter().enumerate() { + let range = encoding::bounds(low, high); + for entry in plan.database.range(txn, &range)? { + let (key, _) = entry?; + key_evaluated(); + if let Some((name, criteria)) = plan.tag + && !tag_value_matches(name == "g", criteria, encoding::tag_value(key)?) + { + continue; + } + let Some((_, id)) = candidate(txn, databases, plan.path, key)? else { + continue; + }; + if matcher.matches(txn, databases, &id)?.is_some() + && !already_counted(txn, databases, counted, gates, now, &id)? + && !earlier_criterion_selects(txn, databases, &plan, index, &id)? + { + total += 1; + } + } + } + Ok(total) +} + +/// The page a filter may materialise: its own limit or the capture's default, and never more +/// than [`MAX_PAGE_KEYS`] however large the limit on the wire is. +fn page_limit(filter: &Filter) -> usize { + count_limit(filter.limit.unwrap_or(DEFAULT_FILTER_LIMIT)) +} + +/// A wire limit clamped to [`MAX_PAGE_KEYS`], so no request chooses the size of an allocation. +fn count_limit(limit: u64) -> usize { + usize::try_from(limit.min(MAX_PAGE_KEYS as u64)).unwrap_or(MAX_PAGE_KEYS) +} + +/// The refusals and short-circuits every read applies before it seeks an index. +/// +/// `true` means the filter can match rows; `false` means a criterion family is present but +/// empty, which is the capture's `1 = 0`. +fn admits(filter: &Filter, gates: &Gates) -> Result { if gates.nip50 && filter .search @@ -643,7 +846,7 @@ fn branch( module: "storage::lmdb::nip50_search", }); } - if filter.ids.as_ref().is_some_and(|ids| ids.is_empty()) + Ok(!(filter.ids.as_ref().is_some_and(|ids| ids.is_empty()) || filter .authors .as_ref() @@ -651,81 +854,159 @@ fn branch( || filter .tags .iter() - .any(|(key, criteria)| generic_tag_name(key).is_some() && criteria.is_empty()) - { + .any(|(key, criteria)| generic_tag_name(key).is_some() && criteria.is_empty()))) +} + +/// One filter's page: the best `limit` keys it matches, ordered as the capture's branch is. +/// +/// The resident set is the page, never the range. A range whose key order is the page's order +/// stops as soon as the page is full (keeping every row at the boundary timestamp, so a tie is +/// the capture's); the other ranges offer their matches to a buffer of at most `2 × limit`, +/// which is cut back to the best `limit` whenever it fills. +fn branch( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + gates: &Gates, + now: i64, + live: bool, + limit: usize, +) -> Result, StorageError> { + if !admits(filter, gates)? { return Ok(Vec::new()); } + let plan = plan(filter, databases); let matcher = Matcher { filter, gates, now, live, + proven: plan.tag.map(|(name, _)| name), }; - let (database, path, ranges) = choose_ranges(filter, databases); let descending = filter.limit.is_some(); - let mut matches: Vec<(i64, EventId)> = Vec::new(); - let mut seen = HashSet::new(); - let mut boundary: Option = None; - - for (low, high) in &ranges { + let mut page = Page::new(descending); + for (low, high) in &plan.ranges { let range = encoding::bounds(low, high); - if path == AccessPath::CreatedAt && descending { - for entry in database.rev_range(txn, &range)? { - let (key, _) = entry?; - let Some((created_at, id)) = key_candidate(path, key)? else { - continue; - }; - if boundary.is_some_and(|bound| created_at < bound) { - break; - } - if matcher.matches(txn, databases, &id)?.is_some() { - push(&mut matches, &mut seen, (created_at, id)); - if limit.is_some_and(|limit| matches.len() >= limit) { - boundary = Some(created_at); - } - } + let mut boundary: Option = None; + for entry in scan(plan.database, txn, &range, plan.ordered && descending)? { + let (key, _) = entry?; + key_evaluated(); + if !tag_key_proves(&plan, key)? { + continue; } - } else { - for entry in database.range(txn, &range)? { - let (key, _) = entry?; - let Some(found) = candidate(txn, databases, path, key)? else { - continue; - }; - // Only the created-at index iterates in page order. `ids`, `authors` and - // `tags` iterate in key order, so an early stop there would drop the newest - // rows of a limited page; those paths scan the whole (time-bounded) range and - // the page is selected below. - if path == AccessPath::CreatedAt && boundary.is_some_and(|bound| found.0 > bound) { - break; - } - if matcher.matches(txn, databases, &found.1)?.is_some() { - push(&mut matches, &mut seen, found); - if path == AccessPath::CreatedAt - && limit.is_some_and(|limit| matches.len() >= limit) - { - boundary = Some(found.0); - } + let Some(found) = candidate(txn, databases, plan.path, key)? else { + continue; + }; + if plan.ordered && boundary.is_some_and(|bound| past(bound, found.0, descending)) { + break; + } + if matcher.matches(txn, databases, &found.1)?.is_some() { + page.offer(found, limit); + if plan.ordered && page.len() >= limit { + boundary = Some(found.0); } } } } + Ok(page.finish(limit)) +} + +/// Whether the key that selected a row satisfies the tag family's criteria. +/// +/// A `tags` key is `name\0value\0…`, so the value the range sought is already in the key: the +/// criterion is checked from it, and the row's body is decoded only for the filter's other tag +/// families. Every other index selects a row without proving a tag criterion. +fn tag_key_proves(plan: &Plan<'_>, key: &[u8]) -> Result { + let Some((name, criteria)) = plan.tag else { + return Ok(true); + }; + let value = encoding::tag_value(key)?; + Ok(tag_value_matches(name == "g", criteria, value)) +} + +/// Whether a key's timestamp is past the timestamp a full page stopped at. +fn past(boundary: i64, created_at: i64, descending: bool) -> bool { + if descending { + created_at < boundary + } else { + created_at > boundary + } +} + +/// A page under construction: at most `2 × limit` entries, cut back to the best `limit`. +struct Page { + entries: Vec<(i64, EventId)>, + seen: HashSet, + descending: bool, +} + +impl Page { + fn new(descending: bool) -> Self { + Self { + entries: Vec::new(), + seen: HashSet::new(), + descending, + } + } + + fn len(&self) -> usize { + self.entries.len() + } + + /// Keeps `key` if it can still be among the best `limit` the page has seen. + fn offer(&mut self, key: (i64, EventId), limit: usize) { + if limit == 0 || !self.seen.insert(key.1) { + return; + } + self.entries.push(key); + if self.entries.len() >= limit.saturating_mul(2) { + self.cut(limit); + } + } + + fn finish(mut self, limit: usize) -> Vec<(i64, EventId)> { + self.cut(limit); + self.entries + } - // A limit on any other index is applied after the page is ordered, exactly as the - // capture's `ORDER BY … LIMIT` is. - sort_page(&mut matches, descending); - if let Some(limit) = limit { - matches.truncate(limit); + fn cut(&mut self, limit: usize) { + sort_page(&mut self.entries, self.descending); + self.entries.truncate(limit); + self.seen = self.entries.iter().map(|(_, id)| *id).collect(); } - Ok(matches) } -fn push(matches: &mut Vec<(i64, EventId)>, seen: &mut HashSet, candidate: (i64, EventId)) { - if seen.insert(candidate.1) { - matches.push(candidate); +/// One range's keys, in the plan's order. +enum Scan<'txn> { + Ascending(heed::RoRange<'txn, Bytes, Bytes>), + Descending(heed::RoRevRange<'txn, Bytes, Bytes>), +} + +impl<'txn> Iterator for Scan<'txn> { + type Item = Result<(&'txn [u8], &'txn [u8]), heed::Error>; + + fn next(&mut self) -> Option { + match self { + Self::Ascending(range) => range.next(), + Self::Descending(range) => range.next(), + } } } +fn scan<'txn>( + database: Database, + txn: &'txn RoTxn<'_>, + range: &(Bound<&[u8]>, Bound<&[u8]>), + descending: bool, +) -> Result, StorageError> { + Ok(if descending { + Scan::Descending(database.rev_range(txn, range)?) + } else { + Scan::Ascending(database.range(txn, range)?) + }) +} + fn sort_page(page: &mut [(i64, EventId)], descending: bool) { if descending { page.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(&right.1))); @@ -734,38 +1015,86 @@ fn sort_page(page: &mut [(i64, EventId)], descending: bool) { } } -/// The index a filter is answered from, and the ranges within it. -fn choose_ranges(filter: &Filter, databases: &Databases) -> IndexRanges { +/// The index a filter is answered from, the ranges within it, and whether a range's key order +/// is the page's order. +fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { if let Some(ids) = &filter.ids && !ids.is_empty() { - return ( - databases.events, - AccessPath::Ids, - encoding::hex_ranges(ids, 0), - ); + return Plan { + database: databases.events, + path: AccessPath::Ids, + ranges: encoding::hex_ranges(ids, 0), + // The `events` key is the id, so the range runs in id order, not time order. + ordered: false, + tag: None, + }; } if let Some(authors) = &filter.authors && !authors.is_empty() { - return ( - databases.events_by_author, - AccessPath::Authors, - encoding::hex_ranges(authors, 4 + 8 + 32), - ); + if let Some(kinds) = &filter.kinds + && !kinds.is_empty() + { + return Plan { + database: databases.events_by_author, + path: AccessPath::Authors, + ranges: author_kind_ranges(authors, kinds), + // A range is `pubkey ‖ kind ‖ created_at ‖ event_id`, so it runs in time order + // only when the criterion is a whole pubkey and the range one author. + ordered: authors.iter().all(|criterion| criterion.len() == 64), + tag: None, + }; + } + // Without kinds the author index runs in kind order before `created_at`, so a page + // ordered by time has no early stop there. The created-at index is the capture's own + // path for this shape — ADR 0003 measured 5 508 rows discarded to return 500 for the + // kind-only form — and its order *is* the page order, so the page stops when full. + return Plan { + database: databases.events_by_created_at, + path: AccessPath::CreatedAt, + ranges: created_at_ranges(filter), + ordered: true, + tag: None, + }; } if let Some((name, criteria)) = tag_path(filter) { - return ( - databases.tags, - AccessPath::Tags, - encoding::tag_ranges(name, criteria), - ); + return Plan { + database: databases.tags, + path: AccessPath::Tags, + ranges: encoding::tag_ranges(name, criteria), + // An exact criterion's range runs `created_at` under one value; a `#g` prefix range + // spans values, so its keys are in value order and the range is walked whole. + ordered: name != "g" || criteria.iter().all(|criterion| !criterion.ends_with('*')), + tag: Some((name, criteria)), + }; } - ( - databases.events_by_created_at, - AccessPath::CreatedAt, - created_at_ranges(filter), - ) + Plan { + database: databases.events_by_created_at, + path: AccessPath::CreatedAt, + ranges: created_at_ranges(filter), + ordered: true, + tag: None, + } +} + +/// One range per author criterion and kind on `events_by_author`, whose key is +/// `pubkey ‖ kind ‖ created_at ‖ event_id`: the `pubkey ‖ kind ‖ created_at` ordering the layout +/// writes is what lets a page for one author and one kind stop as soon as it is full. +fn author_kind_ranges(authors: &[String], kinds: &[Kind]) -> Vec { + let mut ranges = Vec::with_capacity(authors.len() * kinds.len()); + for (low, high) in encoding::hex_ranges(authors, 0) { + for kind in kinds { + let kind = encoding::i32_key(i32::try_from(kind.0).unwrap_or(i32::MAX)); + let mut lower = low.clone(); + lower.extend_from_slice(&kind); + let mut upper = high.clone(); + upper.extend_from_slice(&kind); + upper.extend(std::iter::repeat_n(0xff, 8 + 32)); + ranges.push((lower, upper)); + } + } + ranges } /// The first tag criterion the capture treats as a query: `#` plus one ASCII letter with a @@ -861,6 +1190,8 @@ struct Matcher<'a> { gates: &'a Gates, now: i64, live: bool, + /// The tag family the index range that selected the row has already proved, if any. + proven: Option<&'a str>, } impl Matcher<'_> { @@ -917,19 +1248,27 @@ impl Matcher<'_> { if self.gates.nip56 && self.reported(txn, databases, id, &header.pubkey)? { return Ok(None); } - if self - .filter - .tags - .keys() - .any(|key| generic_tag_name(key).is_some()) - && !tag_criteria_match(&self.filter.tags, &rows::decode_event(value)?) - { + if self.carries_other_tag_criteria() && { + body_decoded(); + !tag_criteria_match(&self.filter.tags, &rows::decode_event(value)?, self.proven) + } { return Ok(None); } Ok(Some(header)) } - /// The capture's `NOT EXISTS (… reports …)`: one point lookup per target column. + /// Whether the filter carries a tag family the row's key has not already proved. A row the + /// `tags` index selected needs no decode when that is the filter's only family. + fn carries_other_tag_criteria(&self) -> bool { + self.filter + .tags + .keys() + .filter_map(|key| generic_tag_name(key)) + .any(|name| Some(name) != self.proven) + } + + /// The capture's `NOT EXISTS (… reports …)`: one lookup per target column, by prefix so the + /// width of `reports.id` is `metadata`'s business and not restated here. fn reported( &self, txn: &RoTxn<'_>, @@ -937,28 +1276,24 @@ impl Matcher<'_> { id: &EventId, pubkey: &Pubkey, ) -> Result { - if has_prefix(databases.reports_actionable_event, txn, id.as_bytes(), 4)? { + if exists_with_prefix(databases.reports_actionable_event, txn, id.as_bytes())? { return Ok(true); } - has_prefix( - databases.reports_actionable_pubkey, - txn, - pubkey.as_bytes(), - 4, - ) + exists_with_prefix(databases.reports_actionable_pubkey, txn, pubkey.as_bytes()) } } -fn has_prefix( +/// Whether `database` holds any key starting with `prefix`. +fn exists_with_prefix( database: Database, txn: &RoTxn<'_>, prefix: &[u8], - suffix: usize, ) -> Result { - let mut upper = prefix.to_vec(); - upper.extend(std::iter::repeat_n(0xff, suffix)); - let range = (Bound::Included(prefix), Bound::Included(upper.as_slice())); - Ok(database.range(txn, &range)?.next().transpose()?.is_some()) + Ok(database + .prefix_iter(txn, prefix)? + .next() + .transpose()? + .is_some()) } /// The capture's `isGenericTagQuery`: `#` plus one ASCII letter. @@ -974,36 +1309,46 @@ pub(crate) fn generic_tag_name(key: &str) -> Option<&str> { Some(&key[1..2]) } -/// Every tag criterion of a filter, under the capture's rules: a value is exact, and only -/// `#g` honours a trailing `*` as a prefix. -fn tag_criteria_match(tags: &BTreeMap>, record: &EventRecord) -> bool { +/// Whether a tag value satisfies one of a family's criteria: an exact value, and only `#g` +/// honours a trailing `*` as a prefix. +/// +/// `event_tags` holds no empty value, so an empty one satisfies nothing. The criterion a `tags` +/// key carries is checked here, from the key, and the same rule answers the decoded record. +fn tag_value_matches(is_geohash: bool, criteria: &[String], value: &str) -> bool { + !value.is_empty() + && criteria.iter().any(|criterion| { + match criterion.strip_suffix('*').filter(|_| is_geohash) { + Some(prefix) => value.starts_with(prefix), + None => value == criterion, + } + }) +} + +/// Whether an event carries a tag called `name` whose value satisfies one of `criteria`. +fn record_has_tag(record: &EventRecord, name: &str, criteria: &[String]) -> bool { + let is_geohash = name == "g"; + record.event.tags.iter().any(|tag| { + tag.first().map(String::as_str) == Some(name) + && tag + .get(1) + .is_some_and(|value| tag_value_matches(is_geohash, criteria, value)) + }) +} + +/// Every tag criterion of a filter, under the capture's rules, except the family `proven`. +fn tag_criteria_match( + tags: &BTreeMap>, + record: &EventRecord, + proven: Option<&str>, +) -> bool { for (key, criteria) in tags { let Some(name) = generic_tag_name(key) else { continue; }; - if criteria.is_empty() { - return false; + if Some(name) == proven { + continue; } - let is_geohash = key == "#g"; - let matched = record.event.tags.iter().any(|tag| { - if tag.first().map(String::as_str) != Some(name) { - return false; - } - let Some(value) = tag.get(1) else { - return false; - }; - // `event_tags` holds no empty value, so the capture cannot match one either. - if value.is_empty() { - return false; - } - criteria.iter().any(|criterion| { - match criterion.strip_suffix('*').filter(|_| is_geohash) { - Some(prefix) => value.starts_with(prefix), - None => value == criterion, - } - }) - }); - if !matched { + if criteria.is_empty() || !record_has_tag(record, name, criteria) { return false; } } @@ -1014,17 +1359,15 @@ fn tag_criteria_match(tags: &BTreeMap>, record: &EventRecord // Streams. // --------------------------------------------------------------------------------------- -/// One batch of keys a stream has read, and where the next batch resumes. +/// Where a reconciliation batch resumes: the range it is on, and the last key it evaluated. struct Position { - ranges: Vec, index: usize, after: Option>, } impl Position { - fn new(filter: &Filter) -> Self { + fn new() -> Self { Self { - ranges: created_at_ranges(filter), index: 0, after: None, } @@ -1034,6 +1377,10 @@ impl Position { /// Reads one page of values in batches of [`STREAM_BATCH_ITEMS`], one read transaction per /// poll, so a stream that is dropped pins no pages. /// +/// It holds the page `find_by_filters` already collected — at most [`MAX_PAGE_KEYS`] ids — and +/// copies each batch's values out of it; it is the reconciliation stream, not this one, that +/// holds only a cursor key. +/// /// shortcut: a stream is not a snapshot — a row written or deleted between two batches may /// appear or vanish; upgrade when a caller needs a stable view, per ADR 0003 trigger 1 and the /// benchmark job, which would buy the snapshot back with a held transaction. @@ -1117,14 +1464,16 @@ impl Stream for ValuesStream { } } -/// Yields the live `(created_at, event_id)` pairs a reconciliation compares, oldest first, -/// in batches of [`STREAM_BATCH_ITEMS`] keys with one read transaction per poll. +/// Yields the live `(created_at, event_id)` pairs a reconciliation compares, oldest first, in +/// batches of at most [`STREAM_BATCH_ITEMS`] items or [`STREAM_BATCH_KEYS`] evaluated keys — +/// whichever comes first — with one read transaction per poll. pub(crate) struct ReconciliationStream { env: Env, databases: Databases, filter: Filter, gates: Gates, now: i64, + range_count: usize, position: Position, pending: Option, buffer: std::vec::IntoIter, @@ -1139,13 +1488,15 @@ impl ReconciliationStream { gates: Gates, now: i64, ) -> Self { + let range_count = stream_plan(filter, &databases).ranges.len(); Self { env, databases, filter: filter.clone(), gates, now, - position: Position::new(filter), + range_count, + position: Position::new(), pending: None, buffer: Vec::new().into_iter(), finished: false, @@ -1193,8 +1544,8 @@ impl Stream for ReconciliationStream { let filter = this.filter.clone(); let gates = this.gates; let now = this.now; - let mut position = std::mem::replace(&mut this.position, Position::new(&this.filter)); - let exhausted = position.index >= position.ranges.len(); + let mut position = std::mem::replace(&mut this.position, Position::new()); + let exhausted = position.index >= this.range_count; if exhausted { this.finished = true; return Poll::Ready(None); @@ -1215,8 +1566,34 @@ impl Stream for ReconciliationStream { } } -/// One batch of a reconciliation scan: up to [`STREAM_BATCH_ITEMS`] live keys, resuming -/// after the last key the previous batch read. +/// The plan a reconciliation scans with. +/// +/// The stream promises the pairs oldest first, and only a single range that runs in `created_at` +/// order gives that without materialising the result: a filter the faster index narrows to one +/// such range (`{authors: [a], kinds: [k]}` or one exact tag value) is scanned there, and +/// anything else — several ranges, or a range that runs in key order — scans the created-at +/// index, whose order *is* the promise. +fn stream_plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { + let chosen = plan(filter, databases); + if chosen.ordered && chosen.ranges.len() == 1 { + return chosen; + } + Plan { + database: databases.events_by_created_at, + path: AccessPath::CreatedAt, + ranges: created_at_ranges(filter), + ordered: true, + tag: None, + } +} + +/// One batch of a reconciliation scan: at most [`STREAM_BATCH_ITEMS`] items or +/// [`STREAM_BATCH_KEYS`] evaluated keys, whichever comes first, resuming after the last key the +/// previous batch read. +/// +/// The key cap is what keeps one poll short however few of the keys it walks match: every +/// transaction a stream opens is closed by the poll that opened it, so no read pins a page while +/// the writer works. fn scan_batch( txn: &RoTxn<'_>, databases: &Databases, @@ -1225,35 +1602,40 @@ fn scan_batch( now: i64, position: &mut Position, ) -> Result, StorageError> { + let plan = stream_plan(filter, databases); let matcher = Matcher { filter, gates, now, live: true, + proven: plan.tag.map(|(name, _)| name), }; let mut items = Vec::new(); - while position.index < position.ranges.len() { - let lower = position.ranges[position.index].0.clone(); - let upper = position.ranges[position.index].1.clone(); - let after = position.after.clone(); - let range = encoding::resume_bounds(after.as_deref(), &lower, &upper); + let mut evaluated = 0; + while position.index < plan.ranges.len() { + let (low, high) = &plan.ranges[position.index]; + let range = encoding::resume_bounds(position.after.as_deref(), low, high); let mut reached_cap = false; - for entry in databases.events_by_created_at.range(txn, &range)? { - let (key, _) = entry?; - if items.len() >= STREAM_BATCH_ITEMS { + for entry in plan.database.range(txn, &range)? { + if items.len() >= STREAM_BATCH_ITEMS || evaluated >= STREAM_BATCH_KEYS { reached_cap = true; break; } + let (key, _) = entry?; + key_evaluated(); // The cursor moves only onto a key this batch evaluated, because the next batch // resumes exclusively after it. position.after = Some(key.to_vec()); - let (created_at, event_id) = - key_candidate(AccessPath::CreatedAt, key)?.ok_or_else(|| { - encoding::invalid( - "events_by_created_at", - "a key without a timestamp".to_owned(), - ) - })?; + evaluated += 1; + if !tag_key_proves(&plan, key)? { + continue; + } + let Some((created_at, event_id)) = key_candidate(plan.path, key)? else { + return Err(encoding::invalid( + "events", + "a key without a timestamp and an event id".to_owned(), + )); + }; if matcher.matches(txn, databases, &event_id)?.is_some() { items.push(ReconciliationItem { created_at, @@ -1365,14 +1747,455 @@ mod tests { let mut tags = BTreeMap::new(); tags.insert("#g".to_owned(), vec!["u4pru*".to_owned()]); - assert!(tag_criteria_match(&tags, &record)); + assert!(tag_criteria_match(&tags, &record, None)); let mut tags = BTreeMap::new(); tags.insert("#e".to_owned(), vec!["u4pru*".to_owned()]); - assert!(!tag_criteria_match(&tags, &record)); + assert!(!tag_criteria_match(&tags, &record, None)); let mut tags = BTreeMap::new(); tags.insert("#em".to_owned(), vec!["nothing".to_owned()]); - assert!(tag_criteria_match(&tags, &record)); + assert!(tag_criteria_match(&tags, &record, None)); + } + + // ----------------------------------------------------------------------------------- + // The cost of a read, and the bound on what it holds. + // ----------------------------------------------------------------------------------- + + use crate::backend::LmdbEnv; + use nostream_core::{Event, Signature}; + + /// An open store on a temporary directory. + fn store() -> (tempfile::TempDir, Env, Databases) { + let directory = tempfile::tempdir().expect("a temporary directory"); + let environment = + LmdbEnv::open(directory.path().join("lmdb"), 8 << 20).expect("an environment"); + let env = environment.env().clone(); + let mut transaction = env.write_txn().expect("a write transaction"); + let databases = Databases::create(&env, &mut transaction).expect("the databases"); + transaction.commit().expect("the commit"); + (directory, env, databases) + } + + /// A signed event of one author, carrying `tags`. + fn signed(secret: u8, kind: u64, created_at: i64, tags: Vec>) -> EventRecord { + let mut wire = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(kind), + tags, + content: format!("{secret}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut wire).expect("the event signs"); + EventRecord::from_event(&wire) + } + + /// Writes the events in one transaction, as `create_many` does. + fn write(env: &Env, databases: &Databases, records: &[EventRecord]) { + let mut transaction = env.write_txn().expect("a write transaction"); + create(&mut transaction, databases, records, Utc::now()).expect("the events store"); + transaction.commit().expect("the commit"); + } + + fn created(records: &[(i64, EventId)]) -> Vec { + records.iter().map(|(created_at, _)| *created_at).collect() + } + + #[test] + fn a_limited_author_page_evaluates_only_the_keys_it_needs() { + // The author index runs in kind order before `created_at`, so this page is answered by + // the created-at index: it stops as soon as it is full instead of walking the author's + // whole timeline (the round-2 shape evaluated every one of the 200). + let _serial = probes::COUNTING.lock().expect("the counting lock"); + let (_directory, env, databases) = store(); + let records: Vec = (1..=200) + .map(|step| signed(7, 1, step, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + limit: Some(2), + ..Filter::default() + }; + + probes::reset(); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + + assert_eq!(created(&page), vec![200, 199]); + assert!( + probes::keys() <= 3, + "a full page stops the walk: {} keys evaluated", + probes::keys() + ); + } + + #[test] + fn a_limited_page_stops_at_the_limit_inside_one_author_and_kind() { + // The `pubkey ‖ kind ‖ created_at` ordering the layout writes is what lets this page + // stop; without it the walk reads every event of the author. + let _serial = probes::COUNTING.lock().expect("the counting lock"); + let (_directory, env, databases) = store(); + let mut records: Vec = (1..=200) + .map(|step| signed(7, 1, step, Vec::new())) + .collect(); + records.extend((1..=200).map(|step| signed(7, 2, step, Vec::new()))); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + kinds: Some(vec![Kind(2)]), + limit: Some(2), + ..Filter::default() + }; + + probes::reset(); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + + assert_eq!(created(&page), vec![200, 199]); + assert!( + probes::keys() <= 3, + "one range runs in time order and stops: {} keys evaluated", + probes::keys() + ); + } + + #[test] + fn a_tag_page_checks_its_criterion_from_the_key_and_decodes_no_body() { + let _serial = probes::COUNTING.lock().expect("the counting lock"); + let (_directory, env, databases) = store(); + let tag = |step: i64| { + vec![ + vec!["t".to_owned(), "perfprobe".to_owned()], + vec!["e".to_owned(), step.to_string()], + ] + }; + let records: Vec = (1..=200) + .map(|step| signed(7, 1, step, tag(step))) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), vec!["perfprobe".to_owned()]); + let filter = Filter { + tags, + limit: Some(2), + ..Filter::default() + }; + probes::reset(); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!(created(&page), vec![200, 199]); + assert_eq!( + probes::decodes(), + 0, + "the key carried the value the criterion sought" + ); + assert!( + probes::keys() <= 3, + "the value's range runs in time order and stops: {} keys evaluated", + probes::keys() + ); + + // A family the key cannot answer is decoded. `#a` sorts first, so the tag index is + // sought by it and `#z` stays unproved. + let (_directory, env, databases) = store(); + let both = |step: i64| { + vec![ + vec!["a".to_owned(), "perfprobe".to_owned()], + vec!["z".to_owned(), format!("other-{step}")], + ] + }; + let records: Vec = (1..=200) + .map(|step| signed(7, 1, step, both(step))) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let mut tags = BTreeMap::new(); + tags.insert("#a".to_owned(), vec!["perfprobe".to_owned()]); + tags.insert( + "#z".to_owned(), + vec!["other-200".to_owned(), "other-199".to_owned()], + ); + let filter = Filter { + tags, + limit: Some(2), + ..Filter::default() + }; + probes::reset(); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!(created(&page), vec![200, 199]); + assert!( + probes::decodes() >= 1, + "the family the key cannot answer is decoded" + ); + } + + #[test] + fn a_wire_limit_cannot_grow_a_page_past_the_backend_bound() { + assert_eq!(count_limit(10), 10); + assert_eq!(count_limit(u64::MAX), MAX_PAGE_KEYS); + + let (_directory, env, databases) = store(); + let records: Vec = (1..=MAX_PAGE_KEYS as i64 + 1) + .map(|step| signed(11, 1, step, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(u64::MAX), + ..Filter::default() + }; + + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + + assert_eq!( + page.len(), + MAX_PAGE_KEYS, + "the page the backend materialises is its own bound, not the wire's" + ); + assert_eq!(page[0].0, MAX_PAGE_KEYS as i64 + 1, "and it is the newest"); + } + + #[test] + fn a_count_holds_no_matches_and_deduplicates_the_union() { + let (_directory, env, databases) = store(); + let records = vec![ + signed(21, 1, 10, Vec::new()), + signed(21, 1, 20, Vec::new()), + signed(22, 1, 30, Vec::new()), + ]; + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let author = records[0].event.pubkey.to_string(); + + let filters = vec![ + Filter { + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }, + Filter { + authors: Some(vec![author.clone()]), + ..Filter::default() + }, + ]; + assert_eq!( + count(&transaction, &databases, &filters, &Gates::default(), 0).expect("the count"), + 3, + "the union of two overlapping filters counts each live id once" + ); + + // A limited branch contributes its own page, and the row a later filter adds is counted + // only when the earlier page does not hold it. + let filters = vec![ + Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(2), + ..Filter::default() + }, + Filter { + authors: Some(vec![author]), + ..Filter::default() + }, + ]; + assert_eq!( + count(&transaction, &databases, &filters, &Gates::default(), 0).expect("the count"), + 3 + ); + } + + #[test] + fn a_count_counts_a_row_once_when_two_tag_values_select_it() { + let (_directory, env, databases) = store(); + let record = signed( + 23, + 1, + 10, + vec![ + vec!["t".to_owned(), "a".to_owned()], + vec!["t".to_owned(), "b".to_owned()], + ], + ); + write(&env, &databases, std::slice::from_ref(&record)); + let transaction = env.read_txn().expect("a read transaction"); + + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), vec!["a".to_owned(), "b".to_owned()]); + let filter = Filter { + tags, + ..Filter::default() + }; + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 1, + "one event under two matching values is one row" + ); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!(page.len(), 1); + } + + #[test] + fn a_reconciliation_scans_the_filter_own_index_when_one_ordered_range_answers_it() { + let _serial = probes::COUNTING.lock().expect("the counting lock"); + let (_directory, env, databases) = store(); + let mut records: Vec = (1..=300) + .map(|step| signed(24, 1, step, Vec::new())) + .collect(); + records.push(signed(25, 1, 400, Vec::new())); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[300].event.pubkey.to_string()]), + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }; + + probes::reset(); + let mut position = Position::new(); + let items = scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"); + + assert_eq!( + items, + vec![ReconciliationItem { + created_at: 400, + event_id: records[300].event.id, + }], + "the filter's own index answers it" + ); + assert!( + probes::keys() <= STREAM_BATCH_ITEMS, + "the narrow range is read, not the whole store: {} keys", + probes::keys() + ); + } + + #[test] + fn a_reconciliation_batch_stops_at_its_key_cap_and_resumes() { + let _serial = probes::COUNTING.lock().expect("the counting lock"); + let (_directory, env, databases) = store(); + let mut records: Vec = (1..=1000) + .map(|step| signed(26, 1, step, Vec::new())) + .collect(); + records.push(signed(27, 4, 2_000, Vec::new())); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + kinds: Some(vec![Kind(4)]), + ..Filter::default() + }; + + // One poll must not hold a read transaction for a whole range: it stops on the keys it + // evaluated, not only on the rows it matched. + probes::reset(); + let mut position = Position::new(); + let first = scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"); + assert!(first.is_empty(), "the only match is past the first batch"); + assert!( + probes::keys() <= STREAM_BATCH_KEYS, + "the batch stopped at the key cap: {} keys", + probes::keys() + ); + assert!(position.after.is_some(), "and it resumes after a key"); + + // Resuming reaches the row, so the cap loses nothing. + let mut found = Vec::new(); + for _ in 0..64 { + found.extend( + scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"), + ); + if position.index == 0 && position.after.is_none() { + break; + } + } + assert_eq!(found.len(), 1); + assert_eq!(found[0].created_at, 2_000); } } diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 031fe3361..64c98c2ec 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -55,12 +55,26 @@ pub const LMDB_MAX_READERS: u32 = 512; /// The capture's default page size (`DEFAULT_FILTER_LIMIT`). pub(crate) const DEFAULT_FILTER_LIMIT: u64 = 500; +/// The largest page a read materialises, however large a limit the wire asks for. +/// +/// `limits.client.subscription.maxLimit` is the relay's own bound, but it is applied only when +/// the deployment sets it, and a request reaches the store before some of those gates; this is +/// the backend's bound, so the size of a read's resident set is never chosen by the client. +/// 5 000 is exactly the shipped `maxLimit` (`resources/default-settings.yaml`), so the clamp +/// cannot shorten a page a configured relay would have served. +pub(crate) const MAX_PAGE_KEYS: usize = 5_000; + /// The retention batch size (`RETENTION_BATCH_SIZE`). pub(crate) const RETENTION_BATCH_SIZE: usize = 1_000; /// How many rows one stream poll copies before it drops its read transaction. pub(crate) const STREAM_BATCH_ITEMS: usize = 256; +/// How many index keys one reconciliation poll evaluates before it drops its read transaction. +/// A filter that matches few of the keys it walks ends a batch on this count rather than on +/// [`STREAM_BATCH_ITEMS`], so the transaction never spans a whole range. +pub(crate) const STREAM_BATCH_KEYS: usize = 256; + /// The settings gates the capture reads per query. The LMDB backend records only whether /// NIP-50 is on: it has no search implementation (ADR 0010, Out of scope). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -149,6 +163,15 @@ impl Databases { } } +/// How much of the fixed map one store has used (`Env::non_free_pages_size`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MapUsage { + /// The ceiling the store never grows past. + pub map_size: usize, + /// The bytes the pages in use occupy, as of the last committed transaction. + pub used_bytes: u64, +} + /// An embedded LMDB event store with its metadata tail, built from the root a caller supplies. #[derive(Debug, Clone)] pub struct LmdbBackend { @@ -159,30 +182,51 @@ pub struct LmdbBackend { impl LmdbBackend { /// Opens (creating if needed) the store under `/lmdb` and creates every database. + /// + /// The usage it reports is what the map's approach to [`LMDB_MAP_SIZE`] is read from: the + /// store never grows, so an operator sees it coming here rather than at the first failed + /// write. pub async fn open(root: impl Into) -> Result { let root = root.into(); // The same rule `data_dir` answers, from the same constant. let path = root.join(crate::backend::LMDB_DIR); - let env = tokio::task::spawn_blocking(move || LmdbEnv::open(path, LMDB_MAP_SIZE)) - .await - .map_err(|_| StorageError::LmdbTaskCancelled)??; - let handle = env.env().clone(); - let databases = tokio::task::spawn_blocking(move || { + let opened = tokio::task::spawn_blocking(move || { + let environment = LmdbEnv::open(&path, LMDB_MAP_SIZE)?; + let handle = environment.env().clone(); let mut transaction = handle.write_txn()?; let databases = Databases::create(&handle, &mut transaction)?; transaction.commit()?; - Ok::(databases) + let used_bytes = handle.non_free_pages_size()?; + Ok::<_, StorageError>((environment, databases, used_bytes)) }) .await .map_err(|_| StorageError::LmdbTaskCancelled)??; + let (environment, databases, used_bytes) = opened; + + tracing::info!( + path = %environment.path().display(), + map_size = LMDB_MAP_SIZE, + used_bytes, + used_percent = used_bytes as f64 / LMDB_MAP_SIZE as f64 * 100.0, + "the lmdb store is open" + ); Ok(Self { - env: env.env().clone(), + env: environment.env().clone(), databases, settings: None, }) } + /// How much of the fixed map the store has used, for an operator's tooling and the log + /// line [`open`](Self::open) writes. + pub fn map_usage(&self) -> Result { + Ok(MapUsage { + map_size: LMDB_MAP_SIZE, + used_bytes: self.env.non_free_pages_size()?, + }) + } + /// Supplies the `nip50`/`nip56` gates. Absent settings mean both features are off. pub fn with_settings(mut self, settings: Arc) -> Self { self.settings = Some(settings); @@ -911,6 +955,20 @@ mod tests { } } + #[tokio::test] + async fn the_store_reports_how_much_of_its_map_it_uses() { + // The map never grows (LMDB_MAP_SIZE), so the approach to it is what an operator has to + // see; before this the first signal was the first failed write. + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let usage = backend.map_usage().expect("the usage"); + assert_eq!(usage.map_size, LMDB_MAP_SIZE); + assert!(usage.used_bytes > 0, "the 29 databases occupy pages"); + assert!(usage.used_bytes <= LMDB_MAP_SIZE as u64); + } + #[tokio::test] async fn absent_settings_leave_both_gates_off() { let directory = tempfile::tempdir().expect("tempdir"); From 731dcb1cc97ca5b58ae4cb2011ed7f27229ee40c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:01:41 -0500 Subject: [PATCH 18/60] docs(storage): correct the read bound and the value layout in ADR 0010 --- crates/nostream-storage/src/repositories.rs | 14 ++- docs/rewrite/decisions/0010-lmdb-backend.md | 111 +++++++++++++++++--- 2 files changed, 104 insertions(+), 21 deletions(-) diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index 2750e13e9..d776245ee 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -22,10 +22,16 @@ pub type StorageFuture<'a, T> = Pin = Pin> + Send + 'a>>; /// The rows a retention pass deleted, expired and kept (`src/@types/repositories.ts` diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index 70392fd53..c53cf01f8 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -37,7 +37,7 @@ opens `/lmdb` itself. The eight tail tables are 22 named databases in the same environment (D2): one primary database per table keyed by its primary key, plus one per query path (`invoices_pending`, `nip05_by_last_checked_at`, `invite_codes_by_created_at`/`_by_expires_at`, -`dvm_jobs_by_status_created_at`, four `reports_*`, three `outbox_*`/`delivery_log_*`), and a +`dvm_jobs_by_status_created_at`, four `reports_*`, four `outbox_*`/`delivery_log_*`), and a `meta` database for the report sequence. **The reason is one write transaction spanning events and tail.** NIP-56's hidden-content @@ -57,13 +57,14 @@ Rows are versioned: one format byte (`ROW_FORMAT_VERSION = 1`), then `serde_json per database rather than per table. ADR 0003 trigger 6 asked for a version marker before a second backend persisted `EventRecord`, and this is it. -## An event value is a 54-byte header, then JSON +## An event value is a 55-byte header, then JSON -`events` holds `1 + 54` bytes of header, then `serde_json` of `EventRecord` (D5): +`events` holds `1 + 54 = 55` bytes of header, then `serde_json` of `EventRecord` (D5): |bytes|field| |---|---| |1|format version (`1`)| +|1|flags: bit 0 is `deleted_at.is_some()`, the soft-delete bit (`FLAG_DELETED`)| |8|`created_at`, big-endian i64| |4|`kind`, big-endian i32| |8|`expires_at`, big-endian i64, `i64::MIN` for NULL| @@ -72,13 +73,29 @@ second backend persisted `EventRecord`, and this is it. The live predicate and the NIP-56 lookup read that header without decoding the body, so `count_by_filters` and reconciliation copy 55 bytes per candidate instead of an event. -The reserved byte keeps the header at the 54 bytes R7, V35 and AC8 pin, and it is where a -future flag goes without a version bump. +The flags byte is header byte 0 — the first byte after the format byte — and it already +carries the soft-delete bit, so a second flag lands there behind `FLAG_DELETED` rather than +in the reserved byte. The reserved byte is the header's last (byte 53 of the 54-byte header), +kept so the header stays the length R7, V35 and AC8 pin; a flag that needs its own bit goes +in the flags byte, and the reserved byte is there for a field that has to be a byte wide. `row_id` stays `None` (D5): it is the Postgres row's `uuid` and nothing reads it through this backend. Text components of a composite key are NUL-terminated except the final one (`text_part`), so `tag_key` is `name NUL value NUL created_at ‖ event_id`, and a tag name or -value carrying `0x00` is refused as `InvalidRow` rather than splitting the key. +value carrying `0x00` is refused as `InvalidRow` rather than splitting the key. The three +component codecs a key reader needs — `text_part`, `text_from` and `stamp_key` — have one +home, `lmdb/encoding.rs`, beside the key builders; the tail modules call them rather than +keeping copies whose rules could drift apart. + +LMDB bounds a key at 511 bytes (`MDB_MAXKEYSIZE`, which this environment does not raise), and +a `tags` key carries the tag value verbatim. **That is the one place this backend refuses a +row Postgres would keep**: a tag value of 470 bytes or more pushes the key past the bound, so +the event's write fails and the relay refuses the event, where the capture stores it. It is a +recorded divergence, not a silent one — the write returns `StorageError::Lmdb` — and it is not +fixed here: the repairs (hashing the value into the key and looking the row up twice, or a +larger `max_key_size` at the open) change the index the benchmark job will measure. On the +read side a criterion too long to name a stored key selects nothing, rather than failing the +whole request. Soft delete is a flag **and** the row's value, never an index removal (D4): `find_by_filters` does not apply the live predicate (`sql.rs:119-140` against `:262-267`), so the row must stay @@ -111,23 +128,79 @@ the writer's mutex and a read transaction must not be held across an `await`. `L therefore has two helpers, `write` and `read`, that own their transaction and commit or drop it inside the closure; no `RwTxn` or `RoTxn` leaves one. -A stream is a key page, not a held transaction. `find_by_filters` collects its union page -inside one blocking closure; the returned `QueryStream` yields values in batches of -`STREAM_BATCH_ITEMS = 256`, each poll opening a read transaction, seeking to the saved -cursor key, copying up to 256 values and dropping it. `find_reconciliation_items` resumes -the same cursor over `events_by_created_at`, reading a candidate's header and decoded record -only to apply the filter's live predicate. **A stream is therefore not a snapshot** (D7): -a row written or deleted between two batches may appear or vanish, where Postgres' portal -holds one snapshot. No comparable behaviour exists to compare, so the divergence is recorded -for the benchmark job, and ADR 0003 trigger 1 ("a long-lived LMDB read transaction hurts -writes") is answered by construction: no transaction outlives a poll, and a dropped stream -pins no pages. +A stream is a key page, not a held transaction, and the two query methods keep different +things — which is what a caller planning memory needs to know: + +- `find_by_filters` collects its union page inside one blocking closure, so the stream **holds + the page**: at most each filter's limit, clamped to `MAX_PAGE_KEYS = 5 000` ids. Each poll + opens a read transaction, copies up to `STREAM_BATCH_ITEMS = 256` values from that page and + drops it; nothing more is read from the index, and the page is the only thing resident. +- `find_reconciliation_items` **holds a cursor key**. Each poll opens a read transaction, + resumes exclusively after the last key the previous batch evaluated, and drops it. A batch + ends at `STREAM_BATCH_ITEMS` items or `STREAM_BATCH_KEYS = 256` evaluated keys, whichever + comes first, so a filter that matches few of the keys it walks cannot make one poll span a + whole range — the cursor resumes exactly where the key cap stopped. + It scans the filter's own index when that index narrows the filter to a single range that + runs in `created_at` order (`{authors: [a], kinds: [k]}`, or one exact tag value); any other + filter scans `events_by_created_at`, because only that range's order is the oldest-first + order the seam promises. Reading a candidate costs its header, plus its decoded record only + for the tag families the key has not already answered. + +**A stream is therefore not a snapshot** (D7): a row written or deleted between two batches +may appear or vanish, where Postgres' portal holds one snapshot. No comparable behaviour +exists to compare, so the divergence is recorded for the benchmark job, and ADR 0003 trigger +1 ("a long-lived LMDB read transaction hurts writes") is answered by construction: no +transaction outlives a poll, and a dropped stream pins no pages. `Env::resize` is `unsafe` and this crate has one audited `unsafe` call, so the map is preallocated at `LMDB_MAP_SIZE = 64 GiB` and never grown (D6). The same single `#[allow(unsafe_code)]` open sets `LMDB_MAX_DBS = 64` (the 29 named databases, comfortably inside) and `LMDB_MAX_READERS = 512`. +Because the map never grows, its approach has to be visible before the first failed write: +`LmdbBackend::open` logs the map size, the bytes the pages in use occupy +(`Env::non_free_pages_size`) and that as a percentage, and `LmdbBackend::map_usage` reports the +same numbers to an operator's tooling. + +## What a read holds, and which index it reads + +The resident set of a read is the page, never the range it was chosen from, and the size of +that page is the backend's to bound: the effective limit is the filter's own or the capture's +default of 500, clamped to `MAX_PAGE_KEYS = 5 000` — the shipped +`limits.client.subscription.maxLimit` — so a wire `limit` cannot choose an allocation, whether +or not a deployment sets `maxLimit`. A filter's branch keeps at most twice that many keys and +cuts back to the best half whenever the buffer fills; a range whose key order *is* the page's +order stops as soon as the page is full, keeping every row at the boundary timestamp so a tie +breaks as the capture's does. + +The index is chosen from the filter, and which ranges can stop early follows from the key: + +| filter | index | stops when full | +|---|---|---| +| `ids` | `events`, a point or prefix range per criterion | never: the key is the id, so the range runs in id order | +| `authors` + `kinds` | `events_by_author`, one range per author criterion and kind | when the author criterion is a whole pubkey, so a range is one `pubkey ‖ kind ‖ created_at` run | +| `authors` alone | `events_by_created_at` | yes — see below | +| a `#` + letter criterion | `tags`, one range per criterion | for an exact value; a `#g` prefix spans values, so its range runs whole | +| anything else | `events_by_created_at` | yes | + +The `authors`-alone row is the capture's own shape rather than the faster index: the author +index orders by kind before `created_at`, so a page ordered by time cannot stop inside it, and +the same query in Postgres walks the created-at index and filters (ADR 0003 records 5 508 rows +discarded to return 500 for the kind-only form). The cost is that a quiet author's page reads +further into the created-at index than the author's own timeline would; the page still stops at +its limit, and the choice belongs to the benchmark job once it can choose with statistics +rather than a rule. + +Two consequences of the same bound: + +- `count_by_filters` has no page, so it holds no matching set. Each filter counts the rows it + visits and skips one the count already holds — an earlier filter's page when that filter had + a limit, or a predicate match when it had none — which is the capture's `UNION`. An unlimited + filter walks its ranges whole, which is what counting every match costs. +- A tag criterion the index selected is checked from the key (`name NUL value NUL …`), so the + event's body is decoded only for the filter's other tag families; a filter with one family + decodes none. + ## Constraints move into Rust The store has no server to enforce the columns' `CHECK` and foreign-key constraints, so the @@ -189,6 +262,10 @@ its op is one of the five above — so the list cannot grow without the diff sho - Retention's batch composition can differ from Postgres' planner-chosen order when more than 1000 rows qualify; both return 1000 rows classified by the same precedence (D13 in `spec.md`, Assumption 10). The corpus is 483 events, so no current entry reaches it. +- A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range, so a + client cannot choose how much a request materialises; a count is exact and holds no set. + The price is that the `ids` and `#g`-prefix paths walk their whole (time-bounded) range, and + an author page without kinds reads the created-at index. - Two JSON values PostgreSQL's `jsonb` considers equal but that serialize differently (`1` against `1.0`) would be distinct deduplication pointers here; the repository builds the value in one shape and the relay passes a single string, so today's callers cannot From 85793c19b628078df9e1630c96f2c3aca9395fb5 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:04:50 -0500 Subject: [PATCH 19/60] refactor(storage): let the tag range be the proof the key carries --- crates/nostream-storage/src/lmdb/events.rs | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index eb8be99db..8214f8316 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -877,6 +877,8 @@ fn branch( } let plan = plan(filter, databases); + // A `tags` range is built from the family's own criteria, so every key in it satisfies that + // family: the matcher skips it and no body is decoded for it. let matcher = Matcher { filter, gates, @@ -892,9 +894,6 @@ fn branch( for entry in scan(plan.database, txn, &range, plan.ordered && descending)? { let (key, _) = entry?; key_evaluated(); - if !tag_key_proves(&plan, key)? { - continue; - } let Some(found) = candidate(txn, databases, plan.path, key)? else { continue; }; @@ -912,19 +911,6 @@ fn branch( Ok(page.finish(limit)) } -/// Whether the key that selected a row satisfies the tag family's criteria. -/// -/// A `tags` key is `name\0value\0…`, so the value the range sought is already in the key: the -/// criterion is checked from it, and the row's body is decoded only for the filter's other tag -/// families. Every other index selects a row without proving a tag criterion. -fn tag_key_proves(plan: &Plan<'_>, key: &[u8]) -> Result { - let Some((name, criteria)) = plan.tag else { - return Ok(true); - }; - let value = encoding::tag_value(key)?; - Ok(tag_value_matches(name == "g", criteria, value)) -} - /// Whether a key's timestamp is past the timestamp a full page stopped at. fn past(boundary: i64, created_at: i64, descending: bool) -> bool { if descending { @@ -1627,9 +1613,6 @@ fn scan_batch( // resumes exclusively after it. position.after = Some(key.to_vec()); evaluated += 1; - if !tag_key_proves(&plan, key)? { - continue; - } let Some((created_at, event_id)) = key_candidate(plan.path, key)? else { return Err(encoding::invalid( "events", From 60fe1bfdff7c35ee44859df2031a7d0520c59951 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:08:07 -0500 Subject: [PATCH 20/60] docs(storage): mark the ranges a page cannot stop inside --- crates/nostream-storage/src/lmdb/events.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 8214f8316..9a77aa6ac 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -1003,6 +1003,10 @@ fn sort_page(page: &mut [(i64, EventId)], descending: bool) { /// The index a filter is answered from, the ranges within it, and whether a range's key order /// is the page's order. +/// +/// shortcut: `ids`, an author prefix and a `#g` prefix name no index that runs in time order, so +/// those pages are bounded in memory but walk their whole time-bounded range; upgrade when a +/// deployment's range outgrows one request, which needs a second index ordered by time. fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { if let Some(ids) = &filter.ids && !ids.is_empty() @@ -1036,6 +1040,10 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { // ordered by time has no early stop there. The created-at index is the capture's own // path for this shape — ADR 0003 measured 5 508 rows discarded to return 500 for the // kind-only form — and its order *is* the page order, so the page stops when full. + // + // shortcut: a quiet author's page therefore scans further into the created-at index + // than the author's own timeline would; upgrade when statistics can choose between the + // two, which is the benchmark job's (ADR 0010). return Plan { database: databases.events_by_created_at, path: AccessPath::CreatedAt, From f260d9c675ef750cbe19164d4419803eb65b7038 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:10:37 -0500 Subject: [PATCH 21/60] test(storage): pin the reconciliation range to the keys it holds --- crates/nostream-storage/src/lmdb/events.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 9a77aa6ac..b744b5d15 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -2126,8 +2126,8 @@ mod tests { "the filter's own index answers it" ); assert!( - probes::keys() <= STREAM_BATCH_ITEMS, - "the narrow range is read, not the whole store: {} keys", + probes::keys() <= 2, + "the range the filter's own index narrows to is read, not the whole store: {} keys", probes::keys() ); } From db5e16427905fb76d3365ae9d5e6badb5902e2e3 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:11:56 -0500 Subject: [PATCH 22/60] docs(storage): say how a tags range proves its family --- docs/rewrite/decisions/0010-lmdb-backend.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index c53cf01f8..605fcab9b 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -197,9 +197,10 @@ Two consequences of the same bound: visits and skips one the count already holds — an earlier filter's page when that filter had a limit, or a predicate match when it had none — which is the capture's `UNION`. An unlimited filter walks its ranges whole, which is what counting every match costs. -- A tag criterion the index selected is checked from the key (`name NUL value NUL …`), so the - event's body is decoded only for the filter's other tag families; a filter with one family - decodes none. +- A `tags` range is built from the family's own criteria, so a key inside it has already proved + that family: the event's body is decoded only for the filter's *other* tag families, and a + filter with one family decodes none. A count reads the value out of the key too + (`name NUL value NUL …`), to count a row two criteria of one family would both select once. ## Constraints move into Rust From f7489675d81d9c188d12e2d7a36cbfb85405b3d3 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:13:28 -0500 Subject: [PATCH 23/60] docs(storage): state the tag key's exact length bound --- crates/nostream-storage/src/lmdb/encoding.rs | 8 ++++---- docs/rewrite/decisions/0010-lmdb-backend.md | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index f9eeaaee8..0101ab238 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -33,10 +33,10 @@ const EVENT_ID_LEN: usize = 32; /// place this backend rejects a row Postgres would keep, and ADR 0010 records it as a /// divergence. /// -/// shortcut: the tag index carries the value verbatim in the key, so a value of 470 bytes or -/// more cannot be indexed and its event is refused; upgrade when an event like that has to be -/// stored, which needs the value hashed into the key plus a second lookup, or a larger -/// `max_key_size` at the open. +/// shortcut: the tag index carries the value verbatim in the key, so a value of 469 bytes or +/// more (with a one-byte tag name) cannot be indexed and its event is refused; upgrade when such +/// an event has to be stored, which needs the value hashed into the key plus a second lookup, +/// or a larger `max_key_size` at the open. const MAX_KEY_LEN: usize = 511; /// An invalid stored value, naming the database it was read from. diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index 605fcab9b..af0a63627 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -89,13 +89,13 @@ keeping copies whose rules could drift apart. LMDB bounds a key at 511 bytes (`MDB_MAXKEYSIZE`, which this environment does not raise), and a `tags` key carries the tag value verbatim. **That is the one place this backend refuses a -row Postgres would keep**: a tag value of 470 bytes or more pushes the key past the bound, so -the event's write fails and the relay refuses the event, where the capture stores it. It is a -recorded divergence, not a silent one — the write returns `StorageError::Lmdb` — and it is not -fixed here: the repairs (hashing the value into the key and looking the row up twice, or a -larger `max_key_size` at the open) change the index the benchmark job will measure. On the -read side a criterion too long to name a stored key selects nothing, rather than failing the -whole request. +row Postgres would keep**: a tag value of 469 bytes or more (with a one-byte tag name) pushes +the key past the bound, so the event's write fails and the relay refuses the event, where the +capture stores it. It is a recorded divergence, not a silent one — the write returns +`StorageError::Lmdb` — and it is not fixed here: the repairs (hashing the value into the key and +looking the row up twice, or a larger `max_key_size` at the open) change the index the benchmark +job will measure. On the read side a criterion too long to name a stored key selects nothing, +rather than failing the whole request. Soft delete is a flag **and** the row's value, never an index removal (D4): `find_by_filters` does not apply the live predicate (`sql.rs:119-140` against `:262-267`), so the row must stay From 760c8c4f72e753ca0288d5bc266a8b6422918b02 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:53:32 -0500 Subject: [PATCH 24/60] fix(storage): compare a hex prefix against its own field --- crates/nostream-storage/src/lmdb/encoding.rs | 38 ++++-- crates/nostream-storage/src/lmdb/events.rs | 84 ++++++++++++- crates/nostream-storage/tests/parity_lmdb.rs | 124 +++++++++++++++++++ docs/rewrite/decisions/0010-lmdb-backend.md | 8 ++ 4 files changed, 244 insertions(+), 10 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index 0101ab238..5b15fd0fb 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -26,6 +26,9 @@ const NO_EXPIRY: i64 = i64::MIN; /// The number of bytes a `created_at ‖ event_id` key appends after the timestamp. const EVENT_ID_LEN: usize = 32; +/// The width of the field a hex criterion is compared over: an id or a pubkey is 32 bytes. +const HEX_FIELD_LEN: usize = 32; + /// LMDB's maximum key length (`MDB_MAXKEYSIZE`, the default this environment is opened with). /// /// It bounds every key this module builds: a tag value longer than the room a `tags` key has @@ -394,12 +397,14 @@ fn hex_digit(digit: u8) -> u8 { /// The inclusive key ranges one hex criterion family covers, over a 32-byte id or pubkey. /// /// A 64-character criterion is one equality range; an even prefix compares its bytes; an odd -/// one compares the bytes of the nibble range `0` to `f`, as `sql.rs` composes it. `suffix` is -/// the number of bytes the index key carries after the 32-byte field. +/// one compares the bytes of the nibble range `0` to `f`, as `sql.rs` composes it. Both bounds +/// are widened to [`HEX_FIELD_LEN`] bytes, so a prefix is compared against its own field and +/// nothing a key carries after it; `suffix` is the number of bytes the index key carries after +/// the field. pub(crate) fn hex_ranges(criteria: &[String], suffix: usize) -> Vec<(Vec, Vec)> { let mut ranges = Vec::with_capacity(criteria.len()); for criterion in criteria { - let (lower, mut upper) = match criterion.len() % 2 { + let (mut lower, mut upper) = match criterion.len() % 2 { 1 => ( hex_bytes(&format!("{criterion}0")), hex_bytes(&format!("{criterion}f")), @@ -409,6 +414,12 @@ pub(crate) fn hex_ranges(criteria: &[String], suffix: usize) -> Vec<(Vec, Ve (prefix.clone(), prefix) } }; + if lower.len() < HEX_FIELD_LEN { + lower.resize(HEX_FIELD_LEN, 0x00); + } + if upper.len() < HEX_FIELD_LEN { + upper.resize(HEX_FIELD_LEN, 0xff); + } upper.extend(std::iter::repeat_n(0xff, suffix)); ranges.push((lower, upper)); } @@ -421,7 +432,7 @@ pub(crate) fn hex_ranges(criteria: &[String], suffix: usize) -> Vec<(Vec, Ve /// builder](hex_ranges) maps to zero does the same — and a walk that reads one key twice would /// count an event twice. Dropping a range another already covers keeps every range a set of /// keys it alone selects. -fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec)> { +pub(crate) fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec)> { ranges.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1))); let mut kept: Vec<(Vec, Vec)> = Vec::with_capacity(ranges.len()); for range in ranges { @@ -502,14 +513,23 @@ mod tests { fn an_even_prefix_is_one_range_and_an_odd_one_is_one_nibble_range() { let ranges = hex_ranges(&["abcd".to_owned(), "abef".to_owned()], 40); assert_eq!(ranges.len(), 2, "two disjoint criteria seek two ranges"); - assert_eq!(ranges[0].0, vec![0xab, 0xcd]); - assert_eq!(ranges[0].1.len(), 2 + 40); - assert_eq!(ranges[1].0, vec![0xab, 0xef]); + let mut low = vec![0xab, 0xcd]; + low.resize(HEX_FIELD_LEN, 0x00); + let mut high = vec![0xab, 0xcd]; + high.resize(HEX_FIELD_LEN, 0xff); + high.extend(std::iter::repeat_n(0xff, 40)); + assert_eq!(ranges[0].0, low); + assert_eq!( + ranges[0].1, high, + "the upper bound widens to the field, so a longer key under the prefix is in range" + ); + assert_eq!(ranges[1].0[..2], [0xab, 0xef]); let odd = hex_ranges(&["abcde".to_owned()], 40); - assert_eq!(odd[0].0, vec![0xab, 0xcd, 0xe0]); + assert_eq!(odd[0].0[..3], [0xab, 0xcd, 0xe0]); assert_eq!(odd[0].1, { let mut high = vec![0xab, 0xcd, 0xef]; + high.resize(HEX_FIELD_LEN, 0xff); high.extend(std::iter::repeat_n(0xff, 40)); high }); @@ -521,7 +541,7 @@ mod tests { // keys twice and count them twice. let nested = hex_ranges(&["abcd".to_owned(), "abcde".to_owned()], 40); assert_eq!(nested.len(), 1); - assert_eq!(nested[0].0, vec![0xab, 0xcd]); + assert_eq!(nested[0].0[..2], [0xab, 0xcd]); // Two criteria that decode to the same bytes are one range too. let duplicate = hex_ranges(&["zz".to_owned(), "z0".to_owned()], 0); diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index b744b5d15..60ae2f53c 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -1075,6 +1075,10 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { /// One range per author criterion and kind on `events_by_author`, whose key is /// `pubkey ‖ kind ‖ created_at ‖ event_id`: the `pubkey ‖ kind ‖ created_at` ordering the layout /// writes is what lets a page for one author and one kind stop as soon as it is full. +/// +/// The author bounds are the index's whole 32-byte pubkey field, so a prefix criterion's kind +/// is compared against the key's own kind bytes; a criterion another already covers, and a +/// repeated kind, seek one range, so a count cannot visit their keys twice. fn author_kind_ranges(authors: &[String], kinds: &[Kind]) -> Vec { let mut ranges = Vec::with_capacity(authors.len() * kinds.len()); for (low, high) in encoding::hex_ranges(authors, 0) { @@ -1088,7 +1092,7 @@ fn author_kind_ranges(authors: &[String], kinds: &[Kind]) -> Vec { ranges.push((lower, upper)); } } - ranges + encoding::disjoint(ranges) } /// The first tag criterion the capture treats as a query: `#` plus one ASCII letter with a @@ -1693,6 +1697,20 @@ mod tests { assert_eq!(page, vec![(10, id(1)), (10, id(2)), (11, id(9))]); } + #[test] + fn a_page_buffer_holds_at_most_twice_its_limit() { + let mut page = Page::new(true); + for step in 0u8..=100 { + page.offer((i64::from(step), id(step)), 3); + assert!( + page.len() <= 6, + "the buffer is cut back to the limit whenever it fills: {} entries", + page.len() + ); + } + assert_eq!(page.finish(3).len(), 3); + } + #[test] fn only_a_hash_and_one_letter_is_a_tag_query() { assert_eq!(generic_tag_name("#e"), Some("e")); @@ -1871,6 +1889,42 @@ mod tests { ); } + #[test] + fn a_full_page_keeps_the_ties_at_its_boundary_timestamp() { + // The created-at index reverses the whole key, so two rows at one instant are scanned + // id-descending: a walk that stopped at the first would keep the larger id, where the + // capture's comparator keeps the smaller (V6). + let (_directory, env, databases) = store(); + let tag = |value: &str| vec![vec!["t".to_owned(), value.to_owned()]]; + let first = signed(9, 1, 500, tag("first")); + let second = signed(9, 1, 500, tag("second")); + let smallest = first.event.id.min(second.event.id); + write(&env, &databases, &[first, second]); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(1), + ..Filter::default() + }; + + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + + assert_eq!( + page, + vec![(500, smallest)], + "the row tied at the boundary is still offered, and the smaller id wins" + ); + } + #[test] fn a_tag_page_checks_its_criterion_from_the_key_and_decodes_no_body() { let _serial = probes::COUNTING.lock().expect("the counting lock"); @@ -2043,6 +2097,34 @@ mod tests { ); } + #[test] + fn a_later_limited_page_does_not_recount_an_earlier_filter_s_rows() { + let (_directory, env, databases) = store(); + let records = vec![ + signed(28, 1, 10, Vec::new()), + signed(28, 1, 20, Vec::new()), + signed(28, 1, 30, Vec::new()), + ]; + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filters = vec![ + Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + ..Filter::default() + }, + Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(2), + ..Filter::default() + }, + ]; + assert_eq!( + count(&transaction, &databases, &filters, &Gates::default(), 0).expect("the count"), + 3, + "the rows the earlier predicate counted are not counted again by the page" + ); + } + #[test] fn a_count_counts_a_row_once_when_two_tag_values_select_it() { let (_directory, env, databases) = store(); diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index 9ced37d05..8653730e2 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -660,6 +660,130 @@ async fn a_limited_author_filter_returns_the_newest_events() { ); } +/// The page and count of one filter: `(rows, count)`. +async fn page_and_count(backend: &LmdbBackend, filter: &Filter) -> (usize, u64) { + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + let count = backend + .count_by_filters(std::slice::from_ref(filter)) + .await + .unwrap(); + (page.len(), count) +} + +#[tokio::test] +async fn an_id_prefix_of_either_parity_selects_the_event_it_prefixes() { + let (_directory, backend) = open_store().await; + let record = event(4, 1, 1_700_000_000); + backend + .create_many(std::slice::from_ref(&record)) + .await + .expect("the event stores"); + let id = record.event.id.to_string(); + + // The capture's `substring(event_id from 1 for n/2)` takes an even prefix's whole bytes, + // and an odd one's final nibble is a range; both select the row (R12). + for length in [8, 9, 16] { + let filter = Filter { + ids: Some(vec![id[..length].to_owned()]), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (1, 1), + "an id prefix of {length} characters names the row" + ); + } + + let mut other = id[..8].to_owned(); + let digit = other.pop().expect("the prefix is not empty"); + other.push(if digit == '0' { '1' } else { '0' }); + let filter = Filter { + ids: Some(vec![other]), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (0, 0), + "the prefix range stops at the prefix's own bytes" + ); +} + +#[tokio::test] +async fn a_prefix_author_with_kinds_selects_the_event() { + let (_directory, backend) = open_store().await; + let record = event(12, 1, 1_700_000_000); + backend + .create_many(std::slice::from_ref(&record)) + .await + .expect("the event stores"); + let author = record.event.pubkey.to_string(); + + // The author index's key is `pubkey ‖ kind ‖ created_at ‖ event_id`, so the kind is at the + // field's own offset whatever the author criterion covers (R12). + for length in [8, 9, 16, 63, 64] { + let filter = Filter { + authors: Some(vec![author[..length].to_owned()]), + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (1, 1), + "an author prefix of {length} characters with the row's kind names it" + ); + + let filter = Filter { + authors: Some(vec![author[..length].to_owned()]), + kinds: Some(vec![Kind(2)]), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (0, 0), + "another kind excludes it at the same offset" + ); + } +} + +#[tokio::test] +async fn a_repeated_kind_counts_its_row_once() { + let (_directory, backend) = open_store().await; + let note = event(13, 1, 1_700_000_000); + let highest = event(14, i32::MAX as u64, 1_700_000_001); + backend + .create_many(&[note.clone(), highest.clone()]) + .await + .expect("the events store"); + + // A duplicated criterion, and two kinds that the key codec maps to the same `i32`, name one + // index range; a count that visited it per criterion would count the row twice (V6, UNION). + for (record, kinds) in [ + (¬e, vec![Kind(1), Kind(1)]), + ( + &highest, + vec![Kind(i32::MAX as u64), Kind(i32::MAX as u64 + 1)], + ), + ] { + let filter = Filter { + authors: Some(vec![record.event.pubkey.to_string()]), + kinds: Some(kinds), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (1, 1), + "the row is one row however its kind is repeated" + ); + } +} + #[tokio::test] async fn a_reconciliation_lists_every_live_row_across_batches() { let (_directory, backend) = open_store().await; diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index af0a63627..42e66b375 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -183,6 +183,14 @@ The index is chosen from the filter, and which ranges can stop early follows fro | a `#` + letter criterion | `tags`, one range per criterion | for an exact value; a `#g` prefix spans values, so its range runs whole | | anything else | `events_by_created_at` | yes | +Every hex criterion's bounds are widened to the field they name — 32 bytes for an id or a +pubkey — before anything the key carries after it is appended. A prefix is therefore compared +against its own field's bytes and never against the kind, the timestamp or the id that follow, +whether the path is `events` or `events_by_author`; an author prefix with `kinds` seeks the +kind at byte 32 as a whole pubkey does, and `hex_ranges` (`encoding.rs`) is the one place that +rule lives. A criterion another already covers, and a repeated kind, seek one range, so a +count that walks ranges whole visits those keys once. + The `authors`-alone row is the capture's own shape rather than the faster index: the author index orders by kind before `created_at`, so a page ordered by time cannot stop inside it, and the same query in Postgres walks the created-at index and filters (ADR 0003 records 5 508 rows From 7a7e45e7c30424a5977ea1925b010184c4133229 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:53:32 -0500 Subject: [PATCH 25/60] fix(storage): move the map usage report off the caller's thread --- crates/nostream-storage/src/lmdb/mod.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 64c98c2ec..718117c6a 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -220,11 +220,19 @@ impl LmdbBackend { /// How much of the fixed map the store has used, for an operator's tooling and the log /// line [`open`](Self::open) writes. - pub fn map_usage(&self) -> Result { - Ok(MapUsage { - map_size: LMDB_MAP_SIZE, - used_bytes: self.env.non_free_pages_size()?, + /// + /// `Env::non_free_pages_size` opens a read transaction and walks every named database, so + /// it runs on the blocking pool (D7) rather than on the caller's runtime worker. + pub async fn map_usage(&self) -> Result { + let env = self.env.clone(); + tokio::task::spawn_blocking(move || { + Ok(MapUsage { + map_size: LMDB_MAP_SIZE, + used_bytes: env.non_free_pages_size()?, + }) }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)? } /// Supplies the `nip50`/`nip56` gates. Absent settings mean both features are off. @@ -963,7 +971,7 @@ mod tests { let backend = LmdbBackend::open(directory.path()) .await .expect("the store opens"); - let usage = backend.map_usage().expect("the usage"); + let usage = backend.map_usage().await.expect("the usage"); assert_eq!(usage.map_size, LMDB_MAP_SIZE); assert!(usage.used_bytes > 0, "the 29 databases occupy pages"); assert!(usage.used_bytes <= LMDB_MAP_SIZE as u64); From 61a6bd40c41b5ddc6f63cc000b1e4a941f9a76c1 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:53:59 -0500 Subject: [PATCH 26/60] docs(storage): give the stream's page bound its filters --- crates/nostream-storage/src/repositories.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index d776245ee..dbf0be6ad 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -26,12 +26,13 @@ pub type StorageFuture<'a, T> = Pin = Pin> + Send + 'a>>; /// The rows a retention pass deleted, expired and kept (`src/@types/repositories.ts` From 514ae250907dd5d8266ca5a947b1bf201ff95617 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 02:58:37 -0500 Subject: [PATCH 27/60] docs(storage): say what the range dedupe covers --- crates/nostream-storage/src/lmdb/encoding.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index 5b15fd0fb..e61537b17 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -428,10 +428,10 @@ pub(crate) fn hex_ranges(criteria: &[String], suffix: usize) -> Vec<(Vec, Ve /// The ranges no key is selected by more than once. /// -/// Two hex criteria can select the same keys — `ab` and `abcd` nest, and a digit the [range -/// builder](hex_ranges) maps to zero does the same — and a walk that reads one key twice would -/// count an event twice. Dropping a range another already covers keeps every range a set of -/// keys it alone selects. +/// Two criteria can select the same keys — `ab` and `abcd` nest, a digit the [range +/// builder](hex_ranges) maps to zero does the same, and a repeated kind composes the same author +/// range twice — and a walk that reads one key twice would count an event twice. Dropping a +/// range another already covers keeps every range a set of keys it alone selects. pub(crate) fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec)> { ranges.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1))); let mut kept: Vec<(Vec, Vec)> = Vec::with_capacity(ranges.len()); From 94d6599d7555bf44a5bc2634c8b836d49f1289e7 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 03:48:10 -0500 Subject: [PATCH 28/60] fix(storage): encode dedup keys without escaping and refuse NUL text --- crates/nostream-storage/src/lmdb/encoding.rs | 211 ++++++++++++++---- crates/nostream-storage/src/lmdb/metadata.rs | 16 +- .../src/lmdb/notifications.rs | 66 ++++-- 3 files changed, 217 insertions(+), 76 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index e61537b17..b9ddd9a28 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -27,14 +27,16 @@ const NO_EXPIRY: i64 = i64::MIN; const EVENT_ID_LEN: usize = 32; /// The width of the field a hex criterion is compared over: an id or a pubkey is 32 bytes. -const HEX_FIELD_LEN: usize = 32; +pub(crate) const HEX_FIELD_LEN: usize = 32; /// LMDB's maximum key length (`MDB_MAXKEYSIZE`, the default this environment is opened with). /// -/// It bounds every key this module builds: a tag value longer than the room a `tags` key has -/// left cannot be stored, so an event carrying one is refused at the write. That is the one -/// place this backend rejects a row Postgres would keep, and ADR 0010 records it as a -/// divergence. +/// It bounds every key this module builds. The `tags` key carries the tag value verbatim, so a +/// value longer than the room the key has left cannot be indexed and its event is refused at +/// the write: that is the one place this backend rejects a row Postgres would keep, and ADR +/// 0010 records it as a divergence. The `replaceable` key is the other variable-length key; its +/// components are length-prefixed ([`replaceable_key`]), which leaves it four bytes of room the +/// `tags` key does not have for the same value, so it refuses nothing the tag index keeps. /// /// shortcut: the tag index carries the value verbatim in the key, so a value of 469 bytes or /// more (with a one-byte tag name) cannot be indexed and its event is refused; upgrade when such @@ -201,12 +203,25 @@ pub(crate) fn check_version(table: &'static str, value: &[u8]) -> Result<(), Sto /// A text component of a composite key, terminated so the next component reads unambiguously. /// /// The terminator is what every key *reader* in this module splits on, so the rule lives here -/// with the writers; a component that carries `0x00` cannot be read back, and the codec that -/// puts a tag into a key ([`tag_key`]) refuses one for that reason. -pub(crate) fn text_part(value: &str) -> Vec { - let mut bytes = value.as_bytes().to_vec(); +/// with the writers: a component that carries `0x00` cannot be read back, and PostgreSQL's +/// `text` cannot hold the byte either, so it is refused rather than truncated — both backends +/// reject the same value. +pub(crate) fn text_part( + value: &str, + table: &'static str, + column: &'static str, +) -> Result, StorageError> { + if value.contains('\0') { + return Err(StorageError::InvalidRow { + table, + column, + value: value.to_owned(), + }); + } + let mut bytes = Vec::with_capacity(value.len() + 1); + bytes.extend_from_slice(value.as_bytes()); bytes.push(0); - bytes + Ok(bytes) } /// The text a composite key's component carries, refusing bytes a `text` column could not hold. @@ -253,8 +268,20 @@ pub(crate) fn author_key( key } -/// `pubkey ‖ kind ‖ dedup_len ‖ canonical JSON`, where `dedup_len` is the JSON's byte length. -// shortcut: the deduplication index uses `serde_json` text as its key, so `1` and `1.0` differ +/// The tag a deduplication key's string element is written behind: its own bytes. +const DEDUP_STRING: u8 = 0; + +/// The tag any other deduplication element is written behind: its `serde_json` text. +const DEDUP_JSON: u8 = 1; + +/// `pubkey ‖ kind ‖ (tag u8 ‖ len u16 ‖ bytes)*`, one element per deduplication value: a string +/// as its own bytes, and anything else as its `serde_json` text behind a different tag. +/// +/// The length prefixes are what keep the key bounded by the value rather than by its JSON +/// escaping — a quote byte is one key byte, not two — and a key past [`MAX_KEY_LEN`] is refused +/// here, naming the column, before LMDB can reject the write. For the same `d` value this key +/// is four bytes shorter than the `tags` key, so the tag index is the one that refuses first. +// shortcut: a non-string element is written as its `serde_json` text, so `1` and `1.0` differ // where `jsonb` would not; upgrade when a caller can pass a hand-built JSON number, which // today's two callers cannot (spec Assumption 11). pub(crate) fn replaceable_key( @@ -262,53 +289,57 @@ pub(crate) fn replaceable_key( kind: i32, deduplication: &[serde_json::Value], ) -> Result, StorageError> { - let json = serde_json::to_vec(deduplication).map_err(|error| StorageError::InvalidRow { - table: "events", - column: "event_deduplication", - value: error.to_string(), - })?; - let length = u16::try_from(json.len()).map_err(|_| StorageError::InvalidRow { - table: "events", - column: "event_deduplication", - value: "the deduplication value is longer than 65535 bytes".to_owned(), - })?; - let mut key = Vec::with_capacity(32 + 4 + 2 + json.len()); + fn invalid(value: String) -> StorageError { + StorageError::InvalidRow { + table: "events", + column: "event_deduplication", + value, + } + } + fn length(len: usize) -> Result { + u16::try_from(len) + .map_err(|_| invalid("a deduplication element is 65536 bytes or longer".to_owned())) + } + + let mut key = Vec::with_capacity(32 + 4 + deduplication.len() * 3); key.extend_from_slice(pubkey.as_bytes()); key.extend_from_slice(&i32_key(kind)); - key.extend_from_slice(&length.to_be_bytes()); - key.extend_from_slice(&json); + for value in deduplication { + match value { + serde_json::Value::String(text) => { + key.push(DEDUP_STRING); + key.extend_from_slice(&length(text.len())?.to_be_bytes()); + key.extend_from_slice(text.as_bytes()); + } + other => { + let json = serde_json::to_vec(other).map_err(|error| invalid(error.to_string()))?; + key.push(DEDUP_JSON); + key.extend_from_slice(&length(json.len())?.to_be_bytes()); + key.extend_from_slice(&json); + } + } + } + if key.len() > MAX_KEY_LEN { + return Err(invalid(format!( + "the deduplication key is {} bytes, more than the {MAX_KEY_LEN} a key may hold", + key.len() + ))); + } Ok(key) } /// `tag_name ‖ 0x00 ‖ tag_value ‖ 0x00 ‖ created_at ‖ event_id`. /// -/// A name or value containing `0x00` is unrepresentable in the capture's `text` columns and -/// would make the separator ambiguous, so it is rejected rather than encoded. +/// A name or value carrying `0x00` is unrepresentable in the capture's `text` columns and would +/// make the separator ambiguous; [`text_part`] owns that refusal, so both components get it. pub(crate) fn tag_key( name: &str, value: &str, created_at: i64, event_id: &EventId, ) -> Result, StorageError> { - if name.contains('\0') { - return Err(StorageError::InvalidRow { - table: "event_tags", - column: "tag_name", - value: name.to_owned(), - }); - } - if value.contains('\0') { - return Err(StorageError::InvalidRow { - table: "event_tags", - column: "tag_value", - value: value.to_owned(), - }); - } - let mut key = Vec::with_capacity(name.len() + value.len() + 2 + 8 + EVENT_ID_LEN); - key.extend_from_slice(name.as_bytes()); - key.push(0); - key.extend_from_slice(value.as_bytes()); - key.push(0); + let mut key = text_part(name, "event_tags", "tag_name")?; + key.extend_from_slice(&text_part(value, "event_tags", "tag_value")?); key.extend_from_slice(&i64_key(created_at)); key.extend_from_slice(event_id.as_bytes()); Ok(key) @@ -683,4 +714,92 @@ mod tests { assert!(!hex_matches(&[0xab, 0xcd], "abce")); assert!(!hex_matches(&[0xab, 0xcd], "abcdef")); } + + #[test] + fn a_deduplication_value_is_not_inflated_by_json_escaping() { + let pubkey = Pubkey::from_byte_array([5; 32]); + // 235 quote bytes are 470 bytes of JSON text once escaped; the key must not grow with + // the escapes, or a value Postgres and the tag index both keep is refused. + let value = "\"".repeat(235); + let key = replaceable_key(&pubkey, 30_023, &[serde_json::Value::String(value.clone())]) + .expect("the pointer"); + assert!( + key.len() <= MAX_KEY_LEN, + "the key is {} bytes for a {}-byte value", + key.len(), + value.len() + ); + } + + #[test] + fn a_deduplication_value_past_the_key_bound_is_refused() { + let pubkey = Pubkey::from_byte_array([5; 32]); + let longest = "x".repeat(MAX_KEY_LEN - 39); + assert!( + replaceable_key(&pubkey, 30_023, &[serde_json::Value::String(longest)]).is_ok(), + "the longest value a pointer key holds is stored" + ); + let too_long = "x".repeat(MAX_KEY_LEN - 39 + 1); + let error = replaceable_key(&pubkey, 30_023, &[serde_json::Value::String(too_long)]) + .expect_err("one byte past the bound"); + assert!(matches!( + error, + StorageError::InvalidRow { + column: "event_deduplication", + .. + } + )); + } + + #[test] + fn the_replaceable_key_never_refuses_a_value_the_tag_key_accepts() { + let pubkey = Pubkey::from_byte_array([5; 32]); + let value = "x".repeat(MAX_KEY_LEN - 43 + 1); + assert!( + tag_key("d", &value, 0, &id(1)).expect("the tag key").len() > MAX_KEY_LEN, + "the tag key is the tighter bound for the same value" + ); + assert!(replaceable_key(&pubkey, 30_023, &[serde_json::Value::String(value)]).is_ok()); + } + + #[test] + fn a_deduplication_string_and_a_number_of_the_same_text_do_not_collide() { + let pubkey = Pubkey::from_byte_array([5; 32]); + let string = replaceable_key( + &pubkey, + 30_023, + &[serde_json::Value::String("1".to_owned())], + ) + .expect("a string element"); + let number = replaceable_key(&pubkey, 30_023, &[serde_json::Value::from(1)]) + .expect("a number element"); + assert_ne!(string, number); + } + + #[test] + fn a_text_component_a_text_column_cannot_hold_is_refused() { + // PostgreSQL's `text` cannot store `0x00`, and the key's separator is the same byte, so + // the codec refuses it where the writer meets it rather than reading it back truncated. + for value in ["a\0b", "\0", "ab\0"] { + let error = text_part(value, "delivery_log_by_outbox", "target_id") + .expect_err("a NUL in a key component"); + assert!(matches!( + error, + StorageError::InvalidRow { + table: "delivery_log_by_outbox", + column: "target_id", + .. + } + )); + } + assert_eq!( + text_part( + "https://example.test", + "delivery_log_by_outbox", + "target_id" + ) + .unwrap(), + b"https://example.test\0".to_vec() + ); + } } diff --git a/crates/nostream-storage/src/lmdb/metadata.rs b/crates/nostream-storage/src/lmdb/metadata.rs index f777ffa6a..2450752d9 100644 --- a/crates/nostream-storage/src/lmdb/metadata.rs +++ b/crates/nostream-storage/src/lmdb/metadata.rs @@ -314,11 +314,15 @@ pub(crate) fn invite_delete_expired( // DVM jobs. // --------------------------------------------------------------------------------------- -fn dvm_status_key(job: &DvmJob, created_at: DateTime) -> Vec { - let mut key = text_part(job.status.as_str()); +fn dvm_status_key(job: &DvmJob, created_at: DateTime) -> Result, StorageError> { + let mut key = text_part( + job.status.as_str(), + "dvm_jobs_by_status_created_at", + "status", + )?; key.extend_from_slice(&stamp_key(created_at)); key.extend_from_slice(job.id.as_bytes()); - key + Ok(key) } pub(crate) fn dvm_find_by_id( @@ -343,11 +347,11 @@ fn dvm_write( if let Some(previous) = previous { databases .dvm_jobs_by_status_created_at - .delete(txn, &dvm_status_key(previous, previous.created_at))?; + .delete(txn, &dvm_status_key(previous, previous.created_at)?)?; } databases .dvm_jobs_by_status_created_at - .put(txn, &dvm_status_key(job, job.created_at), &[])?; + .put(txn, &dvm_status_key(job, job.created_at)?, &[])?; Ok(()) } @@ -442,7 +446,7 @@ pub(crate) fn dvm_find_pending_jobs( ) -> Result, StorageError> { let mut jobs = Vec::new(); for status in [DvmJobStatus::Submitted, DvmJobStatus::PickedUp] { - let prefix = text_part(status.as_str()); + let prefix = text_part(status.as_str(), "dvm_jobs_by_status_created_at", "status")?; for entry in databases .dvm_jobs_by_status_created_at .prefix_iter(txn, &prefix)? diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs index f3ac46f12..241850dc1 100644 --- a/crates/nostream-storage/src/lmdb/notifications.rs +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -36,12 +36,20 @@ fn delivery_created_key(entry: &DeliveryLogEntry) -> Vec { /// `outbox_id ‖ status ‖ target_id ‖ id`, so one entry is addressable and two attempts at /// the same target do not share a key. -fn delivery_outbox_key(outbox_id: &str, entry: &DeliveryLogEntry) -> Vec { - let mut key = text_part(outbox_id); - key.extend_from_slice(&text_part(entry.status.as_str())); - key.extend_from_slice(&text_part(&entry.target_id)); +fn delivery_outbox_key(outbox_id: &str, entry: &DeliveryLogEntry) -> Result, StorageError> { + let mut key = text_part(outbox_id, "delivery_log_by_outbox", "outbox_id")?; + key.extend_from_slice(&text_part( + entry.status.as_str(), + "delivery_log_by_outbox", + "status", + )?); + key.extend_from_slice(&text_part( + &entry.target_id, + "delivery_log_by_outbox", + "target_id", + )?); key.extend_from_slice(entry.id.as_bytes()); - key + Ok(key) } /// The entry id one `delivery_log_by_outbox` key ends with. @@ -107,7 +115,7 @@ pub(crate) fn append( if let Some(outbox_id) = &entry.outbox_id { databases .delivery_log_by_outbox - .put(txn, &delivery_outbox_key(outbox_id, &entry), &[])?; + .put(txn, &delivery_outbox_key(outbox_id, &entry)?, &[])?; } Ok(()) } @@ -148,10 +156,12 @@ pub(crate) fn find_successful_target_ids( databases: &Databases, outbox_id: &str, ) -> Result, StorageError> { - let mut prefix = text_part(outbox_id); + let mut prefix = text_part(outbox_id, "delivery_log_by_outbox", "outbox_id")?; prefix.extend_from_slice(&text_part( crate::repositories::NotificationDeliveryStatus::Success.as_str(), - )); + "delivery_log_by_outbox", + "status", + )?); let mut seen = HashSet::new(); let mut targets = Vec::new(); @@ -203,7 +213,7 @@ pub(crate) fn delete_older_than( if let Some(outbox_id) = &entry.outbox_id { databases .delivery_log_by_outbox - .delete(txn, &delivery_outbox_key(outbox_id, &entry))?; + .delete(txn, &delivery_outbox_key(outbox_id, &entry)?)?; } affected += 1; } @@ -214,18 +224,26 @@ pub(crate) fn delete_older_than( // The outbox. // --------------------------------------------------------------------------------------- -fn outbox_created_key(message: &OutboxMessage) -> Vec { - let mut key = text_part(message.status.as_str()); +fn outbox_created_key(message: &OutboxMessage) -> Result, StorageError> { + let mut key = text_part( + message.status.as_str(), + "outbox_by_status_created_at", + "status", + )?; key.extend_from_slice(&stamp_key(message.created_at)); key.extend_from_slice(message.id.as_bytes()); - key + Ok(key) } -fn outbox_updated_key(message: &OutboxMessage) -> Vec { - let mut key = text_part(message.status.as_str()); +fn outbox_updated_key(message: &OutboxMessage) -> Result, StorageError> { + let mut key = text_part( + message.status.as_str(), + "outbox_by_status_updated_at", + "status", + )?; key.extend_from_slice(&stamp_key(message.updated_at)); key.extend_from_slice(message.id.as_bytes()); - key + Ok(key) } fn outbox_load( @@ -251,17 +269,17 @@ fn outbox_write( if let Some(previous) = previous { databases .outbox_by_status_created_at - .delete(txn, &outbox_created_key(previous))?; + .delete(txn, &outbox_created_key(previous)?)?; databases .outbox_by_status_updated_at - .delete(txn, &outbox_updated_key(previous))?; + .delete(txn, &outbox_updated_key(previous)?)?; } databases .outbox_by_status_created_at - .put(txn, &outbox_created_key(message), &[])?; + .put(txn, &outbox_created_key(message)?, &[])?; databases .outbox_by_status_updated_at - .put(txn, &outbox_updated_key(message), &[])?; + .put(txn, &outbox_updated_key(message)?, &[])?; Ok(()) } @@ -303,7 +321,7 @@ pub(crate) fn claim_batch( let mut candidates = Vec::new(); for status in [OutboxStatus::Pending, OutboxStatus::Processing] { - let prefix = text_part(status.as_str()); + let prefix = text_part(status.as_str(), "outbox_by_status_created_at", "status")?; for entry in databases .outbox_by_status_created_at .prefix_iter(txn, &prefix)? @@ -405,7 +423,7 @@ pub(crate) fn delete_terminal_older_than( let cutoff = encoding::micros(cutoff).timestamp_micros(); let mut terminal = Vec::new(); for status in [OutboxStatus::Delivered, OutboxStatus::Dead] { - let prefix = text_part(status.as_str()); + let prefix = text_part(status.as_str(), "outbox_by_status_updated_at", "status")?; for entry in databases .outbox_by_status_updated_at .prefix_iter(txn, &prefix)? @@ -428,10 +446,10 @@ pub(crate) fn delete_terminal_older_than( databases.outbox.delete(txn, id.as_bytes())?; databases .outbox_by_status_created_at - .delete(txn, &outbox_created_key(&message))?; + .delete(txn, &outbox_created_key(&message)?)?; databases .outbox_by_status_updated_at - .delete(txn, &outbox_updated_key(&message))?; + .delete(txn, &outbox_updated_key(&message)?)?; null_delivery_outbox(txn, databases, &id)?; affected += 1; } @@ -444,7 +462,7 @@ fn null_delivery_outbox( databases: &Databases, outbox_id: &str, ) -> Result<(), StorageError> { - let prefix = text_part(outbox_id); + let prefix = text_part(outbox_id, "delivery_log_by_outbox", "outbox_id")?; let mut named = Vec::new(); for entry in databases.delivery_log_by_outbox.prefix_iter(txn, &prefix)? { let (key, _) = entry?; From 3add554841213bbd126ca575c317aa9d21d2df5d Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 03:48:10 -0500 Subject: [PATCH 29/60] perf(storage): bound a filter's ranges and a stream's id scan --- crates/nostream-storage/src/lmdb/events.rs | 363 +++++++++++++++++++-- 1 file changed, 330 insertions(+), 33 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 60ae2f53c..17480a71b 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -1025,25 +1025,29 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { { if let Some(kinds) = &filter.kinds && !kinds.is_empty() + && let Some(ranges) = author_kind_ranges(authors, kinds) { return Plan { database: databases.events_by_author, path: AccessPath::Authors, - ranges: author_kind_ranges(authors, kinds), + ranges, // A range is `pubkey ‖ kind ‖ created_at ‖ event_id`, so it runs in time order - // only when the criterion is a whole pubkey and the range one author. + // only when every criterion is a whole pubkey and every range one author and + // kind. ordered: authors.iter().all(|criterion| criterion.len() == 64), tag: None, }; } // Without kinds the author index runs in kind order before `created_at`, so a page - // ordered by time has no early stop there. The created-at index is the capture's own - // path for this shape — ADR 0003 measured 5 508 rows discarded to return 500 for the - // kind-only form — and its order *is* the page order, so the page stops when full. + // ordered by time has no early stop there, and the same index is given up when the + // author × kind product is past `MAX_PAGE_KEYS`. The created-at index is the capture's + // own path for this shape — ADR 0003 measured 5 508 rows discarded to return 500 for + // the kind-only form — and its order *is* the page order, so the page stops when full. // - // shortcut: a quiet author's page therefore scans further into the created-at index - // than the author's own timeline would; upgrade when statistics can choose between the - // two, which is the benchmark job's (ADR 0010). + // shortcut: a quiet author's page, and an author × kind filter composing more than + // `MAX_PAGE_KEYS` ranges, read the created-at index with the criteria left to the + // matcher; upgrade when statistics can choose between the two indexes, which is the + // benchmark job's (ADR 0010). return Plan { database: databases.events_by_created_at, path: AccessPath::CreatedAt, @@ -1072,27 +1076,54 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { } } -/// One range per author criterion and kind on `events_by_author`, whose key is -/// `pubkey ‖ kind ‖ created_at ‖ event_id`: the `pubkey ‖ kind ‖ created_at` ordering the layout -/// writes is what lets a page for one author and one kind stop as soon as it is full. +/// One range per author criterion on `events_by_author`, whose key is +/// `pubkey ‖ kind ‖ created_at ‖ event_id`. A whole pubkey composes one range per kind, so the +/// `pubkey ‖ kind ‖ created_at` ordering the layout writes lets a page for one author and one +/// kind stop as soon as it is full; a prefix criterion composes one range over its whole tail +/// instead, because its upper bound runs to the end of the pubkey field and a kind appended +/// after it is compared where no key writes one — every kind under the prefix would fall inside +/// every kind's range, and a count would visit those rows once per kind. /// -/// The author bounds are the index's whole 32-byte pubkey field, so a prefix criterion's kind -/// is compared against the key's own kind bytes; a criterion another already covers, and a -/// repeated kind, seek one range, so a count cannot visit their keys twice. -fn author_kind_ranges(authors: &[String], kinds: &[Kind]) -> Vec { - let mut ranges = Vec::with_capacity(authors.len() * kinds.len()); - for (low, high) in encoding::hex_ranges(authors, 0) { - for kind in kinds { - let kind = encoding::i32_key(i32::try_from(kind.0).unwrap_or(i32::MAX)); - let mut lower = low.clone(); - lower.extend_from_slice(&kind); - let mut upper = high.clone(); - upper.extend_from_slice(&kind); - upper.extend(std::iter::repeat_n(0xff, 8 + 32)); - ranges.push((lower, upper)); +/// A criterion another already covers seeks no range of its own, and a repeated kind is one +/// range, so a count cannot visit a key twice. +/// +/// `None` when the whole-pubkey ranges and kinds compose more than [`MAX_PAGE_KEYS`] ranges: a +/// filter can name a thousand authors and a thousand kinds, and their product is not bounded by +/// the page, so the caller answers it from the created-at index rather than materialising it. +fn author_kind_ranges(authors: &[String], kinds: &[Kind]) -> Option> { + let mut kinds: Vec = kinds + .iter() + .map(|kind| i32::try_from(kind.0).unwrap_or(i32::MAX)) + .collect(); + kinds.sort_unstable(); + kinds.dedup(); + + let author_ranges = encoding::hex_ranges(authors, 0); + let exact = author_ranges + .iter() + .filter(|(low, high)| low == high && low.len() == encoding::HEX_FIELD_LEN) + .count(); + if exact.saturating_mul(kinds.len()) > MAX_PAGE_KEYS { + return None; + } + let mut ranges = + Vec::with_capacity(exact.saturating_mul(kinds.len()) + author_ranges.len() - exact); + for (low, high) in author_ranges { + if low == high && low.len() == encoding::HEX_FIELD_LEN { + for kind in &kinds { + let mut lower = low.clone(); + lower.extend_from_slice(&encoding::i32_key(*kind)); + let mut upper = lower.clone(); + upper.extend(std::iter::repeat_n(0xff, 8 + 32)); + ranges.push((lower, upper)); + } + } else { + let mut upper = high; + upper.extend(std::iter::repeat_n(0xff, 4 + 8 + 32)); + ranges.push((low, upper)); } } - encoding::disjoint(ranges) + Some(encoding::disjoint(ranges)) } /// The first tag criterion the capture treats as a query: `#` plus one ASCII letter with a @@ -1464,7 +1495,8 @@ impl Stream for ValuesStream { /// Yields the live `(created_at, event_id)` pairs a reconciliation compares, oldest first, in /// batches of at most [`STREAM_BATCH_ITEMS`] items or [`STREAM_BATCH_KEYS`] evaluated keys — -/// whichever comes first — with one read transaction per poll. +/// whichever comes first — with one read transaction per poll. An `ids` filter is one batch of +/// the request's own id list, sorted in the poll that reads it. pub(crate) struct ReconciliationStream { env: Env, databases: Databases, @@ -1566,14 +1598,20 @@ impl Stream for ReconciliationStream { /// The plan a reconciliation scans with. /// -/// The stream promises the pairs oldest first, and only a single range that runs in `created_at` -/// order gives that without materialising the result: a filter the faster index narrows to one -/// such range (`{authors: [a], kinds: [k]}` or one exact tag value) is scanned there, and -/// anything else — several ranges, or a range that runs in key order — scans the created-at -/// index, whose order *is* the promise. +/// The stream promises the pairs oldest first. `ids` is answered from the filter's own ranges — +/// one per id, so the work is the request's id list and not the store — and the batch sorts the +/// pairs before it emits them, because an `events` key runs in id order. A filter the faster +/// index narrows to one range that runs in `created_at` order (`{authors: [a], kinds: [k]}` or +/// one exact tag value) is scanned there; anything else scans the created-at index, whose order +/// *is* the promise. +/// +/// shortcut: an author without kinds and a `#g` prefix narrow to no such range, so their stream +/// scans `events_by_created_at` and the criteria are left to the matcher — a poll walks what the +/// store holds rather than what the request names; upgrade when statistics can choose the +/// cheaper index, which is the benchmark job's (ADR 0010). fn stream_plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { let chosen = plan(filter, databases); - if chosen.ordered && chosen.ranges.len() == 1 { + if chosen.path == AccessPath::Ids || (chosen.ordered && chosen.ranges.len() == 1) { return chosen; } Plan { @@ -1608,6 +1646,9 @@ fn scan_batch( live: true, proven: plan.tag.map(|(name, _)| name), }; + if plan.path == AccessPath::Ids { + return scan_ids(txn, databases, &matcher, &plan, position); + } let mut items = Vec::new(); let mut evaluated = 0; while position.index < plan.ranges.len() { @@ -1647,6 +1688,45 @@ fn scan_batch( Ok(items) } +/// The pairs an `ids` filter names, oldest first, with `position` moved past the id list. +/// +/// The ranges are the request's own id list, one per id and already deduplicated, so the work is +/// the request rather than the store, and the list is read whole because an `events` key runs in +/// id order: the pairs are sorted before they are emitted instead of being sought in time order. +fn scan_ids( + txn: &RoTxn<'_>, + databases: &Databases, + matcher: &Matcher<'_>, + plan: &Plan<'_>, + position: &mut Position, +) -> Result, StorageError> { + let mut items = Vec::with_capacity(plan.ranges.len()); + for (low, high) in &plan.ranges { + let range = encoding::bounds(low, high); + for entry in plan.database.range(txn, &range)? { + let (key, _) = entry?; + key_evaluated(); + let Some((created_at, event_id)) = candidate(txn, databases, plan.path, key)? else { + continue; + }; + if matcher.matches(txn, databases, &event_id)?.is_some() { + items.push(ReconciliationItem { + created_at, + event_id, + }); + } + } + } + items.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.event_id.cmp(&right.event_id)) + }); + position.index = plan.ranges.len(); + position.after = None; + Ok(items) +} + /// The `event_id` one `events_by_author` or `events_deleted` key ends with. fn header_id(key: &[u8], start: usize, table: &'static str) -> Result { match key.get(start..) { @@ -2271,4 +2351,221 @@ mod tests { assert_eq!(found.len(), 1); assert_eq!(found[0].created_at, 2_000); } + + // ----------------------------------------------------------------------------------- + // Round 5: the composition bound and the shapes a stream narrows. + // ----------------------------------------------------------------------------------- + + #[test] + fn an_author_and_kind_product_is_bounded_by_the_page() { + // A hundred authors and a hundred kinds are a legal filter; the product is what a + // request must not be able to turn into work this backend materialises. + let authors: Vec = (0..100u32).map(|step| format!("{step:064x}")).collect(); + let kinds: Vec = (0..100u64).map(|step| Kind(10_000 + step)).collect(); + assert!( + author_kind_ranges(&authors, &kinds).is_none(), + "10000 ranges are not a plan" + ); + + // A product the page can hold still composes one range per pair. + let kinds: Vec = (0..49u64).map(|step| Kind(10_000 + step)).collect(); + assert_eq!( + author_kind_ranges(&authors, &kinds) + .expect("a bounded product") + .len(), + authors.len() * kinds.len() + ); + + // At the bound the author index is still the plan; one past it, it is not. + let authors: Vec = (0..MAX_PAGE_KEYS as u32) + .map(|step| format!("{step:064x}")) + .collect(); + assert_eq!( + author_kind_ranges(&authors, &[Kind(1)]) + .expect("a product at the page bound") + .len(), + MAX_PAGE_KEYS + ); + let authors: Vec = (0..=MAX_PAGE_KEYS as u32) + .map(|step| format!("{step:064x}")) + .collect(); + assert!( + author_kind_ranges(&authors, &[Kind(1)]).is_none(), + "a product past the page bound falls back to the created-at index" + ); + } + + #[test] + fn a_prefix_author_counts_a_row_once_whatever_its_kind() { + // A prefix author's bound runs to the end of the pubkey field, so a kind appended to + // it does not narrow the range: the same key is in every kind's range. + let (_directory, env, databases) = store(); + let records = vec![signed(30, 1, 10, Vec::new()), signed(30, 2, 20, Vec::new())]; + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let prefix = records[0].event.pubkey.to_string()[..8].to_owned(); + let filter = Filter { + authors: Some(vec![prefix]), + kinds: Some(vec![Kind(1), Kind(2)]), + ..Filter::default() + }; + + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 2, + "two rows, counted once each" + ); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!(page.len(), 2, "and the page still returns both"); + let plan = plan(&filter, &databases); + assert_eq!( + plan.ranges.len(), + 1, + "a prefix author seeks one range whatever the kinds" + ); + } + + #[test] + fn an_author_and_kind_product_past_the_page_is_answered_from_the_created_at_index() { + let (_directory, env, databases) = store(); + let records = vec![ + signed(32, 10_000, 10, Vec::new()), + signed(33, 10_001, 20, Vec::new()), + ]; + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + + // A hundred authors and a hundred kinds compose ten thousand ranges, so the plan gives + // up the author index; the rows are still the ones the criteria name. + let mut authors: Vec = (0..100u32).map(|step| format!("{step:064x}")).collect(); + authors[0] = records[0].event.pubkey.to_string(); + authors[1] = records[1].event.pubkey.to_string(); + let kinds: Vec = (0..100u64).map(|step| Kind(10_000 + step)).collect(); + let filter = Filter { + authors: Some(authors), + kinds: Some(kinds), + limit: Some(2), + ..Filter::default() + }; + + let plan = plan(&filter, &databases); + assert_eq!(plan.path, AccessPath::CreatedAt); + assert_eq!(plan.ranges.len(), 1); + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 2 + ); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!( + created(&page), + vec![20, 10], + "newest first, the rows the filter names" + ); + } + + #[test] + fn a_reconciliation_reads_the_ids_the_filter_names_not_the_store() { + let _serial = probes::COUNTING.lock().expect("the counting lock"); + let (_directory, env, databases) = store(); + let records: Vec = (1..=300) + .map(|step| signed(29, 1, step, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + ids: Some(vec![ + records[250].event.id.to_string(), + records[12].event.id.to_string(), + records[199].event.id.to_string(), + ]), + ..Filter::default() + }; + + probes::reset(); + let mut position = Position::new(); + let items = scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"); + + assert_eq!( + items, + vec![ + ReconciliationItem { + created_at: 13, + event_id: records[12].event.id, + }, + ReconciliationItem { + created_at: 200, + event_id: records[199].event.id, + }, + ReconciliationItem { + created_at: 251, + event_id: records[250].event.id, + }, + ], + "the request's ids, oldest first" + ); + assert!( + probes::keys() <= 3, + "the id list bounds the work, not the store: {} keys evaluated", + probes::keys() + ); + assert_eq!( + position.index, + stream_plan(&filter, &databases).ranges.len(), + "the whole list is one batch" + ); + } + + #[test] + fn a_quote_only_deduplication_value_stores_without_json_escaping() { + let (_directory, env, databases) = store(); + let value = "\"".repeat(235); + let record = signed(31, 30_023, 1, vec![vec!["d".to_owned(), value.clone()]]) + .with_deduplication(vec![serde_json::Value::String(value)]); + + let mut transaction = env.write_txn().expect("a write transaction"); + let affected = upsert(&mut transaction, &databases, &record, Utc::now()) + .expect("a d value that JSON text would escape twice still stores"); + transaction.commit().expect("the commit"); + assert_eq!(affected, 1); + } } From fbc99a927ea7bd07511f5edd83757adbef7db974 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 03:48:10 -0500 Subject: [PATCH 30/60] test(storage): cover an ids reconciliation and a NUL target id --- crates/nostream-storage/tests/parity_lmdb.rs | 74 +++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index 8653730e2..c96995122 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -27,7 +27,7 @@ use nostream_storage::repositories::{ EventRetentionOptions, FieldUpdate, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, InvoiceUnit, NewDeliveryLogEntry, NotificationDeliveryLogRepository, NotificationDeliveryStatus, NotificationOutboxRepository, NotificationTargetType, - UserRepository, + ReconciliationItem, UserRepository, }; mod common; @@ -494,6 +494,35 @@ async fn a_delivery_log_entry_must_name_an_existing_outbox_row() { ); } +#[tokio::test] +async fn a_target_id_a_text_column_cannot_hold_is_refused_rather_than_truncated() { + let (_directory, backend) = open_store().await; + let message = backend + .enqueue("note.created", &json!({ "id": "note-3" })) + .await + .unwrap(); + let mut entry = one_entry(Some(message.id.clone())); + entry.target_id = "https://example.test/\0hook".to_owned(); + + // `notification_delivery_log.target_id` is a `text` column, and PostgreSQL cannot store + // `0x00` in one, so the capture could never have kept this row; the index key refuses it + // rather than truncating the id at the separator on the way back out. + match backend.append(&entry).await { + Err(StorageError::InvalidRow { column, .. }) => { + assert_eq!(column, "target_id"); + } + other => panic!("a NUL target id is refused, got {other:?}"), + } + assert!( + backend + .find_successful_target_ids(&message.id) + .await + .unwrap() + .is_empty(), + "the refused entry stored nothing to read back" + ); +} + #[tokio::test] async fn deleting_an_outbox_row_nulls_its_delivery_log_entries() { let (_directory, backend) = open_store().await; @@ -817,6 +846,49 @@ async fn a_reconciliation_lists_every_live_row_across_batches() { ); } +#[tokio::test] +async fn a_reconciliation_over_ids_lists_exactly_those_rows_oldest_first() { + let (_directory, backend) = open_store().await; + let records: Vec = (0..300) + .map(|index| event(41, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + ids: Some(vec![ + records[250].event.id.to_string(), + records[12].event.id.to_string(), + records[199].event.id.to_string(), + ]), + ..Filter::default() + }; + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + + assert_eq!( + items, + vec![ + ReconciliationItem { + created_at: 1_700_000_012, + event_id: records[12].event.id, + }, + ReconciliationItem { + created_at: 1_700_000_199, + event_id: records[199].event.id, + }, + ReconciliationItem { + created_at: 1_700_000_250, + event_id: records[250].event.id, + }, + ], + "the request's ids, oldest first, whatever order the ids arrive in" + ); +} + #[tokio::test] async fn an_empty_tag_value_criterion_matches_nothing() { let (_directory, backend) = open_store().await; From 1168af00a9cd1c66bdd5ccb466d1901b2ad6f14e Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 03:48:10 -0500 Subject: [PATCH 31/60] docs(storage): record the key bounds and the stream's ids path --- crates/nostream-storage/src/repositories.rs | 4 ++- docs/rewrite/decisions/0010-lmdb-backend.md | 32 +++++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index dbf0be6ad..1e9e78037 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -29,7 +29,9 @@ pub type StorageFuture<'a, T> = Pin Date: Fri, 6 Nov 2026 03:49:18 -0500 Subject: [PATCH 32/60] docs(storage): mark the id stream's whole-list read --- crates/nostream-storage/src/lmdb/events.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 17480a71b..b0250d5dd 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -1693,6 +1693,10 @@ fn scan_batch( /// The ranges are the request's own id list, one per id and already deduplicated, so the work is /// the request rather than the store, and the list is read whole because an `events` key runs in /// id order: the pairs are sorted before they are emitted instead of being sought in time order. +/// +/// shortcut: the whole list is read in one poll and its pairs are held until the stream ends, so +/// the resident set is the request's own id list; upgrade when a request's list should be paged +/// too, which needs the pairs sorted before emission and therefore somewhere to spill them. fn scan_ids( txn: &RoTxn<'_>, databases: &Databases, From 71162c3f1ef2ac3189144406bc29222810be0ec5 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 03:50:46 -0500 Subject: [PATCH 33/60] docs(storage): say what the page bound counts --- crates/nostream-storage/src/lmdb/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 718117c6a..8d5f63c5d 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -59,9 +59,11 @@ pub(crate) const DEFAULT_FILTER_LIMIT: u64 = 500; /// /// `limits.client.subscription.maxLimit` is the relay's own bound, but it is applied only when /// the deployment sets it, and a request reaches the store before some of those gates; this is -/// the backend's bound, so the size of a read's resident set is never chosen by the client. -/// 5 000 is exactly the shipped `maxLimit` (`resources/default-settings.yaml`), so the clamp -/// cannot shorten a page a configured relay would have served. +/// the backend's bound, so no page a read materialises exceeds it. A multi-filter read holds one +/// such page per filter, and a reconciliation over `ids` holds the request's own id list, whose +/// size the relay's payload limit already bounds. 5 000 is exactly the shipped `maxLimit` +/// (`resources/default-settings.yaml`), so the clamp cannot shorten a page a configured relay +/// would have served. pub(crate) const MAX_PAGE_KEYS: usize = 5_000; /// The retention batch size (`RETENTION_BATCH_SIZE`). From 1388c56cce1f674f2f79ee1ce02270bd5310346c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 04:52:47 -0500 Subject: [PATCH 34/60] fix(storage): give a page one accumulator and bound a prefix id scan --- crates/nostream-storage/src/lmdb/events.rs | 562 +++++++++++++++++-- crates/nostream-storage/tests/parity_lmdb.rs | 59 ++ 2 files changed, 583 insertions(+), 38 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index b0250d5dd..2386823f1 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -607,8 +607,8 @@ struct Plan<'a> { database: Database, path: AccessPath, ranges: Vec, - /// Whether a range's own key order is the page's order, so a range stops once the page is - /// full instead of being walked whole. + /// Whether a range's own key order is the page's order, so a range stops at the page's own + /// worst kept row instead of being walked whole. ordered: bool, /// The tag family the ranges were built from, with its criteria in the criteria's order — /// `tag_ranges` builds one range per criterion. A key in these ranges has already proved @@ -859,10 +859,13 @@ fn admits(filter: &Filter, gates: &Gates) -> Result { /// One filter's page: the best `limit` keys it matches, ordered as the capture's branch is. /// -/// The resident set is the page, never the range. A range whose key order is the page's order -/// stops as soon as the page is full (keeping every row at the boundary timestamp, so a tie is -/// the capture's); the other ranges offer their matches to a buffer of at most `2 × limit`, -/// which is cut back to the best `limit` whenever it fills. +/// The resident set is the page, never the range, and the page is one accumulator across every +/// range the plan seeks — the top `limit` is global to them. A range whose key order is the +/// page's order stops once the page holds its own worst kept row: every row after it in that +/// range is past it, so a later range can no longer arm the stop at the first match it offers. +/// The comparison is strict, so every row tied at the boundary timestamp is still offered and +/// the tie breaks as the capture's does; the other ranges offer their matches to a buffer of at +/// most `2 × limit`, cut back to the best `limit` whenever it fills. fn branch( txn: &RoTxn<'_>, databases: &Databases, @@ -890,7 +893,13 @@ fn branch( let mut page = Page::new(descending); for (low, high) in &plan.ranges { let range = encoding::bounds(low, high); - let mut boundary: Option = None; + // A range starts at the boundary the page already holds: a range whose oldest row is + // past it has nothing to add, and one that is not is walked from its own end. + let mut boundary = if plan.ordered { + page.boundary(limit) + } else { + None + }; for entry in scan(plan.database, txn, &range, plan.ordered && descending)? { let (key, _) = entry?; key_evaluated(); @@ -902,8 +911,8 @@ fn branch( } if matcher.matches(txn, databases, &found.1)?.is_some() { page.offer(found, limit); - if plan.ordered && page.len() >= limit { - boundary = Some(found.0); + if plan.ordered { + boundary = page.boundary(limit); } } } @@ -936,10 +945,6 @@ impl Page { } } - fn len(&self) -> usize { - self.entries.len() - } - /// Keeps `key` if it can still be among the best `limit` the page has seen. fn offer(&mut self, key: (i64, EventId), limit: usize) { if limit == 0 || !self.seen.insert(key.1) { @@ -951,6 +956,19 @@ impl Page { } } + /// The timestamp of the worst row the page would keep, once it holds `limit` of them. + /// + /// A range that runs in the page's order cannot offer a row past it, so the walk stops + /// there. Cutting to the limit first is what makes the boundary the page's own rather than + /// the last row offered, and it is only paid once the page is already full. + fn boundary(&mut self, limit: usize) -> Option { + if limit == 0 || self.entries.len() < limit { + return None; + } + self.cut(limit); + self.entries.last().map(|(created_at, _)| *created_at) + } + fn finish(mut self, limit: usize) -> Vec<(i64, EventId)> { self.cut(limit); self.entries @@ -1495,8 +1513,9 @@ impl Stream for ValuesStream { /// Yields the live `(created_at, event_id)` pairs a reconciliation compares, oldest first, in /// batches of at most [`STREAM_BATCH_ITEMS`] items or [`STREAM_BATCH_KEYS`] evaluated keys — -/// whichever comes first — with one read transaction per poll. An `ids` filter is one batch of -/// the request's own id list, sorted in the poll that reads it. +/// whichever comes first — with one read transaction per poll. An `ids` filter naming exact ids +/// is one batch of the request's own list, sorted in the poll that reads it; a prefix criterion +/// is a byte range and takes the bounded scan. pub(crate) struct ReconciliationStream { env: Env, databases: Databases, @@ -1598,20 +1617,24 @@ impl Stream for ReconciliationStream { /// The plan a reconciliation scans with. /// -/// The stream promises the pairs oldest first. `ids` is answered from the filter's own ranges — -/// one per id, so the work is the request's id list and not the store — and the batch sorts the -/// pairs before it emits them, because an `events` key runs in id order. A filter the faster -/// index narrows to one range that runs in `created_at` order (`{authors: [a], kinds: [k]}` or -/// one exact tag value) is scanned there; anything else scans the created-at index, whose order -/// *is* the promise. +/// The stream promises the pairs oldest first. An `ids` filter whose criteria are all exact is +/// answered from the filter's own ranges — one point per id, so the work is the request's id +/// list and not the store — and the batch sorts the pairs before it emits them, because an +/// `events` key runs in id order. A criterion is exact only when its range is a point: a prefix +/// is a byte range over the id field, and `""` is the whole store, so it is not the request's +/// list and takes the bounded scan instead. A filter the faster index narrows to one range that +/// runs in `created_at` order (`{authors: [a], kinds: [k]}` or one exact tag value) is scanned +/// there; anything else scans the created-at index, whose order *is* the promise. /// -/// shortcut: an author without kinds and a `#g` prefix narrow to no such range, so their stream -/// scans `events_by_created_at` and the criteria are left to the matcher — a poll walks what the -/// store holds rather than what the request names; upgrade when statistics can choose the -/// cheaper index, which is the benchmark job's (ADR 0010). +/// shortcut: an author without kinds, a `#g` prefix and an id prefix narrow to no such range, +/// so their stream scans `events_by_created_at` and the criteria are left to the matcher — a +/// poll walks what the store holds rather than what the request names; upgrade when statistics +/// can choose the cheaper index, which is the benchmark job's (ADR 0010). fn stream_plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { let chosen = plan(filter, databases); - if chosen.path == AccessPath::Ids || (chosen.ordered && chosen.ranges.len() == 1) { + let exact_ids = + chosen.path == AccessPath::Ids && chosen.ranges.iter().all(|(low, high)| low == high); + if exact_ids || (chosen.ordered && chosen.ranges.len() == 1) { return chosen; } Plan { @@ -1688,11 +1711,13 @@ fn scan_batch( Ok(items) } -/// The pairs an `ids` filter names, oldest first, with `position` moved past the id list. +/// The pairs an exact `ids` filter names, oldest first, with `position` moved past the id list. /// -/// The ranges are the request's own id list, one per id and already deduplicated, so the work is -/// the request rather than the store, and the list is read whole because an `events` key runs in -/// id order: the pairs are sorted before they are emitted instead of being sought in time order. +/// The ranges are the request's own list, one point per id and already deduplicated, so the +/// work is the request rather than the store, and the list is read whole because an `events` +/// key runs in id order: the pairs are sorted before they are emitted instead of being sought +/// in time order. [`stream_plan`] keeps a prefix criterion away from here, because its range is +/// the id field's bytes rather than the request. /// /// shortcut: the whole list is read in one poll and its pairs are held until the stream ends, so /// the resident set is the request's own id list; upgrade when a request's list should be paged @@ -1787,9 +1812,9 @@ mod tests { for step in 0u8..=100 { page.offer((i64::from(step), id(step)), 3); assert!( - page.len() <= 6, + page.entries.len() <= 6, "the buffer is cut back to the limit whenever it fills: {} entries", - page.len() + page.entries.len() ); } assert_eq!(page.finish(3).len(), 3); @@ -1901,7 +1926,7 @@ mod tests { // The author index runs in kind order before `created_at`, so this page is answered by // the created-at index: it stops as soon as it is full instead of walking the author's // whole timeline (the round-2 shape evaluated every one of the 200). - let _serial = probes::COUNTING.lock().expect("the counting lock"); + let _serial = counting(); let (_directory, env, databases) = store(); let records: Vec = (1..=200) .map(|step| signed(7, 1, step, Vec::new())) @@ -1938,7 +1963,7 @@ mod tests { fn a_limited_page_stops_at_the_limit_inside_one_author_and_kind() { // The `pubkey ‖ kind ‖ created_at` ordering the layout writes is what lets this page // stop; without it the walk reads every event of the author. - let _serial = probes::COUNTING.lock().expect("the counting lock"); + let _serial = counting(); let (_directory, env, databases) = store(); let mut records: Vec = (1..=200) .map(|step| signed(7, 1, step, Vec::new())) @@ -1978,6 +2003,7 @@ mod tests { // The created-at index reverses the whole key, so two rows at one instant are scanned // id-descending: a walk that stopped at the first would keep the larger id, where the // capture's comparator keeps the smaller (V6). + let _serial = counting(); let (_directory, env, databases) = store(); let tag = |value: &str| vec![vec!["t".to_owned(), value.to_owned()]]; let first = signed(9, 1, 500, tag("first")); @@ -2011,7 +2037,7 @@ mod tests { #[test] fn a_tag_page_checks_its_criterion_from_the_key_and_decodes_no_body() { - let _serial = probes::COUNTING.lock().expect("the counting lock"); + let _serial = counting(); let (_directory, env, databases) = store(); let tag = |step: i64| { vec![ @@ -2100,6 +2126,7 @@ mod tests { #[test] fn a_wire_limit_cannot_grow_a_page_past_the_backend_bound() { + let _serial = counting(); assert_eq!(count_limit(10), 10); assert_eq!(count_limit(u64::MAX), MAX_PAGE_KEYS); @@ -2136,6 +2163,7 @@ mod tests { #[test] fn a_count_holds_no_matches_and_deduplicates_the_union() { + let _serial = counting(); let (_directory, env, databases) = store(); let records = vec![ signed(21, 1, 10, Vec::new()), @@ -2183,6 +2211,7 @@ mod tests { #[test] fn a_later_limited_page_does_not_recount_an_earlier_filter_s_rows() { + let _serial = counting(); let (_directory, env, databases) = store(); let records = vec![ signed(28, 1, 10, Vec::new()), @@ -2211,6 +2240,7 @@ mod tests { #[test] fn a_count_counts_a_row_once_when_two_tag_values_select_it() { + let _serial = counting(); let (_directory, env, databases) = store(); let record = signed( 23, @@ -2257,7 +2287,7 @@ mod tests { #[test] fn a_reconciliation_scans_the_filter_own_index_when_one_ordered_range_answers_it() { - let _serial = probes::COUNTING.lock().expect("the counting lock"); + let _serial = counting(); let (_directory, env, databases) = store(); let mut records: Vec = (1..=300) .map(|step| signed(24, 1, step, Vec::new())) @@ -2300,7 +2330,7 @@ mod tests { #[test] fn a_reconciliation_batch_stops_at_its_key_cap_and_resumes() { - let _serial = probes::COUNTING.lock().expect("the counting lock"); + let _serial = counting(); let (_directory, env, databases) = store(); let mut records: Vec = (1..=1000) .map(|step| signed(26, 1, step, Vec::new())) @@ -2403,6 +2433,7 @@ mod tests { fn a_prefix_author_counts_a_row_once_whatever_its_kind() { // A prefix author's bound runs to the end of the pubkey field, so a kind appended to // it does not narrow the range: the same key is in every kind's range. + let _serial = counting(); let (_directory, env, databases) = store(); let records = vec![signed(30, 1, 10, Vec::new()), signed(30, 2, 20, Vec::new())]; write(&env, &databases, &records); @@ -2447,6 +2478,7 @@ mod tests { #[test] fn an_author_and_kind_product_past_the_page_is_answered_from_the_created_at_index() { + let _serial = counting(); let (_directory, env, databases) = store(); let records = vec![ signed(32, 10_000, 10, Vec::new()), @@ -2501,7 +2533,7 @@ mod tests { #[test] fn a_reconciliation_reads_the_ids_the_filter_names_not_the_store() { - let _serial = probes::COUNTING.lock().expect("the counting lock"); + let _serial = counting(); let (_directory, env, databases) = store(); let records: Vec = (1..=300) .map(|step| signed(29, 1, step, Vec::new())) @@ -2572,4 +2604,458 @@ mod tests { transaction.commit().expect("the commit"); assert_eq!(affected, 1); } + // ----------------------------------------------------------------------------------- + // Round 6: the page across every range a plan seeks, and the id stream's two shapes. + // ----------------------------------------------------------------------------------- + + /// Serialises a test that counts work against every other one, so a clean run means what + /// it says. A test that panicked while holding the lock poisons it, and a poisoned lock + /// must not redden the tests that follow. + fn counting() -> std::sync::MutexGuard<'static, ()> { + probes::COUNTING + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// The page a filter must return, built from the store itself rather than from an index: + /// every row the matcher accepts, in the capture's order, cut to the filter's page. + fn reference_page( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + ) -> Vec<(i64, EventId)> { + let matcher = Matcher { + filter, + gates: &Gates::default(), + now: 0, + live: false, + proven: None, + }; + let mut page = Vec::new(); + for entry in databases.events.iter(txn).expect("the store") { + let (key, _) = entry.expect("a row"); + let id = encoding::event_id_from(key).expect("an event id"); + if let Some(header) = matcher.matches(txn, databases, &id).expect("a match") { + page.push((header.created_at, id)); + } + } + sort_page(&mut page, filter.limit.is_some()); + page.truncate(page_limit(filter)); + page + } + + /// The items a reconciliation must yield for a filter, built from the store itself: every + /// row the matcher accepts, oldest first. + fn reference_items( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + ) -> Vec { + let matcher = Matcher { + filter, + gates: &Gates::default(), + now: 0, + live: true, + proven: None, + }; + let mut items = Vec::new(); + for entry in databases.events.iter(txn).expect("the store") { + let (key, _) = entry.expect("a row"); + let id = encoding::event_id_from(key).expect("an event id"); + if let Some(header) = matcher.matches(txn, databases, &id).expect("a match") { + items.push(ReconciliationItem { + created_at: header.created_at, + event_id: id, + }); + } + } + items.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.event_id.cmp(&right.event_id)) + }); + items + } + + /// The page `branch` returns, the store's own answer, and the count of the same filter all + /// agree. + fn assert_page_agrees( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + expected: &[i64], + what: &str, + ) { + let page = branch( + txn, + databases, + filter, + &Gates::default(), + 0, + false, + page_limit(filter), + ) + .expect("the page"); + assert_eq!( + created(&page), + expected, + "{what}: the page is the capture's newest rows" + ); + assert_eq!( + page, + reference_page(txn, databases, filter), + "{what}: the page is the store's own answer" + ); + assert_eq!( + count( + txn, + databases, + std::slice::from_ref(filter), + &Gates::default(), + 0 + ) + .expect("the count"), + page.len() as u64, + "{what}: the page and the count of the same filter agree" + ); + } + + #[test] + fn a_limited_page_over_several_time_ordered_ranges_holds_the_global_newest_rows() { + // A page is one accumulator across every range the plan seeks: an author × kind + // filter seeks one range per kind, and the top `limit` is global to them, so a later + // range may not stop at the first row it offers. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=40) + .map(|step: i64| signed(7, (step % 3) as u64 + 1, step * 10, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let author = records[0].event.pubkey.to_string(); + let prefix = author[..8].to_owned(); + let two_kinds = &[390, 380, 360, 350, 330]; + let every_kind = &[400, 390, 380, 370, 360]; + + let cases: Vec<(&str, Filter, AccessPath, usize, &[i64])> = vec![ + ( + "one author and two kinds", + Filter { + authors: Some(vec![author.clone()]), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::Authors, + 2, + two_kinds, + ), + ( + "the same kinds in the other order", + Filter { + authors: Some(vec![author.clone()]), + kinds: Some(vec![Kind(3), Kind(1)]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::Authors, + 2, + two_kinds, + ), + ( + "two kinds without an author", + Filter { + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::CreatedAt, + 1, + two_kinds, + ), + ( + "one author without kinds", + Filter { + authors: Some(vec![author]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::CreatedAt, + 1, + every_kind, + ), + ( + "a prefix author and two kinds", + Filter { + authors: Some(vec![prefix]), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::Authors, + 1, + two_kinds, + ), + ]; + for (what, filter, path, ranges, expected) in cases { + let plan = plan(&filter, &databases); + assert_eq!( + (plan.path, plan.ranges.len()), + (path, ranges), + "{what}: the plan" + ); + assert_page_agrees(&transaction, &databases, &filter, expected, what); + } + } + + #[test] + fn a_limited_page_over_two_tag_values_does_not_depend_on_the_criteria_s_order() { + // One criterion is one range on `tags`, and each range runs in `created_at` order + // under its own value, so a page over two values is the global top `limit` of both. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=40) + .map(|step| { + let mut tags = vec![vec!["t".to_owned(), "a".to_owned()]]; + if step % 3 != 1 { + tags.push(vec!["t".to_owned(), "b".to_owned()]); + } + signed(8, 1, step * 10, tags) + }) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let author = records[0].event.pubkey.to_string(); + let expected = [400, 390, 380, 370, 360]; + + for criteria in [ + vec!["a".to_owned(), "b".to_owned()], + vec!["b".to_owned(), "a".to_owned()], + ] { + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), criteria.clone()); + let filter = Filter { + tags: tags.clone(), + limit: Some(5), + ..Filter::default() + }; + assert_eq!( + plan(&filter, &databases).ranges.len(), + 2, + "one range per value" + ); + assert_page_agrees( + &transaction, + &databases, + &filter, + &expected, + "two tag values", + ); + + // A criterion of another family is the matcher's, over the same two ranges. + let filter = Filter { + kinds: Some(vec![Kind(1)]), + tags, + limit: Some(5), + ..Filter::default() + }; + assert_page_agrees( + &transaction, + &databases, + &filter, + &expected, + "a kind and two tag values", + ); + + // An author as well makes the author index the plan; the page is the same one. + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), criteria); + let filter = Filter { + authors: Some(vec![author.clone()]), + kinds: Some(vec![Kind(1)]), + tags, + limit: Some(5), + ..Filter::default() + }; + assert_page_agrees( + &transaction, + &databases, + &filter, + &expected, + "an author, a kind and two tag values", + ); + } + + // A single value's own page, so the two-value case is not "everything, in any order". + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), vec!["b".to_owned()]); + let filter = Filter { + tags, + limit: Some(5), + ..Filter::default() + }; + assert_page_agrees( + &transaction, + &databases, + &filter, + &[390, 380, 360, 350, 330], + "one tag value", + ); + } + + #[test] + fn the_author_index_and_the_created_at_fallback_return_the_same_page() { + // A filter composing more than `MAX_PAGE_KEYS` author × kind ranges is answered from + // the created-at index instead, and both plans must return the same page. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=40) + .map(|step: i64| signed(9, (step % 3) as u64 + 1, step * 10, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let expected = [390, 380, 360, 350, 330]; + let composed = |fakes: u32| -> Vec { + std::iter::once(records[0].event.pubkey.to_string()) + .chain((0..fakes).map(|step| format!("{:064x}", 0x1000 + step))) + .collect() + }; + + let at_bound = Filter { + authors: Some(composed(MAX_PAGE_KEYS as u32 / 2 - 1)), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }; + let chosen = plan(&at_bound, &databases); + assert_eq!( + (chosen.path, chosen.ranges.len()), + (AccessPath::Authors, MAX_PAGE_KEYS) + ); + assert_page_agrees( + &transaction, + &databases, + &at_bound, + &expected, + "at the page bound", + ); + let index_page = branch( + &transaction, + &databases, + &at_bound, + &Gates::default(), + 0, + false, + page_limit(&at_bound), + ) + .expect("the page"); + + let past_bound = Filter { + authors: Some(composed(MAX_PAGE_KEYS as u32 / 2)), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }; + let chosen = plan(&past_bound, &databases); + assert_eq!( + (chosen.path, chosen.ranges.len()), + (AccessPath::CreatedAt, 1) + ); + assert_page_agrees( + &transaction, + &databases, + &past_bound, + &expected, + "past the page bound", + ); + let fallback_page = branch( + &transaction, + &databases, + &past_bound, + &Gates::default(), + 0, + false, + page_limit(&past_bound), + ) + .expect("the page"); + assert_eq!( + index_page, fallback_page, + "the author index and the created-at fallback return the same page" + ); + } + + #[test] + fn a_prefix_id_reconciliation_scans_in_bounded_batches() { + // An exact id list is a point per id, so the list is the bound on the work; a prefix + // criterion is a byte range over the id field, and `""` is the whole store. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=400) + .map(|step| signed(35, 1, step, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let ids = |criteria: &str| Filter { + ids: Some(vec![criteria.to_owned()]), + ..Filter::default() + }; + + let exact = ids(&records[10].event.id.to_string()); + assert_eq!( + stream_plan(&exact, &databases).path, + AccessPath::Ids, + "an exact id list is a point per id" + ); + let prefix = ids(&records[0].event.id.to_string()[..2]); + assert_eq!( + stream_plan(&prefix, &databases).path, + AccessPath::CreatedAt, + "a prefix criterion is a byte range, so it takes the bounded scan" + ); + + for filter in [ids(""), prefix] { + probes::reset(); + let mut position = Position::new(); + let mut items = scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"); + assert!( + probes::keys() <= STREAM_BATCH_KEYS, + "one poll evaluates at most the key cap: {} keys", + probes::keys() + ); + assert!( + position.after.is_some(), + "and it resumes after the last key it evaluated" + ); + + for _ in 0..8 { + if position.index >= stream_plan(&filter, &databases).ranges.len() { + break; + } + items.extend( + scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"), + ); + } + assert_eq!( + items, + reference_items(&transaction, &databases, &filter), + "the bounded scan yields every row the criterion selects, oldest first" + ); + } + } } diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index c96995122..1a9f674f5 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -889,6 +889,65 @@ async fn a_reconciliation_over_ids_lists_exactly_those_rows_oldest_first() { ); } +#[tokio::test] +async fn a_reconciliation_over_an_id_prefix_lists_every_row_the_prefix_selects() { + // A prefix criterion is a byte range over the id field rather than the request's own list, + // so it is answered by the bounded created-at scan; `""` is that range over every row. + let (_directory, backend) = open_store().await; + let records: Vec = (0..400) + .map(|index| event(42, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + ids: Some(vec![String::new()]), + ..Filter::default() + }; + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + assert_eq!( + items.len(), + 400, + "every row the criterion selects arrives across the batches" + ); + assert!( + items + .windows(2) + .all(|pair| (pair[0].created_at, pair[0].event_id) + < (pair[1].created_at, pair[1].event_id)), + "and oldest first" + ); + + let prefix = records[7].event.id.to_string()[..2].to_owned(); + let filter = Filter { + ids: Some(vec![prefix.clone()]), + ..Filter::default() + }; + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + let expected: Vec = records + .iter() + .filter(|record| record.event.id.to_string().starts_with(&prefix)) + .map(|record| ReconciliationItem { + created_at: record.event.created_at, + event_id: record.event.id, + }) + .collect(); + assert!( + !expected.is_empty(), + "the prefix selects the row it came from" + ); + assert_eq!( + items, expected, + "the prefix selects the same rows, oldest first" + ); +} + #[tokio::test] async fn an_empty_tag_value_criterion_matches_nothing() { let (_directory, backend) = open_store().await; From ff7b6165849664d1261aa2f4c3776e0c550b27e7 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Fri, 6 Nov 2026 04:52:49 -0500 Subject: [PATCH 35/60] docs(storage): say which ids read holds the request's list --- crates/nostream-storage/src/lmdb/mod.rs | 5 +++-- crates/nostream-storage/src/repositories.rs | 5 +++-- docs/rewrite/decisions/0010-lmdb-backend.md | 23 ++++++++++++--------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 8d5f63c5d..1aa1310b6 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -60,8 +60,9 @@ pub(crate) const DEFAULT_FILTER_LIMIT: u64 = 500; /// `limits.client.subscription.maxLimit` is the relay's own bound, but it is applied only when /// the deployment sets it, and a request reaches the store before some of those gates; this is /// the backend's bound, so no page a read materialises exceeds it. A multi-filter read holds one -/// such page per filter, and a reconciliation over `ids` holds the request's own id list, whose -/// size the relay's payload limit already bounds. 5 000 is exactly the shipped `maxLimit` +/// such page per filter, and a reconciliation over `ids` holds the request's own id list when +/// its criteria are exact — a prefix criterion is a byte range over the id field, so it takes +/// the bounded scan instead. 5 000 is exactly the shipped `maxLimit` /// (`resources/default-settings.yaml`), so the clamp cannot shorten a page a configured relay /// would have served. pub(crate) const MAX_PAGE_KEYS: usize = 5_000; diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index 1e9e78037..1db5b06e9 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -30,8 +30,9 @@ pub type StorageFuture<'a, T> = Pin Date: Fri, 6 Nov 2026 05:04:10 -0500 Subject: [PATCH 36/60] test(storage): pin an unlimited page over two tag values --- crates/nostream-storage/src/lmdb/events.rs | 50 ++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 2386823f1..bfa5c8812 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -2903,6 +2903,56 @@ mod tests { ); } + #[test] + fn an_unlimited_page_over_two_tag_values_is_the_global_oldest_rows() { + // Without a limit the page keeps the oldest rows, so the same accumulator is read from + // the other end: a later range whose oldest row is older than the page's worst kept row + // must be walked, not stopped at its first match. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=1200) + .map(|step: i64| { + let value = if step % 3 == 0 { "b" } else { "a" }; + signed(36, 1, step, vec![vec!["t".to_owned(), value.to_owned()]]) + }) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let oldest: Vec = (1..=500).collect(); + + for criteria in [ + vec!["a".to_owned(), "b".to_owned()], + vec!["b".to_owned(), "a".to_owned()], + ] { + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), criteria); + let filter = Filter { + tags, + ..Filter::default() + }; + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!( + created(&page), + oldest, + "the page is the 500 oldest rows the values select, in either order" + ); + assert_eq!( + page, + reference_page(&transaction, &databases, &filter), + "and it is the store's own answer" + ); + } + } + #[test] fn the_author_index_and_the_created_at_fallback_return_the_same_page() { // A filter composing more than `MAX_PAGE_KEYS` author × kind ranges is answered from From 6d1372bb27d511527bd1ebd7d78934584deff1eb Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 07:46:59 -0500 Subject: [PATCH 37/60] perf(storage): keep a page as a heap so an offer costs a comparison --- crates/nostream-storage/src/lmdb/events.rs | 316 +++++++++++++++++++-- 1 file changed, 285 insertions(+), 31 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index bfa5c8812..6e7d92210 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -5,7 +5,8 @@ //! `(created_at, event_id)` keys inside one blocking closure, and the stream that returns the //! rows re-seeks its cursor per batch (see [`ValuesStream`] and [`ReconciliationStream`]). -use std::collections::{BTreeMap, HashSet}; +use std::cmp::Ordering; +use std::collections::{BTreeMap, BinaryHeap, HashSet}; use std::future::Future; use std::ops::Bound; use std::pin::Pin; @@ -564,8 +565,8 @@ fn body_decoded() { probes::decode(); } -/// The work counters the cost tests assert on: a page's cost is its keys evaluated and its -/// bodies decoded, and neither is visible in its result. +/// The work counters the cost tests assert on: a page's cost is its keys evaluated, its bodies +/// decoded and the entries it sorted, and none of the three is visible in its result. #[cfg(test)] mod probes { use std::sync::Mutex; @@ -576,10 +577,12 @@ mod probes { static KEYS: AtomicUsize = AtomicUsize::new(0); static DECODES: AtomicUsize = AtomicUsize::new(0); + static SORTED: AtomicUsize = AtomicUsize::new(0); pub(super) fn reset() { KEYS.store(0, Ordering::Relaxed); DECODES.store(0, Ordering::Relaxed); + SORTED.store(0, Ordering::Relaxed); } pub(super) fn key() { @@ -590,6 +593,10 @@ mod probes { DECODES.fetch_add(1, Ordering::Relaxed); } + pub(super) fn sorted(entries: usize) { + SORTED.fetch_add(entries, Ordering::Relaxed); + } + pub(super) fn keys() -> usize { KEYS.load(Ordering::Relaxed) } @@ -597,6 +604,10 @@ mod probes { pub(super) fn decodes() -> usize { DECODES.load(Ordering::Relaxed) } + + pub(super) fn sorted_entries() -> usize { + SORTED.load(Ordering::Relaxed) + } } /// One database's inclusive key range. @@ -864,8 +875,8 @@ fn admits(filter: &Filter, gates: &Gates) -> Result { /// page's order stops once the page holds its own worst kept row: every row after it in that /// range is past it, so a later range can no longer arm the stop at the first match it offers. /// The comparison is strict, so every row tied at the boundary timestamp is still offered and -/// the tie breaks as the capture's does; the other ranges offer their matches to a buffer of at -/// most `2 × limit`, cut back to the best `limit` whenever it fills. +/// the tie breaks as the capture's does. The page is a heap rooted at that worst kept row, so +/// the boundary is one peek and an offer is one comparison until it beats the root. fn branch( txn: &RoTxn<'_>, databases: &Databases, @@ -929,9 +940,10 @@ fn past(boundary: i64, created_at: i64, descending: bool) -> bool { } } -/// A page under construction: at most `2 × limit` entries, cut back to the best `limit`. +/// A page under construction: at most `limit` entries, held so the row the page would drop first +/// is the heap's root. struct Page { - entries: Vec<(i64, EventId)>, + kept: BinaryHeap, seen: HashSet, descending: bool, } @@ -939,7 +951,7 @@ struct Page { impl Page { fn new(descending: bool) -> Self { Self { - entries: Vec::new(), + kept: BinaryHeap::new(), seen: HashSet::new(), descending, } @@ -950,37 +962,83 @@ impl Page { if limit == 0 || !self.seen.insert(key.1) { return; } - self.entries.push(key); - if self.entries.len() >= limit.saturating_mul(2) { - self.cut(limit); + if self.kept.len() < limit { + self.kept.push(Kept { + entry: key, + descending: self.descending, + }); + return; + } + let Some(worst) = self.kept.peek().map(|kept| kept.entry) else { + return; + }; + // A row the page would drop on its way in cannot be in it, and its id must leave `seen` + // with it so that a later range may offer it again. + if page_order(key, worst, self.descending) == Ordering::Less { + if let Some(dropped) = self.kept.pop() { + self.seen.remove(&dropped.entry.1); + } + self.kept.push(Kept { + entry: key, + descending: self.descending, + }); + } else { + self.seen.remove(&key.1); } } /// The timestamp of the worst row the page would keep, once it holds `limit` of them. /// /// A range that runs in the page's order cannot offer a row past it, so the walk stops - /// there. Cutting to the limit first is what makes the boundary the page's own rather than - /// the last row offered, and it is only paid once the page is already full. - fn boundary(&mut self, limit: usize) -> Option { - if limit == 0 || self.entries.len() < limit { + /// there. That row is the heap's root, so the boundary costs one peek and the page is cut to + /// the limit as it fills rather than when the boundary is read. + fn boundary(&self, limit: usize) -> Option { + if limit == 0 || self.kept.len() < limit { return None; } - self.cut(limit); - self.entries.last().map(|(created_at, _)| *created_at) + self.kept.peek().map(|worst| worst.entry.0) + } + + /// The page in the capture's order, which is the one sort the whole page pays. + fn finish(self, limit: usize) -> Vec<(i64, EventId)> { + let mut page: Vec<(i64, EventId)> = self.kept.into_iter().map(|kept| kept.entry).collect(); + #[cfg(test)] + probes::sorted(page.len()); + sort_page(&mut page, self.descending); + page.truncate(limit); + page + } +} + +/// A page entry, ordered by the page's own comparison: the row the page would drop first is the +/// greatest of them, so a max-heap keeps it at the root. Every entry of one heap shares its +/// direction. +#[derive(Clone, Copy)] +struct Kept { + entry: (i64, EventId), + descending: bool, +} + +impl Ord for Kept { + fn cmp(&self, other: &Self) -> Ordering { + page_order(self.entry, other.entry, self.descending) } +} - fn finish(mut self, limit: usize) -> Vec<(i64, EventId)> { - self.cut(limit); - self.entries +impl PartialOrd for Kept { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) } +} - fn cut(&mut self, limit: usize) { - sort_page(&mut self.entries, self.descending); - self.entries.truncate(limit); - self.seen = self.entries.iter().map(|(_, id)| *id).collect(); +impl PartialEq for Kept { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == Ordering::Equal } } +impl Eq for Kept {} + /// One range's keys, in the plan's order. enum Scan<'txn> { Ascending(heed::RoRange<'txn, Bytes, Bytes>), @@ -1011,14 +1069,20 @@ fn scan<'txn>( }) } -fn sort_page(page: &mut [(i64, EventId)], descending: bool) { +/// The capture's page order: newest first when `descending`, oldest first otherwise, the +/// smaller id first on a tie. +fn page_order(left: (i64, EventId), right: (i64, EventId), descending: bool) -> Ordering { if descending { - page.sort_by(|left, right| right.0.cmp(&left.0).then_with(|| left.1.cmp(&right.1))); + right.0.cmp(&left.0).then_with(|| left.1.cmp(&right.1)) } else { - page.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); + left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1)) } } +fn sort_page(page: &mut [(i64, EventId)], descending: bool) { + page.sort_by(|left, right| page_order(*left, *right, descending)); +} + /// The index a filter is answered from, the ranges within it, and whether a range's key order /// is the page's order. /// @@ -1807,14 +1871,14 @@ mod tests { } #[test] - fn a_page_buffer_holds_at_most_twice_its_limit() { + fn a_page_holds_at_most_its_limit() { let mut page = Page::new(true); for step in 0u8..=100 { page.offer((i64::from(step), id(step)), 3); assert!( - page.entries.len() <= 6, - "the buffer is cut back to the limit whenever it fills: {} entries", - page.entries.len() + page.kept.len() <= 3, + "the page is its limit, never a buffer behind it: {} entries", + page.kept.len() ); } assert_eq!(page.finish(3).len(), 3); @@ -1910,6 +1974,22 @@ mod tests { EventRecord::from_event(&wire) } + /// A signed event of one author and its own body, for a corpus `signed` cannot build: more + /// than its 256 authors, or several rows at one `created_at` under one author. + fn signed_by(secret: [u8; 32], created_at: i64, content: &str) -> EventRecord { + let mut wire = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(1), + tags: Vec::new(), + content: content.to_owned(), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&secret, &mut wire).expect("the event signs"); + EventRecord::from_event(&wire) + } + /// Writes the events in one transaction, as `create_many` does. fn write(env: &Env, databases: &Databases, records: &[EventRecord]) { let mut transaction = env.write_txn().expect("a write transaction"); @@ -2720,6 +2800,114 @@ mod tests { ); } + /// The page `branch` returns over a filter whose plan seeks `ranges` time-ordered ranges, + /// with the entries it sorted counted: the accumulator holds the page as a heap rooted at + /// its worst kept row, so the only sort is the one in `finish` and the request sorts its own + /// `limit` rows however many ranges it walks. + fn assert_page_sorts_once( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + ranges: usize, + ) -> Vec<(i64, EventId)> { + let plan = plan(filter, databases); + assert_eq!(plan.ranges.len(), ranges, "the plan seeks its own ranges"); + assert!(plan.ordered, "and a range's key order is the page's"); + probes::reset(); + let page = branch( + txn, + databases, + filter, + &Gates::default(), + 0, + false, + page_limit(filter), + ) + .expect("the page"); + assert_eq!( + probes::sorted_entries(), + page.len(), + "one page sorts its own rows once, whatever the number of ranges" + ); + assert_eq!( + probes::sorted_entries(), + page_limit(filter), + "and a full page is exactly the limit" + ); + assert_eq!( + page, + reference_page(txn, databases, filter), + "the page is the store's own answer" + ); + page + } + + #[test] + fn a_page_sorts_its_own_limit_once_however_many_ranges_it_seeks() { + // The boundary is the page's own worst kept row and the page is a heap rooted at it, so + // an offer costs a comparison with that root and the one sort happens in `finish`. + // Computing the boundary by sorting the page instead costs `limit` entries sorted on + // every accepted offer: 5 000 whole-pubkey ranges at the 500 default sorted 2 828 653 + // entries, and 5 000 rows tied at one timestamp sorted 2 255 500. + let _serial = counting(); + + // One kind-1 event for each of 5 000 authors: one range per author, one key each. + let (_directory, env, databases) = store(); + let records: Vec = (0..MAX_PAGE_KEYS as u32) + .map(|author| { + let mut secret = [0u8; 32]; + secret[..4].copy_from_slice(&author.to_be_bytes()); + secret[31] = 1; + signed_by(secret, i64::from(author), &author.to_string()) + }) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some( + records + .iter() + .map(|record| record.event.pubkey.to_string()) + .collect(), + ), + kinds: Some(vec![Kind(1)]), + limit: Some(DEFAULT_FILTER_LIMIT), + ..Filter::default() + }; + let page = assert_page_sorts_once(&transaction, &databases, &filter, MAX_PAGE_KEYS); + assert_eq!( + created(&page), + (MAX_PAGE_KEYS as i64 - DEFAULT_FILTER_LIMIT as i64..MAX_PAGE_KEYS as i64) + .rev() + .collect::>(), + "the page is the newest `limit` authors, newest first" + ); + + // One author's 5 000 rows at one timestamp: one range, and every row past the limit is + // a match the accumulator considers. + let (_directory, env, databases) = store(); + let records: Vec = (0..MAX_PAGE_KEYS as u32) + .map(|step| signed_by([9; 32], 5_000, &step.to_string())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + kinds: Some(vec![Kind(1)]), + limit: Some(DEFAULT_FILTER_LIMIT), + ..Filter::default() + }; + let page = assert_page_sorts_once(&transaction, &databases, &filter, 1); + assert!( + created(&page).iter().all(|created_at| *created_at == 5_000), + "every row the page holds is at the boundary timestamp" + ); + assert!( + page.windows(2).all(|pair| pair[0].1 < pair[1].1), + "so the tie is broken by the smaller id" + ); + } + #[test] fn a_limited_page_over_several_time_ordered_ranges_holds_the_global_newest_rows() { // A page is one accumulator across every range the plan seeks: an author × kind @@ -2808,6 +2996,72 @@ mod tests { } } + #[test] + fn a_tie_at_the_boundary_timestamp_is_decided_by_a_later_range() { + // A row tied with the page's worst kept row is not past the boundary, so a later range + // still offers it and the smaller id wins the tie. Stopping at the boundary instead of + // after it would keep the earlier range's row. + let _serial = counting(); + let (_directory, env, databases) = store(); + let mut records = vec![ + signed(11, 1, 200, Vec::new()), + signed(11, 3, 200, Vec::new()), + signed(11, 1, 190, Vec::new()), + signed(11, 3, 190, Vec::new()), + ]; + // The tie breaks on the id, so the pair at the boundary timestamp is chosen for it: the + // row of the kind whose range runs second carries the smaller id. + let (first, second) = (0..64) + .map(|step| { + ( + signed(11, 1, 180, vec![vec!["x".to_owned(), format!("a{step}")]]), + signed(11, 3, 180, vec![vec!["x".to_owned(), format!("b{step}")]]), + ) + }) + .find(|(first, second)| second.event.id < first.event.id) + .expect("a pair whose later range carries the smaller id"); + records.push(first); + let tied = second.event.id; + records.push(second); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }; + let chosen = plan(&filter, &databases); + assert_eq!( + (chosen.path, chosen.ranges.len()), + (AccessPath::Authors, 2), + "one range per kind, the first kind's before the second's" + ); + + assert_page_agrees( + &transaction, + &databases, + &filter, + &[200, 200, 190, 190, 180], + "a tie at the boundary", + ); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!( + page.last().map(|(_, id)| *id), + Some(tied), + "the tie goes to the smaller id, from the range that runs later" + ); + } + #[test] fn a_limited_page_over_two_tag_values_does_not_depend_on_the_criteria_s_order() { // One criterion is one range on `tags`, and each range runs in `created_at` order From 1454c1dd134dc782f4563806b974044ec220efd2 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 07:46:59 -0500 Subject: [PATCH 38/60] docs(storage): record the author and kind composition the plan performs --- docs/rewrite/decisions/0010-lmdb-backend.md | 28 +++++++++++++-------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index 76ace43b7..9c4dea8a9 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -184,18 +184,20 @@ The resident set of a read is the page, never the range it was chosen from, and that page is the backend's to bound: the effective limit is the filter's own or the capture's default of 500, clamped to `MAX_PAGE_KEYS = 5 000` — the shipped `limits.client.subscription.maxLimit` — so a wire `limit` cannot choose an allocation, whether -or not a deployment sets `maxLimit`. A filter's branch keeps at most twice that many keys and -cuts back to the best half whenever the buffer fills, and that page is one accumulator across -every range the plan seeks: the top `limit` is global to them. A range whose key order *is* the -page's order stops at the page's own worst kept row — not at the first row the range happened -to offer — keeping every row at that timestamp so a tie breaks as the capture's does. +or not a deployment sets `maxLimit`. A filter's branch holds at most that many rows — the page +itself, kept as a heap rooted at the row it would drop first, so an offer is one comparison +against that root and the page is sorted once, when it is returned — and that page is one +accumulator across every range the plan seeks: the top `limit` is global to them. A range whose +key order *is* the page's order stops at the page's own worst kept row — the heap's root, not +the first row the range happened to offer — keeping every row at that timestamp so a tie breaks +as the capture's does. The index is chosen from the filter, and which ranges can stop early follows from the key: | filter | index | stops when full | |---|---|---| | `ids` | `events`, a point or prefix range per criterion | never: the key is the id, so the range runs in id order | -| `authors` + `kinds` | `events_by_author`, one range per author criterion and kind | when the author criterion is a whole pubkey, so a range is one `pubkey ‖ kind ‖ created_at` run | +| `authors` + `kinds` | `events_by_author`, one range per whole-pubkey criterion and kind, and one whole-tail range per prefix criterion | when every criterion is a whole pubkey, so a range is one `pubkey ‖ kind ‖ created_at` run; a prefix range spans every kind | | `authors` alone | `events_by_created_at` | yes — see below | | a `#` + letter criterion | `tags`, one range per criterion | for an exact value; a `#g` prefix spans values, so its range runs whole | | anything else | `events_by_created_at` | yes | @@ -203,10 +205,16 @@ The index is chosen from the filter, and which ranges can stop early follows fro Every hex criterion's bounds are widened to the field they name — 32 bytes for an id or a pubkey — before anything the key carries after it is appended. A prefix is therefore compared against its own field's bytes and never against the kind, the timestamp or the id that follow, -whether the path is `events` or `events_by_author`; an author prefix with `kinds` seeks the -kind at byte 32 as a whole pubkey does, and `hex_ranges` (`encoding.rs`) is the one place that -rule lives. A criterion another already covers, and a repeated kind, seek one range, so a -count that walks ranges whole visits those keys once. +whether the path is `events` or `events_by_author`. That is why an author prefix with `kinds` +seeks no kind at byte 32: its upper bound runs to the end of the pubkey field, a kind appended +after that bound is compared where no key writes one, and every kind's range would then contain +every other kind's keys — a count visiting each row once per kind. `author_kind_ranges` +(`events.rs`) is where the split is composed, a whole pubkey to one range per kind and a prefix +to one range over its whole tail, and it bounds the composition too: a whole-pubkey × kind +product past `MAX_PAGE_KEYS` composes no range at all and the filter is answered from +`events_by_created_at` instead. `hex_ranges` (`encoding.rs`) is the one place the widening rule +lives. A criterion another already covers, and a repeated kind, seek one range, so a count that +walks ranges whole visits those keys once. The `authors`-alone row is the capture's own shape rather than the faster index: the author index orders by kind before `created_at`, so a page ordered by time cannot stop inside it, and From 9006fece5467937738ba58ee096f8d0e4d1f704d Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 08:22:20 -0500 Subject: [PATCH 39/60] test(storage): quote the corpus the cost test measures The 2 828 653 entry figure came from round 6 performance review's D1 corpus, not from the 5 000-author corpus this test builds, which sorts 2 820 637. Name each corpus so the two are not read as one measurement. --- crates/nostream-storage/src/lmdb/events.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 6e7d92210..d00b823fd 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -2847,8 +2847,10 @@ mod tests { // The boundary is the page's own worst kept row and the page is a heap rooted at it, so // an offer costs a comparison with that root and the one sort happens in `finish`. // Computing the boundary by sorting the page instead costs `limit` entries sorted on - // every accepted offer: 5 000 whole-pubkey ranges at the 500 default sorted 2 828 653 - // entries, and 5 000 rows tied at one timestamp sorted 2 255 500. + // every accepted offer: the 5 000 whole-pubkey ranges built here sorted 2 820 637 + // entries at the 500 default, and the 5 000 rows tied at one timestamp sorted 2 255 500. + // (Round 6's performance review quotes 2 828 653 for a different corpus — 5 001 events + // with one quiet author — so the two figures are not the same measurement.) let _serial = counting(); // One kind-1 event for each of 5 000 authors: one range per author, one key each. From b12e64791b2f3d995fbe6e1f3684c79fe08d862f Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 08:22:22 -0500 Subject: [PATCH 40/60] test(storage): hold the counting lock in the page-holds test Page::finish increments probes::SORTED, and the cost test asserts that counter exactly, so a test that reaches finish without the guard can write it while the cost test measures. --- crates/nostream-storage/src/lmdb/events.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index d00b823fd..c5dd8b643 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -1872,6 +1872,7 @@ mod tests { #[test] fn a_page_holds_at_most_its_limit() { + let _serial = counting(); let mut page = Page::new(true); for step in 0u8..=100 { page.offer((i64::from(step), id(step)), 3); From 51f2fe19b3e0a99743d547a01e99085cc0b9a08c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 09:10:16 -0500 Subject: [PATCH 41/60] docs(storage): record the schema contract the store implements --- docs/rewrite/decisions/0010-lmdb-backend.md | 35 ++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index 9c4dea8a9..c4925810c 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -14,7 +14,8 @@ with the same integration test the Postgres backend passes replayed against it. `spec.md` records ten decisions (D1–D10) with their reasons and their measurements. This document keeps the parts a later reader needs to judge the code: where the state lives, how -a value is shaped, what the seam does not offer, and what the oracle therefore measures. +a value is shaped, what the seam does not offer, which schema it implements, and what the oracle +therefore measures. ## The backend is a feature of the crate, not a crate @@ -256,6 +257,38 @@ schema functions they replace. `confirm_invoice` scales by `sats × 1000`, `msat generated id is a UUID v4 in canonical text form, which is why `uuid` (v4) is the job's only new dependency. +## The schema contract is 37 migrations (D9) + +The store implements the contract both schema oracles pin: 37 migrations +(`crates/nostream-storage/migrations/`, the count `crates/nostream-storage/tests/migrations.rs:25` +asserts as `EXPECTED_MIGRATIONS`) and the captured catalogue's own `subject.migration_count: 37` +(`crates/nostream-storage/tests/fixtures/schema.json`). No table, column or index outside that +contract is added here, and the same tree is what the Postgres arm is migrated from. + +**The seam therefore stays 9 of the capture's 10 repositories.** The tenth is +`user_subscriptions`: it has a TypeScript repository +(`src/repositories/user-subscription-repository.ts`) and no Rust trait, and the capture holds no +entry that exercises it, so it belongs to the job that adds that repository's slice. +`events.first_seen` is inside the contract and carries no index — the sqlx tree declares the +column (`migrations/20220524153400_create_events_table.sql:10`) and indexes it nowhere — so this +backend stores it in the row's JSON body, sets it at create and preserves it across a replace, +and seeks nothing by it. + +The two knex migrations the sqlx tree lacks are closed elsewhere, each by the job that first needs +it: + +- `migrations/20261016_120000_create_user_subscriptions_table.js` creates `user_subscriptions` + and adds three nullable `invoices` columns (`plan_id`, `period_start`, `period_end`) — closed by + the job that adds the capture's tenth repository. +- `migrations/20261025_120000_add_moderation_stats_first_seen_indexes.js` adds two partial + `events` indexes on `first_seen` (`events_live_first_seen_pubkey_idx`, + `events_live_first_seen_remote_address_idx`, built outside a transaction) and changes no column + this store encodes — closed by the job that ports the moderation-stats query. + +Both consequences — nine repositories of ten, and a `first_seen` stored but never indexed — are +stated in `lib.rs`'s "does not own" paragraph, which is where a reader of the crate meets them +first. + ## Oracle: two arms, one executor The fixture is the capture's recording and both backends are compared against the same From 6f5b2e439c3fbbea84837ec9e9634283c776ada8 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 09:10:26 -0500 Subject: [PATCH 42/60] docs(storage): point the NIP-50 gate note at what the ADR holds --- crates/nostream-storage/src/lmdb/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 1aa1310b6..c40f17b8e 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -79,7 +79,8 @@ pub(crate) const STREAM_BATCH_ITEMS: usize = 256; pub(crate) const STREAM_BATCH_KEYS: usize = 256; /// The settings gates the capture reads per query. The LMDB backend records only whether -/// NIP-50 is on: it has no search implementation (ADR 0010, Out of scope). +/// NIP-50 is on: it has no search implementation (ADR 0010, "Oracle: two arms, one executor" — +/// entry 21, out of scope, not a permitted failure). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(crate) struct Gates { pub(crate) nip50: bool, From b426fc5c65685346fdf52720c966f13f24600991 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 09:10:53 -0500 Subject: [PATCH 43/60] docs(storage): correct two ADR references that point at nothing --- docs/rewrite/decisions/0010-lmdb-backend.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index c4925810c..4654090d2 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -52,7 +52,7 @@ dependency and could still tear against the LMDB file. The price is recorded with the decision: 29 named databases maintained by hand, `DISTINCT` and `OFFSET` become cursor walks (`BTreeSet` for `find_actionable_targets`, a skip count for `find_pending_invoices`), and the columns' `CHECK` and foreign-key constraints move into -Rust (§ Constraints, below). +Rust (§ Constraints move into Rust, below). Rows are versioned: one format byte (`ROW_FORMAT_VERSION = 1`), then `serde_json`, the byte per database rather than per table. ADR 0003 trigger 6 asked for a version marker before a @@ -327,8 +327,8 @@ its op is one of the five above — so the list cannot grow without the diff sho - Postgres remains the reference: every query path is written against `sql.rs`'s predicates and orderings, and the shared executor is what proves it. - Retention's batch composition can differ from Postgres' planner-chosen order when more than - 1000 rows qualify; both return 1000 rows classified by the same precedence (D13 in - `spec.md`, Assumption 10). The corpus is 483 events, so no current entry reaches it. + 1000 rows qualify; both return 1000 rows classified by the same precedence (Assumption 10 + in `spec.md`). The corpus is 483 events, so no current entry reaches it. - A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range, so a client cannot choose how much a request materialises; a count is exact and holds no set. The price is that a `#g`-prefix page walks its whole (time-bounded) range, an author page From db95029f356e05ef9a68f8c4ba49881b81707f7c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 09:12:58 -0500 Subject: [PATCH 44/60] docs(storage): cite the sqlx migration tree by its full path --- docs/rewrite/decisions/0010-lmdb-backend.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0010-lmdb-backend.md index 4654090d2..4d774ea23 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0010-lmdb-backend.md @@ -270,9 +270,9 @@ contract is added here, and the same tree is what the Postgres arm is migrated f (`src/repositories/user-subscription-repository.ts`) and no Rust trait, and the capture holds no entry that exercises it, so it belongs to the job that adds that repository's slice. `events.first_seen` is inside the contract and carries no index — the sqlx tree declares the -column (`migrations/20220524153400_create_events_table.sql:10`) and indexes it nowhere — so this -backend stores it in the row's JSON body, sets it at create and preserves it across a replace, -and seeks nothing by it. +column (`crates/nostream-storage/migrations/20220524153400_create_events_table.sql:10`) and +indexes it nowhere — so this backend stores it in the row's JSON body, sets it at create and +preserves it across a replace, and seeks nothing by it. The two knex migrations the sqlx tree lacks are closed elsewhere, each by the job that first needs it: From 2e348bc9a37b128a6bfe3a65c5b9a19ba6f96a3e Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 21:03:26 -0500 Subject: [PATCH 45/60] fix(storage): reduce a user patch's flags as the postgres merge does --- crates/nostream-storage/src/lmdb/users.rs | 90 ++++++++++++++++++++++- 1 file changed, 86 insertions(+), 4 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/users.rs b/crates/nostream-storage/src/lmdb/users.rs index 1bd5f19e6..b9eb2f6c5 100644 --- a/crates/nostream-storage/src/lmdb/users.rs +++ b/crates/nostream-storage/src/lmdb/users.rs @@ -47,8 +47,9 @@ pub(crate) fn find_by_pubkey( Ok(load(txn, databases, pubkey)?.map(StoredUser::user)) } -/// `upsert`: admission, the vanished flag and `updated_at`, plus `tos_accepted_at` only when -/// the patch carries it. `balance` and `created_at` are never touched. +/// `upsert`: the flags the patch carries and `updated_at`, plus `tos_accepted_at` only when the +/// patch carries it. A flag the patch leaves `Absent` is `false` on a new row and left alone on +/// an existing one, as the Postgres merge does. `balance` and `created_at` are never touched. pub(crate) fn upsert( txn: &mut RwTxn<'_>, databases: &Databases, @@ -59,8 +60,12 @@ pub(crate) fn upsert( Some(stored) => stored, None => StoredUser::new(patch.pubkey, encoding::micros(now)), }; - stored.is_admitted = patch.is_admitted; - stored.is_vanished = patch.is_vanished; + if let Some(is_admitted) = patch.is_admitted.value() { + stored.is_admitted = *is_admitted; + } + if let Some(is_vanished) = patch.is_vanished.value() { + stored.is_vanished = *is_vanished; + } stored.updated_at = encoding::micros(now); match patch.tos_accepted_at { FieldUpdate::Absent => {} @@ -146,3 +151,80 @@ pub(crate) fn revoke_admission( store(txn, databases, &stored)?; Ok(1) } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use crate::lmdb::LmdbBackend; + use crate::repositories::UserRepository; + + fn patch( + pubkey: Pubkey, + is_admitted: FieldUpdate, + is_vanished: FieldUpdate, + ) -> UserPatch { + UserPatch { + pubkey, + is_admitted, + is_vanished, + tos_accepted_at: FieldUpdate::Absent, + } + } + + /// A new row takes the capture's column defaults for the flags the patch does not carry. + #[tokio::test] + async fn a_new_row_writes_false_for_a_flag_the_patch_leaves_absent() { + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let pubkey = Pubkey::from_byte_array([1; 32]); + + backend + .upsert(&patch(pubkey, FieldUpdate::Absent, FieldUpdate::Absent)) + .await + .expect("the upsert"); + + let user = backend + .find_by_pubkey(&pubkey) + .await + .expect("the row") + .expect("a row"); + assert!(!user.is_admitted); + assert!(!user.is_vanished); + } + + /// An upsert that only needs the row to exist must not clear an admission or a vanish flag + /// that is already there, which is what the Postgres merge's omitted assignments do. + #[tokio::test] + async fn a_flag_the_patch_leaves_absent_keeps_the_stored_value() { + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let pubkey = Pubkey::from_byte_array([2; 32]); + + backend + .upsert(&patch( + pubkey, + FieldUpdate::Set(true), + FieldUpdate::Set(true), + )) + .await + .expect("the upsert"); + backend + .upsert(&patch(pubkey, FieldUpdate::Absent, FieldUpdate::Absent)) + .await + .expect("the upsert"); + + let user = backend + .find_by_pubkey(&pubkey) + .await + .expect("the row") + .expect("a row"); + assert!(user.is_admitted); + assert!(user.is_vanished); + } +} From 631255c9e2524c630167f554b42dbf9a83540011 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 21:36:03 -0500 Subject: [PATCH 46/60] fix(storage): keep one row per replaceable address in an lmdb batch --- crates/nostream-storage/src/lmdb/encoding.rs | 14 +++-- crates/nostream-storage/src/lmdb/events.rs | 60 +++++++++++++++++--- 2 files changed, 60 insertions(+), 14 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index b9ddd9a28..041125dfd 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -5,7 +5,7 @@ use std::ops::Bound; use chrono::{DateTime, Utc}; -use nostream_core::{EventId, Pubkey}; +use nostream_core::{EventId, Kind, Pubkey}; use crate::error::StorageError; @@ -378,12 +378,14 @@ pub(crate) fn tag_entries(record: &crate::repositories::EventRecord) -> Vec<(&st entries } -/// Whether a kind is replaceable, exactly as the capture's conflict-target predicate says: -/// 0, 3, 41 and the two parameterized ranges. +/// Whether the capture's conflict-target predicate covers a kind: its replaceable and its +/// parameterized-replaceable kinds, which `records::replacement_key` gates the same way. pub(crate) fn replaceable_kind(kind: i32) -> bool { - matches!(kind, 0 | 3 | 41) - || (10_000..20_000).contains(&kind) - || (30_000..40_000).contains(&kind) + let Ok(kind) = u64::try_from(kind) else { + return false; + }; + let kind = Kind(kind); + kind.is_replaceable() || kind.is_parameterized_replaceable() } /// Whether `bytes` starts with `prefix`, compared one nibble at a time with the same digit diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index c5dd8b643..c866d07ce 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -26,6 +26,7 @@ use super::{ STREAM_BATCH_ITEMS, STREAM_BATCH_KEYS, }; use crate::error::StorageError; +use crate::records::{distinct_replacements, replaces}; use crate::repositories::{ EventPurgeCounts, EventRecord, EventRetentionOptions, ReconciliationItem, }; @@ -80,15 +81,19 @@ pub(crate) fn upsert( Ok(u64::from(upsert_one(txn, databases, record, now)?)) } -/// `upsert_many`: the same merge, row by row, in one write transaction. +/// `upsert_many`: one row per replaceable address, then the same merge, in one write +/// transaction. pub(crate) fn upsert_many( txn: &mut RwTxn<'_>, databases: &Databases, records: &[EventRecord], now: DateTime, ) -> Result { + // A batch may hold several versions of one replaceable address: the conflict key's owner + // picks the winner, as Postgres does before its one statement, so the count is rows + // written rather than records passed. let mut affected = 0; - for record in records { + for record in distinct_replacements(records) { if upsert_one(txn, databases, record, now)? { affected += 1; } @@ -122,14 +127,10 @@ fn upsert_one( // A pointer with no row: treat the incoming event as a plain insert. return write_upserted(txn, databases, record, now, kind, &deduplication, None); }; - let stored_header = rows::event_header(value)?; - let incoming_wins = stored_header.created_at < record.event.created_at - || (stored_header.created_at == record.event.created_at - && stored_id > record.event.id); - if !incoming_wins { + let stored_record = rows::decode_event(value)?; + if !replaces(record, &stored_record) { return Ok(false); } - let stored_record = rows::decode_event(value)?; first_seen = stored_record.first_seen; hard_delete(txn, databases, &stored_record)?; databases.replaceable.delete(txn, &pointer)?; @@ -2685,6 +2686,49 @@ mod tests { transaction.commit().expect("the commit"); assert_eq!(affected, 1); } + + // ----------------------------------------------------------------------------------- + // Round 11: one row per replaceable address in a batch. + // ----------------------------------------------------------------------------------- + + /// Two versions of one address reach the same batch in either order: the batch writes the + /// newer row and counts one, which is what `upsert_many` promises its caller. + #[test] + fn a_batch_with_two_versions_of_one_address_writes_one_row() { + let older = signed(7, 0, 1_700_000_000, Vec::new()); + let newer = signed(7, 0, 1_700_000_100, Vec::new()); + assert_eq!(older.event.pubkey, newer.event.pubkey); + + for batch in [ + vec![older.clone(), newer.clone()], + vec![newer.clone(), older.clone()], + ] { + let (_directory, env, databases) = store(); + let mut transaction = env.write_txn().expect("a write transaction"); + let affected = + upsert_many(&mut transaction, &databases, &batch, Utc::now()).expect("the batch"); + transaction.commit().expect("the commit"); + + assert_eq!(affected, 1, "rows written, whatever the order"); + let transaction = env.read_txn().expect("a read transaction"); + assert!( + databases + .events + .get(&transaction, older.event.id.as_bytes()) + .expect("the store") + .is_none(), + "the dropped version is not written" + ); + assert!( + databases + .events + .get(&transaction, newer.event.id.as_bytes()) + .expect("the store") + .is_some(), + "the newer version is the row the batch keeps" + ); + } + } // ----------------------------------------------------------------------------------- // Round 6: the page across every range a plan seeks, and the id stream's two shapes. // ----------------------------------------------------------------------------------- From 861f1714249dd1280391625c21c71147e1d57581 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 22:15:18 -0500 Subject: [PATCH 47/60] fix(storage): decide an lmdb replacement's merge from the header --- crates/nostream-storage/src/lmdb/events.rs | 50 ++++++++++++++++++++-- crates/nostream-storage/src/records.rs | 30 ++++++++++--- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index c866d07ce..e72b540a4 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -26,7 +26,7 @@ use super::{ STREAM_BATCH_ITEMS, STREAM_BATCH_KEYS, }; use crate::error::StorageError; -use crate::records::{distinct_replacements, replaces}; +use crate::records::{distinct_replacements, replaces_fields}; use crate::repositories::{ EventPurgeCounts, EventRecord, EventRetentionOptions, ReconciliationItem, }; @@ -127,10 +127,16 @@ fn upsert_one( // A pointer with no row: treat the incoming event as a plain insert. return write_upserted(txn, databases, record, now, kind, &deduplication, None); }; - let stored_record = rows::decode_event(value)?; - if !replaces(record, &stored_record) { + let header = rows::event_header(value)?; + if !replaces_fields( + record.event.created_at, + record.event.id, + header.created_at, + stored_id, + ) { return Ok(false); } + let stored_record = rows::decode_event(value)?; first_seen = stored_record.first_seen; hard_delete(txn, databases, &stored_record)?; databases.replaceable.delete(txn, &pointer)?; @@ -2729,6 +2735,44 @@ mod tests { ); } } + + // ----------------------------------------------------------------------------------- + // Round 12: the merge predicate reads the header and the pointer, never the body. + // ----------------------------------------------------------------------------------- + + /// A losing upsert decides from the stored value's header and the pointer's id, so it never + /// decodes the body: a stored row whose body is unreadable still loses cleanly. + #[test] + fn a_losing_merge_reads_the_header_and_not_the_body() { + let stored = signed(7, 0, 1_700_000_100, Vec::new()); + let older = signed(7, 0, 1_700_000_000, Vec::new()); + let (_directory, env, databases) = store(); + + let mut transaction = env.write_txn().expect("a write transaction"); + upsert(&mut transaction, &databases, &stored, Utc::now()).expect("the stored version"); + transaction.commit().expect("the commit"); + + // The value keeps its header — where the merge reads the stored `created_at` — and + // loses its body. + let mut transaction = env.write_txn().expect("a write transaction"); + let value = databases + .events + .get(&transaction, stored.event.id.as_bytes()) + .expect("the store") + .expect("the stored row"); + let mut headless = value[..1 + encoding::EVENT_HEADER_LEN].to_vec(); + headless.extend_from_slice(b"{not json"); + databases + .events + .put(&mut transaction, stored.event.id.as_bytes(), &headless) + .expect("the stored value"); + + let affected = upsert(&mut transaction, &databases, &older, Utc::now()) + .expect("the older version loses from the header, never the body"); + transaction.commit().expect("the commit"); + + assert_eq!(affected, 0, "the stored version is still the newer one"); + } // ----------------------------------------------------------------------------------- // Round 6: the page across every range a plan seeks, and the id stream's two shapes. // ----------------------------------------------------------------------------------- diff --git a/crates/nostream-storage/src/records.rs b/crates/nostream-storage/src/records.rs index b6a8f8ad3..f5b55005b 100644 --- a/crates/nostream-storage/src/records.rs +++ b/crates/nostream-storage/src/records.rs @@ -8,7 +8,7 @@ use std::net::IpAddr; use std::str::FromStr; use chrono::{DateTime, Utc}; -use nostream_core::{Event, Kind, Pubkey}; +use nostream_core::{Event, EventId, Kind, Pubkey}; use crate::error::StorageError; use crate::repositories::{ @@ -101,12 +101,30 @@ pub fn replacement_key(record: &EventRecord) -> Option<(Pubkey, u64, Replacement Some((record.event.pubkey, kind.0, key)) } -/// The merge predicate the replaceable upsert applies: the incoming row wins when it is newer, -/// or equal and the smaller id. +/// The merge predicate the replaceable upsert applies, on the two fields it reads: the incoming +/// row wins when it is newer, or equal and the smaller id. +/// +/// The fields are separate from the records because a backend may hold the stored row as a +/// header and a pointer rather than a whole record — LMDB reads `created_at` from the value's +/// fixed header and the id from the `replaceable` pointer, and decodes the body only once the +/// incoming row has won. +pub fn replaces_fields( + created_at: i64, + id: EventId, + current_created_at: i64, + current_id: EventId, +) -> bool { + created_at > current_created_at || (created_at == current_created_at && id < current_id) +} + +/// [`replaces_fields`] applied to the two records a backend holds in memory. pub fn replaces(candidate: &EventRecord, current: &EventRecord) -> bool { - candidate.event.created_at > current.event.created_at - || (candidate.event.created_at == current.event.created_at - && candidate.event.id < current.event.id) + replaces_fields( + candidate.event.created_at, + candidate.event.id, + current.event.created_at, + current.event.id, + ) } /// `records` with at most one row per conflict key, in first-seen order. From dc6db6b8d462c7ff84f8b1e8f2e2526fbfacae64 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 22:25:23 -0500 Subject: [PATCH 48/60] docs(storage): number the lmdb decision record 0011 --- crates/nostream-storage/src/lib.rs | 2 +- crates/nostream-storage/src/lmdb/encoding.rs | 4 ++-- crates/nostream-storage/src/lmdb/events.rs | 4 ++-- crates/nostream-storage/src/lmdb/mod.rs | 4 ++-- .../decisions/{0010-lmdb-backend.md => 0011-lmdb-backend.md} | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) rename docs/rewrite/decisions/{0010-lmdb-backend.md => 0011-lmdb-backend.md} (99%) diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index d8cc9aadb..8bf3ae02c 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -23,7 +23,7 @@ //! complete: Postgres against the 37-migration schema, and LMDB against the same column //! contract for 9 of the capture's 10 repositories — `user_subscriptions` has a TypeScript //! repository and no trait yet, and `events.first_seen` is stored but never indexed -//! (`docs/rewrite/decisions/0010-lmdb-backend.md`). +//! (`docs/rewrite/decisions/0011-lmdb-backend.md`). pub mod backend; pub mod error; diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index 041125dfd..bc19b95c9 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -1,6 +1,6 @@ //! Key and value codecs for the LMDB store: the format byte, the fixed event header, the //! big-endian key layout, timestamp truncation and the hex/tag ranges the event queries seek -//! with. `docs/rewrite/decisions/0010-lmdb-backend.md` records why each shape was chosen. +//! with. `docs/rewrite/decisions/0011-lmdb-backend.md` records why each shape was chosen. use std::ops::Bound; @@ -34,7 +34,7 @@ pub(crate) const HEX_FIELD_LEN: usize = 32; /// It bounds every key this module builds. The `tags` key carries the tag value verbatim, so a /// value longer than the room the key has left cannot be indexed and its event is refused at /// the write: that is the one place this backend rejects a row Postgres would keep, and ADR -/// 0010 records it as a divergence. The `replaceable` key is the other variable-length key; its +/// 0011 records it as a divergence. The `replaceable` key is the other variable-length key; its /// components are length-prefixed ([`replaceable_key`]), which leaves it four bytes of room the /// `tags` key does not have for the same value, so it refuses nothing the tag index keeps. /// diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index e72b540a4..9517f490b 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -1136,7 +1136,7 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { // shortcut: a quiet author's page, and an author × kind filter composing more than // `MAX_PAGE_KEYS` ranges, read the created-at index with the criteria left to the // matcher; upgrade when statistics can choose between the two indexes, which is the - // benchmark job's (ADR 0010). + // benchmark job's (ADR 0011). return Plan { database: databases.events_by_created_at, path: AccessPath::CreatedAt, @@ -1700,7 +1700,7 @@ impl Stream for ReconciliationStream { /// shortcut: an author without kinds, a `#g` prefix and an id prefix narrow to no such range, /// so their stream scans `events_by_created_at` and the criteria are left to the matcher — a /// poll walks what the store holds rather than what the request names; upgrade when statistics -/// can choose the cheaper index, which is the benchmark job's (ADR 0010). +/// can choose the cheaper index, which is the benchmark job's (ADR 0011). fn stream_plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { let chosen = plan(filter, databases); let exact_ids = diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index c40f17b8e..35ac8dfdd 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -1,6 +1,6 @@ //! The LMDB backend: one environment, the 29 named databases, and the nine repository traits. //! Mirrors `src/repositories/` through the seam the Postgres backend also implements, and -//! `docs/rewrite/decisions/0010-lmdb-backend.md` records why each choice below was made. +//! `docs/rewrite/decisions/0011-lmdb-backend.md` records why each choice below was made. //! //! Every heed call runs inside `tokio::task::spawn_blocking`, because `Env::write_txn` blocks //! its thread on the single writer's mutex and a read transaction must not be held across an @@ -79,7 +79,7 @@ pub(crate) const STREAM_BATCH_ITEMS: usize = 256; pub(crate) const STREAM_BATCH_KEYS: usize = 256; /// The settings gates the capture reads per query. The LMDB backend records only whether -/// NIP-50 is on: it has no search implementation (ADR 0010, "Oracle: two arms, one executor" — +/// NIP-50 is on: it has no search implementation (ADR 0011, "Oracle: two arms, one executor" — /// entry 21, out of scope, not a permitted failure). #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub(crate) struct Gates { diff --git a/docs/rewrite/decisions/0010-lmdb-backend.md b/docs/rewrite/decisions/0011-lmdb-backend.md similarity index 99% rename from docs/rewrite/decisions/0010-lmdb-backend.md rename to docs/rewrite/decisions/0011-lmdb-backend.md index 4d774ea23..e8f463efa 100644 --- a/docs/rewrite/decisions/0010-lmdb-backend.md +++ b/docs/rewrite/decisions/0011-lmdb-backend.md @@ -1,4 +1,4 @@ -# ADR 0010: The LMDB event store behind the existing seam (job 024) +# ADR 0011: The LMDB event store behind the existing seam (job 024) - **Status:** accepted (job 024, Phase 2) - **Deciders:** the rewrite team, on the evidence in job 024's recon and spec From eb44918cf2dbf55a4ce86a585cf5277abd86235e Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 23:00:51 -0500 Subject: [PATCH 49/60] fix(storage): match postgres on lmdb replays, bulk merges, tag counts and expiry --- crates/nostream-storage/src/lmdb/encoding.rs | 56 ++- crates/nostream-storage/src/lmdb/events.rs | 394 +++++++++++++++++-- 2 files changed, 406 insertions(+), 44 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index bc19b95c9..03ea6d7a2 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -479,24 +479,32 @@ pub(crate) fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec kept } -/// The inclusive key ranges one tag criterion covers, one range per criterion and in the -/// criteria's order: an exact value is the range from the value's own separator, and a `#g` -/// prefix is the range over the value's first bytes. +/// The inclusive key ranges one tag criterion covers, one range per criterion that fits a key +/// and in the criteria's order, each with the criterion's own position: a criterion that +/// selects nothing leaves the positions after it unshifted, so a count still knows which +/// criteria came before a range. /// -/// A criterion whose own bounds do not fit [`MAX_KEY_LEN`] selects nothing: no stored key can -/// be that long, so seeking it would fail the read rather than answer it. -pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(Vec, Vec)> { +/// An exact value is the range from the value's own separator, and a `#g` prefix is the range +/// over every value that starts with the prefix, padded to the longest key the store can hold +/// so a prefix long enough to leave little room still covers its own values. +/// +/// A criterion whose lower bound does not fit [`MAX_KEY_LEN`] selects nothing: a key carrying +/// it could not be written, so seeking it would fail the read rather than answer it. +pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(usize, Vec, Vec)> { let is_geohash = name == "g"; let mut ranges = Vec::with_capacity(criteria.len()); - for criterion in criteria { + for (position, criterion) in criteria.iter().enumerate() { let mut head = Vec::with_capacity(name.len() + 1 + criterion.len()); head.extend_from_slice(name.as_bytes()); head.push(0); let (lower, upper) = match criterion.strip_suffix('*').filter(|_| is_geohash) { Some(prefix) => { head.extend_from_slice(prefix.as_bytes()); + if head.len() > MAX_KEY_LEN { + continue; + } let mut upper = head.clone(); - upper.extend(std::iter::repeat_n(0xff, 64)); + upper.resize(MAX_KEY_LEN, 0xff); (head, upper) } None => { @@ -508,7 +516,7 @@ pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(Vec, Vec= lower.as_slice()); + assert!( + stored.as_slice() <= upper.as_slice(), + "and the range selects it" + ); } #[test] diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 9517f490b..6d88fdc66 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -8,6 +8,7 @@ use std::cmp::Ordering; use std::collections::{BTreeMap, BinaryHeap, HashSet}; use std::future::Future; +use std::net::IpAddr; use std::ops::Bound; use std::pin::Pin; use std::task::{Context, Poll}; @@ -78,11 +79,12 @@ pub(crate) fn upsert( record: &EventRecord, now: DateTime, ) -> Result { - Ok(u64::from(upsert_one(txn, databases, record, now)?)) + Ok(u64::from(upsert_one(txn, databases, record, now, false)?)) } /// `upsert_many`: one row per replaceable address, then the same merge, in one write -/// transaction. +/// transaction. The capture's bulk merge list omits `remote_address`, so a replacement keeps +/// the address the stored row had, which `postgres/events.rs`'s `upsert_many` does too. pub(crate) fn upsert_many( txn: &mut RwTxn<'_>, databases: &Databases, @@ -94,7 +96,7 @@ pub(crate) fn upsert_many( // written rather than records passed. let mut affected = 0; for record in distinct_replacements(records) { - if upsert_one(txn, databases, record, now)? { + if upsert_one(txn, databases, record, now, true)? { affected += 1; } } @@ -106,18 +108,33 @@ fn upsert_one( databases: &Databases, record: &EventRecord, now: DateTime, + keep_stored_address: bool, ) -> Result { + let kind = kind_of(record)?; + let deduplication = record.deduplication(); + if databases .events .get(txn, record.event.id.as_bytes())? .is_some() { + // A replaceable write conflicts on its address, not on its id: replaying one the store + // already holds updates that row against itself and counts zero (`REPLACEABLE_MERGE` + // never matches), and only an id naming another address violates + // `events_event_id_unique`. + if encoding::replaceable_kind(kind) { + let pointer = encoding::replaceable_key(&record.event.pubkey, kind, &deduplication)?; + if let Some(stored) = databases.replaceable.get(txn, &pointer)? + && encoding::event_id_from(stored)? == record.event.id + { + return Ok(false); + } + } return Err(StorageError::DuplicateEvent); } - let kind = kind_of(record)?; - let deduplication = record.deduplication(); let mut first_seen = None; + let mut remote_address = record.remote_address; if encoding::replaceable_kind(kind) { let pointer = encoding::replaceable_key(&record.event.pubkey, kind, &deduplication)?; @@ -125,7 +142,16 @@ fn upsert_one( let stored_id = encoding::event_id_from(stored)?; let Some(value) = databases.events.get(txn, stored_id.as_bytes())? else { // A pointer with no row: treat the incoming event as a plain insert. - return write_upserted(txn, databases, record, now, kind, &deduplication, None); + return write_upserted( + txn, + databases, + record, + now, + kind, + &deduplication, + None, + remote_address, + ); }; let header = rows::event_header(value)?; if !replaces_fields( @@ -138,6 +164,9 @@ fn upsert_one( } let stored_record = rows::decode_event(value)?; first_seen = stored_record.first_seen; + if keep_stored_address { + remote_address = stored_record.remote_address; + } hard_delete(txn, databases, &stored_record)?; databases.replaceable.delete(txn, &pointer)?; } @@ -151,9 +180,11 @@ fn upsert_one( kind, &deduplication, first_seen, + remote_address, ) } +#[allow(clippy::too_many_arguments)] fn write_upserted( txn: &mut RwTxn<'_>, databases: &Databases, @@ -162,6 +193,7 @@ fn write_upserted( kind: i32, deduplication: &[serde_json::Value], first_seen: Option>, + remote_address: Option, ) -> Result { let stored = EventRecord { row_id: None, @@ -172,7 +204,7 @@ fn write_upserted( deleted_at: None, expires_at: record.expires_at, event_deduplication: Some(deduplication.to_vec()), - remote_address: record.remote_address, + remote_address, }; write_event(txn, databases, &stored)?; if encoding::replaceable_kind(kind) { @@ -429,7 +461,14 @@ pub(crate) fn delete_expired_and_retained( } } if candidates.len() < RETENTION_BATCH_SIZE { - for entry in databases.events_expiring.iter(txn)? { + // A row whose expiry has not passed cannot be classified expired, and one that + // qualifies by age is the created-at pass's business: the index's own order lets this + // walk stop at `now` instead of reading every future expiry under the writer. + let lower = encoding::i64_key(i64::MIN).to_vec(); + let mut upper = encoding::i64_key(now.saturating_sub(1)).to_vec(); + upper.extend([0xff; 32]); + let range = encoding::bounds(&lower, &upper); + for entry in databases.events_expiring.range(txn, &range)? { let (key, _) = entry?; let id = header_id(key, 8, "events_expiring")?; gather( @@ -507,6 +546,7 @@ fn gather( candidates: &mut Vec, seen: &mut HashSet<[u8; 32]>, ) -> Result<(), StorageError> { + key_evaluated(); let Some(value) = databases.events.get(txn, id.as_bytes())? else { return Ok(()); }; @@ -629,9 +669,14 @@ struct Plan<'a> { /// worst kept row instead of being walked whole. ordered: bool, /// The tag family the ranges were built from, with its criteria in the criteria's order — - /// `tag_ranges` builds one range per criterion. A key in these ranges has already proved - /// that family, so the row's body is decoded only for the filter's other tag families. + /// `tag_ranges` builds one range per criterion that fits a key. A key in these ranges has + /// already proved that family, so the row's body is decoded only for the filter's other tag + /// families. tag: Option<(&'a str, &'a [String])>, + /// The criterion each range was built from, in the same order as `ranges`. Empty for a plan + /// that does not read the tags index; a dropped criterion leaves the later positions alone, + /// so a count still knows which criteria came before a range. + tag_positions: Vec, } /// One reconciliation batch still running on the blocking pool, with the cursor it returns. @@ -761,33 +806,57 @@ fn already_counted( Ok(false) } -/// Whether a criterion of the filter's own tag family, other than the one this range was built -/// from, also selects the row. One event can carry two values the family asks for, and the -/// criterion whose range comes first answers for it, so the row is counted once. +/// Whether an earlier criterion — the criterion this range was built from, or one before it — +/// already answers for this row, so counting it here would count it twice. +/// +/// Two ranges select one row when two criteria of the filter's tag family both match one of its +/// tags; one criterion selects it twice when a `#g` prefix matches two of its tags. The first +/// criterion in the criteria's order answers for the row, and within a prefix criterion the +/// smallest matching value does. /// /// This is the one place a count reads an event's tags instead of its key: the key names a -/// single value, and which values the event also carries is in the body. -fn earlier_criterion_selects( +/// single value, and the other values the event carries are in the body. +fn counted_earlier( txn: &RoTxn<'_>, databases: &Databases, plan: &Plan<'_>, index: usize, + value: Option<&str>, id: &EventId, ) -> Result { let Some((name, criteria)) = plan.tag else { return Ok(false); }; - if index == 0 { + let position = plan.tag_positions.get(index).copied().unwrap_or(index); + let prefix = criteria + .get(position) + .and_then(|criterion| criterion.strip_suffix('*').filter(|_| name == "g")); + if position == 0 && prefix.is_none() { return Ok(false); } - let Some(value) = databases.events.get(txn, id.as_bytes())? else { + let Some(stored) = databases.events.get(txn, id.as_bytes())? else { return Ok(false); }; - let record = rows::decode_event(value)?; - Ok(criteria + let record = rows::decode_event(stored)?; + if criteria .iter() - .take(index) - .any(|earlier| record_has_tag(&record, name, std::slice::from_ref(earlier)))) + .take(position) + .any(|earlier| record_has_tag(&record, name, std::slice::from_ref(earlier))) + { + return Ok(true); + } + let (Some(prefix), Some(value)) = (prefix, value) else { + return Ok(false); + }; + Ok(record.event.tags.iter().any(|tag| { + matches!( + (tag.first(), tag.get(1)), + (Some(tag_name), Some(candidate)) + if tag_name == name + && candidate.starts_with(prefix) + && candidate.as_str() < value + ) + })) } /// Counts the live rows one unlimited filter matches that the count does not already hold. @@ -819,17 +888,22 @@ fn count_matches( for entry in plan.database.range(txn, &range)? { let (key, _) = entry?; key_evaluated(); - if let Some((name, criteria)) = plan.tag - && !tag_value_matches(name == "g", criteria, encoding::tag_value(key)?) - { - continue; - } + let value = match plan.tag { + Some((name, criteria)) => { + let value = encoding::tag_value(key)?; + if !tag_value_matches(name == "g", criteria, value) { + continue; + } + Some(value) + } + None => None, + }; let Some((_, id)) = candidate(txn, databases, plan.path, key)? else { continue; }; if matcher.matches(txn, databases, &id)?.is_some() && !already_counted(txn, databases, counted, gates, now, &id)? - && !earlier_criterion_selects(txn, databases, &plan, index, &id)? + && !counted_earlier(txn, databases, &plan, index, value, &id)? { total += 1; } @@ -1107,6 +1181,7 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { // The `events` key is the id, so the range runs in id order, not time order. ordered: false, tag: None, + tag_positions: Vec::new(), }; } if let Some(authors) = &filter.authors @@ -1125,6 +1200,7 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { // kind. ordered: authors.iter().all(|criterion| criterion.len() == 64), tag: None, + tag_positions: Vec::new(), }; } // Without kinds the author index runs in kind order before `created_at`, so a page @@ -1143,17 +1219,24 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { ranges: created_at_ranges(filter), ordered: true, tag: None, + tag_positions: Vec::new(), }; } if let Some((name, criteria)) = tag_path(filter) { + let ranged = encoding::tag_ranges(name, criteria); + let tag_positions = ranged.iter().map(|(position, _, _)| *position).collect(); return Plan { database: databases.tags, path: AccessPath::Tags, - ranges: encoding::tag_ranges(name, criteria), + ranges: ranged + .into_iter() + .map(|(_, low, high)| (low, high)) + .collect(), // An exact criterion's range runs `created_at` under one value; a `#g` prefix range // spans values, so its keys are in value order and the range is walked whole. ordered: name != "g" || criteria.iter().all(|criterion| !criterion.ends_with('*')), tag: Some((name, criteria)), + tag_positions, }; } Plan { @@ -1162,6 +1245,7 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { ranges: created_at_ranges(filter), ordered: true, tag: None, + tag_positions: Vec::new(), } } @@ -1601,15 +1685,23 @@ pub(crate) struct ReconciliationStream { } impl ReconciliationStream { + /// Builds the stream for one filter, or refuses it as every other read does: a non-empty + /// `search` with the NIP-50 gate on is [`StorageError::NotImplemented`], and a criterion + /// family that is present but empty selects nothing (the capture's `1 = 0`). pub(crate) fn new( env: Env, databases: Databases, filter: &Filter, gates: Gates, now: i64, - ) -> Self { - let range_count = stream_plan(filter, &databases).ranges.len(); - Self { + ) -> Result { + let admitted = admits(filter, &gates)?; + let range_count = if admitted { + stream_plan(filter, &databases).ranges.len() + } else { + 0 + }; + Ok(Self { env, databases, filter: filter.clone(), @@ -1619,8 +1711,8 @@ impl ReconciliationStream { position: Position::new(), pending: None, buffer: Vec::new().into_iter(), - finished: false, - } + finished: !admitted, + }) } } @@ -1714,6 +1806,7 @@ fn stream_plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { ranges: created_at_ranges(filter), ordered: true, tag: None, + tag_positions: Vec::new(), } } @@ -2773,6 +2866,241 @@ mod tests { assert_eq!(affected, 0, "the stored version is still the newer one"); } + + // ----------------------------------------------------------------------------------- + // Round 12 rework: the replaceable write paths the PR's reviewer named. + // ----------------------------------------------------------------------------------- + + /// A replay of a replaceable event the store already holds updates that row against + /// itself: the merge predicate never matches, so the count is zero — not the id collision + /// `events_event_id_unique` reports for a kind with no conflict target. + #[test] + fn a_replayed_replaceable_event_counts_zero() { + let record = signed(7, 0, 1_700_000_000, Vec::new()); + let (_directory, env, databases) = store(); + + let mut transaction = env.write_txn().expect("a write transaction"); + assert_eq!( + upsert(&mut transaction, &databases, &record, Utc::now()).expect("the first write"), + 1 + ); + assert_eq!( + upsert(&mut transaction, &databases, &record, Utc::now()).expect("the replay"), + 0, + "the replaceable conflict matches and the merge predicate does not" + ); + transaction.commit().expect("the commit"); + + let transaction = env.read_txn().expect("a read transaction"); + let value = databases + .events + .get(&transaction, record.event.id.as_bytes()) + .expect("the store") + .expect("the row"); + assert_eq!( + rows::decode_event(value).expect("the row").event.id, + record.event.id, + "the replay left the row in place" + ); + } + + /// A kind with no conflict target has only the id to collide on, so a replay is still the + /// capture's `rejected: event already exists`. + #[test] + fn a_duplicate_id_of_a_regular_kind_is_rejected() { + let record = signed(8, 1, 1_700_000_000, Vec::new()); + let (_directory, env, databases) = store(); + + let mut transaction = env.write_txn().expect("a write transaction"); + upsert(&mut transaction, &databases, &record, Utc::now()).expect("the first write"); + let error = upsert(&mut transaction, &databases, &record, Utc::now()) + .expect_err("the second insert of the same id"); + assert!(matches!(error, StorageError::DuplicateEvent)); + } + + /// The capture's bulk merge list omits `remote_address`, so a bulk replacement keeps the + /// address the stored row had — even when the incoming record carries none — while the + /// single-row merge writes the incoming address. + #[test] + fn a_bulk_replacement_keeps_the_stored_source_address() { + let stored = signed(7, 0, 1_700_000_000, Vec::new()) + .with_remote_address(IpAddr::from([10, 0, 0, 1])); + let newer = signed(7, 0, 1_700_000_100, Vec::new()) + .with_remote_address(IpAddr::from([10, 0, 0, 2])); + let anonymous = signed(7, 0, 1_700_000_200, Vec::new()); + + let (_directory, env, databases) = store(); + let mut transaction = env.write_txn().expect("a write transaction"); + upsert_many( + &mut transaction, + &databases, + std::slice::from_ref(&stored), + Utc::now(), + ) + .expect("the stored version"); + upsert_many( + &mut transaction, + &databases, + std::slice::from_ref(&newer), + Utc::now(), + ) + .expect("the bulk replacement"); + upsert_many( + &mut transaction, + &databases, + std::slice::from_ref(&anonymous), + Utc::now(), + ) + .expect("a bulk replacement with no address"); + transaction.commit().expect("the commit"); + + let transaction = env.read_txn().expect("a read transaction"); + let value = databases + .events + .get(&transaction, anonymous.event.id.as_bytes()) + .expect("the store") + .expect("the last row"); + let row = rows::decode_event(value).expect("the row"); + assert_eq!( + row.remote_address, + Some(IpAddr::from([10, 0, 0, 1])), + "the bulk merge preserves the address across both replacements" + ); + + let (_directory, env, databases) = store(); + let mut transaction = env.write_txn().expect("a write transaction"); + upsert(&mut transaction, &databases, &stored, Utc::now()).expect("the stored version"); + upsert(&mut transaction, &databases, &newer, Utc::now()).expect("the single replacement"); + transaction.commit().expect("the commit"); + + let transaction = env.read_txn().expect("a read transaction"); + let value = databases + .events + .get(&transaction, newer.event.id.as_bytes()) + .expect("the store") + .expect("the replacement"); + assert_eq!( + rows::decode_event(value).expect("the row").remote_address, + Some(IpAddr::from([10, 0, 0, 2])), + "the single-row merge writes the incoming address" + ); + } + + /// One event carrying two values a `#g` prefix matches is one row: the tags index holds a + /// key per value, and the capture's query counts rows. + #[test] + fn a_wildcard_tag_range_counts_a_row_once() { + let (_directory, env, databases) = store(); + let record = signed( + 7, + 1, + 1_700_000_000, + vec![ + vec!["g".to_owned(), "u4a".to_owned()], + vec!["g".to_owned(), "u4b".to_owned()], + ], + ); + write(&env, &databases, &[record]); + let transaction = env.read_txn().expect("a read transaction"); + let mut tags = BTreeMap::new(); + tags.insert("#g".to_owned(), vec!["u4*".to_owned()]); + let filter = Filter { + tags, + ..Filter::default() + }; + + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 1, + "one row, though the prefix matches two of its values" + ); + } + + /// A criterion too long to seek leaves the later criteria's positions alone, so a row + /// carrying a later value as well is still counted once. + #[test] + fn a_dropped_criterion_does_not_shift_the_counted_positions() { + let (_directory, env, databases) = store(); + let record = signed( + 7, + 1, + 1_700_000_000, + vec![ + vec!["t".to_owned(), "a".to_owned()], + vec!["t".to_owned(), "b".to_owned()], + ], + ); + write(&env, &databases, &[record]); + let transaction = env.read_txn().expect("a read transaction"); + let mut tags = BTreeMap::new(); + tags.insert( + "#t".to_owned(), + vec!["x".repeat(500), "a".to_owned(), "b".to_owned()], + ); + let filter = Filter { + tags, + ..Filter::default() + }; + + assert_eq!( + plan(&filter, &databases).ranges.len(), + 2, + "the oversized criterion seeks nothing" + ); + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 1, + "the row carries both values and is one row" + ); + } + + /// The expiry pass reads the index no further than the last expired row: a future expiry is + /// not a candidate, and a row that qualifies only by age is the created-at pass's business. + #[test] + fn the_expiry_pass_stops_at_the_last_expired_row() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let now = 1_700_000_000; + let mut records = vec![signed(7, 1, now - 100, Vec::new()).with_expires_at(now - 50)]; + records.extend((0..500).map(|step| { + signed_by([9; 32], now - 10, &format!("future-{step}")) + .with_expires_at(now + 1_000 + step) + })); + write(&env, &databases, &records); + + let options = EventRetentionOptions { + max_days: Some(1), + kind_whitelist: None, + pubkey_whitelist: None, + }; + probes::reset(); + let mut transaction = env.write_txn().expect("a write transaction"); + let counts = delete_expired_and_retained(&mut transaction, &databases, &options, now) + .expect("the pass"); + transaction.commit().expect("the commit"); + + assert_eq!(counts.expired, 1); + assert_eq!(counts.retained, 0, "ten seconds old is not over retention"); + assert_eq!( + probes::keys(), + 1, + "the pass visits the expired row and stops at `now`" + ); + } // ----------------------------------------------------------------------------------- // Round 6: the page across every range a plan seeks, and the id stream's two shapes. // ----------------------------------------------------------------------------------- From 4d7287325c1aabb0ea0d28c992a696777db74032 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 23:00:55 -0500 Subject: [PATCH 50/60] fix(storage): bound lmdb metadata reads and refuse a replayed dvm job --- crates/nostream-storage/src/lmdb/invoices.rs | 4 + crates/nostream-storage/src/lmdb/metadata.rs | 31 ++- crates/nostream-storage/src/lmdb/mod.rs | 2 +- crates/nostream-storage/tests/parity_lmdb.rs | 221 ++++++++++++++++++- 4 files changed, 248 insertions(+), 10 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/invoices.rs b/crates/nostream-storage/src/lmdb/invoices.rs index 09e3ef6c0..4bfd667f4 100644 --- a/crates/nostream-storage/src/lmdb/invoices.rs +++ b/crates/nostream-storage/src/lmdb/invoices.rs @@ -76,6 +76,10 @@ pub(crate) fn upsert( // The insert's column list omits both, so the database fills them with NULL. stored.amount_paid = None; stored.confirmed_at = None; + // Every timestamp column is a PostgreSQL `timestamp` (V38): the caller's instants + // are truncated too, not only the ones this backend stamps. + stored.created_at = encoding::micros(stored.created_at); + stored.expires_at = stored.expires_at.map(encoding::micros); stored.updated_at = encoding::micros(now); write(txn, databases, &stored, None)?; } diff --git a/crates/nostream-storage/src/lmdb/metadata.rs b/crates/nostream-storage/src/lmdb/metadata.rs index 2450752d9..bfcd28072 100644 --- a/crates/nostream-storage/src/lmdb/metadata.rs +++ b/crates/nostream-storage/src/lmdb/metadata.rs @@ -62,6 +62,10 @@ pub(crate) fn nip05_upsert( stored.created_at = previous .as_ref() .map_or_else(|| encoding::micros(now), |row| row.created_at); + // Every timestamp column is a PostgreSQL `timestamp` (V38): the caller's instants are + // truncated too, not only the ones this backend stamps. + stored.last_checked_at = encoding::micros(stored.last_checked_at); + stored.last_verified_at = stored.last_verified_at.map(encoding::micros); stored.updated_at = encoding::micros(now); let value = rows::encode("nip05_verifications", &stored)?; @@ -89,6 +93,9 @@ pub(crate) fn nip05_find_pending( limit: u64, now: DateTime, ) -> Result, StorageError> { + if limit == 0 { + return Ok(Vec::new()); + } let cutoff = encoding::micros(now - Duration::milliseconds(update_frequency_ms)); let mut verifications = Vec::new(); for entry in databases.nip05_by_last_checked_at.iter(txn)? { @@ -251,6 +258,9 @@ pub(crate) fn invite_find_active_codes( limit: u64, now: DateTime, ) -> Result, StorageError> { + if limit == 0 { + return Ok(Vec::new()); + } let now = encoding::micros(now); let mut invites = Vec::new(); for entry in databases.invite_codes_by_created_at.rev_iter(txn)? { @@ -364,6 +374,11 @@ pub(crate) fn dvm_create( kind: Kind, now: DateTime, ) -> Result { + if dvm_find_by_id(txn, databases, id)?.is_some() { + return Err(StorageError::InvalidArgument { + message: "dvm_jobs.id must not already exist", + }); + } let now = encoding::micros(now); let job = DvmJob { id: *id, @@ -444,9 +459,16 @@ pub(crate) fn dvm_find_pending_jobs( limit: u64, kinds: Option<&[Kind]>, ) -> Result, StorageError> { + // Each status range runs in creation order, so the merge only needs the first `limit` + // eligible rows of each: the backlog is not materialised for a small request. + let take = usize::try_from(limit).unwrap_or(usize::MAX); + if take == 0 { + return Ok(Vec::new()); + } let mut jobs = Vec::new(); for status in [DvmJobStatus::Submitted, DvmJobStatus::PickedUp] { let prefix = text_part(status.as_str(), "dvm_jobs_by_status_created_at", "status")?; + let mut taken = 0; for entry in databases .dvm_jobs_by_status_created_at .prefix_iter(txn, &prefix)? @@ -460,6 +482,10 @@ pub(crate) fn dvm_find_pending_jobs( continue; } jobs.push(job); + taken += 1; + if taken >= take { + break; + } } } jobs.sort_by(|left, right| { @@ -467,7 +493,7 @@ pub(crate) fn dvm_find_pending_jobs( .cmp(&right.created_at) .then_with(|| left.id.cmp(&right.id)) }); - jobs.truncate(usize::try_from(limit).unwrap_or(usize::MAX)); + jobs.truncate(take); Ok(jobs) } @@ -603,6 +629,9 @@ pub(crate) fn report_find_actionable( databases: &Databases, limit: u64, ) -> Result, StorageError> { + if limit == 0 { + return Ok(Vec::new()); + } let mut reports = Vec::new(); for entry in databases.reports_actionable_by_created_at.rev_iter(txn)? { let (key, _) = entry?; diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 35ac8dfdd..e9fc7b7d6 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -385,7 +385,7 @@ impl EventRepository for LmdbBackend { &filter, gates, now, - )); + )?); Ok(stream) }) } diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index 1a9f674f5..d7e54c7c9 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -23,11 +23,11 @@ use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; use nostream_storage::LmdbBackend; use nostream_storage::StorageError; use nostream_storage::repositories::{ - CreateInviteCodeOptions, DeliveryLogFilters, EventRecord, EventRepository, + CreateInviteCodeOptions, DeliveryLogFilters, DvmJobStatus, EventRecord, EventRepository, EventRetentionOptions, FieldUpdate, InviteCodeRepository, Invoice, InvoiceRepository, - InvoiceStatus, InvoiceUnit, NewDeliveryLogEntry, NotificationDeliveryLogRepository, - NotificationDeliveryStatus, NotificationOutboxRepository, NotificationTargetType, - ReconciliationItem, UserRepository, + InvoiceStatus, InvoiceUnit, NewDeliveryLogEntry, NewReport, Nip05Verification, + NotificationDeliveryLogRepository, NotificationDeliveryStatus, NotificationOutboxRepository, + NotificationTargetType, ReconciliationItem, ReportType, UserRepository, }; mod common; @@ -423,6 +423,8 @@ async fn a_nanosecond_instant_reads_back_at_microsecond_precision() { let now = Utc::now(); let payee = Pubkey::from_byte_array([11; 32]); backend.admit_user(&payee, now).await.unwrap(); + // Three digits past the microsecond, which no `timestamp` column can hold (V38). + let instant = DateTime::::from_timestamp_nanos(1_700_000_000_123_456_789); let invoice = Invoice { id: "invoice-micros".to_owned(), pubkey: payee, @@ -433,15 +435,28 @@ async fn a_nanosecond_instant_reads_back_at_microsecond_precision() { status: InvoiceStatus::Pending, description: "test".to_owned(), confirmed_at: None, - expires_at: None, + expires_at: Some(instant), updated_at: now, - created_at: now, + created_at: instant, verify_url: None, }; InvoiceRepository::upsert(&backend, &invoice).await.unwrap(); - // Three digits past the microsecond, which no `timestamp` column can hold (V38). - let instant = DateTime::::from_timestamp_nanos(1_700_000_000_123_456_789); + let stored = backend.find_by_id("invoice-micros").await.unwrap().unwrap(); + assert_eq!( + stored.created_at.timestamp_subsec_nanos(), + 123_456_000, + "the insert truncates the caller's created_at (V38)" + ); + assert_eq!( + stored + .expires_at + .expect("the expiry") + .timestamp_subsec_nanos(), + 123_456_000, + "and its expires_at" + ); + backend .confirm_invoice("invoice-micros", 1, instant) .await @@ -460,6 +475,44 @@ async fn a_nanosecond_instant_reads_back_at_microsecond_precision() { "a nanosecond instant reads back at microsecond precision (V38)" ); assert_eq!(confirmed_at.timestamp_subsec_nanos(), 123_456_000); + + // The NIP-05 row keeps the caller's last check and verification at the same precision. + use nostream_storage::repositories::Nip05VerificationRepository; + + let watched = Pubkey::from_byte_array([12; 32]); + Nip05VerificationRepository::upsert( + &backend, + &Nip05Verification { + pubkey: watched, + nip05: "micros@example.test".to_owned(), + domain: "example.test".to_owned(), + is_verified: true, + last_verified_at: Some(instant), + last_checked_at: instant, + failure_count: 0, + created_at: instant, + updated_at: instant, + }, + ) + .await + .unwrap(); + let verification = Nip05VerificationRepository::find_by_pubkey(&backend, &watched) + .await + .unwrap() + .unwrap(); + assert_eq!( + verification.last_checked_at.timestamp_subsec_nanos(), + 123_456_000, + "the NIP-05 row truncates last_checked_at" + ); + assert_eq!( + verification + .last_verified_at + .expect("the verification") + .timestamp_subsec_nanos(), + 123_456_000, + "and last_verified_at" + ); } #[tokio::test] @@ -1174,3 +1227,155 @@ async fn a_retention_pass_takes_at_most_one_batch_of_candidates() { "the next pass takes the row the batch ceiling left behind" ); } + +/// A `LIMIT 0` returns no rows, in every finder whose walk arms its bound after the first +/// match: Postgres answers an empty list, so a bound of zero must not return one. +#[tokio::test] +async fn a_zero_limit_returns_no_rows() { + use nostream_storage::repositories::{Nip05VerificationRepository, ReportRepository}; + + let (_directory, backend) = open_store().await; + let now = Utc::now(); + + let watched = Pubkey::from_byte_array([21; 32]); + Nip05VerificationRepository::upsert( + &backend, + &Nip05Verification { + pubkey: watched, + nip05: "pending@example.test".to_owned(), + domain: "example.test".to_owned(), + is_verified: false, + last_verified_at: None, + last_checked_at: now - Duration::hours(2), + failure_count: 0, + created_at: now - Duration::hours(2), + updated_at: now - Duration::hours(2), + }, + ) + .await + .unwrap(); + assert!( + backend + .find_pending_verifications(0, 3, 1) + .await + .unwrap() + .len() + == 1, + "the row the zero limit must not return" + ); + assert!( + backend + .find_pending_verifications(0, 3, 0) + .await + .unwrap() + .is_empty(), + "a zero limit returns no pending verification" + ); + + InviteCodeRepository::create( + &backend, + "invite-zero", + &CreateInviteCodeOptions { + expires_at: Some(now + Duration::hours(1)), + remaining_uses: FieldUpdate::Set(3), + created_by: None, + }, + ) + .await + .unwrap(); + assert_eq!(backend.find_active_codes(1).await.unwrap().len(), 1); + assert!( + backend.find_active_codes(0).await.unwrap().is_empty(), + "a zero limit returns no invite code" + ); + + ReportRepository::create( + &backend, + &NewReport { + event_id: EventId::from_byte_array([22; 32]), + reporter_pubkey: Pubkey::from_byte_array([23; 32]), + reported_pubkey: Some(Pubkey::from_byte_array([24; 32])), + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + }, + ) + .await + .unwrap(); + assert_eq!(backend.find_actionable(1).await.unwrap().len(), 1); + assert!( + backend.find_actionable(0).await.unwrap().is_empty(), + "a zero limit returns no report" + ); + + backend.append(&one_entry(None)).await.unwrap(); + let filters = DeliveryLogFilters::default(); + assert_eq!(backend.find_recent(1, &filters).await.unwrap().len(), 1); + assert!( + backend.find_recent(0, &filters).await.unwrap().is_empty(), + "a zero limit returns no delivery-log entry" + ); +} + +/// `dvm_jobs.id` is the primary key: creating a row that already exists is refused instead of +/// overwriting the worker and result a later state wrote. +#[tokio::test] +async fn creating_a_job_that_exists_is_refused() { + use nostream_storage::repositories::DvmJobRepository; + + let (_directory, backend) = open_store().await; + let id = EventId::from_byte_array([31; 32]); + let requester = Pubkey::from_byte_array([32; 32]); + + DvmJobRepository::create(&backend, &id, &requester, Kind(5_000)) + .await + .expect("the job"); + assert!(backend.assign_worker(&id, 3).await.unwrap()); + + let error = DvmJobRepository::create(&backend, &id, &requester, Kind(5_000)) + .await + .expect_err("a second create of the same id"); + assert!( + matches!(error, StorageError::InvalidArgument { .. }), + "the primary key refuses the row, got {error:?}" + ); + + let stored = DvmJobRepository::find_by_id(&backend, &id) + .await + .unwrap() + .expect("the job"); + assert_eq!(stored.status, DvmJobStatus::PickedUp); + assert_eq!( + stored.worker_index, + Some(3), + "the refused create changed nothing" + ); +} + +/// A non-empty `search` with the NIP-50 gate on is refused by every query method, the +/// reconciliation stream included: this backend cannot search, and returning a broader filter's +/// rows would answer a question it did not ask. +#[tokio::test] +async fn reconciliation_refuses_a_search_the_gate_enables() { + let (_directory, backend) = open_store().await; + let filter = Filter { + search: Some("nostr".to_owned()), + ..Filter::default() + }; + + let error = backend + .find_reconciliation_items(&filter) + .await + .err() + .expect("the refusal"); + assert!( + matches!( + error, + StorageError::NotImplemented { + module: "storage::lmdb::nip50_search" + } + ), + "a search the gate enables is NotImplemented, got {error:?}" + ); +} From fd159c52c6579b23b08f9f939a14c011475b8cac Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sat, 7 Nov 2026 23:00:55 -0500 Subject: [PATCH 51/60] fix(storage): bound the lmdb claim and the delivery-log index key --- .../src/lmdb/notifications.rs | 180 ++++++++++++++++-- 1 file changed, 162 insertions(+), 18 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs index 241850dc1..524235766 100644 --- a/crates/nostream-storage/src/lmdb/notifications.rs +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -24,6 +24,38 @@ fn new_id() -> String { uuid::Uuid::new_v4().to_string() } +/// Counts the outbox rows a claim loaded. Only the tests read the count, so a release build +/// drops it. +fn probe_load() { + #[cfg(test)] + probes::load(); +} + +/// The work counter the claim's cost test asserts on: a claim is bounded by the rows it +/// returns, never by the backlog behind them. +#[cfg(test)] +mod probes { + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Serialises the tests that count loads, so one cannot read another's numbers. + pub(super) static COUNTING: Mutex<()> = Mutex::new(()); + + static LOADS: AtomicUsize = AtomicUsize::new(0); + + pub(super) fn reset() { + LOADS.store(0, Ordering::Relaxed); + } + + pub(super) fn load() { + LOADS.fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn loads() -> usize { + LOADS.load(Ordering::Relaxed) + } +} + // --------------------------------------------------------------------------------------- // The delivery log. // --------------------------------------------------------------------------------------- @@ -34,8 +66,10 @@ fn delivery_created_key(entry: &DeliveryLogEntry) -> Vec { key } -/// `outbox_id ‖ status ‖ target_id ‖ id`, so one entry is addressable and two attempts at -/// the same target do not share a key. +/// `outbox_id ‖ status ‖ id`, so one entry is addressable and two attempts do not share a key. +/// +/// The target is in the row, not the key: a target id may be longer than an LMDB key holds, +/// and the index only has to find the entry, whose row carries the id to report. fn delivery_outbox_key(outbox_id: &str, entry: &DeliveryLogEntry) -> Result, StorageError> { let mut key = text_part(outbox_id, "delivery_log_by_outbox", "outbox_id")?; key.extend_from_slice(&text_part( @@ -43,11 +77,6 @@ fn delivery_outbox_key(outbox_id: &str, entry: &DeliveryLogEntry) -> Result Result, StorageError> { + if limit == 0 { + return Ok(Vec::new()); + } let mut entries = Vec::new(); for entry in databases.delivery_log_by_created_at.rev_iter(txn)? { let (key, _) = entry?; @@ -167,17 +208,12 @@ pub(crate) fn find_successful_target_ids( let mut targets = Vec::new(); for entry in databases.delivery_log_by_outbox.prefix_iter(txn, &prefix)? { let (key, _) = entry?; - let rest = &key[prefix.len()..]; - let Some(end) = rest.iter().position(|byte| *byte == 0) else { - return Err(StorageError::InvalidRow { - table: "delivery_log_by_outbox", - column: "key", - value: "a key without a target terminator".to_owned(), - }); + let id = delivery_outbox_id(key)?; + let Some(row) = delivery_load(txn, databases, id)? else { + continue; }; - let target_id = text_from(&rest[..end], "delivery_log_by_outbox")?; - if seen.insert(target_id.to_owned()) { - targets.push(target_id.to_owned()); + if seen.insert(row.target_id.clone()) { + targets.push(row.target_id); } } Ok(targets) @@ -320,8 +356,16 @@ pub(crate) fn claim_batch( let stale = now - Duration::milliseconds(NOTIFICATION_OUTBOX_PROCESSING_LEASE_MS); let mut candidates = Vec::new(); + // Each status range runs in creation order, so the merge only needs the first `limit` + // eligible rows of each: the global first `limit` cannot need more from either one. The + // backlog is never materialised: a small claim reads a small slice whatever it holds. + let take = usize::try_from(limit).unwrap_or(usize::MAX); + if take == 0 { + return Ok(Vec::new()); + } for status in [OutboxStatus::Pending, OutboxStatus::Processing] { let prefix = text_part(status.as_str(), "outbox_by_status_created_at", "status")?; + let mut taken = 0; for entry in databases .outbox_by_status_created_at .prefix_iter(txn, &prefix)? @@ -331,6 +375,7 @@ pub(crate) fn claim_batch( let Some(message) = outbox_load(txn, databases, id)? else { continue; }; + probe_load(); let due = match status { OutboxStatus::Pending => message.available_at <= now, OutboxStatus::Processing => message.updated_at < stale, @@ -338,6 +383,10 @@ pub(crate) fn claim_batch( }; if due { candidates.push(message); + taken += 1; + if taken >= take { + break; + } } } } @@ -346,7 +395,7 @@ pub(crate) fn claim_batch( .cmp(&right.created_at) .then_with(|| left.id.cmp(&right.id)) }); - candidates.truncate(usize::try_from(limit).unwrap_or(usize::MAX)); + candidates.truncate(take); let mut claimed = Vec::with_capacity(candidates.len()); for message in candidates { @@ -508,3 +557,98 @@ fn truncate_bytes(value: &str, limit: usize) -> String { } value[..end].to_owned() } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use crate::backend::LmdbEnv; + use crate::repositories::{NotificationDeliveryStatus, NotificationTargetType}; + use heed::Env; + + fn store() -> (tempfile::TempDir, Env, Databases) { + let directory = tempfile::tempdir().expect("a temporary directory"); + let environment = + LmdbEnv::open(directory.path().join("lmdb"), 8 << 20).expect("an environment"); + let env = environment.env().clone(); + let mut transaction = env.write_txn().expect("a write transaction"); + let databases = Databases::create(&env, &mut transaction).expect("the databases"); + transaction.commit().expect("the commit"); + (directory, env, databases) + } + + fn now() -> DateTime { + DateTime::::from_timestamp(1_700_000_000, 0).expect("an instant") + } + + /// A claim is bounded by the rows it returns: a small request over a large backlog reads + /// the head of each status range, not the backlog. + #[test] + fn a_small_claim_does_not_load_the_backlog() { + let _serial = probes::COUNTING + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let (_directory, env, databases) = store(); + let now = now(); + + let mut transaction = env.write_txn().expect("a write transaction"); + for step in 0..200 { + enqueue( + &mut transaction, + &databases, + "test", + &serde_json::json!({ "step": step }), + now, + ) + .expect("a message"); + } + transaction.commit().expect("the commit"); + + let mut transaction = env.write_txn().expect("a write transaction"); + probes::reset(); + let claimed = claim_batch(&mut transaction, &databases, 5, now).expect("the claim"); + let loads = probes::loads(); + transaction.commit().expect("the commit"); + + assert_eq!(claimed.len(), 5, "the claim is the request's size"); + assert!( + loads <= 10, + "a claim reads the head of each status range, not the backlog: {loads} rows loaded" + ); + } + + /// A target id may be longer than an LMDB key holds: the delivery index keys the entry, and + /// the row carries the target, so the success is recorded and reported. + #[test] + fn a_long_target_id_still_reaches_its_delivery_record() { + let (_directory, env, databases) = store(); + let now = now(); + let target = "t".repeat(430); + + let mut transaction = env.write_txn().expect("a write transaction"); + let message = enqueue( + &mut transaction, + &databases, + "test", + &serde_json::json!({}), + now, + ) + .expect("a message"); + let entry = NewDeliveryLogEntry { + outbox_id: Some(message.id.clone()), + event_type: "test".to_owned(), + target_id: target.clone(), + target_type: NotificationTargetType::Telegram, + status: NotificationDeliveryStatus::Success, + attempt_number: 1, + error_snippet: None, + }; + append(&mut transaction, &databases, &entry, now).expect("the log row"); + let targets = find_successful_target_ids(&transaction, &databases, &message.id) + .expect("the successful targets"); + transaction.commit().expect("the commit"); + + assert_eq!(targets, vec![target]); + } +} From 50d09ea351088d86dc17f50690ae85764e75837b Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 00:19:49 -0500 Subject: [PATCH 52/60] fix(storage): count an lmdb tag walk by the id in its key --- crates/nostream-storage/src/lmdb/encoding.rs | 14 +- crates/nostream-storage/src/lmdb/events.rs | 163 +++++++++---------- 2 files changed, 84 insertions(+), 93 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index 03ea6d7a2..585ef192a 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -480,9 +480,7 @@ pub(crate) fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec } /// The inclusive key ranges one tag criterion covers, one range per criterion that fits a key -/// and in the criteria's order, each with the criterion's own position: a criterion that -/// selects nothing leaves the positions after it unshifted, so a count still knows which -/// criteria came before a range. +/// and in the criteria's order. /// /// An exact value is the range from the value's own separator, and a `#g` prefix is the range /// over every value that starts with the prefix, padded to the longest key the store can hold @@ -490,10 +488,10 @@ pub(crate) fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec /// /// A criterion whose lower bound does not fit [`MAX_KEY_LEN`] selects nothing: a key carrying /// it could not be written, so seeking it would fail the read rather than answer it. -pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(usize, Vec, Vec)> { +pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(Vec, Vec)> { let is_geohash = name == "g"; let mut ranges = Vec::with_capacity(criteria.len()); - for (position, criterion) in criteria.iter().enumerate() { + for criterion in criteria { let mut head = Vec::with_capacity(name.len() + 1 + criterion.len()); head.extend_from_slice(name.as_bytes()); head.push(0); @@ -516,7 +514,7 @@ pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(usize, Vec } }; if lower.len() <= MAX_KEY_LEN && upper.len() <= MAX_KEY_LEN { - ranges.push((position, lower, upper)); + ranges.push((lower, upper)); } } ranges @@ -613,9 +611,9 @@ mod tests { #[test] fn a_geohash_criterion_is_a_prefix_range_and_another_tag_is_exact() { let geohash = tag_ranges("g", &["u4pru*".to_owned()]); - assert_eq!(geohash[0].1, b"g\0u4pru".to_vec()); + assert_eq!(geohash[0].0, b"g\0u4pru".to_vec()); let exact = tag_ranges("e", &["u4pru*".to_owned()]); - assert_eq!(exact[0].1, b"e\0u4pru*\0".to_vec()); + assert_eq!(exact[0].0, b"e\0u4pru*\0".to_vec()); } #[test] diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 6d88fdc66..e40a7018f 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -671,12 +671,8 @@ struct Plan<'a> { /// The tag family the ranges were built from, with its criteria in the criteria's order — /// `tag_ranges` builds one range per criterion that fits a key. A key in these ranges has /// already proved that family, so the row's body is decoded only for the filter's other tag - /// families. + /// families, and a count holds the ids its walk has counted rather than decoding one. tag: Option<(&'a str, &'a [String])>, - /// The criterion each range was built from, in the same order as `ranges`. Empty for a plan - /// that does not read the tags index; a dropped criterion leaves the later positions alone, - /// so a count still knows which criteria came before a range. - tag_positions: Vec, } /// One reconciliation batch still running on the blocking pool, with the cursor it returns. @@ -806,63 +802,15 @@ fn already_counted( Ok(false) } -/// Whether an earlier criterion — the criterion this range was built from, or one before it — -/// already answers for this row, so counting it here would count it twice. -/// -/// Two ranges select one row when two criteria of the filter's tag family both match one of its -/// tags; one criterion selects it twice when a `#g` prefix matches two of its tags. The first -/// criterion in the criteria's order answers for the row, and within a prefix criterion the -/// smallest matching value does. -/// -/// This is the one place a count reads an event's tags instead of its key: the key names a -/// single value, and the other values the event carries are in the body. -fn counted_earlier( - txn: &RoTxn<'_>, - databases: &Databases, - plan: &Plan<'_>, - index: usize, - value: Option<&str>, - id: &EventId, -) -> Result { - let Some((name, criteria)) = plan.tag else { - return Ok(false); - }; - let position = plan.tag_positions.get(index).copied().unwrap_or(index); - let prefix = criteria - .get(position) - .and_then(|criterion| criterion.strip_suffix('*').filter(|_| name == "g")); - if position == 0 && prefix.is_none() { - return Ok(false); - } - let Some(stored) = databases.events.get(txn, id.as_bytes())? else { - return Ok(false); - }; - let record = rows::decode_event(stored)?; - if criteria - .iter() - .take(position) - .any(|earlier| record_has_tag(&record, name, std::slice::from_ref(earlier))) - { - return Ok(true); - } - let (Some(prefix), Some(value)) = (prefix, value) else { - return Ok(false); - }; - Ok(record.event.tags.iter().any(|tag| { - matches!( - (tag.first(), tag.get(1)), - (Some(tag_name), Some(candidate)) - if tag_name == name - && candidate.starts_with(prefix) - && candidate.as_str() < value - ) - })) -} - /// Counts the live rows one unlimited filter matches that the count does not already hold. /// /// Every range is walked whole — a count has no page to stop at — and a row two of them select -/// is counted once, at the first criterion's range. +/// is counted once, at the first key that selects it. +/// +/// A `tags` key names one value of one row, so a row's keys within this walk are one per +/// matching value and per matching criterion, in criteria order and then value order: the first +/// one is the row's smallest matching key, and the id it ends with answers both a second +/// criterion and a second value of a `#g` prefix without reading the body. fn count_matches( txn: &RoTxn<'_>, databases: &Databases, @@ -882,28 +830,37 @@ fn count_matches( live: true, proven: plan.tag.map(|(name, _)| name), }; + // A single exact criterion names one key per row, so only a second criterion or a `#g` + // prefix can walk one row twice, and only then is the id set paid for. + // + // shortcut: such a walk holds the id of every row it counts for its length, against a body + // decode per key; upgrade when a single count's match set outgrows the request's memory + // budget, which needs an index order that visits a row once. + let mut counted_ids = HashSet::new(); + let dedupe = plan.tag.is_some_and(|(name, criteria)| { + criteria.len() > 1 + || (name == "g" && criteria.iter().any(|criterion| criterion.ends_with('*'))) + }); let mut total = 0; - for (index, (low, high)) in plan.ranges.iter().enumerate() { + for (low, high) in &plan.ranges { let range = encoding::bounds(low, high); for entry in plan.database.range(txn, &range)? { let (key, _) = entry?; key_evaluated(); - let value = match plan.tag { - Some((name, criteria)) => { - let value = encoding::tag_value(key)?; - if !tag_value_matches(name == "g", criteria, value) { - continue; - } - Some(value) + if let Some((name, criteria)) = plan.tag { + let value = encoding::tag_value(key)?; + if !tag_value_matches(name == "g", criteria, value) { + continue; } - None => None, - }; + } let Some((_, id)) = candidate(txn, databases, plan.path, key)? else { continue; }; + if dedupe && !counted_ids.insert(id) { + continue; + } if matcher.matches(txn, databases, &id)?.is_some() && !already_counted(txn, databases, counted, gates, now, &id)? - && !counted_earlier(txn, databases, &plan, index, value, &id)? { total += 1; } @@ -1181,7 +1138,6 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { // The `events` key is the id, so the range runs in id order, not time order. ordered: false, tag: None, - tag_positions: Vec::new(), }; } if let Some(authors) = &filter.authors @@ -1200,7 +1156,6 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { // kind. ordered: authors.iter().all(|criterion| criterion.len() == 64), tag: None, - tag_positions: Vec::new(), }; } // Without kinds the author index runs in kind order before `created_at`, so a page @@ -1219,24 +1174,17 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { ranges: created_at_ranges(filter), ordered: true, tag: None, - tag_positions: Vec::new(), }; } if let Some((name, criteria)) = tag_path(filter) { - let ranged = encoding::tag_ranges(name, criteria); - let tag_positions = ranged.iter().map(|(position, _, _)| *position).collect(); return Plan { database: databases.tags, path: AccessPath::Tags, - ranges: ranged - .into_iter() - .map(|(_, low, high)| (low, high)) - .collect(), + ranges: encoding::tag_ranges(name, criteria), // An exact criterion's range runs `created_at` under one value; a `#g` prefix range // spans values, so its keys are in value order and the range is walked whole. ordered: name != "g" || criteria.iter().all(|criterion| !criterion.ends_with('*')), tag: Some((name, criteria)), - tag_positions, }; } Plan { @@ -1245,7 +1193,6 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { ranges: created_at_ranges(filter), ordered: true, tag: None, - tag_positions: Vec::new(), } } @@ -1806,7 +1753,6 @@ fn stream_plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { ranges: created_at_ranges(filter), ordered: true, tag: None, - tag_positions: Vec::new(), } } @@ -3023,10 +2969,51 @@ mod tests { ); } - /// A criterion too long to seek leaves the later criteria's positions alone, so a row - /// carrying a later value as well is still counted once. + /// A count walks keys, not bodies: the id a tag key ends with says whether the row is + /// already counted, so a wildcard count decodes nothing however many values it matches. + #[test] + fn a_wildcard_count_decodes_no_body() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let record = signed( + 7, + 1, + 1_700_000_000, + vec![ + vec!["g".to_owned(), "u4a".to_owned()], + vec!["g".to_owned(), "u4b".to_owned()], + ], + ); + write(&env, &databases, &[record]); + let transaction = env.read_txn().expect("a read transaction"); + let mut tags = BTreeMap::new(); + tags.insert("#g".to_owned(), vec!["u4*".to_owned()]); + let filter = Filter { + tags, + ..Filter::default() + }; + + probes::reset(); + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 1 + ); + assert_eq!(probes::decodes(), 0, "a count decodes no body"); + assert_eq!(probes::keys(), 2, "both keys the prefix selects are read"); + } + + /// A criterion too long to seek selects nothing, so a row carrying only later criteria is + /// still counted — once, at the first criterion's range that selects it. #[test] - fn a_dropped_criterion_does_not_shift_the_counted_positions() { + fn a_dropped_criterion_does_not_count_a_row_twice() { + let _serial = counting(); let (_directory, env, databases) = store(); let record = signed( 7, @@ -3054,6 +3041,7 @@ mod tests { 2, "the oversized criterion seeks nothing" ); + probes::reset(); assert_eq!( count( &transaction, @@ -3066,6 +3054,11 @@ mod tests { 1, "the row carries both values and is one row" ); + assert_eq!( + probes::decodes(), + 0, + "two criteria are deduplicated by the id, not the body" + ); } /// The expiry pass reads the index no further than the last expired row: a future expiry is From 08a75634b6c3ac2272dc1b5ee97cb910849d48e9 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 00:19:54 -0500 Subject: [PATCH 53/60] refactor(storage): pass the merge predicate its created-at and id pairs --- crates/nostream-storage/src/lmdb/events.rs | 6 ++---- crates/nostream-storage/src/records.rs | 15 ++++----------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index e40a7018f..4549ad1a2 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -155,10 +155,8 @@ fn upsert_one( }; let header = rows::event_header(value)?; if !replaces_fields( - record.event.created_at, - record.event.id, - header.created_at, - stored_id, + (record.event.created_at, record.event.id), + (header.created_at, stored_id), ) { return Ok(false); } diff --git a/crates/nostream-storage/src/records.rs b/crates/nostream-storage/src/records.rs index f5b55005b..3019a551e 100644 --- a/crates/nostream-storage/src/records.rs +++ b/crates/nostream-storage/src/records.rs @@ -108,22 +108,15 @@ pub fn replacement_key(record: &EventRecord) -> Option<(Pubkey, u64, Replacement /// header and a pointer rather than a whole record — LMDB reads `created_at` from the value's /// fixed header and the id from the `replaceable` pointer, and decodes the body only once the /// incoming row has won. -pub fn replaces_fields( - created_at: i64, - id: EventId, - current_created_at: i64, - current_id: EventId, -) -> bool { - created_at > current_created_at || (created_at == current_created_at && id < current_id) +pub fn replaces_fields(candidate: (i64, EventId), current: (i64, EventId)) -> bool { + candidate.0 > current.0 || (candidate.0 == current.0 && candidate.1 < current.1) } /// [`replaces_fields`] applied to the two records a backend holds in memory. pub fn replaces(candidate: &EventRecord, current: &EventRecord) -> bool { replaces_fields( - candidate.event.created_at, - candidate.event.id, - current.event.created_at, - current.event.id, + (candidate.event.created_at, candidate.event.id), + (current.event.created_at, current.event.id), ) } From c6a238167c51aa4e97bf3fdd172a63a514ff15f3 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 00:19:59 -0500 Subject: [PATCH 54/60] refactor(storage): give the text-column NUL rule one owner --- crates/nostream-storage/src/lmdb/encoding.rs | 58 ++++++++++++++----- .../src/lmdb/notifications.rs | 12 +--- crates/nostream-storage/tests/parity_lmdb.rs | 4 +- 3 files changed, 48 insertions(+), 26 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index 585ef192a..503eb9442 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -200,17 +200,16 @@ pub(crate) fn check_version(table: &'static str, value: &[u8]) -> Result<(), Sto } } -/// A text component of a composite key, terminated so the next component reads unambiguously. +/// Refuses a value a `text` column cannot hold. /// -/// The terminator is what every key *reader* in this module splits on, so the rule lives here -/// with the writers: a component that carries `0x00` cannot be read back, and PostgreSQL's -/// `text` cannot hold the byte either, so it is refused rather than truncated — both backends -/// reject the same value. -pub(crate) fn text_part( - value: &str, +/// PostgreSQL's `text` cannot store `0x00`, and this module's key separator is the same byte, so +/// one check serves a key component and a row's own text column: both backends reject the same +/// value rather than truncating it. +pub(crate) fn refuse_nul( table: &'static str, column: &'static str, -) -> Result, StorageError> { + value: &str, +) -> Result<(), StorageError> { if value.contains('\0') { return Err(StorageError::InvalidRow { table, @@ -218,6 +217,20 @@ pub(crate) fn text_part( value: value.to_owned(), }); } + Ok(()) +} + +/// A text component of a composite key, terminated so the next component reads unambiguously. +/// +/// The terminator is what every key *reader* in this module splits on, so the rule lives here +/// with the writers: a component that carries `0x00` cannot be read back, and the value is +/// refused by [`refuse_nul`] rather than truncated. +pub(crate) fn text_part( + value: &str, + table: &'static str, + column: &'static str, +) -> Result, StorageError> { + refuse_nul(table, column, value)?; let mut bytes = Vec::with_capacity(value.len() + 1); bytes.extend_from_slice(value.as_bytes()); bytes.push(0); @@ -623,8 +636,7 @@ mod tests { let prefix = "u".repeat(447); let ranges = tag_ranges("g", &[format!("{prefix}*")]); assert_eq!(ranges.len(), 1, "the prefix names a range"); - let (position, lower, upper) = &ranges[0]; - assert_eq!(*position, 0); + let (lower, upper) = &ranges[0]; assert_eq!(upper.len(), MAX_KEY_LEN, "the upper bound is a legal key"); let mut expected = b"g\0".to_vec(); expected.extend_from_slice(prefix.as_bytes()); @@ -811,16 +823,26 @@ mod tests { } #[test] - fn a_text_component_a_text_column_cannot_hold_is_refused() { - // PostgreSQL's `text` cannot store `0x00`, and the key's separator is the same byte, so - // the codec refuses it where the writer meets it rather than reading it back truncated. + fn a_value_a_text_column_cannot_hold_is_refused() { + // PostgreSQL's `text` cannot store `0x00`, so the check refuses it wherever a writer meets + // the value: a key component here, a row's own text column in the delivery log. for value in ["a\0b", "\0", "ab\0"] { - let error = text_part(value, "delivery_log_by_outbox", "target_id") + let error = text_part(value, "delivery_log_by_outbox", "outbox_id") .expect_err("a NUL in a key component"); assert!(matches!( error, StorageError::InvalidRow { table: "delivery_log_by_outbox", + column: "outbox_id", + .. + } + )); + let error = refuse_nul("notification_delivery_log", "target_id", value) + .expect_err("a NUL in a text column"); + assert!(matches!( + error, + StorageError::InvalidRow { + table: "notification_delivery_log", column: "target_id", .. } @@ -830,10 +852,16 @@ mod tests { text_part( "https://example.test", "delivery_log_by_outbox", - "target_id" + "outbox_id" ) .unwrap(), b"https://example.test\0".to_vec() ); + refuse_nul( + "notification_delivery_log", + "target_id", + "https://example.test", + ) + .expect("a target a text column can hold"); } } diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs index 524235766..3c73734c8 100644 --- a/crates/nostream-storage/src/lmdb/notifications.rs +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -134,15 +134,9 @@ pub(crate) fn append( error_snippet: entry.error_snippet.clone(), created_at: encoding::micros(now), }; - // `target_id` is a `text` column, which cannot hold `0x00`: the capture could never have - // stored this row, and the index key no longer carries the value to refuse it on. - if entry.target_id.contains('\0') { - return Err(StorageError::InvalidRow { - table: "notification_delivery_log", - column: "target_id", - value: entry.target_id.clone(), - }); - } + // The outbox key no longer carries the target, so the row's own text column is where the + // codec's refusal is met. + encoding::refuse_nul("notification_delivery_log", "target_id", &entry.target_id)?; let value = rows::encode("notification_delivery_log", &entry)?; databases .notification_delivery_log diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index d7e54c7c9..5cc20f545 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -558,8 +558,8 @@ async fn a_target_id_a_text_column_cannot_hold_is_refused_rather_than_truncated( entry.target_id = "https://example.test/\0hook".to_owned(); // `notification_delivery_log.target_id` is a `text` column, and PostgreSQL cannot store - // `0x00` in one, so the capture could never have kept this row; the index key refuses it - // rather than truncating the id at the separator on the way back out. + // `0x00` in one, so the capture could never have kept this row; `append` refuses it through + // `encoding::refuse_nul` rather than writing a value the other backend cannot hold. match backend.append(&entry).await { Err(StorageError::InvalidRow { column, .. }) => { assert_eq!(column, "target_id"); From fb73314108857abc563ba96075834efdb0506c7b Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 00:20:03 -0500 Subject: [PATCH 55/60] docs(storage): make the lmdb decision record match the code --- docs/rewrite/decisions/0011-lmdb-backend.md | 38 ++++++++++++--------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/docs/rewrite/decisions/0011-lmdb-backend.md b/docs/rewrite/decisions/0011-lmdb-backend.md index e8f463efa..0afda946b 100644 --- a/docs/rewrite/decisions/0011-lmdb-backend.md +++ b/docs/rewrite/decisions/0011-lmdb-backend.md @@ -19,9 +19,9 @@ therefore measures. ## The backend is a feature of the crate, not a crate -`lmdb = ["dep:heed", "dep:uuid"]`, default on beside `postgres` (D1). heed was already a -hard dependency, the seams' own ADR says the `--no-default-features` build is what this -follow-up uses (ADR 0003, "Backend coupling"), and a second crate would repeat five +`lmdb = ["dep:heed", "dep:uuid", "dep:tracing"]`, default on beside `postgres` (D1). heed was +already a hard dependency, the seams' own ADR says the `--no-default-features` build is what +this follow-up uses (ADR 0003, "Backend coupling"), and a second crate would repeat five manifest fields plus `[lints] workspace = true` to isolate code that shares every type in `nostream-storage`. The cost is that `--no-default-features` now links no driver at all — which is also the point: `cargo check -p nostream-storage --no-default-features --all-targets` @@ -82,13 +82,15 @@ in the flags byte, and the reserved byte is there for a field that has to be a b `row_id` stays `None` (D5): it is the Postgres row's `uuid` and nothing reads it through this backend. Text components of a composite key are NUL-terminated except the final one -(`text_part`), so `tag_key` is `name NUL value NUL created_at ‖ event_id`. A component carrying -`0x00` is refused as `InvalidRow` by `text_part` itself rather than splitting the key, and -PostgreSQL's `text` cannot hold that byte either, so both backends reject the same value — -including a delivery log's `target_id`, which the capture could not have stored. The three -component codecs a key reader needs — `text_part`, `text_from` and `stamp_key` — have one -home, `lmdb/encoding.rs`, beside the key builders; the tail modules call them rather than -keeping copies whose rules could drift apart. +(`text_part`), so `tag_key` is `name NUL value NUL created_at ‖ event_id`. A value carrying +`0x00` is refused as `InvalidRow` by `refuse_nul` rather than splitting the key, and +PostgreSQL's `text` cannot hold that byte either, so both backends reject the same value. A +row's own text column meets the same check: the outbox key stopped carrying the target, so +`append` refuses `notification_delivery_log.target_id` through `refuse_nul` rather than +writing a value the capture could not have stored. The three component codecs a key reader +needs — `text_part`, `text_from` and `stamp_key` — have one home, `lmdb/encoding.rs`, beside +the key builders; the tail modules call them rather than keeping copies whose rules could +drift apart. LMDB bounds a key at 511 bytes (`MDB_MAXKEYSIZE`, which this environment does not raise), and a `tags` key carries the tag value verbatim. **That is the one place this backend refuses a @@ -227,14 +229,18 @@ rather than a rule. Two consequences of the same bound: -- `count_by_filters` has no page, so it holds no matching set. Each filter counts the rows it - visits and skips one the count already holds — an earlier filter's page when that filter had - a limit, or a predicate match when it had none — which is the capture's `UNION`. An unlimited - filter walks its ranges whole, which is what counting every match costs. +- `count_by_filters` has no page, so it materialises no result set: each filter counts the rows + it visits and skips one the count already holds — an earlier filter's page when that filter + had a limit, or a predicate match when it had none — which is the capture's `UNION`. An + unlimited filter walks its ranges whole, which is what counting every match costs. - A `tags` range is built from the family's own criteria, so a key inside it has already proved that family: the event's body is decoded only for the filter's *other* tag families, and a - filter with one family decodes none. A count reads the value out of the key too - (`name NUL value NUL …`), to count a row two criteria of one family would both select once. + filter with one family decodes none. A count may hold the id each key ends with for the length + of its tag walk, so a row two criteria — or two values of a `#g` prefix — both select is + counted at its smallest matching key and skipped at the rest, without a body decode. Only + those two shapes hold the set: a single exact criterion names one key per row. It is the + count's one resident cost, per distinct match, and the index has no order that would let a + walk visit each row once instead. ## Constraints move into Rust From 4e5cd43d52117600ac86c393c7b1f73eb04df54d Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 01:06:07 -0500 Subject: [PATCH 56/60] fix(storage): hold an lmdb count's ids only for the rows it matches --- crates/nostream-storage/src/lmdb/events.rs | 33 +++++++++++++--------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 4549ad1a2..d28185c8a 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -669,7 +669,7 @@ struct Plan<'a> { /// The tag family the ranges were built from, with its criteria in the criteria's order — /// `tag_ranges` builds one range per criterion that fits a key. A key in these ranges has /// already proved that family, so the row's body is decoded only for the filter's other tag - /// families, and a count holds the ids its walk has counted rather than decoding one. + /// families, and a `tags` count holds one id per row its matcher accepts rather than decoding. tag: Option<(&'a str, &'a [String])>, } @@ -722,10 +722,12 @@ pub(crate) fn page( /// The count of the live ids the filters match, deduplicated across filters. /// -/// A count has no page, so it holds no candidate set: each filter counts the rows it visits and -/// skips one the count already holds — a row of an earlier filter's page when that filter had a -/// limit (a page is at most [`MAX_PAGE_KEYS`] ids), or one its predicate matches when it had -/// none — which is the capture's `UNION`. +/// A count has no page, so it materialises no result set: each filter counts the rows it visits +/// and skips one the count already holds — a row of an earlier filter's page when that filter had +/// a limit (a page is at most [`MAX_PAGE_KEYS`] ids), or one its predicate matches when it had +/// none — which is the capture's `UNION`. A `tags` walk whose criteria can select one row twice +/// also holds the ids of the rows it matches, for the length of that walk; no +/// [`MAX_PAGE_KEYS`] bounds that set. pub(crate) fn count( txn: &RoTxn<'_>, databases: &Databases, @@ -806,9 +808,9 @@ fn already_counted( /// is counted once, at the first key that selects it. /// /// A `tags` key names one value of one row, so a row's keys within this walk are one per -/// matching value and per matching criterion, in criteria order and then value order: the first -/// one is the row's smallest matching key, and the id it ends with answers both a second -/// criterion and a second value of a `#g` prefix without reading the body. +/// matching value and per matching criterion. Whichever key the walk reaches first answers for +/// the row — ranges run in the criteria's order, so that is not necessarily the smallest of its +/// matching values — and the id it ends with answers the rest without reading the body. fn count_matches( txn: &RoTxn<'_>, databases: &Databases, @@ -831,9 +833,9 @@ fn count_matches( // A single exact criterion names one key per row, so only a second criterion or a `#g` // prefix can walk one row twice, and only then is the id set paid for. // - // shortcut: such a walk holds the id of every row it counts for its length, against a body - // decode per key; upgrade when a single count's match set outgrows the request's memory - // budget, which needs an index order that visits a row once. + // shortcut: such a walk holds one id per row its matcher accepts for the walk's length, + // against a body decode per key; upgrade when a single count's match set outgrows the + // request's memory budget, which needs an index order that visits a row once. let mut counted_ids = HashSet::new(); let dedupe = plan.tag.is_some_and(|(name, criteria)| { criteria.len() > 1 @@ -854,12 +856,15 @@ fn count_matches( let Some((_, id)) = candidate(txn, databases, plan.path, key)? else { continue; }; + if matcher.matches(txn, databases, &id)?.is_none() { + continue; + } + // The verdict is the row's, not the key's, so a second key is decided the same way + // and only a row this filter matches takes a slot in the set. if dedupe && !counted_ids.insert(id) { continue; } - if matcher.matches(txn, databases, &id)?.is_some() - && !already_counted(txn, databases, counted, gates, now, &id)? - { + if !already_counted(txn, databases, counted, gates, now, &id)? { total += 1; } } From 785e1ea7e1c85f27116d079a1f78f1b79e8780f3 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 01:06:13 -0500 Subject: [PATCH 57/60] test(storage): count the decode an lmdb count no longer makes --- crates/nostream-storage/src/lmdb/events.rs | 62 +++------------------- crates/nostream-storage/src/lmdb/rows.rs | 28 ++++++++++ 2 files changed, 35 insertions(+), 55 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index d28185c8a..7d725de27 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -604,14 +604,9 @@ fn key_evaluated() { probes::key(); } -/// Counts an event body a read decoded. Only the tests read the count. -fn body_decoded() { - #[cfg(test)] - probes::decode(); -} - /// The work counters the cost tests assert on: a page's cost is its keys evaluated, its bodies -/// decoded and the entries it sorted, and none of the three is visible in its result. +/// decoded — [`rows::probes`] counts every event-body decode, wherever it happens — and the +/// entries it sorted, and none of the three is visible in its result. #[cfg(test)] mod probes { use std::sync::Mutex; @@ -621,12 +616,11 @@ mod probes { pub(super) static COUNTING: Mutex<()> = Mutex::new(()); static KEYS: AtomicUsize = AtomicUsize::new(0); - static DECODES: AtomicUsize = AtomicUsize::new(0); static SORTED: AtomicUsize = AtomicUsize::new(0); pub(super) fn reset() { KEYS.store(0, Ordering::Relaxed); - DECODES.store(0, Ordering::Relaxed); + super::rows::probes::reset(); SORTED.store(0, Ordering::Relaxed); } @@ -634,10 +628,6 @@ mod probes { KEYS.fetch_add(1, Ordering::Relaxed); } - pub(super) fn decode() { - DECODES.fetch_add(1, Ordering::Relaxed); - } - pub(super) fn sorted(entries: usize) { SORTED.fetch_add(entries, Ordering::Relaxed); } @@ -647,7 +637,7 @@ mod probes { } pub(super) fn decodes() -> usize { - DECODES.load(Ordering::Relaxed) + super::rows::probes::decodes() } pub(super) fn sorted_entries() -> usize { @@ -1400,10 +1390,9 @@ impl Matcher<'_> { if self.gates.nip56 && self.reported(txn, databases, id, &header.pubkey)? { return Ok(None); } - if self.carries_other_tag_criteria() && { - body_decoded(); - !tag_criteria_match(&self.filter.tags, &rows::decode_event(value)?, self.proven) - } { + if self.carries_other_tag_criteria() + && !tag_criteria_match(&self.filter.tags, &rows::decode_event(value)?, self.proven) + { return Ok(None); } Ok(Some(header)) @@ -2935,43 +2924,6 @@ mod tests { ); } - /// One event carrying two values a `#g` prefix matches is one row: the tags index holds a - /// key per value, and the capture's query counts rows. - #[test] - fn a_wildcard_tag_range_counts_a_row_once() { - let (_directory, env, databases) = store(); - let record = signed( - 7, - 1, - 1_700_000_000, - vec![ - vec!["g".to_owned(), "u4a".to_owned()], - vec!["g".to_owned(), "u4b".to_owned()], - ], - ); - write(&env, &databases, &[record]); - let transaction = env.read_txn().expect("a read transaction"); - let mut tags = BTreeMap::new(); - tags.insert("#g".to_owned(), vec!["u4*".to_owned()]); - let filter = Filter { - tags, - ..Filter::default() - }; - - assert_eq!( - count( - &transaction, - &databases, - std::slice::from_ref(&filter), - &Gates::default(), - 0 - ) - .expect("the count"), - 1, - "one row, though the prefix matches two of its values" - ); - } - /// A count walks keys, not bodies: the id a tag key ends with says whether the row is /// already counted, so a wildcard count decodes nothing however many values it matches. #[test] diff --git a/crates/nostream-storage/src/lmdb/rows.rs b/crates/nostream-storage/src/lmdb/rows.rs index 013f56c11..e40e48310 100644 --- a/crates/nostream-storage/src/lmdb/rows.rs +++ b/crates/nostream-storage/src/lmdb/rows.rs @@ -106,6 +106,8 @@ pub(crate) fn encode_event(record: &EventRecord) -> Result, StorageError /// Decodes one event value. `row_id` is backend-defined and this backend writes none. pub(crate) fn decode_event(value: &[u8]) -> Result { + #[cfg(test)] + probes::decode(); let body = encoding::body("events", value)?; let mut record: EventRecord = serde_json::from_slice(body).map_err(|error| StorageError::InvalidRow { @@ -117,6 +119,32 @@ pub(crate) fn decode_event(value: &[u8]) -> Result { Ok(record) } +/// The body-decode counter the read tests assert on: every `serde_json` decode of an event body +/// passes through [`decode_event`], whatever reader asked for it. +/// +/// The count is per thread, so a test reads its own read's decodes and not a sibling's write +/// path; the blocking-pool callers never assert on it. +#[cfg(test)] +pub(crate) mod probes { + use std::cell::Cell; + + thread_local! { + static DECODES: Cell = const { Cell::new(0) }; + } + + pub(crate) fn reset() { + DECODES.with(|count| count.set(0)); + } + + pub(crate) fn decode() { + DECODES.with(|count| count.set(count.get() + 1)); + } + + pub(crate) fn decodes() -> usize { + DECODES.with(Cell::get) + } +} + /// Decodes one event value's fixed header, without its JSON body. pub(crate) fn event_header(value: &[u8]) -> Result { encoding::event_header("events", value) From 94a981540e47182bc9983e12e40a9ba5e2b91d47 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 01:06:19 -0500 Subject: [PATCH 58/60] docs(storage): qualify the lmdb record's residency and ordering claims --- docs/rewrite/decisions/0011-lmdb-backend.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/rewrite/decisions/0011-lmdb-backend.md b/docs/rewrite/decisions/0011-lmdb-backend.md index 0afda946b..5e6c39489 100644 --- a/docs/rewrite/decisions/0011-lmdb-backend.md +++ b/docs/rewrite/decisions/0011-lmdb-backend.md @@ -195,6 +195,12 @@ key order *is* the page's order stops at the page's own worst kept row — the h the first row the range happened to offer — keeping every row at that timestamp so a tie breaks as the capture's does. +One read keeps a second thing resident, and it is a count's rather than a page's: a `tags` count +whose criteria can select one row twice holds the id of each row its matcher accepts, for the +length of that filter's tag walk. Nothing bounds that set — `MAX_PAGE_KEYS` bounds a page — so it +grows with the rows the walk matches, and the same bullet's index that would visit each row once +is what would remove it. + The index is chosen from the filter, and which ranges can stop early follows from the key: | filter | index | stops when full | @@ -232,15 +238,17 @@ Two consequences of the same bound: - `count_by_filters` has no page, so it materialises no result set: each filter counts the rows it visits and skips one the count already holds — an earlier filter's page when that filter had a limit, or a predicate match when it had none — which is the capture's `UNION`. An - unlimited filter walks its ranges whole, which is what counting every match costs. + unlimited filter walks its ranges whole, which is what counting every match costs, and a + `tags` walk that can select one row twice also holds that row's id (see above). - A `tags` range is built from the family's own criteria, so a key inside it has already proved that family: the event's body is decoded only for the filter's *other* tag families, and a filter with one family decodes none. A count may hold the id each key ends with for the length of its tag walk, so a row two criteria — or two values of a `#g` prefix — both select is - counted at its smallest matching key and skipped at the rest, without a body decode. Only - those two shapes hold the set: a single exact criterion names one key per row. It is the - count's one resident cost, per distinct match, and the index has no order that would let a - walk visit each row once instead. + counted at the first key the walk reaches and skipped at the rest, without a body decode: the + ranges run in the criteria's order, so that key is not necessarily the row's smallest matching + one. Only those two shapes hold the set, whose population is one id per row its matcher + accepts: a single exact criterion names one key per row. It is the count's one resident cost, + and the index has no order that would let a walk visit each row once instead. ## Constraints move into Rust @@ -336,7 +344,8 @@ its op is one of the five above — so the list cannot grow without the diff sho 1000 rows qualify; both return 1000 rows classified by the same precedence (Assumption 10 in `spec.md`). The corpus is 483 events, so no current entry reaches it. - A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range, so a - client cannot choose how much a request materialises; a count is exact and holds no set. + client cannot choose how much a request materialises; a count materialises no result set and + holds ids only for a tag walk that can select a row twice, which no `MAX_PAGE_KEYS` bounds. The price is that a `#g`-prefix page walks its whole (time-bounded) range, an author page without kinds reads the created-at index, and so does an author × kind filter whose composition passes `MAX_PAGE_KEYS` ranges; an `ids` read whose criteria are exact is bounded From a98ad600a8d45fc4c96e5636f9195a6c93f440cf Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 07:42:53 -0500 Subject: [PATCH 59/60] fix(storage): bound an lmdb count's id set and check it before the matcher --- crates/nostream-storage/src/lmdb/encoding.rs | 62 +++- crates/nostream-storage/src/lmdb/events.rs | 306 ++++++++++++++++--- 2 files changed, 309 insertions(+), 59 deletions(-) diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs index 503eb9442..d60eafe0d 100644 --- a/crates/nostream-storage/src/lmdb/encoding.rs +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -492,6 +492,30 @@ pub(crate) fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec kept } +/// One inclusive key range a plan seeks, with the tag criterion that built it. +/// +/// A `tags` walk can reach one row more than once — two criteria, or two values of one `#g` +/// prefix — so a count past its id cap asks the criterion which keys of the row came earlier in +/// the walk. The criterion travels with the range it built, so the pairing cannot drift. +pub(crate) struct KeyRange<'a> { + /// The criterion this range was built from, and the family's criteria before it in the + /// search order. `None` for a range another index built. + pub(crate) tag: Option<(&'a [String], &'a str)>, + pub(crate) low: Vec, + pub(crate) high: Vec, +} + +impl<'a> KeyRange<'a> { + /// A range another index built: no tag criterion answers for the keys in it. + pub(crate) fn plain((low, high): (Vec, Vec)) -> Self { + Self { + tag: None, + low, + high, + } + } +} + /// The inclusive key ranges one tag criterion covers, one range per criterion that fits a key /// and in the criteria's order. /// @@ -499,12 +523,15 @@ pub(crate) fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec /// over every value that starts with the prefix, padded to the longest key the store can hold /// so a prefix long enough to leave little room still covers its own values. /// +/// Each range carries the criterion that built it, preceded by the family's earlier criteria, +/// so a count can ask which keys of one row this walk has already reached. +/// /// A criterion whose lower bound does not fit [`MAX_KEY_LEN`] selects nothing: a key carrying /// it could not be written, so seeking it would fail the read rather than answer it. -pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(Vec, Vec)> { +pub(crate) fn tag_ranges<'a>(name: &str, criteria: &'a [String]) -> Vec> { let is_geohash = name == "g"; let mut ranges = Vec::with_capacity(criteria.len()); - for criterion in criteria { + for (index, criterion) in criteria.iter().enumerate() { let mut head = Vec::with_capacity(name.len() + 1 + criterion.len()); head.extend_from_slice(name.as_bytes()); head.push(0); @@ -527,7 +554,11 @@ pub(crate) fn tag_ranges(name: &str, criteria: &[String]) -> Vec<(Vec, Vec, Vec); - /// The index one filter is answered from, the ranges within it, and what a key proves. struct Plan<'a> { database: Database, path: AccessPath, - ranges: Vec, + ranges: Vec>, /// Whether a range's own key order is the page's order, so a range stops at the page's own /// worst kept row instead of being walked whole. ordered: bool, /// The tag family the ranges were built from, with its criteria in the criteria's order — - /// `tag_ranges` builds one range per criterion that fits a key. A key in these ranges has - /// already proved that family, so the row's body is decoded only for the filter's other tag - /// families, and a `tags` count holds one id per row its matcher accepts rather than decoding. + /// `tag_ranges` builds one range per criterion that fits a key, carrying that criterion. A + /// key in these ranges has already proved that family, so the row's body is decoded only for + /// the filter's other tag families, and a `tags` count holds one id per row its matcher + /// accepts until the set is full, then reads a repeated key's body instead. tag: Option<(&'a str, &'a [String])>, } @@ -716,8 +714,9 @@ pub(crate) fn page( /// and skips one the count already holds — a row of an earlier filter's page when that filter had /// a limit (a page is at most [`MAX_PAGE_KEYS`] ids), or one its predicate matches when it had /// none — which is the capture's `UNION`. A `tags` walk whose criteria can select one row twice -/// also holds the ids of the rows it matches, for the length of that walk; no -/// [`MAX_PAGE_KEYS`] bounds that set. +/// also holds the ids of the rows it matches, at most [`MAX_PAGE_KEYS`] of them; past that the +/// walk reads a row's body to tell an earlier key of the walk from a first one, so the set stops +/// growing and the total stays exact. pub(crate) fn count( txn: &RoTxn<'_>, databases: &Databases, @@ -800,7 +799,8 @@ fn already_counted( /// A `tags` key names one value of one row, so a row's keys within this walk are one per /// matching value and per matching criterion. Whichever key the walk reaches first answers for /// the row — ranges run in the criteria's order, so that is not necessarily the smallest of its -/// matching values — and the id it ends with answers the rest without reading the body. +/// matching values — and the id it ends with answers the rest until the set holds +/// [`MAX_PAGE_KEYS`] of them. fn count_matches( txn: &RoTxn<'_>, databases: &Databases, @@ -808,6 +808,20 @@ fn count_matches( counted: &Counted<'_>, gates: &Gates, now: i64, +) -> Result { + count_matches_bounded(txn, databases, filter, counted, gates, now, MAX_PAGE_KEYS) +} + +/// [`count_matches`], with the number of ids the walk may hold given explicitly, so a test can +/// compare the bounded walk against an unbounded one over one corpus. +fn count_matches_bounded( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + counted: &Counted<'_>, + gates: &Gates, + now: i64, + cap: usize, ) -> Result { if !admits(filter, gates)? { return Ok(0); @@ -823,37 +837,53 @@ fn count_matches( // A single exact criterion names one key per row, so only a second criterion or a `#g` // prefix can walk one row twice, and only then is the id set paid for. // - // shortcut: such a walk holds one id per row its matcher accepts for the walk's length, - // against a body decode per key; upgrade when a single count's match set outgrows the - // request's memory budget, which needs an index order that visits a row once. + // shortcut: the set holds at most `MAX_PAGE_KEYS` ids; a repeated key past that reads the + // row's body to ask whether an earlier key of this walk already answered for it, against a + // slot per repeated row. Upgrade when a count past the cap must decode nothing, which needs + // an index order that visits a row once. let mut counted_ids = HashSet::new(); let dedupe = plan.tag.is_some_and(|(name, criteria)| { criteria.len() > 1 || (name == "g" && criteria.iter().any(|criterion| criterion.ends_with('*'))) }); let mut total = 0; - for (low, high) in &plan.ranges { - let range = encoding::bounds(low, high); - for entry in plan.database.range(txn, &range)? { + for range in &plan.ranges { + let bounds = encoding::bounds(&range.low, &range.high); + for entry in plan.database.range(txn, &bounds)? { let (key, _) = entry?; key_evaluated(); - if let Some((name, criteria)) = plan.tag { - let value = encoding::tag_value(key)?; - if !tag_value_matches(name == "g", criteria, value) { - continue; + let value = match plan.tag { + Some((name, criteria)) => { + let value = encoding::tag_value(key)?; + if !tag_value_matches(name == "g", criteria, value) { + continue; + } + Some(value) } - } + None => None, + }; let Some((_, id)) = candidate(txn, databases, plan.path, key)? else { continue; }; - if matcher.matches(txn, databases, &id)?.is_none() { + // The set holds only rows this walk accepted, so a hit is a row it already counted: + // no second test and no second decode. + if dedupe && counted_ids.contains(&id) { continue; } - // The verdict is the row's, not the key's, so a second key is decided the same way - // and only a row this filter matches takes a slot in the set. - if dedupe && !counted_ids.insert(id) { + if matcher.matches(txn, databases, &id)?.is_none() { continue; } + if dedupe { + // The verdict is the row's, not the key's, so a second key is decided the same + // way and only a row this filter matches takes a slot in the set. + if counted_ids.len() < cap { + counted_ids.insert(id); + } else if let (Some((name, _)), Some((earlier, criterion))) = (plan.tag, range.tag) + && counted_earlier(txn, databases, name, earlier, criterion, value, &id)? + { + continue; + } + } if !already_counted(txn, databases, counted, gates, now, &id)? { total += 1; } @@ -862,6 +892,59 @@ fn count_matches( Ok(total) } +/// Whether an earlier key of this walk already selects the row `id`, so counting it here would +/// count the row twice — the check a count makes once its id set is full. +/// +/// Two ranges select one row when two criteria of the family match its tags; one range selects +/// it twice when a `#g` prefix matches two of its tags. The family's criteria before `criterion` +/// are passed in their order, so the first of them that matches answers for the row, and within +/// one prefix range the walk reaches the smallest matching value first. +/// +/// This is the one place a count reads a row's body instead of its key: the key names a single +/// value, and the other values the row carries are only in the body. +fn counted_earlier( + txn: &RoTxn<'_>, + databases: &Databases, + name: &str, + earlier: &[String], + criterion: &str, + value: Option<&str>, + id: &EventId, +) -> Result { + let is_geohash = name == "g"; + let prefix = criterion.strip_suffix('*').filter(|_| is_geohash); + // A first exact criterion is the row's only key in this walk, so nothing earlier answers + // for it and the body is not read. + if earlier.is_empty() && prefix.is_none() { + return Ok(false); + } + let Some(stored) = databases.events.get(txn, id.as_bytes())? else { + return Ok(false); + }; + let record = rows::decode_event(stored)?; + if earlier.iter().any(|earlier_criterion| { + record_has_tag(&record, name, std::slice::from_ref(earlier_criterion)) + }) { + return Ok(true); + } + if prefix.is_none() { + return Ok(false); + } + let Some(value) = value else { + return Ok(false); + }; + Ok(record.event.tags.iter().any(|tag| { + matches!( + (tag.first(), tag.get(1)), + (Some(tag_name), Some(candidate)) + if tag_name == name + && !candidate.is_empty() + && matching_value(is_geohash, criterion, candidate).is_some() + && candidate.as_str() < value + ) + })) +} + /// The page a filter may materialise: its own limit or the capture's default, and never more /// than [`MAX_PAGE_KEYS`] however large the limit on the wire is. fn page_limit(filter: &Filter) -> usize { @@ -933,8 +1016,8 @@ fn branch( }; let descending = filter.limit.is_some(); let mut page = Page::new(descending); - for (low, high) in &plan.ranges { - let range = encoding::bounds(low, high); + for range in &plan.ranges { + let bounds = encoding::bounds(&range.low, &range.high); // A range starts at the boundary the page already holds: a range whose oldest row is // past it has nothing to add, and one that is not is walked from its own end. let mut boundary = if plan.ordered { @@ -942,7 +1025,7 @@ fn branch( } else { None }; - for entry in scan(plan.database, txn, &range, plan.ordered && descending)? { + for entry in scan(plan.database, txn, &bounds, plan.ordered && descending)? { let (key, _) = entry?; key_evaluated(); let Some(found) = candidate(txn, databases, plan.path, key)? else { @@ -1127,7 +1210,7 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { return Plan { database: databases.events, path: AccessPath::Ids, - ranges: encoding::hex_ranges(ids, 0), + ranges: plain_ranges(encoding::hex_ranges(ids, 0)), // The `events` key is the id, so the range runs in id order, not time order. ordered: false, tag: None, @@ -1143,7 +1226,7 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { return Plan { database: databases.events_by_author, path: AccessPath::Authors, - ranges, + ranges: plain_ranges(ranges), // A range is `pubkey ‖ kind ‖ created_at ‖ event_id`, so it runs in time order // only when every criterion is a whole pubkey and every range one author and // kind. @@ -1164,7 +1247,7 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { return Plan { database: databases.events_by_created_at, path: AccessPath::CreatedAt, - ranges: created_at_ranges(filter), + ranges: plain_ranges(created_at_ranges(filter)), ordered: true, tag: None, }; @@ -1183,12 +1266,18 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { Plan { database: databases.events_by_created_at, path: AccessPath::CreatedAt, - ranges: created_at_ranges(filter), + ranges: plain_ranges(created_at_ranges(filter)), ordered: true, tag: None, } } +/// The ranges of a plan that reads an index a tag family did not build, so no criterion answers +/// for the keys in them. +fn plain_ranges<'a>(ranges: Vec<(Vec, Vec)>) -> Vec> { + ranges.into_iter().map(encoding::KeyRange::plain).collect() +} + /// One range per author criterion on `events_by_author`, whose key is /// `pubkey ‖ kind ‖ created_at ‖ event_id`. A whole pubkey composes one range per kind, so the /// `pubkey ‖ kind ‖ created_at` ordering the layout writes lets a page for one author and one @@ -1203,7 +1292,7 @@ fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { /// `None` when the whole-pubkey ranges and kinds compose more than [`MAX_PAGE_KEYS`] ranges: a /// filter can name a thousand authors and a thousand kinds, and their product is not bounded by /// the page, so the caller answers it from the created-at index rather than materialising it. -fn author_kind_ranges(authors: &[String], kinds: &[Kind]) -> Option> { +fn author_kind_ranges(authors: &[String], kinds: &[Kind]) -> Option, Vec)>> { let mut kinds: Vec = kinds .iter() .map(|kind| i32::try_from(kind.0).unwrap_or(i32::MAX)) @@ -1450,19 +1539,25 @@ pub(crate) fn generic_tag_name(key: &str) -> Option<&str> { Some(&key[1..2]) } -/// Whether a tag value satisfies one of a family's criteria: an exact value, and only `#g` -/// honours a trailing `*` as a prefix. +/// The value a criterion selects, when it selects `value`: an exact equality, and only `#g` +/// honours a trailing `*` as a prefix. The one place the wildcard rule is written. +fn matching_value<'v>(is_geohash: bool, criterion: &str, value: &'v str) -> Option<&'v str> { + let matches = match criterion.strip_suffix('*').filter(|_| is_geohash) { + Some(prefix) => value.starts_with(prefix), + None => value == criterion, + }; + matches.then_some(value) +} + +/// Whether a tag value satisfies one of a family's criteria. /// /// `event_tags` holds no empty value, so an empty one satisfies nothing. The criterion a `tags` /// key carries is checked here, from the key, and the same rule answers the decoded record. fn tag_value_matches(is_geohash: bool, criteria: &[String], value: &str) -> bool { !value.is_empty() - && criteria.iter().any(|criterion| { - match criterion.strip_suffix('*').filter(|_| is_geohash) { - Some(prefix) => value.starts_with(prefix), - None => value == criterion, - } - }) + && criteria + .iter() + .any(|criterion| matching_value(is_geohash, criterion, value).is_some()) } /// Whether an event carries a tag called `name` whose value satisfies one of `criteria`. @@ -1735,14 +1830,14 @@ impl Stream for ReconciliationStream { fn stream_plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { let chosen = plan(filter, databases); let exact_ids = - chosen.path == AccessPath::Ids && chosen.ranges.iter().all(|(low, high)| low == high); + chosen.path == AccessPath::Ids && chosen.ranges.iter().all(|range| range.low == range.high); if exact_ids || (chosen.ordered && chosen.ranges.len() == 1) { return chosen; } Plan { database: databases.events_by_created_at, path: AccessPath::CreatedAt, - ranges: created_at_ranges(filter), + ranges: plain_ranges(created_at_ranges(filter)), ordered: true, tag: None, } @@ -1777,10 +1872,10 @@ fn scan_batch( let mut items = Vec::new(); let mut evaluated = 0; while position.index < plan.ranges.len() { - let (low, high) = &plan.ranges[position.index]; - let range = encoding::resume_bounds(position.after.as_deref(), low, high); + let range = &plan.ranges[position.index]; + let bounds = encoding::resume_bounds(position.after.as_deref(), &range.low, &range.high); let mut reached_cap = false; - for entry in plan.database.range(txn, &range)? { + for entry in plan.database.range(txn, &bounds)? { if items.len() >= STREAM_BATCH_ITEMS || evaluated >= STREAM_BATCH_KEYS { reached_cap = true; break; @@ -1832,9 +1927,9 @@ fn scan_ids( position: &mut Position, ) -> Result, StorageError> { let mut items = Vec::with_capacity(plan.ranges.len()); - for (low, high) in &plan.ranges { - let range = encoding::bounds(low, high); - for entry in plan.database.range(txn, &range)? { + for range in &plan.ranges { + let bounds = encoding::bounds(&range.low, &range.high); + for entry in plan.database.range(txn, &bounds)? { let (key, _) = entry?; key_evaluated(); let Some((created_at, event_id)) = candidate(txn, databases, plan.path, key)? else { @@ -1988,9 +2083,14 @@ mod tests { /// An open store on a temporary directory. fn store() -> (tempfile::TempDir, Env, Databases) { + store_with_map(8 << 20) + } + + /// An open store on a temporary directory, with room for a corpus past `MAX_PAGE_KEYS`. + fn store_with_map(map_size: usize) -> (tempfile::TempDir, Env, Databases) { let directory = tempfile::tempdir().expect("a temporary directory"); let environment = - LmdbEnv::open(directory.path().join("lmdb"), 8 << 20).expect("an environment"); + LmdbEnv::open(directory.path().join("lmdb"), map_size).expect("an environment"); let env = environment.env().clone(); let mut transaction = env.write_txn().expect("a write transaction"); let databases = Databases::create(&env, &mut transaction).expect("the databases"); @@ -3016,6 +3116,112 @@ mod tests { ); } + /// A count's total does not depend on how many ids its walk may hold: past the cap a + /// repeated key reads the row's body to tell an earlier key from a first one, and this runs + /// that walk and the one the set alone answers over one corpus. + #[test] + fn a_count_past_its_id_cap_totals_what_the_unbounded_walk_totals() { + let _serial = counting(); + let (_directory, env, databases) = store_with_map(32 << 20); + // Past `MAX_PAGE_KEYS`, so every filter below crosses the cap: the first 5 000 ids fill + // the set and the rest of the walk takes the body check. + let rows = MAX_PAGE_KEYS as i64 + 200; + let records: Vec = (0..rows) + .map(|step| { + signed( + (step % 200 + 1) as u8, + 1, + step, + vec![ + vec!["t".to_owned(), "a".to_owned()], + vec!["t".to_owned(), "b".to_owned()], + vec!["g".to_owned(), "u4a".to_owned()], + vec!["g".to_owned(), "u4b".to_owned()], + ], + ) + }) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let tags = |families: &[(&str, &[&str])]| -> BTreeMap> { + families + .iter() + .map(|(key, criteria)| { + ( + (*key).to_owned(), + criteria.iter().map(|value| (*value).to_owned()).collect(), + ) + }) + .collect() + }; + let cases = [ + ("two criteria", tags(&[("#t", &["a", "b"])])), + ("a repeated criterion", tags(&[("#t", &["a", "a"])])), + ("a prefix over two values", tags(&[("#g", &["u4*"])])), + ( + "a prefix and an exact value it covers", + tags(&[("#g", &["u4*", "u4b"])]), + ), + ( + "a value only the later criterion names", + tags(&[("#t", &["zz", "a"])]), + ), + ]; + for (what, tags) in cases { + let filter = Filter { + tags, + ..Filter::default() + }; + let counted = Counted::default(); + probes::reset(); + let capped = count_matches_bounded( + &transaction, + &databases, + &filter, + &counted, + &Gates::default(), + 0, + MAX_PAGE_KEYS, + ) + .expect("the bounded count"); + let decodes = probes::decodes(); + probes::reset(); + let uncapped = count_matches_bounded( + &transaction, + &databases, + &filter, + &counted, + &Gates::default(), + 0, + usize::MAX, + ) + .expect("the unbounded count"); + assert_eq!( + capped, uncapped, + "{what}: the cap changes how, not how many" + ); + assert_eq!(capped, rows as u64, "{what}: every row matches"); + assert_eq!(probes::decodes(), 0, "{what}: the set decodes no body"); + assert!( + decodes > 0, + "{what}: the walk past the cap really took the body check" + ); + probes::reset(); + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + capped, + "{what}: the count's own path agrees with the bounded walk" + ); + } + } + /// The expiry pass reads the index no further than the last expired row: a future expiry is /// not a candidate, and a row that qualifies only by age is the created-at pass's business. #[test] From f31011f6830d400767d15f01d04a2f7d3d05335e Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Sun, 8 Nov 2026 07:42:54 -0500 Subject: [PATCH 60/60] docs(storage): record the count's id cap and what it costs --- docs/rewrite/decisions/0011-lmdb-backend.md | 41 ++++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/docs/rewrite/decisions/0011-lmdb-backend.md b/docs/rewrite/decisions/0011-lmdb-backend.md index 5e6c39489..c8f34cf4e 100644 --- a/docs/rewrite/decisions/0011-lmdb-backend.md +++ b/docs/rewrite/decisions/0011-lmdb-backend.md @@ -196,10 +196,16 @@ the first row the range happened to offer — keeping every row at that timestam as the capture's does. One read keeps a second thing resident, and it is a count's rather than a page's: a `tags` count -whose criteria can select one row twice holds the id of each row its matcher accepts, for the -length of that filter's tag walk. Nothing bounds that set — `MAX_PAGE_KEYS` bounds a page — so it -grows with the rows the walk matches, and the same bullet's index that would visit each row once -is what would remove it. +whose criteria can select one row twice holds an id per row its matcher accepts, and that set is +bounded by `MAX_PAGE_KEYS` — the same 5 000. Below the cap the walk reads no body; at the cap it +stops inserting and decides each further key with a bounded-memory check instead: a repeated key +reads the row's body and asks whether an earlier key of the walk already answered for it, an +earlier criterion that matches one of its tags or, within one `#g` prefix range, a smaller value +the prefix matches. The total is therefore the same at every corpus size, and the set is a +constant 270 352 B once full (measured live: 5 000 ids at a `HashSet` capacity of 7 168 slots, +against 1 081 360 B for the same 20 000-row corpus with the set unbounded). The price of the +cap is a body decode per repeated key past it; an index order that visits each row once is what +would remove that too. The index is chosen from the filter, and which ranges can stop early follows from the key: @@ -239,16 +245,21 @@ Two consequences of the same bound: it visits and skips one the count already holds — an earlier filter's page when that filter had a limit, or a predicate match when it had none — which is the capture's `UNION`. An unlimited filter walks its ranges whole, which is what counting every match costs, and a - `tags` walk that can select one row twice also holds that row's id (see above). + `tags` walk that can select one row twice also holds that row's id, up to `MAX_PAGE_KEYS` + ids (see above). The slot is filled before the cross-filter check, so the set also holds rows + an earlier filter already counts; the same cap bounds that residue. - A `tags` range is built from the family's own criteria, so a key inside it has already proved that family: the event's body is decoded only for the filter's *other* tag families, and a - filter with one family decodes none. A count may hold the id each key ends with for the length - of its tag walk, so a row two criteria — or two values of a `#g` prefix — both select is - counted at the first key the walk reaches and skipped at the rest, without a body decode: the + filter with one family decodes none. The id a key ends with answers the rest of the walk, so a + row two criteria — or two values of a `#g` prefix — both select is counted at the first key + the walk reaches and skipped at the rest without a body decode, while the set has room: the ranges run in the criteria's order, so that key is not necessarily the row's smallest matching - one. Only those two shapes hold the set, whose population is one id per row its matcher - accepts: a single exact criterion names one key per row. It is the count's one resident cost, - and the index has no order that would let a walk visit each row once instead. + one. Once the set is full the walk reads the body of a repeated key instead and asks whether + an earlier key of the walk already answered for it — an earlier criterion, or a smaller value + of the same prefix — so the count stays exact and its resident set stays a constant. Only those + two shapes hold the set, whose population is one id per row its matcher accepts: a single exact + criterion names one key per row. The tags index has no order that visits each row once, which + is why the cap trades a slot past it for a body decode. ## Constraints move into Rust @@ -343,9 +354,11 @@ its op is one of the five above — so the list cannot grow without the diff sho - Retention's batch composition can differ from Postgres' planner-chosen order when more than 1000 rows qualify; both return 1000 rows classified by the same precedence (Assumption 10 in `spec.md`). The corpus is 483 events, so no current entry reaches it. -- A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range, so a - client cannot choose how much a request materialises; a count materialises no result set and - holds ids only for a tag walk that can select a row twice, which no `MAX_PAGE_KEYS` bounds. +- A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range; a count + materialises no result set, and the ids it holds for a tag walk that can select a row twice + are bounded by the same `MAX_PAGE_KEYS` — 270 352 B of resident set once full, however large + the corpus, with a body decode per repeated key past the cap. So a client cannot choose how + much a request materialises. The price is that a `#g`-prefix page walks its whole (time-bounded) range, an author page without kinds reads the created-at index, and so does an author × kind filter whose composition passes `MAX_PAGE_KEYS` ranges; an `ids` read whose criteria are exact is bounded