diff --git a/.changeset/storage-lmdb-backend.md b/.changeset/storage-lmdb-backend.md new file mode 100644 index 000000000..a845151cc --- /dev/null +++ b/.changeset/storage-lmdb-backend.md @@ -0,0 +1,2 @@ +--- +--- diff --git a/CLAUDE.md b/CLAUDE.md index 0e885d22c..034dd0b06 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,7 +47,10 @@ cargo test -p nostream-storage --test migrations --test parity -- --ignored --te `PARITY_ADMIN_URL` names the administrative database (CI sets `postgres://postgres:postgres@localhost:5432/postgres`); the targets create, migrate and drop their own scratch databases. `scripts/storage-parity.sh` runs both plus the -TypeScript Cucumber suite against the sqlx-migrated schema — the complete local oracle. +TypeScript Cucumber suite against the sqlx-migrated schema — the complete local oracle. The +LMDB arm of that oracle needs no database: `cargo test -p nostream-storage --test parity_lmdb +-- --nocapture` replays the same fixture on a temporary store and runs inside the fifth +validation command above. - **`--workspace`, never a single crate.** A crate-local `cargo test -p` proves nothing about a workspace lint that another member trips. diff --git a/Cargo.lock b/Cargo.lock index 18987eb41..ef6843886 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1353,6 +1353,8 @@ dependencies = [ "tempfile", "thiserror", "tokio", + "tracing", + "uuid", ] [[package]] @@ -2796,6 +2798,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" diff --git a/Cargo.toml b/Cargo.toml index aeab00ba8..852232b2f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ tracing = "0.1" tracing-opentelemetry = "0.34" tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } url = "2" +uuid = { version = "1", features = ["v4"] } [workspace.lints.rust] # `deny` rather than `forbid`: heed's LMDB `EnvOpenOptions::open` is an `unsafe fn` diff --git a/crates/nostream-storage/Cargo.toml b/crates/nostream-storage/Cargo.toml index 41536ea2d..516868329 100644 --- a/crates/nostream-storage/Cargo.toml +++ b/crates/nostream-storage/Cargo.toml @@ -5,19 +5,22 @@ edition.workspace = true rust-version.workspace = true license.workspace = true repository.workspace = true -description = "Repository traits, backend selection and the Postgres storage backend" +description = "Repository traits, backend selection and the Postgres and LMDB storage backends" [features] -# The Postgres backend and its driver. On by default because Postgres is the reference -# implementation; a build that only needs the trait seam (the LMDB follow-up) can turn it -# off and lose the sqlx link entirely. -default = ["postgres"] +# Both backends are on by default: Postgres is the reference implementation and the LMDB +# backend is what the rewrite exists to provide. A build that only needs the trait seam +# turns both off and links no driver at all. +default = ["postgres", "lmdb"] postgres = ["dep:sqlx"] +# The LMDB backend, its environment and its row encodings. `uuid` is only used to mint +# the canonical text ids the outbox and delivery-log columns generate. +lmdb = ["dep:heed", "dep:uuid", "dep:tracing"] [dependencies] chrono.workspace = true futures-core.workspace = true -heed.workspace = true +heed = { workspace = true, optional = true } nostream-config.workspace = true nostream-core.workspace = true serde.workspace = true @@ -25,6 +28,8 @@ serde_json.workspace = true sqlx = { workspace = true, optional = true } thiserror.workspace = true tokio.workspace = true +tracing = { workspace = true, optional = true } +uuid = { workspace = true, optional = true } [dev-dependencies] serde_yaml.workspace = true diff --git a/crates/nostream-storage/src/backend.rs b/crates/nostream-storage/src/backend.rs index 81e2d2218..980ec522f 100644 --- a/crates/nostream-storage/src/backend.rs +++ b/crates/nostream-storage/src/backend.rs @@ -1,47 +1,33 @@ -//! Backend selection and the two backends. Mirrors `src/utils/event-store.ts` (the selection -//! half) and the storage boundary in `src/@types/adapters.ts`. +//! Backend selection and the LMDB environment. Mirrors `src/utils/event-store.ts` (the +//! selection half) and the storage boundary in `src/@types/adapters.ts`. use std::path::{Path, PathBuf}; -use heed::types::Str; +use nostream_core::EventStoreBackend; -use crate::error::StorageError; -use heed::{Database, Env, EnvOpenOptions}; - -/// The storage backends the rewrite can be configured with (`src/constants/base.ts` -/// `EventStoreBackend`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Backend { - Postgres, - Lmdb, -} +#[cfg(feature = "lmdb")] +use heed::{Env, EnvOpenOptions}; -impl Backend { - /// The settings spelling of this backend. - pub const fn as_str(self) -> &'static str { - match self { - Self::Postgres => "postgres", - Self::Lmdb => "lmdb", - } - } +#[cfg(feature = "lmdb")] +use crate::error::StorageError; +#[cfg(feature = "lmdb")] +use crate::lmdb::{LMDB_MAX_DBS, LMDB_MAX_READERS}; - /// Parses a settings value. - pub fn parse(value: &str) -> Option { - match value { - "postgres" => Some(Self::Postgres), - "lmdb" => Some(Self::Lmdb), - _ => None, - } - } -} +/// The directory one backend's data lives in, under the root a caller supplies. +/// `src/@types/adapters.ts` names the LMDB store as a directory of its own beside the +/// database it replaces. +pub const LMDB_DIR: &str = "lmdb"; -impl std::fmt::Display for Backend { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(self.as_str()) +/// Resolves the data directory under `root` for one backend. A backend that keeps its data +/// somewhere other than a directory of its own answers `None`. +pub fn data_dir(root: &Path, backend: EventStoreBackend) -> Option { + match backend { + EventStoreBackend::Lmdb => Some(root.join(LMDB_DIR)), + EventStoreBackend::Postgres => None, } } -/// An opened LMDB environment with one named database of string keys and values. +/// An opened LMDB environment, with the named databases the backend creates. /// /// The constructor is the crate's single `unsafe` boundary. heed 0.20.5 marks /// `EnvOpenOptions::open` unsafe because an LMDB environment memory-maps a file shared with @@ -51,34 +37,36 @@ impl std::fmt::Display for Backend { /// and `Error::BadOpenOptions` when they differ (heed-0.20.5 `src/env.rs`, `OPENED_ENV`). /// heed's documented preconditions, and where each stands today: /// -/// - **No long-lived transactions** — upheld by the code: `put`/`get` open a transaction, -/// use it, and commit/drop it before returning; nothing holds one across an await. +/// - **No long-lived transactions** — upheld by the code: every repository call opens a +/// transaction, uses it, and commits or drops it inside one `spawn_blocking` closure; +/// nothing holds one across an await. /// - **No aborting the process with an active transaction** — relied on the operator for -/// SIGKILL-class exits; the Phase-2 daemon drain path must ensure a clean shutdown +/// SIGKILL-class exits; the daemon drain path must ensure a clean shutdown /// (SIGINT/SIGTERM drain through `ShutdownState` does not abort mid-transaction). -/// - **No network filesystems** — relied on the operator today: `open` accepts any path and -/// only `create_dir_all` before mapping; the Phase-2 backend factory must refuse a remote -/// location before this slice is wired into settings. +/// - **No network filesystems** — relied on the operator: `open` accepts any path and only +/// `create_dir_all` before mapping; the daemon's backend factory must refuse a remote +/// location before this environment is wired into settings. /// - **`NO_LOCK` requires self-managed concurrency; a broken lock file means /// synchronization issues and possible UB** — split responsibility. The code guarantees /// its half: no env flags are passed (`EnvOpenOptions::new()` only), so `NO_LOCK` is /// never in play and this crate never reads, writes or removes the lock file. The -/// operator must supply the rest: the data directory's lock file must not be broken -/// (stale locks left by a crashed process on a filesystem without proper flock support, -/// manual deletion, two processes on different hosts sharing a volume — LMDB cannot -/// recover from any of those, per heed's contract). +/// operator must supply the rest: the data directory's lock file must not be broken. /// - **One `map_size` per path for the life of the process** — upheld by heed + propagated: /// heed's per-process `OPENED_ENV` registry returns the same `Env` for a second open with /// equal options and `Error::BadOpenOptions` when they differ, and `open` surfaces that -/// error to the caller rather than opening a second environment. +/// error to the caller rather than opening a second environment. `Env::resize` is unsafe +/// as well, so the map is sized once and never grown. +#[cfg(feature = "lmdb")] #[derive(Debug, Clone)] pub struct LmdbEnv { env: Env, path: PathBuf, } +#[cfg(feature = "lmdb")] impl LmdbEnv { - /// Opens (creating if needed) an LMDB environment at `path` with `map_size` bytes. + /// Opens (creating if needed) an LMDB environment at `path` with `map_size` bytes, room + /// for the backend's named databases and its reader slots. pub fn open(path: impl Into, map_size: usize) -> Result { let path = path.into(); std::fs::create_dir_all(&path).map_err(|source| StorageError::CreateDirectory { @@ -87,40 +75,26 @@ impl LmdbEnv { })?; // # Safety: the preconditions on `LmdbEnv` above, each marked upheld or relied-on: - // no env flags (LMDB's locking runs), transactions never escape the call, and - // heed's per-process dedup turns a second open into the same Env or - // BadOpenOptions. The network-filesystem precondition rests on the operator - // passing a local path; see the list above. + // no env flags (LMDB's locking runs), transactions never escape the closure they are + // opened in, and heed's per-process dedup turns a second open into the same Env or + // BadOpenOptions. The network-filesystem precondition rests on the operator passing a + // local path; see the list above. #[allow(unsafe_code)] - let env = - unsafe { EnvOpenOptions::new().map_size(map_size).open(&path) }.map_err(|source| { - StorageError::OpenLmdb { - path: path.clone(), - source, - } - })?; + let env = unsafe { + EnvOpenOptions::new() + .map_size(map_size) + .max_dbs(LMDB_MAX_DBS) + .max_readers(LMDB_MAX_READERS) + .open(&path) + } + .map_err(|source| StorageError::OpenLmdb { + path: path.clone(), + source, + })?; Ok(Self { env, path }) } - /// Writes one value under one key, committing immediately. - pub fn put(&self, key: &str, value: &str) -> Result<(), StorageError> { - let mut transaction = self.env.write_txn()?; - let database: Database = self.env.create_database(&mut transaction, None)?; - database.put(&mut transaction, key, value)?; - transaction.commit()?; - Ok(()) - } - - /// Reads one value; `Ok(None)` means the key is absent. - pub fn get(&self, key: &str) -> Result, StorageError> { - let transaction = self.env.read_txn()?; - let Some(database) = self.env.open_database::(&transaction, None)? else { - return Ok(None); - }; - Ok(database.get(&transaction, key)?.map(str::to_owned)) - } - - /// The environment handle, for the Phase-2 repository implementations. + /// The environment handle, for the repository implementations. pub fn env(&self) -> &Env { &self.env } @@ -130,13 +104,3 @@ impl LmdbEnv { &self.path } } - -/// Resolves the database path under `root` for one backend. -/// -/// Phase 1 stub — job 005; will pick the per-backend data directory; see `src/@types/adapters.ts`. -pub fn data_dir(root: &Path, backend: Backend) -> Result { - let _ = (root, backend); - Err(StorageError::NotImplemented { - module: "storage::backend::data_dir", - }) -} diff --git a/crates/nostream-storage/src/error.rs b/crates/nostream-storage/src/error.rs index c2688f326..e0f1edbc5 100644 --- a/crates/nostream-storage/src/error.rs +++ b/crates/nostream-storage/src/error.rs @@ -49,10 +49,12 @@ pub enum StorageError { ValueOutOfRange { column: &'static str, value: String }, /// An LMDB environment or transaction failed. + #[cfg(feature = "lmdb")] #[error("lmdb operation failed: {0}")] Lmdb(#[from] heed::Error), /// An LMDB environment could not be opened at the requested path. + #[cfg(feature = "lmdb")] #[error("failed to open the lmdb environment at {path}: {source}")] OpenLmdb { path: std::path::PathBuf, @@ -60,6 +62,11 @@ pub enum StorageError { source: heed::Error, }, + /// An LMDB blocking task was cancelled before it returned (the runtime is shutting down). + #[cfg(feature = "lmdb")] + #[error("the lmdb task did not finish")] + LmdbTaskCancelled, + /// A value must be valid UTF-8 to be stored through the string codecs. #[error("lmdb values must be valid utf-8")] NotUtf8, diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index 707227afb..8bf3ae02c 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -1,11 +1,11 @@ //! `nostream-storage` owns the persistence seam: the nine repository traits, the streaming -//! query surface, the Postgres backend with its embedded migrations, the backend selection -//! and the LMDB environment. +//! query surface, the Postgres backend with its embedded migrations, the LMDB backend, the +//! backend selection and the LMDB environment. //! //! It owns these TypeScript paths from the capture commit //! (`242a3001d77149e4d70151ec03060c4ef44a1547`): //! -//! - `src/repositories/` (9 files, ported to `src/postgres/`) +//! - `src/repositories/` (9 of the 10 files, ported to `src/postgres/` and `src/lmdb/`) //! - `src/database/client.ts`, `src/database/transaction.ts` (`src/postgres/mod.rs`) //! - the schema: `migrations/` (37 knex migrations, ported to `migrations/*.sql`) //! - `src/utils/event-store.ts` (the mirror of `src/utils/hidden-content-cache.ts` is the @@ -16,21 +16,30 @@ //! - `src/factories/event-repository-factory.ts` //! //! What it deliberately does **not** own: the relay handlers that call these traits, the -//! payments service, and the LMDB backend. The cache is not a cache server: no Redis client -//! is added and `src/cache/client.ts` / `src/adapters/redis-adapter.ts` are not ported — the -//! decision and what would overturn it are ADR -//! `0010-services-worker-model-and-cache.md`. `LmdbEnv`'s environment slice and -//! `backend::data_dir` remain labelled stubs; the Postgres backend is complete. +//! payments service, the daemon's backend factory and the `nostream-cli migrate pg-to-lmdb` +//! tool. The cache is not a cache server: no Redis client is added and +//! `src/cache/client.ts` / `src/adapters/redis-adapter.ts` are not ported — the decision and +//! what would overturn it are ADR `0010-services-worker-model-and-cache.md`. Both backends are +//! complete: Postgres against the 37-migration schema, and LMDB against the same column +//! contract for 9 of the capture's 10 repositories — `user_subscriptions` has a TypeScript +//! repository and no trait yet, and `events.first_seen` is stored but never indexed +//! (`docs/rewrite/decisions/0011-lmdb-backend.md`). pub mod backend; pub mod error; pub mod event_store; +#[cfg(feature = "lmdb")] +pub mod lmdb; #[cfg(feature = "postgres")] pub mod postgres; pub mod records; pub mod repositories; -pub use backend::{Backend, LmdbEnv}; +#[cfg(feature = "lmdb")] +pub use backend::LmdbEnv; +pub use backend::data_dir; pub use error::StorageError; +#[cfg(feature = "lmdb")] +pub use lmdb::{LmdbBackend, MapUsage}; #[cfg(feature = "postgres")] pub use postgres::{PostgresBackend, PostgresTransaction}; diff --git a/crates/nostream-storage/src/lmdb/encoding.rs b/crates/nostream-storage/src/lmdb/encoding.rs new file mode 100644 index 000000000..d60eafe0d --- /dev/null +++ b/crates/nostream-storage/src/lmdb/encoding.rs @@ -0,0 +1,911 @@ +//! Key and value codecs for the LMDB store: the format byte, the fixed event header, the +//! big-endian key layout, timestamp truncation and the hex/tag ranges the event queries seek +//! with. `docs/rewrite/decisions/0011-lmdb-backend.md` records why each shape was chosen. + +use std::ops::Bound; + +use chrono::{DateTime, Utc}; +use nostream_core::{EventId, Kind, Pubkey}; + +use crate::error::StorageError; + +/// The first byte of every value in every database (ADR 0003 revisit trigger 6): a row +/// written by a later field set cannot be decoded as this one. +pub(crate) const ROW_FORMAT_VERSION: u8 = 1; + +/// The fixed event header before the JSON body: flags, `created_at`, `kind`, `expires_at`, +/// `pubkey` and one reserved byte, so a live test reads at most 55 bytes of a value. +pub(crate) const EVENT_HEADER_LEN: usize = 54; + +/// The header's "this row is soft-deleted" bit. +const FLAG_DELETED: u8 = 1; + +/// The header's "no expiry" sentinel: `expires_at` is a fixed `i64` slot. +const NO_EXPIRY: i64 = i64::MIN; + +/// The number of bytes a `created_at ‖ event_id` key appends after the timestamp. +const EVENT_ID_LEN: usize = 32; + +/// The width of the field a hex criterion is compared over: an id or a pubkey is 32 bytes. +pub(crate) const HEX_FIELD_LEN: usize = 32; + +/// LMDB's maximum key length (`MDB_MAXKEYSIZE`, the default this environment is opened with). +/// +/// It bounds every key this module builds. The `tags` key carries the tag value verbatim, so a +/// value longer than the room the key has left cannot be indexed and its event is refused at +/// the write: that is the one place this backend rejects a row Postgres would keep, and ADR +/// 0011 records it as a divergence. The `replaceable` key is the other variable-length key; its +/// components are length-prefixed ([`replaceable_key`]), which leaves it four bytes of room the +/// `tags` key does not have for the same value, so it refuses nothing the tag index keeps. +/// +/// shortcut: the tag index carries the value verbatim in the key, so a value of 469 bytes or +/// more (with a one-byte tag name) cannot be indexed and its event is refused; upgrade when such +/// an event has to be stored, which needs the value hashed into the key plus a second lookup, +/// or a larger `max_key_size` at the open. +const MAX_KEY_LEN: usize = 511; + +/// An invalid stored value, naming the database it was read from. +pub(crate) fn invalid(table: &'static str, message: String) -> StorageError { + StorageError::InvalidRow { + table, + column: "value", + value: message, + } +} + +/// The instant a row carries, truncated to the microsecond precision every PostgreSQL +/// `timestamp` column has, so a value written by either backend reads back the same. +pub(crate) fn micros(value: DateTime) -> DateTime { + DateTime::::from_timestamp_micros(value.timestamp_micros()).unwrap_or(value) +} + +/// An `i64` ordered by its bytes: the sign bit is flipped so a negative value sorts before +/// zero rather than after every positive one. +pub(crate) fn i64_key(value: i64) -> [u8; 8] { + ((value as u64) ^ (1 << 63)).to_be_bytes() +} + +/// The inverse of [`i64_key`]. +pub(crate) fn i64_from_key(bytes: &[u8]) -> Result { + let raw = u64::from_be_bytes(fixed::<8>("events", bytes)?); + Ok((raw ^ (1 << 63)) as i64) +} + +/// An `i32` ordered by its bytes, sign-flipped for the same reason as [`i64_key`]. +pub(crate) fn i32_key(value: i32) -> [u8; 4] { + ((value as u32) ^ (1 << 31)).to_be_bytes() +} + +/// The inverse of [`i32_key`]. +pub(crate) fn i32_from_key(bytes: &[u8]) -> Result { + let raw = u32::from_be_bytes(fixed::<4>("events", bytes)?); + Ok((raw ^ (1 << 31)) as i32) +} + +fn fixed(table: &'static str, bytes: &[u8]) -> Result<[u8; N], StorageError> { + bytes + .try_into() + .map_err(|_| invalid(table, format!("{N} bytes expected, {} found", bytes.len()))) +} + +/// One event value's fixed header, decoded without touching the JSON body. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct EventHeader { + pub deleted: bool, + pub created_at: i64, + pub kind: i32, + pub expires_at: Option, + pub pubkey: Pubkey, +} + +/// Encodes the fixed header of an event value. +pub(crate) fn header_bytes(header: &EventHeader) -> [u8; EVENT_HEADER_LEN] { + let mut bytes = [0u8; EVENT_HEADER_LEN]; + if header.deleted { + bytes[0] = FLAG_DELETED; + } + bytes[1..9].copy_from_slice(&header.created_at.to_be_bytes()); + bytes[9..13].copy_from_slice(&header.kind.to_be_bytes()); + bytes[13..21].copy_from_slice(&header.expires_at.unwrap_or(NO_EXPIRY).to_be_bytes()); + bytes[21..53].copy_from_slice(header.pubkey.as_bytes()); + // bytes[53] is reserved and written as zero. + bytes +} + +/// Checks the format byte and returns the value's body (the version byte and header gone). +pub(crate) fn body<'a>(table: &'static str, value: &'a [u8]) -> Result<&'a [u8], StorageError> { + check_version(table, value)?; + let start = 1 + EVENT_HEADER_LEN; + value + .get(start..) + .filter(|body| !body.is_empty()) + .ok_or_else(|| { + invalid( + table, + format!("the header is truncated at {} bytes", value.len()), + ) + }) +} + +/// The 32-byte event id an index key or a database value carries. +pub(crate) fn event_id_from(bytes: &[u8]) -> Result { + Ok(EventId::from_byte_array(fixed::<32>("events", bytes)?)) +} + +/// The 32-byte pubkey an index key carries. +pub(crate) fn pubkey_from(bytes: &[u8]) -> Result { + Ok(Pubkey::from_byte_array(fixed::<32>("events", bytes)?)) +} + +/// One key range as the borrowed pair heed's `range` accepts. An empty bound is +/// [`Bound::Unbounded`]: LMDB answers `MDB_BAD_VALSIZE` for a zero-length seek key, and the +/// created-at path uses the empty bound to mean "from the first key". +pub(crate) fn bounds<'a>(lower: &'a [u8], upper: &'a [u8]) -> (Bound<&'a [u8]>, Bound<&'a [u8]>) { + let start = if lower.is_empty() { + Bound::Unbounded + } else { + Bound::Included(lower) + }; + let end = if upper.is_empty() { + Bound::Unbounded + } else { + Bound::Included(upper) + }; + (start, end) +} + +/// One key range that resumes after `after`, exclusive, when a previous batch stopped there. +pub(crate) fn resume_bounds<'a>( + after: Option<&'a [u8]>, + lower: &'a [u8], + upper: &'a [u8], +) -> (Bound<&'a [u8]>, Bound<&'a [u8]>) { + let (start, end) = bounds(lower, upper); + let start = match after { + Some(after) => Bound::Excluded(after), + None => start, + }; + (start, end) +} + +/// Decodes an event value's fixed header, without its JSON body. +pub(crate) fn event_header(table: &'static str, value: &[u8]) -> Result { + check_version(table, value)?; + let end = 1 + EVENT_HEADER_LEN; + let header = value.get(1..end).ok_or_else(|| { + invalid( + table, + format!("the header is truncated at {} bytes", value.len()), + ) + })?; + let expires_at = i64::from_be_bytes(fixed::<8>(table, &header[13..21])?); + Ok(EventHeader { + deleted: header[0] & FLAG_DELETED != 0, + created_at: i64::from_be_bytes(fixed::<8>(table, &header[1..9])?), + kind: i32::from_be_bytes(fixed::<4>(table, &header[9..13])?), + expires_at: (expires_at != NO_EXPIRY).then_some(expires_at), + pubkey: Pubkey::from_byte_array(fixed::<32>(table, &header[21..53])?), + }) +} + +/// Rejects a value written by a different row format, and one too short to carry a version. +pub(crate) fn check_version(table: &'static str, value: &[u8]) -> Result<(), StorageError> { + match value.split_first() { + Some((version, _)) if *version == ROW_FORMAT_VERSION => Ok(()), + Some((version, _)) => Err(invalid( + table, + format!("unsupported format version {version}"), + )), + None => Err(invalid(table, "the value is empty".to_owned())), + } +} + +/// Refuses a value a `text` column cannot hold. +/// +/// PostgreSQL's `text` cannot store `0x00`, and this module's key separator is the same byte, so +/// one check serves a key component and a row's own text column: both backends reject the same +/// value rather than truncating it. +pub(crate) fn refuse_nul( + table: &'static str, + column: &'static str, + value: &str, +) -> Result<(), StorageError> { + if value.contains('\0') { + return Err(StorageError::InvalidRow { + table, + column, + value: value.to_owned(), + }); + } + Ok(()) +} + +/// A text component of a composite key, terminated so the next component reads unambiguously. +/// +/// The terminator is what every key *reader* in this module splits on, so the rule lives here +/// with the writers: a component that carries `0x00` cannot be read back, and the value is +/// refused by [`refuse_nul`] rather than truncated. +pub(crate) fn text_part( + value: &str, + table: &'static str, + column: &'static str, +) -> Result, StorageError> { + refuse_nul(table, column, value)?; + let mut bytes = Vec::with_capacity(value.len() + 1); + bytes.extend_from_slice(value.as_bytes()); + bytes.push(0); + Ok(bytes) +} + +/// The text a composite key's component carries, refusing bytes a `text` column could not hold. +pub(crate) fn text_from<'a>(bytes: &'a [u8], table: &'static str) -> Result<&'a str, StorageError> { + std::str::from_utf8(bytes).map_err(|_| StorageError::InvalidRow { + table, + column: "key", + value: "not utf-8".to_owned(), + }) +} + +/// A `created_at`-style timestamp component of a tail index key, in the microseconds every +/// PostgreSQL `timestamp` column carries. +pub(crate) fn stamp_key(value: DateTime) -> [u8; 8] { + i64_key(value.timestamp_micros()) +} + +/// `created_at ‖ event_id`. +pub(crate) fn created_at_key(created_at: i64, event_id: &EventId) -> Vec { + let mut key = Vec::with_capacity(8 + EVENT_ID_LEN); + key.extend_from_slice(&i64_key(created_at)); + key.extend_from_slice(event_id.as_bytes()); + key +} + +/// `expires_at ‖ event_id` and `deleted_at ‖ event_id`, which have the same shape as a +/// `created_at` key. +pub(crate) fn time_key(value: i64, event_id: &EventId) -> Vec { + created_at_key(value, event_id) +} + +/// `pubkey ‖ kind ‖ created_at ‖ event_id`. +pub(crate) fn author_key( + pubkey: &Pubkey, + kind: i32, + created_at: i64, + event_id: &EventId, +) -> Vec { + let mut key = Vec::with_capacity(32 + 4 + 8 + EVENT_ID_LEN); + key.extend_from_slice(pubkey.as_bytes()); + key.extend_from_slice(&i32_key(kind)); + key.extend_from_slice(&i64_key(created_at)); + key.extend_from_slice(event_id.as_bytes()); + key +} + +/// The tag a deduplication key's string element is written behind: its own bytes. +const DEDUP_STRING: u8 = 0; + +/// The tag any other deduplication element is written behind: its `serde_json` text. +const DEDUP_JSON: u8 = 1; + +/// `pubkey ‖ kind ‖ (tag u8 ‖ len u16 ‖ bytes)*`, one element per deduplication value: a string +/// as its own bytes, and anything else as its `serde_json` text behind a different tag. +/// +/// The length prefixes are what keep the key bounded by the value rather than by its JSON +/// escaping — a quote byte is one key byte, not two — and a key past [`MAX_KEY_LEN`] is refused +/// here, naming the column, before LMDB can reject the write. For the same `d` value this key +/// is four bytes shorter than the `tags` key, so the tag index is the one that refuses first. +// shortcut: a non-string element is written as its `serde_json` text, so `1` and `1.0` differ +// where `jsonb` would not; upgrade when a caller can pass a hand-built JSON number, which +// today's two callers cannot (spec Assumption 11). +pub(crate) fn replaceable_key( + pubkey: &Pubkey, + kind: i32, + deduplication: &[serde_json::Value], +) -> Result, StorageError> { + fn invalid(value: String) -> StorageError { + StorageError::InvalidRow { + table: "events", + column: "event_deduplication", + value, + } + } + fn length(len: usize) -> Result { + u16::try_from(len) + .map_err(|_| invalid("a deduplication element is 65536 bytes or longer".to_owned())) + } + + let mut key = Vec::with_capacity(32 + 4 + deduplication.len() * 3); + key.extend_from_slice(pubkey.as_bytes()); + key.extend_from_slice(&i32_key(kind)); + for value in deduplication { + match value { + serde_json::Value::String(text) => { + key.push(DEDUP_STRING); + key.extend_from_slice(&length(text.len())?.to_be_bytes()); + key.extend_from_slice(text.as_bytes()); + } + other => { + let json = serde_json::to_vec(other).map_err(|error| invalid(error.to_string()))?; + key.push(DEDUP_JSON); + key.extend_from_slice(&length(json.len())?.to_be_bytes()); + key.extend_from_slice(&json); + } + } + } + if key.len() > MAX_KEY_LEN { + return Err(invalid(format!( + "the deduplication key is {} bytes, more than the {MAX_KEY_LEN} a key may hold", + key.len() + ))); + } + Ok(key) +} + +/// `tag_name ‖ 0x00 ‖ tag_value ‖ 0x00 ‖ created_at ‖ event_id`. +/// +/// A name or value carrying `0x00` is unrepresentable in the capture's `text` columns and would +/// make the separator ambiguous; [`text_part`] owns that refusal, so both components get it. +pub(crate) fn tag_key( + name: &str, + value: &str, + created_at: i64, + event_id: &EventId, +) -> Result, StorageError> { + let mut key = text_part(name, "event_tags", "tag_name")?; + key.extend_from_slice(&text_part(value, "event_tags", "tag_value")?); + key.extend_from_slice(&i64_key(created_at)); + key.extend_from_slice(event_id.as_bytes()); + Ok(key) +} + +/// The tag value a [`tag_key`] carries: the bytes between its two `0x00` separators. +/// +/// A query that walked the `tags` index has already been told the value by the key, so it does +/// not have to decode the event's body to check the criterion that selected the row. +pub(crate) fn tag_value(key: &[u8]) -> Result<&str, StorageError> { + let Some(name_end) = key.iter().position(|byte| *byte == 0) else { + return Err(invalid("tags", "a key without a name separator".to_owned())); + }; + let rest = &key[name_end + 1..]; + let Some(value_end) = rest.iter().position(|byte| *byte == 0) else { + return Err(invalid( + "tags", + "a key without a value separator".to_owned(), + )); + }; + text_from(&rest[..value_end], "tags") +} + +/// The tag rows the capture's trigger writes: one per element whose name is exactly one +/// character and whose value is present and non-empty, and never beyond the second element. +pub(crate) fn tag_entries(record: &crate::repositories::EventRecord) -> Vec<(&str, &str)> { + let mut entries = Vec::new(); + for tag in &record.event.tags { + let (Some(name), Some(value)) = (tag.first(), tag.get(1)) else { + continue; + }; + if name.chars().count() == 1 && !value.is_empty() { + entries.push((name.as_str(), value.as_str())); + } + } + entries +} + +/// Whether the capture's conflict-target predicate covers a kind: its replaceable and its +/// parameterized-replaceable kinds, which `records::replacement_key` gates the same way. +pub(crate) fn replaceable_kind(kind: i32) -> bool { + let Ok(kind) = u64::try_from(kind) else { + return false; + }; + let kind = Kind(kind); + kind.is_replaceable() || kind.is_parameterized_replaceable() +} + +/// Whether `bytes` starts with `prefix`, compared one nibble at a time with the same digit +/// mapping [`hex_bytes`] gives the range that selected the row, so a criterion means the same +/// thing to the matcher as to the index it was sought in. +pub(crate) fn hex_matches(bytes: &[u8], prefix: &str) -> bool { + if prefix.len() > bytes.len() * 2 { + return false; + } + prefix.bytes().enumerate().all(|(index, character)| { + let byte = bytes[index / 2]; + let nibble = if index % 2 == 0 { + byte >> 4 + } else { + byte & 0x0f + }; + nibble == hex_digit(character) + }) +} + +/// Hex to bytes, mapping an invalid digit to zero exactly as the SQL composer does. +pub(crate) fn hex_bytes(value: &str) -> Vec { + let mut bytes = Vec::with_capacity(value.len() / 2); + let digits = value.as_bytes(); + let mut index = 0; + while index + 1 < digits.len() { + bytes.push((hex_digit(digits[index]) << 4) | hex_digit(digits[index + 1])); + index += 2; + } + bytes +} + +fn hex_digit(digit: u8) -> u8 { + match digit { + b'0'..=b'9' => digit - b'0', + b'a'..=b'f' => digit - b'a' + 10, + b'A'..=b'F' => digit - b'A' + 10, + _ => 0, + } +} + +/// The inclusive key ranges one hex criterion family covers, over a 32-byte id or pubkey. +/// +/// A 64-character criterion is one equality range; an even prefix compares its bytes; an odd +/// one compares the bytes of the nibble range `0` to `f`, as `sql.rs` composes it. Both bounds +/// are widened to [`HEX_FIELD_LEN`] bytes, so a prefix is compared against its own field and +/// nothing a key carries after it; `suffix` is the number of bytes the index key carries after +/// the field. +pub(crate) fn hex_ranges(criteria: &[String], suffix: usize) -> Vec<(Vec, Vec)> { + let mut ranges = Vec::with_capacity(criteria.len()); + for criterion in criteria { + let (mut lower, mut upper) = match criterion.len() % 2 { + 1 => ( + hex_bytes(&format!("{criterion}0")), + hex_bytes(&format!("{criterion}f")), + ), + _ => { + let prefix = hex_bytes(criterion); + (prefix.clone(), prefix) + } + }; + if lower.len() < HEX_FIELD_LEN { + lower.resize(HEX_FIELD_LEN, 0x00); + } + if upper.len() < HEX_FIELD_LEN { + upper.resize(HEX_FIELD_LEN, 0xff); + } + upper.extend(std::iter::repeat_n(0xff, suffix)); + ranges.push((lower, upper)); + } + disjoint(ranges) +} + +/// The ranges no key is selected by more than once. +/// +/// Two criteria can select the same keys — `ab` and `abcd` nest, a digit the [range +/// builder](hex_ranges) maps to zero does the same, and a repeated kind composes the same author +/// range twice — and a walk that reads one key twice would count an event twice. Dropping a +/// range another already covers keeps every range a set of keys it alone selects. +pub(crate) fn disjoint(mut ranges: Vec<(Vec, Vec)>) -> Vec<(Vec, Vec)> { + ranges.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| right.1.cmp(&left.1))); + let mut kept: Vec<(Vec, Vec)> = Vec::with_capacity(ranges.len()); + for range in ranges { + let covered = kept + .last() + .is_some_and(|last| last.0 <= range.0 && range.1 <= last.1); + if !covered { + kept.push(range); + } + } + kept +} + +/// One inclusive key range a plan seeks, with the tag criterion that built it. +/// +/// A `tags` walk can reach one row more than once — two criteria, or two values of one `#g` +/// prefix — so a count past its id cap asks the criterion which keys of the row came earlier in +/// the walk. The criterion travels with the range it built, so the pairing cannot drift. +pub(crate) struct KeyRange<'a> { + /// The criterion this range was built from, and the family's criteria before it in the + /// search order. `None` for a range another index built. + pub(crate) tag: Option<(&'a [String], &'a str)>, + pub(crate) low: Vec, + pub(crate) high: Vec, +} + +impl<'a> KeyRange<'a> { + /// A range another index built: no tag criterion answers for the keys in it. + pub(crate) fn plain((low, high): (Vec, Vec)) -> Self { + Self { + tag: None, + low, + high, + } + } +} + +/// The inclusive key ranges one tag criterion covers, one range per criterion that fits a key +/// and in the criteria's order. +/// +/// An exact value is the range from the value's own separator, and a `#g` prefix is the range +/// over every value that starts with the prefix, padded to the longest key the store can hold +/// so a prefix long enough to leave little room still covers its own values. +/// +/// Each range carries the criterion that built it, preceded by the family's earlier criteria, +/// so a count can ask which keys of one row this walk has already reached. +/// +/// A criterion whose lower bound does not fit [`MAX_KEY_LEN`] selects nothing: a key carrying +/// it could not be written, so seeking it would fail the read rather than answer it. +pub(crate) fn tag_ranges<'a>(name: &str, criteria: &'a [String]) -> Vec> { + let is_geohash = name == "g"; + let mut ranges = Vec::with_capacity(criteria.len()); + for (index, criterion) in criteria.iter().enumerate() { + let mut head = Vec::with_capacity(name.len() + 1 + criterion.len()); + head.extend_from_slice(name.as_bytes()); + head.push(0); + let (lower, upper) = match criterion.strip_suffix('*').filter(|_| is_geohash) { + Some(prefix) => { + head.extend_from_slice(prefix.as_bytes()); + if head.len() > MAX_KEY_LEN { + continue; + } + let mut upper = head.clone(); + upper.resize(MAX_KEY_LEN, 0xff); + (head, upper) + } + None => { + head.extend_from_slice(criterion.as_bytes()); + head.push(0); + let mut upper = head.clone(); + upper.extend(std::iter::repeat_n(0xff, 8 + EVENT_ID_LEN)); + (head, upper) + } + }; + if lower.len() <= MAX_KEY_LEN && upper.len() <= MAX_KEY_LEN { + ranges.push(KeyRange { + tag: Some((&criteria[..index], criterion)), + low: lower, + high: upper, + }); + } + } + ranges +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + fn id(byte: u8) -> EventId { + EventId::from_byte_array([byte; 32]) + } + + #[test] + fn a_timestamp_key_sorts_numerically() { + let low = created_at_key(-5, &id(1)); + let zero = created_at_key(0, &id(1)); + let high = created_at_key(1_700_000_000, &id(1)); + assert!(low < zero); + assert!(zero < high); + assert_eq!(i64_from_key(&high[..8]).unwrap(), 1_700_000_000); + assert_eq!(i64_from_key(&low[..8]).unwrap(), -5); + } + + #[test] + fn a_kind_key_sorts_numerically() { + assert!(i32_key(-1) < i32_key(0)); + assert!(i32_key(0) < i32_key(1)); + assert_eq!(i32_from_key(&i32_key(30_023)).unwrap(), 30_023); + } + + #[test] + fn an_even_prefix_is_one_range_and_an_odd_one_is_one_nibble_range() { + let ranges = hex_ranges(&["abcd".to_owned(), "abef".to_owned()], 40); + assert_eq!(ranges.len(), 2, "two disjoint criteria seek two ranges"); + let mut low = vec![0xab, 0xcd]; + low.resize(HEX_FIELD_LEN, 0x00); + let mut high = vec![0xab, 0xcd]; + high.resize(HEX_FIELD_LEN, 0xff); + high.extend(std::iter::repeat_n(0xff, 40)); + assert_eq!(ranges[0].0, low); + assert_eq!( + ranges[0].1, high, + "the upper bound widens to the field, so a longer key under the prefix is in range" + ); + assert_eq!(ranges[1].0[..2], [0xab, 0xef]); + + let odd = hex_ranges(&["abcde".to_owned()], 40); + assert_eq!(odd[0].0[..3], [0xab, 0xcd, 0xe0]); + assert_eq!(odd[0].1, { + let mut high = vec![0xab, 0xcd, 0xef]; + high.resize(HEX_FIELD_LEN, 0xff); + high.extend(std::iter::repeat_n(0xff, 40)); + high + }); + } + + #[test] + fn a_criterion_another_range_covers_seeks_no_range_of_its_own() { + // `abcd` selects every key `abcde` does, so a walk that read both would visit those + // keys twice and count them twice. + let nested = hex_ranges(&["abcd".to_owned(), "abcde".to_owned()], 40); + assert_eq!(nested.len(), 1); + assert_eq!(nested[0].0[..2], [0xab, 0xcd]); + + // Two criteria that decode to the same bytes are one range too. + let duplicate = hex_ranges(&["zz".to_owned(), "z0".to_owned()], 0); + assert_eq!(duplicate.len(), 1, "both criteria map to the zero digit"); + } + + #[test] + fn a_tag_key_names_the_value_that_selected_it() { + let key = tag_key("t", "perfprobe", 1_700_000_000, &id(9)).unwrap(); + assert_eq!(tag_value(&key).unwrap(), "perfprobe"); + // A value shorter than the key's tail is still read up to its own separator. + let key = tag_key("t", "a", 0, &id(1)).unwrap(); + assert_eq!(tag_value(&key).unwrap(), "a"); + assert!(tag_value(b"t").is_err()); + assert!(tag_value(b"t\0a").is_err()); + } + + #[test] + fn a_criterion_too_long_for_a_key_seeks_nothing() { + // A key carrying it could not be written, so no stored row can match it; seeking it + // would fail the read with LMDB's key-length error instead of answering it. + let long = "x".repeat(MAX_KEY_LEN); + assert!(tag_ranges("t", std::slice::from_ref(&long)).is_empty()); + let short = "x".repeat(64); + assert_eq!(tag_ranges("t", std::slice::from_ref(&short)).len(), 1); + } + + #[test] + fn a_geohash_criterion_is_a_prefix_range_and_another_tag_is_exact() { + let criteria = vec!["u4pru*".to_owned()]; + let geohash = tag_ranges("g", &criteria); + assert_eq!(geohash[0].low, b"g\0u4pru".to_vec()); + let exact = tag_ranges("e", &criteria); + assert_eq!(exact[0].low, b"e\0u4pru*\0".to_vec()); + } + + #[test] + fn a_tag_range_carries_the_criterion_that_built_it() { + // A criterion too long to seek builds no range, but it still precedes the criteria + // after it, so a count asks it too — it matches nothing the store holds. + let criteria = vec!["x".repeat(MAX_KEY_LEN), "a".to_owned(), "b".to_owned()]; + let ranges = tag_ranges("t", &criteria); + assert_eq!(ranges.len(), 2, "the oversized criterion seeks nothing"); + assert_eq!(ranges[0].tag, Some((&criteria[..1], "a"))); + assert_eq!(ranges[1].tag, Some((&criteria[..2], "b"))); + } + + #[test] + fn a_long_geohash_prefix_still_seeks_the_values_it_names() { + // The prefix's own key fits; padding it with a fixed 64 bytes would not, and dropping + // the range would miss every value the prefix names. + let prefix = "u".repeat(447); + let criteria = vec![format!("{prefix}*")]; + let ranges = tag_ranges("g", &criteria); + assert_eq!(ranges.len(), 1, "the prefix names a range"); + let (lower, upper) = (&ranges[0].low, &ranges[0].high); + assert_eq!(upper.len(), MAX_KEY_LEN, "the upper bound is a legal key"); + let mut expected = b"g\0".to_vec(); + expected.extend_from_slice(prefix.as_bytes()); + assert_eq!(lower, &expected); + + // The key of a stored value that is exactly the prefix: the separator, the timestamp + // and the id follow it, and the whole key is 490 bytes. + let mut stored = lower.clone(); + stored.extend_from_slice(&[0; 1 + 8 + EVENT_ID_LEN]); + assert!(stored.len() <= MAX_KEY_LEN, "such an event can be stored"); + assert!(stored.as_slice() >= lower.as_slice()); + assert!( + stored.as_slice() <= upper.as_slice(), + "and the range selects it" + ); + } + + #[test] + fn a_nanosecond_instant_is_truncated_to_microseconds() { + let instant = DateTime::::from_timestamp_nanos(1_700_000_000_123_456_789); + let truncated = micros(instant); + assert_eq!(truncated.timestamp_micros(), 1_700_000_000_123_456); + assert_eq!( + truncated.timestamp_subsec_nanos(), + 123_456_000, + "the sub-microsecond digits are dropped, as a PostgreSQL timestamp drops them (V38)" + ); + } + + #[test] + fn the_header_round_trips_without_its_body() { + let header = EventHeader { + deleted: true, + created_at: 1_700_000_000, + kind: 30_023, + expires_at: Some(2_000_000_000), + pubkey: Pubkey::from_byte_array([7; 32]), + }; + let mut value = vec![ROW_FORMAT_VERSION]; + value.extend_from_slice(&header_bytes(&header)); + assert_eq!(event_header("events", &value).unwrap(), header); + } + + #[test] + fn an_unsupported_format_version_names_the_version() { + let error = check_version("events", &[2, 0]).expect_err("version 2"); + match error { + StorageError::InvalidRow { value, .. } => { + assert_eq!(value, "unsupported format version 2"); + } + other => panic!("expected InvalidRow, got {other:?}"), + } + } + + #[test] + fn a_truncated_header_is_an_invalid_row() { + let error = event_header("events", &[ROW_FORMAT_VERSION, 0]).expect_err("truncated"); + assert!(matches!(error, StorageError::InvalidRow { .. })); + } + + #[test] + fn a_tag_name_or_value_carrying_a_separator_is_rejected() { + let error = tag_key("e", "a\0b", 0, &id(1)).expect_err("a separator in the value"); + assert!(matches!( + error, + StorageError::InvalidRow { + column: "tag_value", + .. + } + )); + let error = tag_key("e\0", "ab", 0, &id(1)).expect_err("a separator in the name"); + assert!(matches!( + error, + StorageError::InvalidRow { + column: "tag_name", + .. + } + )); + } + + #[test] + fn only_the_trigger_s_tag_shape_is_indexed() { + use nostream_core::Event; + + let event: Event = serde_json::from_value(serde_json::json!({ + "id": "0d684e8ec2431de586aa3cafbee2f6d308d19b28805e53deabcac3220e9136a5", + "pubkey": "f43c1f9bff677b8f27b602725ea0ad51af221344f69a6b352a74991a4479bac3", + "created_at": 1660407625i64, + "kind": 1, + "tags": [["e", "x", "y"], ["", "x"], ["e", ""], ["ab", "c"], ["p", "q"]], + "content": "", + "sig": "ab".repeat(64) + })) + .expect("a wire event"); + let record = crate::repositories::EventRecord::from_event(&event); + assert_eq!(tag_entries(&record), vec![("e", "x"), ("p", "q")]); + } + + #[test] + fn the_replaceable_kind_set_matches_the_conflict_target() { + for kind in [0, 3, 41, 10_000, 19_999, 30_000, 39_999] { + assert!(replaceable_kind(kind), "{kind} is replaceable"); + } + for kind in [1, 2, 4, 9_999, 20_000, 29_999, 40_000] { + assert!(!replaceable_kind(kind), "{kind} is not replaceable"); + } + } + + #[test] + fn a_hex_prefix_compares_the_nibbles_the_range_builder_sought() { + assert!(hex_matches(&[0xab, 0xcd], "abcd")); + assert!(hex_matches(&[0xab, 0xcd], "abc")); + assert!( + hex_matches(&[0xab, 0xcd], "ABC"), + "an uppercase digit names the same nibble the range was built from" + ); + assert!( + hex_matches(&[0x0a], "z"), + "an invalid digit maps to zero, as the capture's decode does" + ); + assert!(!hex_matches(&[0xab, 0xcd], "abce")); + assert!(!hex_matches(&[0xab, 0xcd], "abcdef")); + } + + #[test] + fn a_deduplication_value_is_not_inflated_by_json_escaping() { + let pubkey = Pubkey::from_byte_array([5; 32]); + // 235 quote bytes are 470 bytes of JSON text once escaped; the key must not grow with + // the escapes, or a value Postgres and the tag index both keep is refused. + let value = "\"".repeat(235); + let key = replaceable_key(&pubkey, 30_023, &[serde_json::Value::String(value.clone())]) + .expect("the pointer"); + assert!( + key.len() <= MAX_KEY_LEN, + "the key is {} bytes for a {}-byte value", + key.len(), + value.len() + ); + } + + #[test] + fn a_deduplication_value_past_the_key_bound_is_refused() { + let pubkey = Pubkey::from_byte_array([5; 32]); + let longest = "x".repeat(MAX_KEY_LEN - 39); + assert!( + replaceable_key(&pubkey, 30_023, &[serde_json::Value::String(longest)]).is_ok(), + "the longest value a pointer key holds is stored" + ); + let too_long = "x".repeat(MAX_KEY_LEN - 39 + 1); + let error = replaceable_key(&pubkey, 30_023, &[serde_json::Value::String(too_long)]) + .expect_err("one byte past the bound"); + assert!(matches!( + error, + StorageError::InvalidRow { + column: "event_deduplication", + .. + } + )); + } + + #[test] + fn the_replaceable_key_never_refuses_a_value_the_tag_key_accepts() { + let pubkey = Pubkey::from_byte_array([5; 32]); + let value = "x".repeat(MAX_KEY_LEN - 43 + 1); + assert!( + tag_key("d", &value, 0, &id(1)).expect("the tag key").len() > MAX_KEY_LEN, + "the tag key is the tighter bound for the same value" + ); + assert!(replaceable_key(&pubkey, 30_023, &[serde_json::Value::String(value)]).is_ok()); + } + + #[test] + fn a_deduplication_string_and_a_number_of_the_same_text_do_not_collide() { + let pubkey = Pubkey::from_byte_array([5; 32]); + let string = replaceable_key( + &pubkey, + 30_023, + &[serde_json::Value::String("1".to_owned())], + ) + .expect("a string element"); + let number = replaceable_key(&pubkey, 30_023, &[serde_json::Value::from(1)]) + .expect("a number element"); + assert_ne!(string, number); + } + + #[test] + fn a_value_a_text_column_cannot_hold_is_refused() { + // PostgreSQL's `text` cannot store `0x00`, so the check refuses it wherever a writer meets + // the value: a key component here, a row's own text column in the delivery log. + for value in ["a\0b", "\0", "ab\0"] { + let error = text_part(value, "delivery_log_by_outbox", "outbox_id") + .expect_err("a NUL in a key component"); + assert!(matches!( + error, + StorageError::InvalidRow { + table: "delivery_log_by_outbox", + column: "outbox_id", + .. + } + )); + let error = refuse_nul("notification_delivery_log", "target_id", value) + .expect_err("a NUL in a text column"); + assert!(matches!( + error, + StorageError::InvalidRow { + table: "notification_delivery_log", + column: "target_id", + .. + } + )); + } + assert_eq!( + text_part( + "https://example.test", + "delivery_log_by_outbox", + "outbox_id" + ) + .unwrap(), + b"https://example.test\0".to_vec() + ); + refuse_nul( + "notification_delivery_log", + "target_id", + "https://example.test", + ) + .expect("a target a text column can hold"); + } +} diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs new file mode 100644 index 000000000..f5fe51d25 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -0,0 +1,3938 @@ +//! `EventRepository` against LMDB: the seven event databases, the write paths the capture's +//! `INSERT`/`ON CONFLICT` statements define, and the filter evaluation `sql.rs` composes. +//! +//! A read never holds a transaction across an await: a query materialises its page of +//! `(created_at, event_id)` keys inside one blocking closure, and the stream that returns the +//! rows re-seeks its cursor per batch (see [`ValuesStream`] and [`ReconciliationStream`]). + +use std::cmp::Ordering; +use std::collections::{BTreeMap, BinaryHeap, HashSet}; +use std::future::Future; +use std::net::IpAddr; +use std::ops::Bound; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use chrono::{DateTime, Utc}; +use futures_core::Stream; +use heed::types::Bytes; +use heed::{Database, Env, RoTxn, RwTxn}; +use nostream_core::{EventId, Filter, Kind, KindSelector, Pubkey}; +use tokio::task::JoinHandle; + +use super::encoding::{self, EventHeader}; +use super::rows; +use super::{ + DEFAULT_FILTER_LIMIT, Databases, Gates, MAX_PAGE_KEYS, RETENTION_BATCH_SIZE, + STREAM_BATCH_ITEMS, STREAM_BATCH_KEYS, +}; +use crate::error::StorageError; +use crate::records::{distinct_replacements, replaces_fields}; +use crate::repositories::{ + EventPurgeCounts, EventRecord, EventRetentionOptions, ReconciliationItem, +}; + +/// The kind of the capture's `REQUEST_TO_VANISH`, which every retention whitelist keeps and +/// every soft delete of one pubkey's ids spares. +const REQUEST_TO_VANISH: i32 = 62; + +// --------------------------------------------------------------------------------------- +// Write paths. +// --------------------------------------------------------------------------------------- + +/// `create`/`create_many`: an id that is already stored writes nothing and counts nothing. +pub(crate) fn create( + txn: &mut RwTxn<'_>, + databases: &Databases, + records: &[EventRecord], + now: DateTime, +) -> Result { + let mut affected = 0; + for record in records { + if databases + .events + .get(txn, record.event.id.as_bytes())? + .is_some() + { + continue; + } + let stored = EventRecord { + row_id: None, + event: record.event.clone(), + first_seen: Some(now), + deleted_at: None, + expires_at: record.expires_at, + // `INSERT_COLUMNS` omits the deduplication column, so the capture stores NULL. + event_deduplication: None, + remote_address: record.remote_address, + }; + write_event(txn, databases, &stored)?; + affected += 1; + } + Ok(affected) +} + +/// `upsert`: a replaceable event merges through its pointer, and a stored id is a duplicate. +pub(crate) fn upsert( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, + now: DateTime, +) -> Result { + Ok(u64::from(upsert_one(txn, databases, record, now, false)?)) +} + +/// `upsert_many`: one row per replaceable address, then the same merge, in one write +/// transaction. The capture's bulk merge list omits `remote_address`, so a replacement keeps +/// the address the stored row had, which `postgres/events.rs`'s `upsert_many` does too. +pub(crate) fn upsert_many( + txn: &mut RwTxn<'_>, + databases: &Databases, + records: &[EventRecord], + now: DateTime, +) -> Result { + // A batch may hold several versions of one replaceable address: the conflict key's owner + // picks the winner, as Postgres does before its one statement, so the count is rows + // written rather than records passed. + let mut affected = 0; + for record in distinct_replacements(records) { + if upsert_one(txn, databases, record, now, true)? { + affected += 1; + } + } + Ok(affected) +} + +fn upsert_one( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, + now: DateTime, + keep_stored_address: bool, +) -> Result { + let kind = kind_of(record)?; + let deduplication = record.deduplication(); + + if databases + .events + .get(txn, record.event.id.as_bytes())? + .is_some() + { + // A replaceable write conflicts on its address, not on its id: replaying one the store + // already holds updates that row against itself and counts zero (`REPLACEABLE_MERGE` + // never matches), and only an id naming another address violates + // `events_event_id_unique`. + if encoding::replaceable_kind(kind) { + let pointer = encoding::replaceable_key(&record.event.pubkey, kind, &deduplication)?; + if let Some(stored) = databases.replaceable.get(txn, &pointer)? + && encoding::event_id_from(stored)? == record.event.id + { + return Ok(false); + } + } + return Err(StorageError::DuplicateEvent); + } + + let mut first_seen = None; + let mut remote_address = record.remote_address; + + if encoding::replaceable_kind(kind) { + let pointer = encoding::replaceable_key(&record.event.pubkey, kind, &deduplication)?; + if let Some(stored) = databases.replaceable.get(txn, &pointer)? { + let stored_id = encoding::event_id_from(stored)?; + let Some(value) = databases.events.get(txn, stored_id.as_bytes())? else { + // A pointer with no row: treat the incoming event as a plain insert. + return write_upserted( + txn, + databases, + record, + now, + kind, + &deduplication, + None, + remote_address, + ); + }; + let header = rows::event_header(value)?; + if !replaces_fields( + (record.event.created_at, record.event.id), + (header.created_at, stored_id), + ) { + return Ok(false); + } + let stored_record = rows::decode_event(value)?; + first_seen = stored_record.first_seen; + if keep_stored_address { + remote_address = stored_record.remote_address; + } + hard_delete(txn, databases, &stored_record)?; + databases.replaceable.delete(txn, &pointer)?; + } + } + + write_upserted( + txn, + databases, + record, + now, + kind, + &deduplication, + first_seen, + remote_address, + ) +} + +#[allow(clippy::too_many_arguments)] +fn write_upserted( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, + now: DateTime, + kind: i32, + deduplication: &[serde_json::Value], + first_seen: Option>, + remote_address: Option, +) -> Result { + let stored = EventRecord { + row_id: None, + event: record.event.clone(), + // `DO UPDATE` never writes `first_seen`, so a merged row keeps the one it had. + first_seen: Some(first_seen.unwrap_or(now)), + // The capture's merge list clears `deleted_at`. + deleted_at: None, + expires_at: record.expires_at, + event_deduplication: Some(deduplication.to_vec()), + remote_address, + }; + write_event(txn, databases, &stored)?; + if encoding::replaceable_kind(kind) { + databases.replaceable.put( + txn, + &encoding::replaceable_key(&record.event.pubkey, kind, deduplication)?, + record.event.id.as_bytes(), + )?; + } + Ok(true) +} + +/// Writes an event and every index entry it needs. +fn write_event( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, +) -> Result<(), StorageError> { + let kind = kind_of(record)?; + let id = record.event.id; + let created_at = record.event.created_at; + + put_value(txn, databases, record)?; + databases + .events_by_created_at + .put(txn, &encoding::created_at_key(created_at, &id), &[])?; + databases.events_by_author.put( + txn, + &encoding::author_key(&record.event.pubkey, kind, created_at, &id), + &[], + )?; + for (name, value) in encoding::tag_entries(record) { + databases + .tags + .put(txn, &encoding::tag_key(name, value, created_at, &id)?, &[])?; + } + if let Some(expires_at) = record.expires_at { + databases + .events_expiring + .put(txn, &encoding::time_key(expires_at, &id), &[])?; + } + Ok(()) +} + +/// Writes only the `events` value, which is what a soft delete changes. +fn put_value( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, +) -> Result<(), StorageError> { + let value = rows::encode_event(record)?; + databases + .events + .put(txn, record.event.id.as_bytes(), &value)?; + Ok(()) +} + +/// Removes an event from every database, which is what a replacement and a retention pass do. +fn hard_delete( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, +) -> Result<(), StorageError> { + let kind = kind_of(record)?; + let id = record.event.id; + let created_at = record.event.created_at; + + databases.events.delete(txn, id.as_bytes())?; + databases + .events_by_created_at + .delete(txn, &encoding::created_at_key(created_at, &id))?; + databases.events_by_author.delete( + txn, + &encoding::author_key(&record.event.pubkey, kind, created_at, &id), + )?; + if let Some(deleted_at) = record.deleted_at { + databases + .events_deleted + .delete(txn, &encoding::time_key(deleted_at.timestamp(), &id))?; + } + if let Some(expires_at) = record.expires_at { + databases + .events_expiring + .delete(txn, &encoding::time_key(expires_at, &id))?; + } + for (name, value) in encoding::tag_entries(record) { + databases + .tags + .delete(txn, &encoding::tag_key(name, value, created_at, &id)?)?; + } + if let Some(deduplication) = &record.event_deduplication + && encoding::replaceable_kind(kind) + { + let pointer = encoding::replaceable_key(&record.event.pubkey, kind, deduplication)?; + if let Some(stored) = databases.replaceable.get(txn, &pointer)? + && stored == &id.as_bytes()[..] + { + databases.replaceable.delete(txn, &pointer)?; + } + } + Ok(()) +} + +/// Marks one event soft-deleted: the header bit, the JSON field and the deleted index. Every +/// read index is left alone, which is what keeps the row reachable by `find_by_filters`. +fn soft_delete( + txn: &mut RwTxn<'_>, + databases: &Databases, + record: &EventRecord, + now: DateTime, +) -> Result<(), StorageError> { + let mut record = record.clone(); + record.deleted_at = Some(now); + put_value(txn, databases, &record)?; + databases.events_deleted.put( + txn, + &encoding::time_key(now.timestamp(), &record.event.id), + &[], + )?; + Ok(()) +} + +/// `delete_by_pubkey_and_ids`: a soft delete that never touches kind 62 or a deleted row. +pub(crate) fn delete_by_pubkey_and_ids( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + ids: &[EventId], + now: DateTime, +) -> Result { + let mut affected = 0; + for id in ids { + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + continue; + }; + let record = rows::decode_event(value)?; + if record.event.pubkey != *pubkey + || kind_of(&record)? == REQUEST_TO_VANISH + || record.deleted_at.is_some() + { + continue; + } + soft_delete(txn, databases, &record, now)?; + affected += 1; + } + Ok(affected) +} + +/// `delete_by_pubkey_except_kinds`: every other live row of one author, soft-deleted. +pub(crate) fn delete_by_pubkey_except_kinds( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + excluded_kinds: &[Kind], + now: DateTime, +) -> Result { + let excluded: Vec = excluded_kinds + .iter() + .map(|kind| i32::try_from(kind.0).unwrap_or(i32::MAX)) + .collect(); + let mut keys = Vec::new(); + for entry in databases + .events_by_author + .prefix_iter(txn, pubkey.as_bytes())? + { + let (key, _) = entry?; + keys.push(key.to_vec()); + } + + let mut affected = 0; + for key in keys { + let header = author_key_header(&key)?; + if excluded.contains(&header.kind) { + continue; + } + let id = header.event_id; + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + continue; + }; + let record = rows::decode_event(value)?; + if record.deleted_at.is_some() { + continue; + } + soft_delete(txn, databases, &record, now)?; + affected += 1; + } + Ok(affected) +} + +/// `has_active_request_to_vanish`: a live kind-62 row exists for the pubkey. +pub(crate) fn has_active_request_to_vanish( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result { + let mut prefix = pubkey.as_bytes().to_vec(); + prefix.extend_from_slice(&encoding::i32_key(REQUEST_TO_VANISH)); + for entry in databases.events_by_author.prefix_iter(txn, &prefix)? { + let (key, _) = entry?; + let id = header_id(key, 44, "events_by_author")?; + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + continue; + }; + if !rows::event_header(value)?.deleted { + return Ok(true); + } + } + Ok(false) +} + +/// `delete_expired_and_retained`: one hard-delete batch of at most +/// [`RETENTION_BATCH_SIZE`] rows, gathered in the capture's index order and classified +/// deleted, then expired, then over-retention. +// shortcut: retention takes its 1000 candidates in a fixed order (deleted, expiring, +// created_at) where Postgres' planner chooses; upgrade when a deployment exceeds 1000 +// qualifying rows per pass and the mix, not the total, starts to matter (spec Assumption 10). +pub(crate) fn delete_expired_and_retained( + txn: &mut RwTxn<'_>, + databases: &Databases, + options: &EventRetentionOptions, + now: i64, +) -> Result { + let Some(max_days) = options.max_days.filter(|days| *days > 0) else { + return Ok(EventPurgeCounts::default()); + }; + let retention_limit = now.saturating_sub(max_days.saturating_mul(86_400)); + let kinds = retention_kinds(options.kind_whitelist.as_deref()); + let pubkeys: Vec<[u8; 32]> = options + .pubkey_whitelist + .iter() + .flatten() + .map(|pubkey| encoding::hex_bytes(&pubkey.0)) + .filter_map(|bytes| bytes.try_into().ok()) + .collect(); + + let mut candidates = Vec::new(); + let mut seen = HashSet::new(); + for entry in databases.events_deleted.iter(txn)? { + let (key, _) = entry?; + let id = header_id(key, 8, "events_deleted")?; + gather( + txn, + databases, + id, + now, + retention_limit, + &kinds, + &pubkeys, + &mut candidates, + &mut seen, + )?; + if candidates.len() >= RETENTION_BATCH_SIZE { + break; + } + } + if candidates.len() < RETENTION_BATCH_SIZE { + // A row whose expiry has not passed cannot be classified expired, and one that + // qualifies by age is the created-at pass's business: the index's own order lets this + // walk stop at `now` instead of reading every future expiry under the writer. + let lower = encoding::i64_key(i64::MIN).to_vec(); + let mut upper = encoding::i64_key(now.saturating_sub(1)).to_vec(); + upper.extend([0xff; 32]); + let range = encoding::bounds(&lower, &upper); + for entry in databases.events_expiring.range(txn, &range)? { + let (key, _) = entry?; + let id = header_id(key, 8, "events_expiring")?; + gather( + txn, + databases, + id, + now, + retention_limit, + &kinds, + &pubkeys, + &mut candidates, + &mut seen, + )?; + if candidates.len() >= RETENTION_BATCH_SIZE { + break; + } + } + } + if candidates.len() < RETENTION_BATCH_SIZE { + let lower = encoding::i64_key(i64::MIN).to_vec(); + let mut upper = encoding::i64_key(retention_limit.saturating_sub(1)).to_vec(); + upper.extend([0xff; 32]); + let range = encoding::bounds(&lower, &upper); + for entry in databases.events_by_created_at.range(txn, &range)? { + let (key, _) = entry?; + let id = header_id(key, 8, "events_by_created_at")?; + if candidates.len() >= RETENTION_BATCH_SIZE { + break; + } + gather( + txn, + databases, + id, + now, + retention_limit, + &kinds, + &pubkeys, + &mut candidates, + &mut seen, + )?; + } + } + + let mut counts = EventPurgeCounts::default(); + for id in candidates { + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + continue; + }; + let header = rows::event_header(value)?; + if header.deleted { + counts.deleted += 1; + } else if header.expires_at.is_some_and(|expires_at| expires_at < now) { + counts.expired += 1; + } else if header.created_at < retention_limit { + counts.retained += 1; + } else { + continue; + } + let record = rows::decode_event(value)?; + hard_delete(txn, databases, &record)?; + } + Ok(counts) +} + +/// Adds one retention candidate when it qualifies and is not protected, deduplicating. +#[allow(clippy::too_many_arguments)] +fn gather( + txn: &RoTxn<'_>, + databases: &Databases, + id: EventId, + now: i64, + retention_limit: i64, + kinds: &[KindSelector], + pubkeys: &[[u8; 32]], + candidates: &mut Vec, + seen: &mut HashSet<[u8; 32]>, +) -> Result<(), StorageError> { + key_evaluated(); + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + return Ok(()); + }; + let header = rows::event_header(value)?; + if whitelisted(header.kind, kinds) || pubkeys.contains(header.pubkey.as_bytes()) { + return Ok(()); + } + let qualifies = header.deleted + || header.expires_at.is_some_and(|expires_at| expires_at < now) + || header.created_at < retention_limit; + if qualifies && seen.insert(*id.as_bytes()) { + candidates.push(id); + } + Ok(()) +} + +/// The capture's deduplicated whitelist, which always keeps kind 62. +fn retention_kinds(configured: Option<&[KindSelector]>) -> Vec { + let mut whitelist: Vec = Vec::new(); + for selector in configured.unwrap_or_default() { + if !whitelist.contains(selector) { + whitelist.push(*selector); + } + } + let vanish = KindSelector::Single(Kind(REQUEST_TO_VANISH as u64)); + if !whitelist.contains(&vanish) { + whitelist.push(vanish); + } + whitelist +} + +fn whitelisted(kind: i32, selectors: &[KindSelector]) -> bool { + selectors.iter().any(|selector| match selector { + KindSelector::Single(selector) => i32::try_from(selector.0).unwrap_or(i32::MAX) == kind, + KindSelector::Range([start, end]) => { + let start = i32::try_from(start.0).unwrap_or(i32::MAX); + let end = i32::try_from(end.0).unwrap_or(i32::MAX); + kind >= start && kind <= end + } + }) +} + +fn kind_of(record: &EventRecord) -> Result { + i32::try_from(record.event.kind.0).map_err(|_| StorageError::ValueOutOfRange { + column: "event_kind", + value: record.event.kind.0.to_string(), + }) +} + +// --------------------------------------------------------------------------------------- +// Read paths. +// --------------------------------------------------------------------------------------- + +/// Counts a key a read evaluated. Only the tests read the count, so a release build drops it. +fn key_evaluated() { + #[cfg(test)] + probes::key(); +} + +/// The work counters the cost tests assert on: a page's cost is its keys evaluated, its bodies +/// decoded — [`rows::probes`] counts every event-body decode, wherever it happens — and the +/// entries it sorted, and none of the three is visible in its result. +#[cfg(test)] +mod probes { + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Serialises the tests that count work, so one cannot read another's numbers. + pub(super) static COUNTING: Mutex<()> = Mutex::new(()); + + static KEYS: AtomicUsize = AtomicUsize::new(0); + static SORTED: AtomicUsize = AtomicUsize::new(0); + + pub(super) fn reset() { + KEYS.store(0, Ordering::Relaxed); + super::rows::probes::reset(); + SORTED.store(0, Ordering::Relaxed); + } + + pub(super) fn key() { + KEYS.fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn sorted(entries: usize) { + SORTED.fetch_add(entries, Ordering::Relaxed); + } + + pub(super) fn keys() -> usize { + KEYS.load(Ordering::Relaxed) + } + + pub(super) fn decodes() -> usize { + super::rows::probes::decodes() + } + + pub(super) fn sorted_entries() -> usize { + SORTED.load(Ordering::Relaxed) + } +} + +/// The index one filter is answered from, the ranges within it, and what a key proves. +struct Plan<'a> { + database: Database, + path: AccessPath, + ranges: Vec>, + /// Whether a range's own key order is the page's order, so a range stops at the page's own + /// worst kept row instead of being walked whole. + ordered: bool, + /// The tag family the ranges were built from, with its criteria in the criteria's order — + /// `tag_ranges` builds one range per criterion that fits a key, carrying that criterion. A + /// key in these ranges has already proved that family, so the row's body is decoded only for + /// the filter's other tag families, and a `tags` count holds one id per row its matcher + /// accepts until the set is full, then reads a repeated key's body instead. + tag: Option<(&'a str, &'a [String])>, +} + +/// One reconciliation batch still running on the blocking pool, with the cursor it returns. +type PendingBatch = JoinHandle, Position), StorageError>>; + +/// The index one filter is answered from, and how its keys are read back. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AccessPath { + Ids, + Authors, + Tags, + CreatedAt, +} + +/// The page of keys one filter matches, in the capture's order. +pub(crate) fn page( + txn: &RoTxn<'_>, + databases: &Databases, + filters: &[Filter], + gates: &Gates, + now: i64, +) -> Result, StorageError> { + if filters.is_empty() { + return Err(StorageError::InvalidArgument { + message: "Filters cannot be empty", + }); + } + + let mut union = Vec::new(); + let mut seen = HashSet::new(); + for filter in filters { + for candidate in branch( + txn, + databases, + filter, + gates, + now, + false, + page_limit(filter), + )? { + if seen.insert(candidate) { + union.push(candidate); + } + } + } + sort_page(&mut union, filters[0].limit.is_some()); + Ok(union) +} + +/// The count of the live ids the filters match, deduplicated across filters. +/// +/// A count has no page, so it materialises no result set: each filter counts the rows it visits +/// and skips one the count already holds — a row of an earlier filter's page when that filter had +/// a limit (a page is at most [`MAX_PAGE_KEYS`] ids), or one its predicate matches when it had +/// none — which is the capture's `UNION`. A `tags` walk whose criteria can select one row twice +/// also holds the ids of the rows it matches, at most [`MAX_PAGE_KEYS`] of them; past that the +/// walk reads a row's body to tell an earlier key of the walk from a first one, so the set stops +/// growing and the total stays exact. +pub(crate) fn count( + txn: &RoTxn<'_>, + databases: &Databases, + filters: &[Filter], + gates: &Gates, + now: i64, +) -> Result { + if filters.is_empty() { + return Err(StorageError::InvalidArgument { + message: "Filters cannot be empty", + }); + } + + let mut counted = Counted::default(); + let mut total = 0; + for filter in filters { + total += match filter.limit { + Some(limit) => { + let page = branch(txn, databases, filter, gates, now, true, count_limit(limit))?; + let mut fresh = 0; + for (_, id) in &page { + if !already_counted(txn, databases, &counted, gates, now, id)? { + fresh += 1; + } + } + counted.pages.push(page.iter().map(|(_, id)| *id).collect()); + fresh + } + None => { + let fresh = count_matches(txn, databases, filter, &counted, gates, now)?; + counted.predicates.push(filter); + fresh + } + }; + } + Ok(total) +} + +/// What a count has already counted: bounded by [`MAX_PAGE_KEYS`] per limited filter. +#[derive(Default)] +struct Counted<'a> { + /// The ids of every earlier filter that had a limit: its whole page. + pages: Vec>, + /// Every earlier filter without a limit, whose page is every row its predicate matches. + predicates: Vec<&'a Filter>, +} + +/// Whether the count already holds `id`, so counting it again would count the row twice. +fn already_counted( + txn: &RoTxn<'_>, + databases: &Databases, + counted: &Counted<'_>, + gates: &Gates, + now: i64, + id: &EventId, +) -> Result { + if counted.pages.iter().any(|page| page.contains(id)) { + return Ok(true); + } + for filter in &counted.predicates { + let matcher = Matcher { + filter, + gates, + now, + live: true, + proven: None, + }; + if matcher.matches(txn, databases, id)?.is_some() { + return Ok(true); + } + } + Ok(false) +} + +/// Counts the live rows one unlimited filter matches that the count does not already hold. +/// +/// Every range is walked whole — a count has no page to stop at — and a row two of them select +/// is counted once, at the first key that selects it. +/// +/// A `tags` key names one value of one row, so a row's keys within this walk are one per +/// matching value and per matching criterion. Whichever key the walk reaches first answers for +/// the row — ranges run in the criteria's order, so that is not necessarily the smallest of its +/// matching values — and the id it ends with answers the rest until the set holds +/// [`MAX_PAGE_KEYS`] of them. +fn count_matches( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + counted: &Counted<'_>, + gates: &Gates, + now: i64, +) -> Result { + count_matches_bounded(txn, databases, filter, counted, gates, now, MAX_PAGE_KEYS) +} + +/// [`count_matches`], with the number of ids the walk may hold given explicitly, so a test can +/// compare the bounded walk against an unbounded one over one corpus. +fn count_matches_bounded( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + counted: &Counted<'_>, + gates: &Gates, + now: i64, + cap: usize, +) -> Result { + if !admits(filter, gates)? { + return Ok(0); + } + let plan = plan(filter, databases); + let matcher = Matcher { + filter, + gates, + now, + live: true, + proven: plan.tag.map(|(name, _)| name), + }; + // A single exact criterion names one key per row, so only a second criterion or a `#g` + // prefix can walk one row twice, and only then is the id set paid for. + // + // shortcut: the set holds at most `MAX_PAGE_KEYS` ids; a repeated key past that reads the + // row's body to ask whether an earlier key of this walk already answered for it, against a + // slot per repeated row. Upgrade when a count past the cap must decode nothing, which needs + // an index order that visits a row once. + let mut counted_ids = HashSet::new(); + let dedupe = plan.tag.is_some_and(|(name, criteria)| { + criteria.len() > 1 + || (name == "g" && criteria.iter().any(|criterion| criterion.ends_with('*'))) + }); + let mut total = 0; + for range in &plan.ranges { + let bounds = encoding::bounds(&range.low, &range.high); + for entry in plan.database.range(txn, &bounds)? { + let (key, _) = entry?; + key_evaluated(); + let value = match plan.tag { + Some((name, criteria)) => { + let value = encoding::tag_value(key)?; + if !tag_value_matches(name == "g", criteria, value) { + continue; + } + Some(value) + } + None => None, + }; + let Some((_, id)) = candidate(txn, databases, plan.path, key)? else { + continue; + }; + // The set holds only rows this walk accepted, so a hit is a row it already counted: + // no second test and no second decode. + if dedupe && counted_ids.contains(&id) { + continue; + } + if matcher.matches(txn, databases, &id)?.is_none() { + continue; + } + if dedupe { + // The verdict is the row's, not the key's, so a second key is decided the same + // way and only a row this filter matches takes a slot in the set. + if counted_ids.len() < cap { + counted_ids.insert(id); + } else if let (Some((name, _)), Some((earlier, criterion))) = (plan.tag, range.tag) + && counted_earlier(txn, databases, name, earlier, criterion, value, &id)? + { + continue; + } + } + if !already_counted(txn, databases, counted, gates, now, &id)? { + total += 1; + } + } + } + Ok(total) +} + +/// Whether an earlier key of this walk already selects the row `id`, so counting it here would +/// count the row twice — the check a count makes once its id set is full. +/// +/// Two ranges select one row when two criteria of the family match its tags; one range selects +/// it twice when a `#g` prefix matches two of its tags. The family's criteria before `criterion` +/// are passed in their order, so the first of them that matches answers for the row, and within +/// one prefix range the walk reaches the smallest matching value first. +/// +/// This is the one place a count reads a row's body instead of its key: the key names a single +/// value, and the other values the row carries are only in the body. +fn counted_earlier( + txn: &RoTxn<'_>, + databases: &Databases, + name: &str, + earlier: &[String], + criterion: &str, + value: Option<&str>, + id: &EventId, +) -> Result { + let is_geohash = name == "g"; + let prefix = criterion.strip_suffix('*').filter(|_| is_geohash); + // A first exact criterion is the row's only key in this walk, so nothing earlier answers + // for it and the body is not read. + if earlier.is_empty() && prefix.is_none() { + return Ok(false); + } + let Some(stored) = databases.events.get(txn, id.as_bytes())? else { + return Ok(false); + }; + let record = rows::decode_event(stored)?; + if earlier.iter().any(|earlier_criterion| { + record_has_tag(&record, name, std::slice::from_ref(earlier_criterion)) + }) { + return Ok(true); + } + if prefix.is_none() { + return Ok(false); + } + let Some(value) = value else { + return Ok(false); + }; + Ok(record.event.tags.iter().any(|tag| { + matches!( + (tag.first(), tag.get(1)), + (Some(tag_name), Some(candidate)) + if tag_name == name + && !candidate.is_empty() + && matching_value(is_geohash, criterion, candidate).is_some() + && candidate.as_str() < value + ) + })) +} + +/// The page a filter may materialise: its own limit or the capture's default, and never more +/// than [`MAX_PAGE_KEYS`] however large the limit on the wire is. +fn page_limit(filter: &Filter) -> usize { + count_limit(filter.limit.unwrap_or(DEFAULT_FILTER_LIMIT)) +} + +/// A wire limit clamped to [`MAX_PAGE_KEYS`], so no request chooses the size of an allocation. +fn count_limit(limit: u64) -> usize { + usize::try_from(limit.min(MAX_PAGE_KEYS as u64)).unwrap_or(MAX_PAGE_KEYS) +} + +/// The refusals and short-circuits every read applies before it seeks an index. +/// +/// `true` means the filter can match rows; `false` means a criterion family is present but +/// empty, which is the capture's `1 = 0`. +fn admits(filter: &Filter, gates: &Gates) -> Result { + if gates.nip50 + && filter + .search + .as_deref() + .is_some_and(|search| !search.trim().is_empty()) + { + return Err(StorageError::NotImplemented { + module: "storage::lmdb::nip50_search", + }); + } + Ok(!(filter.ids.as_ref().is_some_and(|ids| ids.is_empty()) + || filter + .authors + .as_ref() + .is_some_and(|authors| authors.is_empty()) + || filter + .tags + .iter() + .any(|(key, criteria)| generic_tag_name(key).is_some() && criteria.is_empty()))) +} + +/// One filter's page: the best `limit` keys it matches, ordered as the capture's branch is. +/// +/// The resident set is the page, never the range, and the page is one accumulator across every +/// range the plan seeks — the top `limit` is global to them. A range whose key order is the +/// page's order stops once the page holds its own worst kept row: every row after it in that +/// range is past it, so a later range can no longer arm the stop at the first match it offers. +/// The comparison is strict, so every row tied at the boundary timestamp is still offered and +/// the tie breaks as the capture's does. The page is a heap rooted at that worst kept row, so +/// the boundary is one peek and an offer is one comparison until it beats the root. +fn branch( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + gates: &Gates, + now: i64, + live: bool, + limit: usize, +) -> Result, StorageError> { + if !admits(filter, gates)? { + return Ok(Vec::new()); + } + + let plan = plan(filter, databases); + // A `tags` range is built from the family's own criteria, so every key in it satisfies that + // family: the matcher skips it and no body is decoded for it. + let matcher = Matcher { + filter, + gates, + now, + live, + proven: plan.tag.map(|(name, _)| name), + }; + let descending = filter.limit.is_some(); + let mut page = Page::new(descending); + for range in &plan.ranges { + let bounds = encoding::bounds(&range.low, &range.high); + // A range starts at the boundary the page already holds: a range whose oldest row is + // past it has nothing to add, and one that is not is walked from its own end. + let mut boundary = if plan.ordered { + page.boundary(limit) + } else { + None + }; + for entry in scan(plan.database, txn, &bounds, plan.ordered && descending)? { + let (key, _) = entry?; + key_evaluated(); + let Some(found) = candidate(txn, databases, plan.path, key)? else { + continue; + }; + if plan.ordered && boundary.is_some_and(|bound| past(bound, found.0, descending)) { + break; + } + if matcher.matches(txn, databases, &found.1)?.is_some() { + page.offer(found, limit); + if plan.ordered { + boundary = page.boundary(limit); + } + } + } + } + Ok(page.finish(limit)) +} + +/// Whether a key's timestamp is past the timestamp a full page stopped at. +fn past(boundary: i64, created_at: i64, descending: bool) -> bool { + if descending { + created_at < boundary + } else { + created_at > boundary + } +} + +/// A page under construction: at most `limit` entries, held so the row the page would drop first +/// is the heap's root. +struct Page { + kept: BinaryHeap, + seen: HashSet, + descending: bool, +} + +impl Page { + fn new(descending: bool) -> Self { + Self { + kept: BinaryHeap::new(), + seen: HashSet::new(), + descending, + } + } + + /// Keeps `key` if it can still be among the best `limit` the page has seen. + fn offer(&mut self, key: (i64, EventId), limit: usize) { + if limit == 0 || !self.seen.insert(key.1) { + return; + } + if self.kept.len() < limit { + self.kept.push(Kept { + entry: key, + descending: self.descending, + }); + return; + } + let Some(worst) = self.kept.peek().map(|kept| kept.entry) else { + return; + }; + // A row the page would drop on its way in cannot be in it, and its id must leave `seen` + // with it so that a later range may offer it again. + if page_order(key, worst, self.descending) == Ordering::Less { + if let Some(dropped) = self.kept.pop() { + self.seen.remove(&dropped.entry.1); + } + self.kept.push(Kept { + entry: key, + descending: self.descending, + }); + } else { + self.seen.remove(&key.1); + } + } + + /// The timestamp of the worst row the page would keep, once it holds `limit` of them. + /// + /// A range that runs in the page's order cannot offer a row past it, so the walk stops + /// there. That row is the heap's root, so the boundary costs one peek and the page is cut to + /// the limit as it fills rather than when the boundary is read. + fn boundary(&self, limit: usize) -> Option { + if limit == 0 || self.kept.len() < limit { + return None; + } + self.kept.peek().map(|worst| worst.entry.0) + } + + /// The page in the capture's order, which is the one sort the whole page pays. + fn finish(self, limit: usize) -> Vec<(i64, EventId)> { + let mut page: Vec<(i64, EventId)> = self.kept.into_iter().map(|kept| kept.entry).collect(); + #[cfg(test)] + probes::sorted(page.len()); + sort_page(&mut page, self.descending); + page.truncate(limit); + page + } +} + +/// A page entry, ordered by the page's own comparison: the row the page would drop first is the +/// greatest of them, so a max-heap keeps it at the root. Every entry of one heap shares its +/// direction. +#[derive(Clone, Copy)] +struct Kept { + entry: (i64, EventId), + descending: bool, +} + +impl Ord for Kept { + fn cmp(&self, other: &Self) -> Ordering { + page_order(self.entry, other.entry, self.descending) + } +} + +impl PartialOrd for Kept { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl PartialEq for Kept { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == Ordering::Equal + } +} + +impl Eq for Kept {} + +/// One range's keys, in the plan's order. +enum Scan<'txn> { + Ascending(heed::RoRange<'txn, Bytes, Bytes>), + Descending(heed::RoRevRange<'txn, Bytes, Bytes>), +} + +impl<'txn> Iterator for Scan<'txn> { + type Item = Result<(&'txn [u8], &'txn [u8]), heed::Error>; + + fn next(&mut self) -> Option { + match self { + Self::Ascending(range) => range.next(), + Self::Descending(range) => range.next(), + } + } +} + +fn scan<'txn>( + database: Database, + txn: &'txn RoTxn<'_>, + range: &(Bound<&[u8]>, Bound<&[u8]>), + descending: bool, +) -> Result, StorageError> { + Ok(if descending { + Scan::Descending(database.rev_range(txn, range)?) + } else { + Scan::Ascending(database.range(txn, range)?) + }) +} + +/// The capture's page order: newest first when `descending`, oldest first otherwise, the +/// smaller id first on a tie. +fn page_order(left: (i64, EventId), right: (i64, EventId), descending: bool) -> Ordering { + if descending { + right.0.cmp(&left.0).then_with(|| left.1.cmp(&right.1)) + } else { + left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1)) + } +} + +fn sort_page(page: &mut [(i64, EventId)], descending: bool) { + page.sort_by(|left, right| page_order(*left, *right, descending)); +} + +/// The index a filter is answered from, the ranges within it, and whether a range's key order +/// is the page's order. +/// +/// shortcut: `ids`, an author prefix and a `#g` prefix name no index that runs in time order, so +/// those pages are bounded in memory but walk their whole time-bounded range; upgrade when a +/// deployment's range outgrows one request, which needs a second index ordered by time. +fn plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { + if let Some(ids) = &filter.ids + && !ids.is_empty() + { + return Plan { + database: databases.events, + path: AccessPath::Ids, + ranges: plain_ranges(encoding::hex_ranges(ids, 0)), + // The `events` key is the id, so the range runs in id order, not time order. + ordered: false, + tag: None, + }; + } + if let Some(authors) = &filter.authors + && !authors.is_empty() + { + if let Some(kinds) = &filter.kinds + && !kinds.is_empty() + && let Some(ranges) = author_kind_ranges(authors, kinds) + { + return Plan { + database: databases.events_by_author, + path: AccessPath::Authors, + ranges: plain_ranges(ranges), + // A range is `pubkey ‖ kind ‖ created_at ‖ event_id`, so it runs in time order + // only when every criterion is a whole pubkey and every range one author and + // kind. + ordered: authors.iter().all(|criterion| criterion.len() == 64), + tag: None, + }; + } + // Without kinds the author index runs in kind order before `created_at`, so a page + // ordered by time has no early stop there, and the same index is given up when the + // author × kind product is past `MAX_PAGE_KEYS`. The created-at index is the capture's + // own path for this shape — ADR 0003 measured 5 508 rows discarded to return 500 for + // the kind-only form — and its order *is* the page order, so the page stops when full. + // + // shortcut: a quiet author's page, and an author × kind filter composing more than + // `MAX_PAGE_KEYS` ranges, read the created-at index with the criteria left to the + // matcher; upgrade when statistics can choose between the two indexes, which is the + // benchmark job's (ADR 0011). + return Plan { + database: databases.events_by_created_at, + path: AccessPath::CreatedAt, + ranges: plain_ranges(created_at_ranges(filter)), + ordered: true, + tag: None, + }; + } + if let Some((name, criteria)) = tag_path(filter) { + return Plan { + database: databases.tags, + path: AccessPath::Tags, + ranges: encoding::tag_ranges(name, criteria), + // An exact criterion's range runs `created_at` under one value; a `#g` prefix range + // spans values, so its keys are in value order and the range is walked whole. + ordered: name != "g" || criteria.iter().all(|criterion| !criterion.ends_with('*')), + tag: Some((name, criteria)), + }; + } + Plan { + database: databases.events_by_created_at, + path: AccessPath::CreatedAt, + ranges: plain_ranges(created_at_ranges(filter)), + ordered: true, + tag: None, + } +} + +/// The ranges of a plan that reads an index a tag family did not build, so no criterion answers +/// for the keys in them. +fn plain_ranges<'a>(ranges: Vec<(Vec, Vec)>) -> Vec> { + ranges.into_iter().map(encoding::KeyRange::plain).collect() +} + +/// One range per author criterion on `events_by_author`, whose key is +/// `pubkey ‖ kind ‖ created_at ‖ event_id`. A whole pubkey composes one range per kind, so the +/// `pubkey ‖ kind ‖ created_at` ordering the layout writes lets a page for one author and one +/// kind stop as soon as it is full; a prefix criterion composes one range over its whole tail +/// instead, because its upper bound runs to the end of the pubkey field and a kind appended +/// after it is compared where no key writes one — every kind under the prefix would fall inside +/// every kind's range, and a count would visit those rows once per kind. +/// +/// A criterion another already covers seeks no range of its own, and a repeated kind is one +/// range, so a count cannot visit a key twice. +/// +/// `None` when the whole-pubkey ranges and kinds compose more than [`MAX_PAGE_KEYS`] ranges: a +/// filter can name a thousand authors and a thousand kinds, and their product is not bounded by +/// the page, so the caller answers it from the created-at index rather than materialising it. +fn author_kind_ranges(authors: &[String], kinds: &[Kind]) -> Option, Vec)>> { + let mut kinds: Vec = kinds + .iter() + .map(|kind| i32::try_from(kind.0).unwrap_or(i32::MAX)) + .collect(); + kinds.sort_unstable(); + kinds.dedup(); + + let author_ranges = encoding::hex_ranges(authors, 0); + let exact = author_ranges + .iter() + .filter(|(low, high)| low == high && low.len() == encoding::HEX_FIELD_LEN) + .count(); + if exact.saturating_mul(kinds.len()) > MAX_PAGE_KEYS { + return None; + } + let mut ranges = + Vec::with_capacity(exact.saturating_mul(kinds.len()) + author_ranges.len() - exact); + for (low, high) in author_ranges { + if low == high && low.len() == encoding::HEX_FIELD_LEN { + for kind in &kinds { + let mut lower = low.clone(); + lower.extend_from_slice(&encoding::i32_key(*kind)); + let mut upper = lower.clone(); + upper.extend(std::iter::repeat_n(0xff, 8 + 32)); + ranges.push((lower, upper)); + } + } else { + let mut upper = high; + upper.extend(std::iter::repeat_n(0xff, 4 + 8 + 32)); + ranges.push((low, upper)); + } + } + Some(encoding::disjoint(ranges)) +} + +/// The first tag criterion the capture treats as a query: `#` plus one ASCII letter with a +/// non-empty list. +fn tag_path(filter: &Filter) -> Option<(&str, &[String])> { + filter.tags.iter().find_map(|(key, criteria)| { + generic_tag_name(key) + .filter(|_| !criteria.is_empty()) + .map(|name| (name, criteria.as_slice())) + }) +} + +fn created_at_ranges(filter: &Filter) -> Vec<(Vec, Vec)> { + let lower = match filter.since { + Some(since) => encoding::i64_key(since).to_vec(), + None => Vec::new(), + }; + let upper = match filter.until { + Some(until) => { + let mut upper = encoding::i64_key(until).to_vec(); + upper.extend([0xff; 32]); + upper + } + None => vec![0xff; 64], + }; + vec![(lower, upper)] +} + +/// The `(created_at, event_id)` one index key carries, when the key carries it. +fn key_candidate(path: AccessPath, key: &[u8]) -> Result, StorageError> { + match path { + AccessPath::Ids => Ok(None), + AccessPath::Authors => { + if key.len() != 76 { + return Err(encoding::invalid( + "events_by_author", + format!("a key of {} bytes", key.len()), + )); + } + Ok(Some(( + encoding::i64_from_key(&key[36..44])?, + encoding::event_id_from(&key[44..76])?, + ))) + } + AccessPath::Tags => { + if key.len() < 42 { + return Err(encoding::invalid( + "tags", + format!("a key of {} bytes", key.len()), + )); + } + let start = key.len() - 40; + Ok(Some(( + encoding::i64_from_key(&key[start..start + 8])?, + encoding::event_id_from(&key[start + 8..])?, + ))) + } + AccessPath::CreatedAt => { + if key.len() != 40 { + return Err(encoding::invalid( + "events_by_created_at", + format!("a key of {} bytes", key.len()), + )); + } + Ok(Some(( + encoding::i64_from_key(&key[..8])?, + encoding::event_id_from(&key[8..])?, + ))) + } + } +} + +/// A candidate whose `created_at` is only in the value, read from the header. +fn candidate( + txn: &RoTxn<'_>, + databases: &Databases, + path: AccessPath, + key: &[u8], +) -> Result, StorageError> { + if let Some(candidate) = key_candidate(path, key)? { + return Ok(Some(candidate)); + } + let id = encoding::event_id_from(key)?; + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + return Ok(None); + }; + Ok(Some((rows::event_header(value)?.created_at, id))) +} + +/// One filter's criteria, evaluated against a stored event. +struct Matcher<'a> { + filter: &'a Filter, + gates: &'a Gates, + now: i64, + live: bool, + /// The tag family the index range that selected the row has already proved, if any. + proven: Option<&'a str>, +} + +impl Matcher<'_> { + fn matches( + &self, + txn: &RoTxn<'_>, + databases: &Databases, + id: &EventId, + ) -> Result, StorageError> { + let Some(value) = databases.events.get(txn, id.as_bytes())? else { + return Ok(None); + }; + let header = rows::event_header(value)?; + + if self.live + && (header.deleted + || header + .expires_at + .is_some_and(|expires_at| expires_at <= self.now)) + { + return Ok(None); + } + if let Some(ids) = &self.filter.ids + && !ids + .iter() + .any(|criterion| encoding::hex_matches(id.as_bytes(), criterion)) + { + return Ok(None); + } + if let Some(authors) = &self.filter.authors + && !authors + .iter() + .any(|criterion| encoding::hex_matches(header.pubkey.as_bytes(), criterion)) + { + return Ok(None); + } + if let Some(kinds) = &self.filter.kinds + && !kinds + .iter() + .any(|kind| i32::try_from(kind.0).unwrap_or(i32::MAX) == header.kind) + { + return Ok(None); + } + if let Some(since) = self.filter.since + && header.created_at < since + { + return Ok(None); + } + if let Some(until) = self.filter.until + && header.created_at > until + { + return Ok(None); + } + if self.gates.nip56 && self.reported(txn, databases, id, &header.pubkey)? { + return Ok(None); + } + if self.carries_other_tag_criteria() + && !tag_criteria_match(&self.filter.tags, &rows::decode_event(value)?, self.proven) + { + return Ok(None); + } + Ok(Some(header)) + } + + /// Whether the filter carries a tag family the row's key has not already proved. A row the + /// `tags` index selected needs no decode when that is the filter's only family. + fn carries_other_tag_criteria(&self) -> bool { + self.filter + .tags + .keys() + .filter_map(|key| generic_tag_name(key)) + .any(|name| Some(name) != self.proven) + } + + /// The capture's `NOT EXISTS (… reports …)`: one lookup per target column, by prefix so the + /// width of `reports.id` is `metadata`'s business and not restated here. + fn reported( + &self, + txn: &RoTxn<'_>, + databases: &Databases, + id: &EventId, + pubkey: &Pubkey, + ) -> Result { + if exists_with_prefix(databases.reports_actionable_event, txn, id.as_bytes())? { + return Ok(true); + } + exists_with_prefix(databases.reports_actionable_pubkey, txn, pubkey.as_bytes()) + } +} + +/// Whether `database` holds any key starting with `prefix`. +fn exists_with_prefix( + database: Database, + txn: &RoTxn<'_>, + prefix: &[u8], +) -> Result { + Ok(database + .prefix_iter(txn, prefix)? + .next() + .transpose()? + .is_some()) +} + +/// The capture's `isGenericTagQuery`: `#` plus one ASCII letter. +pub(crate) fn generic_tag_name(key: &str) -> Option<&str> { + let mut characters = key.chars(); + if characters.next() != Some('#') { + return None; + } + let name = characters.next()?; + if characters.next().is_some() || !name.is_ascii_alphabetic() { + return None; + } + Some(&key[1..2]) +} + +/// The value a criterion selects, when it selects `value`: an exact equality, and only `#g` +/// honours a trailing `*` as a prefix. The one place the wildcard rule is written. +fn matching_value<'v>(is_geohash: bool, criterion: &str, value: &'v str) -> Option<&'v str> { + let matches = match criterion.strip_suffix('*').filter(|_| is_geohash) { + Some(prefix) => value.starts_with(prefix), + None => value == criterion, + }; + matches.then_some(value) +} + +/// Whether a tag value satisfies one of a family's criteria. +/// +/// `event_tags` holds no empty value, so an empty one satisfies nothing. The criterion a `tags` +/// key carries is checked here, from the key, and the same rule answers the decoded record. +fn tag_value_matches(is_geohash: bool, criteria: &[String], value: &str) -> bool { + !value.is_empty() + && criteria + .iter() + .any(|criterion| matching_value(is_geohash, criterion, value).is_some()) +} + +/// Whether an event carries a tag called `name` whose value satisfies one of `criteria`. +fn record_has_tag(record: &EventRecord, name: &str, criteria: &[String]) -> bool { + let is_geohash = name == "g"; + record.event.tags.iter().any(|tag| { + tag.first().map(String::as_str) == Some(name) + && tag + .get(1) + .is_some_and(|value| tag_value_matches(is_geohash, criteria, value)) + }) +} + +/// Every tag criterion of a filter, under the capture's rules, except the family `proven`. +fn tag_criteria_match( + tags: &BTreeMap>, + record: &EventRecord, + proven: Option<&str>, +) -> bool { + for (key, criteria) in tags { + let Some(name) = generic_tag_name(key) else { + continue; + }; + if Some(name) == proven { + continue; + } + if criteria.is_empty() || !record_has_tag(record, name, criteria) { + return false; + } + } + true +} + +// --------------------------------------------------------------------------------------- +// Streams. +// --------------------------------------------------------------------------------------- + +/// Where a reconciliation batch resumes: the range it is on, and the last key it evaluated. +struct Position { + index: usize, + after: Option>, +} + +impl Position { + fn new() -> Self { + Self { + index: 0, + after: None, + } + } +} + +/// Reads one page of values in batches of [`STREAM_BATCH_ITEMS`], one read transaction per +/// poll, so a stream that is dropped pins no pages. +/// +/// It holds the page `find_by_filters` already collected — at most [`MAX_PAGE_KEYS`] ids — and +/// copies each batch's values out of it; it is the reconciliation stream, not this one, that +/// holds only a cursor key. +/// +/// shortcut: a stream is not a snapshot — a row written or deleted between two batches may +/// appear or vanish; upgrade when a caller needs a stable view, per ADR 0003 trigger 1 and the +/// benchmark job, which would buy the snapshot back with a held transaction. +pub(crate) struct ValuesStream { + env: Env, + databases: Databases, + keys: std::vec::IntoIter, + pending: Option, StorageError>>>, + buffer: std::vec::IntoIter, + finished: bool, +} + +impl ValuesStream { + pub(crate) fn new(env: Env, databases: Databases, page: &[(i64, EventId)]) -> Self { + Self { + env, + databases, + keys: page + .iter() + .map(|(_, id)| *id) + .collect::>() + .into_iter(), + pending: None, + buffer: Vec::new().into_iter(), + finished: false, + } + } +} + +impl Stream for ValuesStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + if let Some(record) = this.buffer.next() { + return Poll::Ready(Some(Ok(record))); + } + if let Some(handle) = this.pending.as_mut() { + match Pin::new(handle).poll(context) { + Poll::Pending => return Poll::Pending, + Poll::Ready(joined) => { + this.pending = None; + match joined { + Ok(Ok(records)) => this.buffer = records.into_iter(), + Ok(Err(error)) => { + this.finished = true; + return Poll::Ready(Some(Err(error))); + } + Err(_) => { + this.finished = true; + return Poll::Ready(Some(Err(StorageError::LmdbTaskCancelled))); + } + } + } + } + continue; + } + if this.finished { + return Poll::Ready(None); + } + let batch: Vec = this.keys.by_ref().take(STREAM_BATCH_ITEMS).collect(); + if batch.is_empty() { + this.finished = true; + return Poll::Ready(None); + } + let env = this.env.clone(); + let database = this.databases.events; + this.pending = Some(tokio::task::spawn_blocking(move || { + let transaction = env.read_txn()?; + let mut records = Vec::with_capacity(batch.len()); + for id in batch { + let Some(value) = database.get(&transaction, id.as_bytes())? else { + continue; + }; + records.push(rows::decode_event(value)?); + } + Ok(records) + })); + } + } +} + +/// Yields the live `(created_at, event_id)` pairs a reconciliation compares, oldest first, in +/// batches of at most [`STREAM_BATCH_ITEMS`] items or [`STREAM_BATCH_KEYS`] evaluated keys — +/// whichever comes first — with one read transaction per poll. An `ids` filter naming exact ids +/// is one batch of the request's own list, sorted in the poll that reads it; a prefix criterion +/// is a byte range and takes the bounded scan. +pub(crate) struct ReconciliationStream { + env: Env, + databases: Databases, + filter: Filter, + gates: Gates, + now: i64, + range_count: usize, + position: Position, + pending: Option, + buffer: std::vec::IntoIter, + finished: bool, +} + +impl ReconciliationStream { + /// Builds the stream for one filter, or refuses it as every other read does: a non-empty + /// `search` with the NIP-50 gate on is [`StorageError::NotImplemented`], and a criterion + /// family that is present but empty selects nothing (the capture's `1 = 0`). + pub(crate) fn new( + env: Env, + databases: Databases, + filter: &Filter, + gates: Gates, + now: i64, + ) -> Result { + let admitted = admits(filter, &gates)?; + let range_count = if admitted { + stream_plan(filter, &databases).ranges.len() + } else { + 0 + }; + Ok(Self { + env, + databases, + filter: filter.clone(), + gates, + now, + range_count, + position: Position::new(), + pending: None, + buffer: Vec::new().into_iter(), + finished: !admitted, + }) + } +} + +impl Stream for ReconciliationStream { + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + if let Some(item) = this.buffer.next() { + return Poll::Ready(Some(Ok(item))); + } + if let Some(handle) = this.pending.as_mut() { + match Pin::new(handle).poll(context) { + Poll::Pending => return Poll::Pending, + Poll::Ready(joined) => { + this.pending = None; + match joined { + Ok(Ok((items, position))) => { + this.position = position; + this.buffer = items.into_iter(); + } + Ok(Err(error)) => { + this.finished = true; + return Poll::Ready(Some(Err(error))); + } + Err(_) => { + this.finished = true; + return Poll::Ready(Some(Err(StorageError::LmdbTaskCancelled))); + } + } + } + } + continue; + } + if this.finished { + return Poll::Ready(None); + } + let env = this.env.clone(); + let databases = this.databases; + let filter = this.filter.clone(); + let gates = this.gates; + let now = this.now; + let mut position = std::mem::replace(&mut this.position, Position::new()); + let exhausted = position.index >= this.range_count; + if exhausted { + this.finished = true; + return Poll::Ready(None); + } + this.pending = Some(tokio::task::spawn_blocking(move || { + let transaction = env.read_txn()?; + let items = scan_batch( + &transaction, + &databases, + &filter, + &gates, + now, + &mut position, + )?; + Ok((items, position)) + })); + } + } +} + +/// The plan a reconciliation scans with. +/// +/// The stream promises the pairs oldest first. An `ids` filter whose criteria are all exact is +/// answered from the filter's own ranges — one point per id, so the work is the request's id +/// list and not the store — and the batch sorts the pairs before it emits them, because an +/// `events` key runs in id order. A criterion is exact only when its range is a point: a prefix +/// is a byte range over the id field, and `""` is the whole store, so it is not the request's +/// list and takes the bounded scan instead. A filter the faster index narrows to one range that +/// runs in `created_at` order (`{authors: [a], kinds: [k]}` or one exact tag value) is scanned +/// there; anything else scans the created-at index, whose order *is* the promise. +/// +/// shortcut: an author without kinds, a `#g` prefix and an id prefix narrow to no such range, +/// so their stream scans `events_by_created_at` and the criteria are left to the matcher — a +/// poll walks what the store holds rather than what the request names; upgrade when statistics +/// can choose the cheaper index, which is the benchmark job's (ADR 0011). +fn stream_plan<'a>(filter: &'a Filter, databases: &Databases) -> Plan<'a> { + let chosen = plan(filter, databases); + let exact_ids = + chosen.path == AccessPath::Ids && chosen.ranges.iter().all(|range| range.low == range.high); + if exact_ids || (chosen.ordered && chosen.ranges.len() == 1) { + return chosen; + } + Plan { + database: databases.events_by_created_at, + path: AccessPath::CreatedAt, + ranges: plain_ranges(created_at_ranges(filter)), + ordered: true, + tag: None, + } +} + +/// One batch of a reconciliation scan: at most [`STREAM_BATCH_ITEMS`] items or +/// [`STREAM_BATCH_KEYS`] evaluated keys, whichever comes first, resuming after the last key the +/// previous batch read. +/// +/// The key cap is what keeps one poll short however few of the keys it walks match: every +/// transaction a stream opens is closed by the poll that opened it, so no read pins a page while +/// the writer works. +fn scan_batch( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + gates: &Gates, + now: i64, + position: &mut Position, +) -> Result, StorageError> { + let plan = stream_plan(filter, databases); + let matcher = Matcher { + filter, + gates, + now, + live: true, + proven: plan.tag.map(|(name, _)| name), + }; + if plan.path == AccessPath::Ids { + return scan_ids(txn, databases, &matcher, &plan, position); + } + let mut items = Vec::new(); + let mut evaluated = 0; + while position.index < plan.ranges.len() { + let range = &plan.ranges[position.index]; + let bounds = encoding::resume_bounds(position.after.as_deref(), &range.low, &range.high); + let mut reached_cap = false; + for entry in plan.database.range(txn, &bounds)? { + if items.len() >= STREAM_BATCH_ITEMS || evaluated >= STREAM_BATCH_KEYS { + reached_cap = true; + break; + } + let (key, _) = entry?; + key_evaluated(); + // The cursor moves only onto a key this batch evaluated, because the next batch + // resumes exclusively after it. + position.after = Some(key.to_vec()); + evaluated += 1; + let Some((created_at, event_id)) = key_candidate(plan.path, key)? else { + return Err(encoding::invalid( + "events", + "a key without a timestamp and an event id".to_owned(), + )); + }; + if matcher.matches(txn, databases, &event_id)?.is_some() { + items.push(ReconciliationItem { + created_at, + event_id, + }); + } + } + if reached_cap { + break; + } + position.index += 1; + position.after = None; + } + Ok(items) +} + +/// The pairs an exact `ids` filter names, oldest first, with `position` moved past the id list. +/// +/// The ranges are the request's own list, one point per id and already deduplicated, so the +/// work is the request rather than the store, and the list is read whole because an `events` +/// key runs in id order: the pairs are sorted before they are emitted instead of being sought +/// in time order. [`stream_plan`] keeps a prefix criterion away from here, because its range is +/// the id field's bytes rather than the request. +/// +/// shortcut: the whole list is read in one poll and its pairs are held until the stream ends, so +/// the resident set is the request's own id list; upgrade when a request's list should be paged +/// too, which needs the pairs sorted before emission and therefore somewhere to spill them. +fn scan_ids( + txn: &RoTxn<'_>, + databases: &Databases, + matcher: &Matcher<'_>, + plan: &Plan<'_>, + position: &mut Position, +) -> Result, StorageError> { + let mut items = Vec::with_capacity(plan.ranges.len()); + for range in &plan.ranges { + let bounds = encoding::bounds(&range.low, &range.high); + for entry in plan.database.range(txn, &bounds)? { + let (key, _) = entry?; + key_evaluated(); + let Some((created_at, event_id)) = candidate(txn, databases, plan.path, key)? else { + continue; + }; + if matcher.matches(txn, databases, &event_id)?.is_some() { + items.push(ReconciliationItem { + created_at, + event_id, + }); + } + } + } + items.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.event_id.cmp(&right.event_id)) + }); + position.index = plan.ranges.len(); + position.after = None; + Ok(items) +} + +/// The `event_id` one `events_by_author` or `events_deleted` key ends with. +fn header_id(key: &[u8], start: usize, table: &'static str) -> Result { + match key.get(start..) { + Some(bytes) => encoding::event_id_from(bytes), + None => Err(encoding::invalid( + table, + format!("a key of {} bytes", key.len()), + )), + } +} + +/// The kind and event id an `events_by_author` key carries. +struct AuthorKeyHeader { + kind: i32, + event_id: EventId, +} + +fn author_key_header(key: &[u8]) -> Result { + if key.len() != 76 { + return Err(encoding::invalid( + "events_by_author", + format!("a key of {} bytes", key.len()), + )); + } + Ok(AuthorKeyHeader { + kind: encoding::i32_from_key(&key[32..36])?, + event_id: encoding::event_id_from(&key[44..76])?, + }) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + fn id(byte: u8) -> EventId { + EventId::from_byte_array([byte; 32]) + } + + #[test] + fn a_page_sorts_newest_first_with_the_smaller_id_first_on_a_tie() { + let mut page = vec![(10, id(2)), (10, id(1)), (11, id(9))]; + sort_page(&mut page, true); + assert_eq!(page, vec![(11, id(9)), (10, id(1)), (10, id(2))]); + + sort_page(&mut page, false); + assert_eq!(page, vec![(10, id(1)), (10, id(2)), (11, id(9))]); + } + + #[test] + fn a_page_holds_at_most_its_limit() { + let _serial = counting(); + let mut page = Page::new(true); + for step in 0u8..=100 { + page.offer((i64::from(step), id(step)), 3); + assert!( + page.kept.len() <= 3, + "the page is its limit, never a buffer behind it: {} entries", + page.kept.len() + ); + } + assert_eq!(page.finish(3).len(), 3); + } + + #[test] + fn only_a_hash_and_one_letter_is_a_tag_query() { + assert_eq!(generic_tag_name("#e"), Some("e")); + assert_eq!(generic_tag_name("#1"), None); + assert_eq!(generic_tag_name("#ee"), None); + assert_eq!(generic_tag_name("e"), None); + assert_eq!(generic_tag_name("#"), None); + } + + #[test] + fn the_retention_whitelist_always_keeps_kind_62_and_deduplicates() { + let whitelist = retention_kinds(Some(&[ + KindSelector::Single(Kind(1)), + KindSelector::Single(Kind(1)), + ])); + assert_eq!( + whitelist, + vec![ + KindSelector::Single(Kind(1)), + KindSelector::Single(Kind(62)), + ] + ); + assert!(whitelisted(62, &whitelist)); + assert!(whitelisted(1, &[KindSelector::Range([Kind(1), Kind(9)])])); + assert!(!whitelisted(10, &[KindSelector::Range([Kind(1), Kind(9)])])); + } + + #[test] + fn a_geohash_wildcard_is_a_prefix_and_another_tag_treats_it_literally() { + use nostream_core::Event; + + let event: Event = serde_json::from_value(serde_json::json!({ + "id": "0d684e8ec2431de586aa3cafbee2f6d308d19b28805e53deabcac3220e9136a5", + "pubkey": "f43c1f9bff677b8f27b602725ea0ad51af221344f69a6b352a74991a4479bac3", + "created_at": 1660407625i64, + "kind": 1, + "tags": [["g", "u4pruyd"], ["e", "other"]], + "content": "", + "sig": "ab".repeat(64) + })) + .expect("a wire event"); + let record = EventRecord::from_event(&event); + + let mut tags = BTreeMap::new(); + tags.insert("#g".to_owned(), vec!["u4pru*".to_owned()]); + assert!(tag_criteria_match(&tags, &record, None)); + + let mut tags = BTreeMap::new(); + tags.insert("#e".to_owned(), vec!["u4pru*".to_owned()]); + assert!(!tag_criteria_match(&tags, &record, None)); + + let mut tags = BTreeMap::new(); + tags.insert("#em".to_owned(), vec!["nothing".to_owned()]); + assert!(tag_criteria_match(&tags, &record, None)); + } + + // ----------------------------------------------------------------------------------- + // The cost of a read, and the bound on what it holds. + // ----------------------------------------------------------------------------------- + + use crate::backend::LmdbEnv; + use nostream_core::{Event, Signature}; + + /// An open store on a temporary directory. + fn store() -> (tempfile::TempDir, Env, Databases) { + store_with_map(8 << 20) + } + + /// An open store on a temporary directory, with room for a corpus past `MAX_PAGE_KEYS`. + fn store_with_map(map_size: usize) -> (tempfile::TempDir, Env, Databases) { + let directory = tempfile::tempdir().expect("a temporary directory"); + let environment = + LmdbEnv::open(directory.path().join("lmdb"), map_size).expect("an environment"); + let env = environment.env().clone(); + let mut transaction = env.write_txn().expect("a write transaction"); + let databases = Databases::create(&env, &mut transaction).expect("the databases"); + transaction.commit().expect("the commit"); + (directory, env, databases) + } + + /// A signed event of one author, carrying `tags`. + fn signed(secret: u8, kind: u64, created_at: i64, tags: Vec>) -> EventRecord { + let mut wire = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(kind), + tags, + content: format!("{secret}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut wire).expect("the event signs"); + EventRecord::from_event(&wire) + } + + /// A signed event of one author and its own body, for a corpus `signed` cannot build: more + /// than its 256 authors, or several rows at one `created_at` under one author. + fn signed_by(secret: [u8; 32], created_at: i64, content: &str) -> EventRecord { + let mut wire = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(1), + tags: Vec::new(), + content: content.to_owned(), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&secret, &mut wire).expect("the event signs"); + EventRecord::from_event(&wire) + } + + /// Writes the events in one transaction, as `create_many` does. + fn write(env: &Env, databases: &Databases, records: &[EventRecord]) { + let mut transaction = env.write_txn().expect("a write transaction"); + create(&mut transaction, databases, records, Utc::now()).expect("the events store"); + transaction.commit().expect("the commit"); + } + + fn created(records: &[(i64, EventId)]) -> Vec { + records.iter().map(|(created_at, _)| *created_at).collect() + } + + #[test] + fn a_limited_author_page_evaluates_only_the_keys_it_needs() { + // The author index runs in kind order before `created_at`, so this page is answered by + // the created-at index: it stops as soon as it is full instead of walking the author's + // whole timeline (the round-2 shape evaluated every one of the 200). + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=200) + .map(|step| signed(7, 1, step, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + limit: Some(2), + ..Filter::default() + }; + + probes::reset(); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + + assert_eq!(created(&page), vec![200, 199]); + assert!( + probes::keys() <= 3, + "a full page stops the walk: {} keys evaluated", + probes::keys() + ); + } + + #[test] + fn a_limited_page_stops_at_the_limit_inside_one_author_and_kind() { + // The `pubkey ‖ kind ‖ created_at` ordering the layout writes is what lets this page + // stop; without it the walk reads every event of the author. + let _serial = counting(); + let (_directory, env, databases) = store(); + let mut records: Vec = (1..=200) + .map(|step| signed(7, 1, step, Vec::new())) + .collect(); + records.extend((1..=200).map(|step| signed(7, 2, step, Vec::new()))); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + kinds: Some(vec![Kind(2)]), + limit: Some(2), + ..Filter::default() + }; + + probes::reset(); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + + assert_eq!(created(&page), vec![200, 199]); + assert!( + probes::keys() <= 3, + "one range runs in time order and stops: {} keys evaluated", + probes::keys() + ); + } + + #[test] + fn a_full_page_keeps_the_ties_at_its_boundary_timestamp() { + // The created-at index reverses the whole key, so two rows at one instant are scanned + // id-descending: a walk that stopped at the first would keep the larger id, where the + // capture's comparator keeps the smaller (V6). + let _serial = counting(); + let (_directory, env, databases) = store(); + let tag = |value: &str| vec![vec!["t".to_owned(), value.to_owned()]]; + let first = signed(9, 1, 500, tag("first")); + let second = signed(9, 1, 500, tag("second")); + let smallest = first.event.id.min(second.event.id); + write(&env, &databases, &[first, second]); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(1), + ..Filter::default() + }; + + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + + assert_eq!( + page, + vec![(500, smallest)], + "the row tied at the boundary is still offered, and the smaller id wins" + ); + } + + #[test] + fn a_tag_page_checks_its_criterion_from_the_key_and_decodes_no_body() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let tag = |step: i64| { + vec![ + vec!["t".to_owned(), "perfprobe".to_owned()], + vec!["e".to_owned(), step.to_string()], + ] + }; + let records: Vec = (1..=200) + .map(|step| signed(7, 1, step, tag(step))) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), vec!["perfprobe".to_owned()]); + let filter = Filter { + tags, + limit: Some(2), + ..Filter::default() + }; + probes::reset(); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!(created(&page), vec![200, 199]); + assert_eq!( + probes::decodes(), + 0, + "the key carried the value the criterion sought" + ); + assert!( + probes::keys() <= 3, + "the value's range runs in time order and stops: {} keys evaluated", + probes::keys() + ); + + // A family the key cannot answer is decoded. `#a` sorts first, so the tag index is + // sought by it and `#z` stays unproved. + let (_directory, env, databases) = store(); + let both = |step: i64| { + vec![ + vec!["a".to_owned(), "perfprobe".to_owned()], + vec!["z".to_owned(), format!("other-{step}")], + ] + }; + let records: Vec = (1..=200) + .map(|step| signed(7, 1, step, both(step))) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let mut tags = BTreeMap::new(); + tags.insert("#a".to_owned(), vec!["perfprobe".to_owned()]); + tags.insert( + "#z".to_owned(), + vec!["other-200".to_owned(), "other-199".to_owned()], + ); + let filter = Filter { + tags, + limit: Some(2), + ..Filter::default() + }; + probes::reset(); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!(created(&page), vec![200, 199]); + assert!( + probes::decodes() >= 1, + "the family the key cannot answer is decoded" + ); + } + + #[test] + fn a_wire_limit_cannot_grow_a_page_past_the_backend_bound() { + let _serial = counting(); + assert_eq!(count_limit(10), 10); + assert_eq!(count_limit(u64::MAX), MAX_PAGE_KEYS); + + let (_directory, env, databases) = store(); + let records: Vec = (1..=MAX_PAGE_KEYS as i64 + 1) + .map(|step| signed(11, 1, step, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(u64::MAX), + ..Filter::default() + }; + + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + + assert_eq!( + page.len(), + MAX_PAGE_KEYS, + "the page the backend materialises is its own bound, not the wire's" + ); + assert_eq!(page[0].0, MAX_PAGE_KEYS as i64 + 1, "and it is the newest"); + } + + #[test] + fn a_count_holds_no_matches_and_deduplicates_the_union() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let records = vec![ + signed(21, 1, 10, Vec::new()), + signed(21, 1, 20, Vec::new()), + signed(22, 1, 30, Vec::new()), + ]; + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let author = records[0].event.pubkey.to_string(); + + let filters = vec![ + Filter { + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }, + Filter { + authors: Some(vec![author.clone()]), + ..Filter::default() + }, + ]; + assert_eq!( + count(&transaction, &databases, &filters, &Gates::default(), 0).expect("the count"), + 3, + "the union of two overlapping filters counts each live id once" + ); + + // A limited branch contributes its own page, and the row a later filter adds is counted + // only when the earlier page does not hold it. + let filters = vec![ + Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(2), + ..Filter::default() + }, + Filter { + authors: Some(vec![author]), + ..Filter::default() + }, + ]; + assert_eq!( + count(&transaction, &databases, &filters, &Gates::default(), 0).expect("the count"), + 3 + ); + } + + #[test] + fn a_later_limited_page_does_not_recount_an_earlier_filter_s_rows() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let records = vec![ + signed(28, 1, 10, Vec::new()), + signed(28, 1, 20, Vec::new()), + signed(28, 1, 30, Vec::new()), + ]; + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filters = vec![ + Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + ..Filter::default() + }, + Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(2), + ..Filter::default() + }, + ]; + assert_eq!( + count(&transaction, &databases, &filters, &Gates::default(), 0).expect("the count"), + 3, + "the rows the earlier predicate counted are not counted again by the page" + ); + } + + #[test] + fn a_count_counts_a_row_once_when_two_tag_values_select_it() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let record = signed( + 23, + 1, + 10, + vec![ + vec!["t".to_owned(), "a".to_owned()], + vec!["t".to_owned(), "b".to_owned()], + ], + ); + write(&env, &databases, std::slice::from_ref(&record)); + let transaction = env.read_txn().expect("a read transaction"); + + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), vec!["a".to_owned(), "b".to_owned()]); + let filter = Filter { + tags, + ..Filter::default() + }; + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 1, + "one event under two matching values is one row" + ); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!(page.len(), 1); + } + + #[test] + fn a_reconciliation_scans_the_filter_own_index_when_one_ordered_range_answers_it() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let mut records: Vec = (1..=300) + .map(|step| signed(24, 1, step, Vec::new())) + .collect(); + records.push(signed(25, 1, 400, Vec::new())); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[300].event.pubkey.to_string()]), + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }; + + probes::reset(); + let mut position = Position::new(); + let items = scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"); + + assert_eq!( + items, + vec![ReconciliationItem { + created_at: 400, + event_id: records[300].event.id, + }], + "the filter's own index answers it" + ); + assert!( + probes::keys() <= 2, + "the range the filter's own index narrows to is read, not the whole store: {} keys", + probes::keys() + ); + } + + #[test] + fn a_reconciliation_batch_stops_at_its_key_cap_and_resumes() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let mut records: Vec = (1..=1000) + .map(|step| signed(26, 1, step, Vec::new())) + .collect(); + records.push(signed(27, 4, 2_000, Vec::new())); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + kinds: Some(vec![Kind(4)]), + ..Filter::default() + }; + + // One poll must not hold a read transaction for a whole range: it stops on the keys it + // evaluated, not only on the rows it matched. + probes::reset(); + let mut position = Position::new(); + let first = scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"); + assert!(first.is_empty(), "the only match is past the first batch"); + assert!( + probes::keys() <= STREAM_BATCH_KEYS, + "the batch stopped at the key cap: {} keys", + probes::keys() + ); + assert!(position.after.is_some(), "and it resumes after a key"); + + // Resuming reaches the row, so the cap loses nothing. + let mut found = Vec::new(); + for _ in 0..64 { + found.extend( + scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"), + ); + if position.index == 0 && position.after.is_none() { + break; + } + } + assert_eq!(found.len(), 1); + assert_eq!(found[0].created_at, 2_000); + } + + // ----------------------------------------------------------------------------------- + // Round 5: the composition bound and the shapes a stream narrows. + // ----------------------------------------------------------------------------------- + + #[test] + fn an_author_and_kind_product_is_bounded_by_the_page() { + // A hundred authors and a hundred kinds are a legal filter; the product is what a + // request must not be able to turn into work this backend materialises. + let authors: Vec = (0..100u32).map(|step| format!("{step:064x}")).collect(); + let kinds: Vec = (0..100u64).map(|step| Kind(10_000 + step)).collect(); + assert!( + author_kind_ranges(&authors, &kinds).is_none(), + "10000 ranges are not a plan" + ); + + // A product the page can hold still composes one range per pair. + let kinds: Vec = (0..49u64).map(|step| Kind(10_000 + step)).collect(); + assert_eq!( + author_kind_ranges(&authors, &kinds) + .expect("a bounded product") + .len(), + authors.len() * kinds.len() + ); + + // At the bound the author index is still the plan; one past it, it is not. + let authors: Vec = (0..MAX_PAGE_KEYS as u32) + .map(|step| format!("{step:064x}")) + .collect(); + assert_eq!( + author_kind_ranges(&authors, &[Kind(1)]) + .expect("a product at the page bound") + .len(), + MAX_PAGE_KEYS + ); + let authors: Vec = (0..=MAX_PAGE_KEYS as u32) + .map(|step| format!("{step:064x}")) + .collect(); + assert!( + author_kind_ranges(&authors, &[Kind(1)]).is_none(), + "a product past the page bound falls back to the created-at index" + ); + } + + #[test] + fn a_prefix_author_counts_a_row_once_whatever_its_kind() { + // A prefix author's bound runs to the end of the pubkey field, so a kind appended to + // it does not narrow the range: the same key is in every kind's range. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records = vec![signed(30, 1, 10, Vec::new()), signed(30, 2, 20, Vec::new())]; + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let prefix = records[0].event.pubkey.to_string()[..8].to_owned(); + let filter = Filter { + authors: Some(vec![prefix]), + kinds: Some(vec![Kind(1), Kind(2)]), + ..Filter::default() + }; + + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 2, + "two rows, counted once each" + ); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!(page.len(), 2, "and the page still returns both"); + let plan = plan(&filter, &databases); + assert_eq!( + plan.ranges.len(), + 1, + "a prefix author seeks one range whatever the kinds" + ); + } + + #[test] + fn an_author_and_kind_product_past_the_page_is_answered_from_the_created_at_index() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let records = vec![ + signed(32, 10_000, 10, Vec::new()), + signed(33, 10_001, 20, Vec::new()), + ]; + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + + // A hundred authors and a hundred kinds compose ten thousand ranges, so the plan gives + // up the author index; the rows are still the ones the criteria name. + let mut authors: Vec = (0..100u32).map(|step| format!("{step:064x}")).collect(); + authors[0] = records[0].event.pubkey.to_string(); + authors[1] = records[1].event.pubkey.to_string(); + let kinds: Vec = (0..100u64).map(|step| Kind(10_000 + step)).collect(); + let filter = Filter { + authors: Some(authors), + kinds: Some(kinds), + limit: Some(2), + ..Filter::default() + }; + + let plan = plan(&filter, &databases); + assert_eq!(plan.path, AccessPath::CreatedAt); + assert_eq!(plan.ranges.len(), 1); + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 2 + ); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!( + created(&page), + vec![20, 10], + "newest first, the rows the filter names" + ); + } + + #[test] + fn a_reconciliation_reads_the_ids_the_filter_names_not_the_store() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=300) + .map(|step| signed(29, 1, step, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + ids: Some(vec![ + records[250].event.id.to_string(), + records[12].event.id.to_string(), + records[199].event.id.to_string(), + ]), + ..Filter::default() + }; + + probes::reset(); + let mut position = Position::new(); + let items = scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"); + + assert_eq!( + items, + vec![ + ReconciliationItem { + created_at: 13, + event_id: records[12].event.id, + }, + ReconciliationItem { + created_at: 200, + event_id: records[199].event.id, + }, + ReconciliationItem { + created_at: 251, + event_id: records[250].event.id, + }, + ], + "the request's ids, oldest first" + ); + assert!( + probes::keys() <= 3, + "the id list bounds the work, not the store: {} keys evaluated", + probes::keys() + ); + assert_eq!( + position.index, + stream_plan(&filter, &databases).ranges.len(), + "the whole list is one batch" + ); + } + + #[test] + fn a_quote_only_deduplication_value_stores_without_json_escaping() { + let (_directory, env, databases) = store(); + let value = "\"".repeat(235); + let record = signed(31, 30_023, 1, vec![vec!["d".to_owned(), value.clone()]]) + .with_deduplication(vec![serde_json::Value::String(value)]); + + let mut transaction = env.write_txn().expect("a write transaction"); + let affected = upsert(&mut transaction, &databases, &record, Utc::now()) + .expect("a d value that JSON text would escape twice still stores"); + transaction.commit().expect("the commit"); + assert_eq!(affected, 1); + } + + // ----------------------------------------------------------------------------------- + // Round 11: one row per replaceable address in a batch. + // ----------------------------------------------------------------------------------- + + /// Two versions of one address reach the same batch in either order: the batch writes the + /// newer row and counts one, which is what `upsert_many` promises its caller. + #[test] + fn a_batch_with_two_versions_of_one_address_writes_one_row() { + let older = signed(7, 0, 1_700_000_000, Vec::new()); + let newer = signed(7, 0, 1_700_000_100, Vec::new()); + assert_eq!(older.event.pubkey, newer.event.pubkey); + + for batch in [ + vec![older.clone(), newer.clone()], + vec![newer.clone(), older.clone()], + ] { + let (_directory, env, databases) = store(); + let mut transaction = env.write_txn().expect("a write transaction"); + let affected = + upsert_many(&mut transaction, &databases, &batch, Utc::now()).expect("the batch"); + transaction.commit().expect("the commit"); + + assert_eq!(affected, 1, "rows written, whatever the order"); + let transaction = env.read_txn().expect("a read transaction"); + assert!( + databases + .events + .get(&transaction, older.event.id.as_bytes()) + .expect("the store") + .is_none(), + "the dropped version is not written" + ); + assert!( + databases + .events + .get(&transaction, newer.event.id.as_bytes()) + .expect("the store") + .is_some(), + "the newer version is the row the batch keeps" + ); + } + } + + // ----------------------------------------------------------------------------------- + // Round 12: the merge predicate reads the header and the pointer, never the body. + // ----------------------------------------------------------------------------------- + + /// A losing upsert decides from the stored value's header and the pointer's id, so it never + /// decodes the body: a stored row whose body is unreadable still loses cleanly. + #[test] + fn a_losing_merge_reads_the_header_and_not_the_body() { + let stored = signed(7, 0, 1_700_000_100, Vec::new()); + let older = signed(7, 0, 1_700_000_000, Vec::new()); + let (_directory, env, databases) = store(); + + let mut transaction = env.write_txn().expect("a write transaction"); + upsert(&mut transaction, &databases, &stored, Utc::now()).expect("the stored version"); + transaction.commit().expect("the commit"); + + // The value keeps its header — where the merge reads the stored `created_at` — and + // loses its body. + let mut transaction = env.write_txn().expect("a write transaction"); + let value = databases + .events + .get(&transaction, stored.event.id.as_bytes()) + .expect("the store") + .expect("the stored row"); + let mut headless = value[..1 + encoding::EVENT_HEADER_LEN].to_vec(); + headless.extend_from_slice(b"{not json"); + databases + .events + .put(&mut transaction, stored.event.id.as_bytes(), &headless) + .expect("the stored value"); + + let affected = upsert(&mut transaction, &databases, &older, Utc::now()) + .expect("the older version loses from the header, never the body"); + transaction.commit().expect("the commit"); + + assert_eq!(affected, 0, "the stored version is still the newer one"); + } + + // ----------------------------------------------------------------------------------- + // Round 12 rework: the replaceable write paths the PR's reviewer named. + // ----------------------------------------------------------------------------------- + + /// A replay of a replaceable event the store already holds updates that row against + /// itself: the merge predicate never matches, so the count is zero — not the id collision + /// `events_event_id_unique` reports for a kind with no conflict target. + #[test] + fn a_replayed_replaceable_event_counts_zero() { + let record = signed(7, 0, 1_700_000_000, Vec::new()); + let (_directory, env, databases) = store(); + + let mut transaction = env.write_txn().expect("a write transaction"); + assert_eq!( + upsert(&mut transaction, &databases, &record, Utc::now()).expect("the first write"), + 1 + ); + assert_eq!( + upsert(&mut transaction, &databases, &record, Utc::now()).expect("the replay"), + 0, + "the replaceable conflict matches and the merge predicate does not" + ); + transaction.commit().expect("the commit"); + + let transaction = env.read_txn().expect("a read transaction"); + let value = databases + .events + .get(&transaction, record.event.id.as_bytes()) + .expect("the store") + .expect("the row"); + assert_eq!( + rows::decode_event(value).expect("the row").event.id, + record.event.id, + "the replay left the row in place" + ); + } + + /// A kind with no conflict target has only the id to collide on, so a replay is still the + /// capture's `rejected: event already exists`. + #[test] + fn a_duplicate_id_of_a_regular_kind_is_rejected() { + let record = signed(8, 1, 1_700_000_000, Vec::new()); + let (_directory, env, databases) = store(); + + let mut transaction = env.write_txn().expect("a write transaction"); + upsert(&mut transaction, &databases, &record, Utc::now()).expect("the first write"); + let error = upsert(&mut transaction, &databases, &record, Utc::now()) + .expect_err("the second insert of the same id"); + assert!(matches!(error, StorageError::DuplicateEvent)); + } + + /// The capture's bulk merge list omits `remote_address`, so a bulk replacement keeps the + /// address the stored row had — even when the incoming record carries none — while the + /// single-row merge writes the incoming address. + #[test] + fn a_bulk_replacement_keeps_the_stored_source_address() { + let stored = signed(7, 0, 1_700_000_000, Vec::new()) + .with_remote_address(IpAddr::from([10, 0, 0, 1])); + let newer = signed(7, 0, 1_700_000_100, Vec::new()) + .with_remote_address(IpAddr::from([10, 0, 0, 2])); + let anonymous = signed(7, 0, 1_700_000_200, Vec::new()); + + let (_directory, env, databases) = store(); + let mut transaction = env.write_txn().expect("a write transaction"); + upsert_many( + &mut transaction, + &databases, + std::slice::from_ref(&stored), + Utc::now(), + ) + .expect("the stored version"); + upsert_many( + &mut transaction, + &databases, + std::slice::from_ref(&newer), + Utc::now(), + ) + .expect("the bulk replacement"); + upsert_many( + &mut transaction, + &databases, + std::slice::from_ref(&anonymous), + Utc::now(), + ) + .expect("a bulk replacement with no address"); + transaction.commit().expect("the commit"); + + let transaction = env.read_txn().expect("a read transaction"); + let value = databases + .events + .get(&transaction, anonymous.event.id.as_bytes()) + .expect("the store") + .expect("the last row"); + let row = rows::decode_event(value).expect("the row"); + assert_eq!( + row.remote_address, + Some(IpAddr::from([10, 0, 0, 1])), + "the bulk merge preserves the address across both replacements" + ); + + let (_directory, env, databases) = store(); + let mut transaction = env.write_txn().expect("a write transaction"); + upsert(&mut transaction, &databases, &stored, Utc::now()).expect("the stored version"); + upsert(&mut transaction, &databases, &newer, Utc::now()).expect("the single replacement"); + transaction.commit().expect("the commit"); + + let transaction = env.read_txn().expect("a read transaction"); + let value = databases + .events + .get(&transaction, newer.event.id.as_bytes()) + .expect("the store") + .expect("the replacement"); + assert_eq!( + rows::decode_event(value).expect("the row").remote_address, + Some(IpAddr::from([10, 0, 0, 2])), + "the single-row merge writes the incoming address" + ); + } + + /// A count walks keys, not bodies: the id a tag key ends with says whether the row is + /// already counted, so a wildcard count decodes nothing however many values it matches. + #[test] + fn a_wildcard_count_decodes_no_body() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let record = signed( + 7, + 1, + 1_700_000_000, + vec![ + vec!["g".to_owned(), "u4a".to_owned()], + vec!["g".to_owned(), "u4b".to_owned()], + ], + ); + write(&env, &databases, &[record]); + let transaction = env.read_txn().expect("a read transaction"); + let mut tags = BTreeMap::new(); + tags.insert("#g".to_owned(), vec!["u4*".to_owned()]); + let filter = Filter { + tags, + ..Filter::default() + }; + + probes::reset(); + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 1 + ); + assert_eq!(probes::decodes(), 0, "a count decodes no body"); + assert_eq!(probes::keys(), 2, "both keys the prefix selects are read"); + } + + /// A criterion too long to seek selects nothing, so a row carrying only later criteria is + /// still counted — once, at the first criterion's range that selects it. + #[test] + fn a_dropped_criterion_does_not_count_a_row_twice() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let record = signed( + 7, + 1, + 1_700_000_000, + vec![ + vec!["t".to_owned(), "a".to_owned()], + vec!["t".to_owned(), "b".to_owned()], + ], + ); + write(&env, &databases, &[record]); + let transaction = env.read_txn().expect("a read transaction"); + let mut tags = BTreeMap::new(); + tags.insert( + "#t".to_owned(), + vec!["x".repeat(500), "a".to_owned(), "b".to_owned()], + ); + let filter = Filter { + tags, + ..Filter::default() + }; + + assert_eq!( + plan(&filter, &databases).ranges.len(), + 2, + "the oversized criterion seeks nothing" + ); + probes::reset(); + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + 1, + "the row carries both values and is one row" + ); + assert_eq!( + probes::decodes(), + 0, + "two criteria are deduplicated by the id, not the body" + ); + } + + /// A count's total does not depend on how many ids its walk may hold: past the cap a + /// repeated key reads the row's body to tell an earlier key from a first one, and this runs + /// that walk and the one the set alone answers over one corpus. + #[test] + fn a_count_past_its_id_cap_totals_what_the_unbounded_walk_totals() { + let _serial = counting(); + let (_directory, env, databases) = store_with_map(32 << 20); + // Past `MAX_PAGE_KEYS`, so every filter below crosses the cap: the first 5 000 ids fill + // the set and the rest of the walk takes the body check. + let rows = MAX_PAGE_KEYS as i64 + 200; + let records: Vec = (0..rows) + .map(|step| { + signed( + (step % 200 + 1) as u8, + 1, + step, + vec![ + vec!["t".to_owned(), "a".to_owned()], + vec!["t".to_owned(), "b".to_owned()], + vec!["g".to_owned(), "u4a".to_owned()], + vec!["g".to_owned(), "u4b".to_owned()], + ], + ) + }) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let tags = |families: &[(&str, &[&str])]| -> BTreeMap> { + families + .iter() + .map(|(key, criteria)| { + ( + (*key).to_owned(), + criteria.iter().map(|value| (*value).to_owned()).collect(), + ) + }) + .collect() + }; + let cases = [ + ("two criteria", tags(&[("#t", &["a", "b"])])), + ("a repeated criterion", tags(&[("#t", &["a", "a"])])), + ("a prefix over two values", tags(&[("#g", &["u4*"])])), + ( + "a prefix and an exact value it covers", + tags(&[("#g", &["u4*", "u4b"])]), + ), + ( + "a value only the later criterion names", + tags(&[("#t", &["zz", "a"])]), + ), + ]; + for (what, tags) in cases { + let filter = Filter { + tags, + ..Filter::default() + }; + let counted = Counted::default(); + probes::reset(); + let capped = count_matches_bounded( + &transaction, + &databases, + &filter, + &counted, + &Gates::default(), + 0, + MAX_PAGE_KEYS, + ) + .expect("the bounded count"); + let decodes = probes::decodes(); + probes::reset(); + let uncapped = count_matches_bounded( + &transaction, + &databases, + &filter, + &counted, + &Gates::default(), + 0, + usize::MAX, + ) + .expect("the unbounded count"); + assert_eq!( + capped, uncapped, + "{what}: the cap changes how, not how many" + ); + assert_eq!(capped, rows as u64, "{what}: every row matches"); + assert_eq!(probes::decodes(), 0, "{what}: the set decodes no body"); + assert!( + decodes > 0, + "{what}: the walk past the cap really took the body check" + ); + probes::reset(); + assert_eq!( + count( + &transaction, + &databases, + std::slice::from_ref(&filter), + &Gates::default(), + 0 + ) + .expect("the count"), + capped, + "{what}: the count's own path agrees with the bounded walk" + ); + } + } + + /// The expiry pass reads the index no further than the last expired row: a future expiry is + /// not a candidate, and a row that qualifies only by age is the created-at pass's business. + #[test] + fn the_expiry_pass_stops_at_the_last_expired_row() { + let _serial = counting(); + let (_directory, env, databases) = store(); + let now = 1_700_000_000; + let mut records = vec![signed(7, 1, now - 100, Vec::new()).with_expires_at(now - 50)]; + records.extend((0..500).map(|step| { + signed_by([9; 32], now - 10, &format!("future-{step}")) + .with_expires_at(now + 1_000 + step) + })); + write(&env, &databases, &records); + + let options = EventRetentionOptions { + max_days: Some(1), + kind_whitelist: None, + pubkey_whitelist: None, + }; + probes::reset(); + let mut transaction = env.write_txn().expect("a write transaction"); + let counts = delete_expired_and_retained(&mut transaction, &databases, &options, now) + .expect("the pass"); + transaction.commit().expect("the commit"); + + assert_eq!(counts.expired, 1); + assert_eq!(counts.retained, 0, "ten seconds old is not over retention"); + assert_eq!( + probes::keys(), + 1, + "the pass visits the expired row and stops at `now`" + ); + } + // ----------------------------------------------------------------------------------- + // Round 6: the page across every range a plan seeks, and the id stream's two shapes. + // ----------------------------------------------------------------------------------- + + /// Serialises a test that counts work against every other one, so a clean run means what + /// it says. A test that panicked while holding the lock poisons it, and a poisoned lock + /// must not redden the tests that follow. + fn counting() -> std::sync::MutexGuard<'static, ()> { + probes::COUNTING + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + /// The page a filter must return, built from the store itself rather than from an index: + /// every row the matcher accepts, in the capture's order, cut to the filter's page. + fn reference_page( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + ) -> Vec<(i64, EventId)> { + let matcher = Matcher { + filter, + gates: &Gates::default(), + now: 0, + live: false, + proven: None, + }; + let mut page = Vec::new(); + for entry in databases.events.iter(txn).expect("the store") { + let (key, _) = entry.expect("a row"); + let id = encoding::event_id_from(key).expect("an event id"); + if let Some(header) = matcher.matches(txn, databases, &id).expect("a match") { + page.push((header.created_at, id)); + } + } + sort_page(&mut page, filter.limit.is_some()); + page.truncate(page_limit(filter)); + page + } + + /// The items a reconciliation must yield for a filter, built from the store itself: every + /// row the matcher accepts, oldest first. + fn reference_items( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + ) -> Vec { + let matcher = Matcher { + filter, + gates: &Gates::default(), + now: 0, + live: true, + proven: None, + }; + let mut items = Vec::new(); + for entry in databases.events.iter(txn).expect("the store") { + let (key, _) = entry.expect("a row"); + let id = encoding::event_id_from(key).expect("an event id"); + if let Some(header) = matcher.matches(txn, databases, &id).expect("a match") { + items.push(ReconciliationItem { + created_at: header.created_at, + event_id: id, + }); + } + } + items.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.event_id.cmp(&right.event_id)) + }); + items + } + + /// The page `branch` returns, the store's own answer, and the count of the same filter all + /// agree. + fn assert_page_agrees( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + expected: &[i64], + what: &str, + ) { + let page = branch( + txn, + databases, + filter, + &Gates::default(), + 0, + false, + page_limit(filter), + ) + .expect("the page"); + assert_eq!( + created(&page), + expected, + "{what}: the page is the capture's newest rows" + ); + assert_eq!( + page, + reference_page(txn, databases, filter), + "{what}: the page is the store's own answer" + ); + assert_eq!( + count( + txn, + databases, + std::slice::from_ref(filter), + &Gates::default(), + 0 + ) + .expect("the count"), + page.len() as u64, + "{what}: the page and the count of the same filter agree" + ); + } + + /// The page `branch` returns over a filter whose plan seeks `ranges` time-ordered ranges, + /// with the entries it sorted counted: the accumulator holds the page as a heap rooted at + /// its worst kept row, so the only sort is the one in `finish` and the request sorts its own + /// `limit` rows however many ranges it walks. + fn assert_page_sorts_once( + txn: &RoTxn<'_>, + databases: &Databases, + filter: &Filter, + ranges: usize, + ) -> Vec<(i64, EventId)> { + let plan = plan(filter, databases); + assert_eq!(plan.ranges.len(), ranges, "the plan seeks its own ranges"); + assert!(plan.ordered, "and a range's key order is the page's"); + probes::reset(); + let page = branch( + txn, + databases, + filter, + &Gates::default(), + 0, + false, + page_limit(filter), + ) + .expect("the page"); + assert_eq!( + probes::sorted_entries(), + page.len(), + "one page sorts its own rows once, whatever the number of ranges" + ); + assert_eq!( + probes::sorted_entries(), + page_limit(filter), + "and a full page is exactly the limit" + ); + assert_eq!( + page, + reference_page(txn, databases, filter), + "the page is the store's own answer" + ); + page + } + + #[test] + fn a_page_sorts_its_own_limit_once_however_many_ranges_it_seeks() { + // The boundary is the page's own worst kept row and the page is a heap rooted at it, so + // an offer costs a comparison with that root and the one sort happens in `finish`. + // Computing the boundary by sorting the page instead costs `limit` entries sorted on + // every accepted offer: the 5 000 whole-pubkey ranges built here sorted 2 820 637 + // entries at the 500 default, and the 5 000 rows tied at one timestamp sorted 2 255 500. + // (Round 6's performance review quotes 2 828 653 for a different corpus — 5 001 events + // with one quiet author — so the two figures are not the same measurement.) + let _serial = counting(); + + // One kind-1 event for each of 5 000 authors: one range per author, one key each. + let (_directory, env, databases) = store(); + let records: Vec = (0..MAX_PAGE_KEYS as u32) + .map(|author| { + let mut secret = [0u8; 32]; + secret[..4].copy_from_slice(&author.to_be_bytes()); + secret[31] = 1; + signed_by(secret, i64::from(author), &author.to_string()) + }) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some( + records + .iter() + .map(|record| record.event.pubkey.to_string()) + .collect(), + ), + kinds: Some(vec![Kind(1)]), + limit: Some(DEFAULT_FILTER_LIMIT), + ..Filter::default() + }; + let page = assert_page_sorts_once(&transaction, &databases, &filter, MAX_PAGE_KEYS); + assert_eq!( + created(&page), + (MAX_PAGE_KEYS as i64 - DEFAULT_FILTER_LIMIT as i64..MAX_PAGE_KEYS as i64) + .rev() + .collect::>(), + "the page is the newest `limit` authors, newest first" + ); + + // One author's 5 000 rows at one timestamp: one range, and every row past the limit is + // a match the accumulator considers. + let (_directory, env, databases) = store(); + let records: Vec = (0..MAX_PAGE_KEYS as u32) + .map(|step| signed_by([9; 32], 5_000, &step.to_string())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + kinds: Some(vec![Kind(1)]), + limit: Some(DEFAULT_FILTER_LIMIT), + ..Filter::default() + }; + let page = assert_page_sorts_once(&transaction, &databases, &filter, 1); + assert!( + created(&page).iter().all(|created_at| *created_at == 5_000), + "every row the page holds is at the boundary timestamp" + ); + assert!( + page.windows(2).all(|pair| pair[0].1 < pair[1].1), + "so the tie is broken by the smaller id" + ); + } + + #[test] + fn a_limited_page_over_several_time_ordered_ranges_holds_the_global_newest_rows() { + // A page is one accumulator across every range the plan seeks: an author × kind + // filter seeks one range per kind, and the top `limit` is global to them, so a later + // range may not stop at the first row it offers. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=40) + .map(|step: i64| signed(7, (step % 3) as u64 + 1, step * 10, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let author = records[0].event.pubkey.to_string(); + let prefix = author[..8].to_owned(); + let two_kinds = &[390, 380, 360, 350, 330]; + let every_kind = &[400, 390, 380, 370, 360]; + + let cases: Vec<(&str, Filter, AccessPath, usize, &[i64])> = vec![ + ( + "one author and two kinds", + Filter { + authors: Some(vec![author.clone()]), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::Authors, + 2, + two_kinds, + ), + ( + "the same kinds in the other order", + Filter { + authors: Some(vec![author.clone()]), + kinds: Some(vec![Kind(3), Kind(1)]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::Authors, + 2, + two_kinds, + ), + ( + "two kinds without an author", + Filter { + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::CreatedAt, + 1, + two_kinds, + ), + ( + "one author without kinds", + Filter { + authors: Some(vec![author]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::CreatedAt, + 1, + every_kind, + ), + ( + "a prefix author and two kinds", + Filter { + authors: Some(vec![prefix]), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }, + AccessPath::Authors, + 1, + two_kinds, + ), + ]; + for (what, filter, path, ranges, expected) in cases { + let plan = plan(&filter, &databases); + assert_eq!( + (plan.path, plan.ranges.len()), + (path, ranges), + "{what}: the plan" + ); + assert_page_agrees(&transaction, &databases, &filter, expected, what); + } + } + + #[test] + fn a_tie_at_the_boundary_timestamp_is_decided_by_a_later_range() { + // A row tied with the page's worst kept row is not past the boundary, so a later range + // still offers it and the smaller id wins the tie. Stopping at the boundary instead of + // after it would keep the earlier range's row. + let _serial = counting(); + let (_directory, env, databases) = store(); + let mut records = vec![ + signed(11, 1, 200, Vec::new()), + signed(11, 3, 200, Vec::new()), + signed(11, 1, 190, Vec::new()), + signed(11, 3, 190, Vec::new()), + ]; + // The tie breaks on the id, so the pair at the boundary timestamp is chosen for it: the + // row of the kind whose range runs second carries the smaller id. + let (first, second) = (0..64) + .map(|step| { + ( + signed(11, 1, 180, vec![vec!["x".to_owned(), format!("a{step}")]]), + signed(11, 3, 180, vec![vec!["x".to_owned(), format!("b{step}")]]), + ) + }) + .find(|(first, second)| second.event.id < first.event.id) + .expect("a pair whose later range carries the smaller id"); + records.push(first); + let tied = second.event.id; + records.push(second); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }; + let chosen = plan(&filter, &databases); + assert_eq!( + (chosen.path, chosen.ranges.len()), + (AccessPath::Authors, 2), + "one range per kind, the first kind's before the second's" + ); + + assert_page_agrees( + &transaction, + &databases, + &filter, + &[200, 200, 190, 190, 180], + "a tie at the boundary", + ); + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!( + page.last().map(|(_, id)| *id), + Some(tied), + "the tie goes to the smaller id, from the range that runs later" + ); + } + + #[test] + fn a_limited_page_over_two_tag_values_does_not_depend_on_the_criteria_s_order() { + // One criterion is one range on `tags`, and each range runs in `created_at` order + // under its own value, so a page over two values is the global top `limit` of both. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=40) + .map(|step| { + let mut tags = vec![vec!["t".to_owned(), "a".to_owned()]]; + if step % 3 != 1 { + tags.push(vec!["t".to_owned(), "b".to_owned()]); + } + signed(8, 1, step * 10, tags) + }) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let author = records[0].event.pubkey.to_string(); + let expected = [400, 390, 380, 370, 360]; + + for criteria in [ + vec!["a".to_owned(), "b".to_owned()], + vec!["b".to_owned(), "a".to_owned()], + ] { + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), criteria.clone()); + let filter = Filter { + tags: tags.clone(), + limit: Some(5), + ..Filter::default() + }; + assert_eq!( + plan(&filter, &databases).ranges.len(), + 2, + "one range per value" + ); + assert_page_agrees( + &transaction, + &databases, + &filter, + &expected, + "two tag values", + ); + + // A criterion of another family is the matcher's, over the same two ranges. + let filter = Filter { + kinds: Some(vec![Kind(1)]), + tags, + limit: Some(5), + ..Filter::default() + }; + assert_page_agrees( + &transaction, + &databases, + &filter, + &expected, + "a kind and two tag values", + ); + + // An author as well makes the author index the plan; the page is the same one. + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), criteria); + let filter = Filter { + authors: Some(vec![author.clone()]), + kinds: Some(vec![Kind(1)]), + tags, + limit: Some(5), + ..Filter::default() + }; + assert_page_agrees( + &transaction, + &databases, + &filter, + &expected, + "an author, a kind and two tag values", + ); + } + + // A single value's own page, so the two-value case is not "everything, in any order". + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), vec!["b".to_owned()]); + let filter = Filter { + tags, + limit: Some(5), + ..Filter::default() + }; + assert_page_agrees( + &transaction, + &databases, + &filter, + &[390, 380, 360, 350, 330], + "one tag value", + ); + } + + #[test] + fn an_unlimited_page_over_two_tag_values_is_the_global_oldest_rows() { + // Without a limit the page keeps the oldest rows, so the same accumulator is read from + // the other end: a later range whose oldest row is older than the page's worst kept row + // must be walked, not stopped at its first match. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=1200) + .map(|step: i64| { + let value = if step % 3 == 0 { "b" } else { "a" }; + signed(36, 1, step, vec![vec!["t".to_owned(), value.to_owned()]]) + }) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let oldest: Vec = (1..=500).collect(); + + for criteria in [ + vec!["a".to_owned(), "b".to_owned()], + vec!["b".to_owned(), "a".to_owned()], + ] { + let mut tags = BTreeMap::new(); + tags.insert("#t".to_owned(), criteria); + let filter = Filter { + tags, + ..Filter::default() + }; + let page = branch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + false, + page_limit(&filter), + ) + .expect("the page"); + assert_eq!( + created(&page), + oldest, + "the page is the 500 oldest rows the values select, in either order" + ); + assert_eq!( + page, + reference_page(&transaction, &databases, &filter), + "and it is the store's own answer" + ); + } + } + + #[test] + fn the_author_index_and_the_created_at_fallback_return_the_same_page() { + // A filter composing more than `MAX_PAGE_KEYS` author × kind ranges is answered from + // the created-at index instead, and both plans must return the same page. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=40) + .map(|step: i64| signed(9, (step % 3) as u64 + 1, step * 10, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let expected = [390, 380, 360, 350, 330]; + let composed = |fakes: u32| -> Vec { + std::iter::once(records[0].event.pubkey.to_string()) + .chain((0..fakes).map(|step| format!("{:064x}", 0x1000 + step))) + .collect() + }; + + let at_bound = Filter { + authors: Some(composed(MAX_PAGE_KEYS as u32 / 2 - 1)), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }; + let chosen = plan(&at_bound, &databases); + assert_eq!( + (chosen.path, chosen.ranges.len()), + (AccessPath::Authors, MAX_PAGE_KEYS) + ); + assert_page_agrees( + &transaction, + &databases, + &at_bound, + &expected, + "at the page bound", + ); + let index_page = branch( + &transaction, + &databases, + &at_bound, + &Gates::default(), + 0, + false, + page_limit(&at_bound), + ) + .expect("the page"); + + let past_bound = Filter { + authors: Some(composed(MAX_PAGE_KEYS as u32 / 2)), + kinds: Some(vec![Kind(1), Kind(3)]), + limit: Some(5), + ..Filter::default() + }; + let chosen = plan(&past_bound, &databases); + assert_eq!( + (chosen.path, chosen.ranges.len()), + (AccessPath::CreatedAt, 1) + ); + assert_page_agrees( + &transaction, + &databases, + &past_bound, + &expected, + "past the page bound", + ); + let fallback_page = branch( + &transaction, + &databases, + &past_bound, + &Gates::default(), + 0, + false, + page_limit(&past_bound), + ) + .expect("the page"); + assert_eq!( + index_page, fallback_page, + "the author index and the created-at fallback return the same page" + ); + } + + #[test] + fn a_prefix_id_reconciliation_scans_in_bounded_batches() { + // An exact id list is a point per id, so the list is the bound on the work; a prefix + // criterion is a byte range over the id field, and `""` is the whole store. + let _serial = counting(); + let (_directory, env, databases) = store(); + let records: Vec = (1..=400) + .map(|step| signed(35, 1, step, Vec::new())) + .collect(); + write(&env, &databases, &records); + let transaction = env.read_txn().expect("a read transaction"); + let ids = |criteria: &str| Filter { + ids: Some(vec![criteria.to_owned()]), + ..Filter::default() + }; + + let exact = ids(&records[10].event.id.to_string()); + assert_eq!( + stream_plan(&exact, &databases).path, + AccessPath::Ids, + "an exact id list is a point per id" + ); + let prefix = ids(&records[0].event.id.to_string()[..2]); + assert_eq!( + stream_plan(&prefix, &databases).path, + AccessPath::CreatedAt, + "a prefix criterion is a byte range, so it takes the bounded scan" + ); + + for filter in [ids(""), prefix] { + probes::reset(); + let mut position = Position::new(); + let mut items = scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"); + assert!( + probes::keys() <= STREAM_BATCH_KEYS, + "one poll evaluates at most the key cap: {} keys", + probes::keys() + ); + assert!( + position.after.is_some(), + "and it resumes after the last key it evaluated" + ); + + for _ in 0..8 { + if position.index >= stream_plan(&filter, &databases).ranges.len() { + break; + } + items.extend( + scan_batch( + &transaction, + &databases, + &filter, + &Gates::default(), + 0, + &mut position, + ) + .expect("the batch"), + ); + } + assert_eq!( + items, + reference_items(&transaction, &databases, &filter), + "the bounded scan yields every row the criterion selects, oldest first" + ); + } + } +} diff --git a/crates/nostream-storage/src/lmdb/invoices.rs b/crates/nostream-storage/src/lmdb/invoices.rs new file mode 100644 index 000000000..4bfd667f4 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/invoices.rs @@ -0,0 +1,197 @@ +//! `InvoiceRepository` against LMDB. Mirrors `src/repositories/invoice-repository.ts`. +//! +//! Two databases: `invoices` by id, and `invoices_pending` (`created_at ‖ id`) holding only +//! the pending rows, which is the partial index `find_pending_invoices` pages through. + +use chrono::{DateTime, Utc}; +use heed::{RoTxn, RwTxn}; + +use super::Databases; +use super::encoding; +use super::rows; +use super::users; +use crate::error::StorageError; +use crate::repositories::{Invoice, InvoiceStatus}; + +/// `findById`: one row by primary key. +pub(crate) fn find_by_id( + txn: &RoTxn<'_>, + databases: &Databases, + id: &str, +) -> Result, StorageError> { + let Some(value) = databases.invoices.get(txn, id.as_bytes())? else { + return Ok(None); + }; + rows::decode("invoices", value).map(Some) +} + +/// `created_at ‖ id`, the key of the pending index. +fn pending_key(created_at: DateTime, id: &str) -> Vec { + let mut key = encoding::i64_key(created_at.timestamp_micros()).to_vec(); + key.extend_from_slice(id.as_bytes()); + key +} + +/// Writes a row and moves its pending entry, when the status or the row changed. +fn write( + txn: &mut RwTxn<'_>, + databases: &Databases, + invoice: &Invoice, + previous: Option<&Invoice>, +) -> Result<(), StorageError> { + let value = rows::encode("invoices", invoice)?; + databases.invoices.put(txn, invoice.id.as_bytes(), &value)?; + if let Some(previous) = previous + && previous.status == InvoiceStatus::Pending + { + databases + .invoices_pending + .delete(txn, &pending_key(previous.created_at, &previous.id))?; + } + if invoice.status == InvoiceStatus::Pending { + databases + .invoices_pending + .put(txn, &pending_key(invoice.created_at, &invoice.id), &[])?; + } + Ok(()) +} + +/// `upsert`: inserts the row, or updates only `status` and `updated_at` — the capture's +/// merge list, which never carries `amount_paid` or `confirmed_at`. +pub(crate) fn upsert( + txn: &mut RwTxn<'_>, + databases: &Databases, + invoice: &Invoice, + now: DateTime, +) -> Result { + match find_by_id(txn, databases, &invoice.id)? { + Some(mut stored) => { + let previous = stored.clone(); + stored.status = invoice.status; + stored.updated_at = encoding::micros(now); + write(txn, databases, &stored, Some(&previous))?; + } + None => { + let mut stored = invoice.clone(); + // The insert's column list omits both, so the database fills them with NULL. + stored.amount_paid = None; + stored.confirmed_at = None; + // Every timestamp column is a PostgreSQL `timestamp` (V38): the caller's instants + // are truncated too, not only the ones this backend stamps. + stored.created_at = encoding::micros(stored.created_at); + stored.expires_at = stored.expires_at.map(encoding::micros); + stored.updated_at = encoding::micros(now); + write(txn, databases, &stored, None)?; + } + } + Ok(1) +} + +/// `updateStatus`: updates one row and returns it. +pub(crate) fn update_status( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &str, + status: InvoiceStatus, + now: DateTime, +) -> Result, StorageError> { + let Some(mut stored) = find_by_id(txn, databases, id)? else { + return Ok(None); + }; + let previous = stored.clone(); + stored.status = status; + stored.updated_at = encoding::micros(now); + write(txn, databases, &stored, Some(&previous))?; + Ok(Some(stored)) +} + +/// The balance credit one unit is worth (`scale_balance_addition_with_unit`). +fn unit_scale(unit: crate::repositories::InvoiceUnit) -> i64 { + use crate::repositories::InvoiceUnit; + match unit { + InvoiceUnit::Msats => 1, + InvoiceUnit::Sats => 1_000, + InvoiceUnit::Btc => 100_000_000 * 1_000, + } +} + +/// `confirmInvoice`: the schema function's body, with `int8` overflow answered as +/// [`StorageError::ValueOutOfRange`]. +pub(crate) fn confirm_invoice( + txn: &mut RwTxn<'_>, + databases: &Databases, + invoice_id: &str, + amount_received: i64, + confirmed_at: DateTime, + now: DateTime, +) -> Result<(), StorageError> { + let Some(stored) = find_by_id(txn, databases, invoice_id)? else { + // The capture's `SELECT … INTO` leaves `payee` NULL and both updates match nothing. + return Ok(()); + }; + if stored.confirmed_at.is_some() { + return Ok(()); + } + + let credit = amount_received + .checked_mul(unit_scale(stored.unit)) + .ok_or_else(|| StorageError::ValueOutOfRange { + column: "users.balance", + value: amount_received.to_string(), + })?; + let payee = users::load(txn, databases, &stored.pubkey)?; + let balance = match &payee { + Some(payee) => Some(payee.balance.checked_add(credit).ok_or_else(|| { + StorageError::ValueOutOfRange { + column: "users.balance", + value: payee.balance.to_string(), + } + })?), + None => None, + }; + + let mut confirmed = stored.clone(); + confirmed.confirmed_at = Some(encoding::micros(confirmed_at)); + confirmed.amount_paid = Some(amount_received); + confirmed.updated_at = encoding::micros(now); + write(txn, databases, &confirmed, Some(&stored))?; + + if let (Some(mut payee), Some(balance)) = (payee, balance) { + payee.balance = balance; + users::store(txn, databases, &payee)?; + } + Ok(()) +} + +/// `findPendingInvoices`: the pending rows, oldest first, paged. +pub(crate) fn find_pending_invoices( + txn: &RoTxn<'_>, + databases: &Databases, + offset: u64, + limit: u64, +) -> Result, StorageError> { + let mut invoices = Vec::new(); + let mut skipped = 0; + for entry in databases.invoices_pending.iter(txn)? { + let (key, _) = entry?; + if skipped < offset { + skipped += 1; + continue; + } + if invoices.len() as u64 >= limit { + break; + } + let id = key.get(8..).ok_or_else(|| { + encoding::invalid("invoices_pending", format!("a key of {} bytes", key.len())) + })?; + let id = std::str::from_utf8(id).map_err(|_| StorageError::InvalidRow { + table: "invoices_pending", + column: "id", + value: "not utf-8".to_owned(), + })?; + if let Some(value) = databases.invoices.get(txn, id.as_bytes())? { + invoices.push(rows::decode("invoices", value)?); + } + } + Ok(invoices) +} diff --git a/crates/nostream-storage/src/lmdb/metadata.rs b/crates/nostream-storage/src/lmdb/metadata.rs new file mode 100644 index 000000000..bfcd28072 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/metadata.rs @@ -0,0 +1,716 @@ +//! `Nip05VerificationRepository`, `InviteCodeRepository`, `DvmJobRepository` and +//! `ReportRepository` against LMDB. Mirrors the four `src/repositories/*` files. + +use std::collections::BTreeSet; + +use chrono::{DateTime, Duration, Utc}; +use heed::{RoTxn, RwTxn}; +use nostream_core::{EventId, Kind, Pubkey}; + +use super::Databases; +use super::encoding; +use super::encoding::{stamp_key, text_from, text_part}; +use super::rows; +use crate::error::StorageError; +use crate::repositories::{ + ActionableTarget, CreateInviteCodeOptions, DvmJob, DvmJobStatus, DvmJobUpdate, FieldUpdate, + InviteCode, NewReport, Nip05Verification, Report, +}; + +/// The `meta` key holding the `reports.id` sequence (`reports_id_seq`). +const REPORT_SEQUENCE: &[u8] = b"reports.next_id"; + +/// The width of `reports.id` in every `reports_*` index key, pinned so a widening is a compile +/// error here rather than a prefix lookup that silently seeks a narrower range. +const REPORT_ID_LEN: usize = 4; + +// --------------------------------------------------------------------------------------- +// NIP-05 verifications. +// --------------------------------------------------------------------------------------- + +fn nip05_key(pubkey: &Pubkey) -> Vec { + pubkey.as_bytes().to_vec() +} + +/// `last_checked_at ‖ pubkey`, the key of the pending-check index. +fn nip05_pending_key(verification: &Nip05Verification) -> Vec { + let mut key = stamp_key(verification.last_checked_at).to_vec(); + key.extend_from_slice(verification.pubkey.as_bytes()); + key +} + +pub(crate) fn nip05_find_by_pubkey( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result, StorageError> { + let Some(value) = databases.nip05.get(txn, &nip05_key(pubkey))? else { + return Ok(None); + }; + rows::decode("nip05_verifications", value).map(Some) +} + +/// `Nip05VerificationRepository::upsert`: every column but `pubkey` and `created_at` moves. +pub(crate) fn nip05_upsert( + txn: &mut RwTxn<'_>, + databases: &Databases, + verification: &Nip05Verification, + now: DateTime, +) -> Result { + let previous = nip05_find_by_pubkey(txn, databases, &verification.pubkey)?; + let mut stored = verification.clone(); + stored.created_at = previous + .as_ref() + .map_or_else(|| encoding::micros(now), |row| row.created_at); + // Every timestamp column is a PostgreSQL `timestamp` (V38): the caller's instants are + // truncated too, not only the ones this backend stamps. + stored.last_checked_at = encoding::micros(stored.last_checked_at); + stored.last_verified_at = stored.last_verified_at.map(encoding::micros); + stored.updated_at = encoding::micros(now); + + let value = rows::encode("nip05_verifications", &stored)?; + databases + .nip05 + .put(txn, &nip05_key(&stored.pubkey), &value)?; + if let Some(previous) = &previous { + databases + .nip05_by_last_checked_at + .delete(txn, &nip05_pending_key(previous))?; + } + databases + .nip05_by_last_checked_at + .put(txn, &nip05_pending_key(&stored), &[])?; + Ok(1) +} + +/// `Nip05VerificationRepository::find_pending_verifications`: rows last checked before the +/// cutoff with fewer than `max_failures`, oldest check first. +pub(crate) fn nip05_find_pending( + txn: &RoTxn<'_>, + databases: &Databases, + update_frequency_ms: i64, + max_failures: i32, + limit: u64, + now: DateTime, +) -> Result, StorageError> { + if limit == 0 { + return Ok(Vec::new()); + } + let cutoff = encoding::micros(now - Duration::milliseconds(update_frequency_ms)); + let mut verifications = Vec::new(); + for entry in databases.nip05_by_last_checked_at.iter(txn)? { + let (key, _) = entry?; + let last_checked_at = encoding::i64_from_key(&key[..8])?; + if last_checked_at >= cutoff.timestamp_micros() { + break; + } + let pubkey = encoding::pubkey_from(&key[8..])?; + let Some(verification) = nip05_find_by_pubkey(txn, databases, &pubkey)? else { + continue; + }; + if verification.failure_count >= max_failures { + continue; + } + verifications.push(verification); + if verifications.len() as u64 >= limit { + break; + } + } + Ok(verifications) +} + +/// `Nip05VerificationRepository::delete_by_pubkey`. +pub(crate) fn nip05_delete_by_pubkey( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result { + let Some(verification) = nip05_find_by_pubkey(txn, databases, pubkey)? else { + return Ok(0); + }; + databases.nip05.delete(txn, &nip05_key(pubkey))?; + databases + .nip05_by_last_checked_at + .delete(txn, &nip05_pending_key(&verification))?; + Ok(1) +} + +// --------------------------------------------------------------------------------------- +// Invite codes. +// --------------------------------------------------------------------------------------- + +fn invite_created_key(invite: &InviteCode) -> Vec { + let mut key = stamp_key(invite.created_at).to_vec(); + key.extend_from_slice(invite.code.as_bytes()); + key +} + +fn invite_expires_key(invite: &InviteCode, expires_at: DateTime) -> Vec { + let mut key = stamp_key(expires_at).to_vec(); + key.extend_from_slice(invite.code.as_bytes()); + key +} + +pub(crate) fn invite_find_by_code( + txn: &RoTxn<'_>, + databases: &Databases, + code: &str, +) -> Result, StorageError> { + let Some(value) = databases.invite_codes.get(txn, code.as_bytes())? else { + return Ok(None); + }; + rows::decode("invite_codes", value).map(Some) +} + +fn invite_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + invite: &InviteCode, +) -> Result<(), StorageError> { + let value = rows::encode("invite_codes", invite)?; + databases + .invite_codes + .put(txn, invite.code.as_bytes(), &value)?; + databases + .invite_codes_by_created_at + .put(txn, &invite_created_key(invite), &[])?; + if let Some(expires_at) = invite.expires_at { + databases.invite_codes_by_expires_at.put( + txn, + &invite_expires_key(invite, expires_at), + &[], + )?; + } + Ok(()) +} + +/// `InviteCodeRepository::create`: the row the capture returns, minted with its options. +pub(crate) fn invite_create( + txn: &mut RwTxn<'_>, + databases: &Databases, + code: &str, + options: &CreateInviteCodeOptions, + now: DateTime, +) -> Result { + let remaining_uses = match &options.remaining_uses { + FieldUpdate::Absent => Some(1), + FieldUpdate::Clear => None, + FieldUpdate::Set(uses) => { + if *uses < 0 { + return Err(StorageError::InvalidArgument { + message: "invite_codes.remaining_uses must not be negative", + }); + } + Some(*uses) + } + }; + if invite_find_by_code(txn, databases, code)?.is_some() { + return Err(StorageError::InvalidArgument { + message: "invite code already exists", + }); + } + + let invite = InviteCode { + code: code.to_owned(), + created_by: options.created_by, + claimed_by: None, + expires_at: options.expires_at.map(encoding::micros), + remaining_uses, + created_at: encoding::micros(now), + updated_at: encoding::micros(now), + }; + invite_write(txn, databases, &invite)?; + Ok(invite) +} + +/// `InviteCodeRepository::claim_code`: one conditional update, so exactly one caller wins. +pub(crate) fn invite_claim_code( + txn: &mut RwTxn<'_>, + databases: &Databases, + code: &str, + pubkey: &Pubkey, + now: DateTime, +) -> Result { + let Some(mut invite) = invite_find_by_code(txn, databases, code)? else { + return Ok(false); + }; + let now = encoding::micros(now); + if invite + .expires_at + .is_some_and(|expires_at| expires_at <= now) + || invite.remaining_uses.is_some_and(|uses| uses <= 0) + { + return Ok(false); + } + + invite.remaining_uses = invite.remaining_uses.map(|uses| uses - 1); + invite.claimed_by = invite.claimed_by.or(Some(*pubkey)); + invite.updated_at = now; + let value = rows::encode("invite_codes", &invite)?; + databases.invite_codes.put(txn, code.as_bytes(), &value)?; + Ok(true) +} + +/// `InviteCodeRepository::find_active_codes`: unexpired and claimable, newest first. +pub(crate) fn invite_find_active_codes( + txn: &RoTxn<'_>, + databases: &Databases, + limit: u64, + now: DateTime, +) -> Result, StorageError> { + if limit == 0 { + return Ok(Vec::new()); + } + let now = encoding::micros(now); + let mut invites = Vec::new(); + for entry in databases.invite_codes_by_created_at.rev_iter(txn)? { + let (key, _) = entry?; + let code = text_from(&key[8..], "invite_codes_by_created_at")?; + let Some(invite) = invite_find_by_code(txn, databases, code)? else { + continue; + }; + if invite + .expires_at + .is_some_and(|expires_at| expires_at <= now) + || invite.remaining_uses.is_some_and(|uses| uses <= 0) + { + continue; + } + invites.push(invite); + if invites.len() as u64 >= limit { + break; + } + } + Ok(invites) +} + +/// `InviteCodeRepository::delete_expired_codes`. +pub(crate) fn invite_delete_expired( + txn: &mut RwTxn<'_>, + databases: &Databases, + now: DateTime, +) -> Result { + let now = encoding::micros(now); + let mut expired = Vec::new(); + for entry in databases.invite_codes_by_expires_at.iter(txn)? { + let (key, _) = entry?; + let expires_at = encoding::i64_from_key(&key[..8])?; + if expires_at > now.timestamp_micros() { + break; + } + expired.push(text_from(&key[8..], "invite_codes_by_expires_at")?.to_owned()); + } + + let mut affected = 0; + for code in expired { + let Some(invite) = invite_find_by_code(txn, databases, &code)? else { + continue; + }; + databases.invite_codes.delete(txn, code.as_bytes())?; + databases + .invite_codes_by_created_at + .delete(txn, &invite_created_key(&invite))?; + if let Some(expires_at) = invite.expires_at { + databases + .invite_codes_by_expires_at + .delete(txn, &invite_expires_key(&invite, expires_at))?; + } + affected += 1; + } + Ok(affected) +} + +// --------------------------------------------------------------------------------------- +// DVM jobs. +// --------------------------------------------------------------------------------------- + +fn dvm_status_key(job: &DvmJob, created_at: DateTime) -> Result, StorageError> { + let mut key = text_part( + job.status.as_str(), + "dvm_jobs_by_status_created_at", + "status", + )?; + key.extend_from_slice(&stamp_key(created_at)); + key.extend_from_slice(job.id.as_bytes()); + Ok(key) +} + +pub(crate) fn dvm_find_by_id( + txn: &RoTxn<'_>, + databases: &Databases, + id: &EventId, +) -> Result, StorageError> { + let Some(value) = databases.dvm_jobs.get(txn, id.as_bytes())? else { + return Ok(None); + }; + rows::decode("dvm_jobs", value).map(Some) +} + +fn dvm_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + job: &DvmJob, + previous: Option<&DvmJob>, +) -> Result<(), StorageError> { + let value = rows::encode("dvm_jobs", job)?; + databases.dvm_jobs.put(txn, job.id.as_bytes(), &value)?; + if let Some(previous) = previous { + databases + .dvm_jobs_by_status_created_at + .delete(txn, &dvm_status_key(previous, previous.created_at)?)?; + } + databases + .dvm_jobs_by_status_created_at + .put(txn, &dvm_status_key(job, job.created_at)?, &[])?; + Ok(()) +} + +/// `DvmJobRepository::create`: a submitted job with every optional column empty. +pub(crate) fn dvm_create( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &EventId, + requester: &Pubkey, + kind: Kind, + now: DateTime, +) -> Result { + if dvm_find_by_id(txn, databases, id)?.is_some() { + return Err(StorageError::InvalidArgument { + message: "dvm_jobs.id must not already exist", + }); + } + let now = encoding::micros(now); + let job = DvmJob { + id: *id, + requester_pubkey: *requester, + kind, + worker_index: None, + status: DvmJobStatus::Submitted, + result_event_id: None, + error: None, + picked_up_at: None, + completed_at: None, + created_at: now, + updated_at: now, + }; + dvm_write(txn, databases, &job, None)?; + Ok(job) +} + +/// `DvmJobRepository::assign_worker`: a conditional update from `submitted`. +pub(crate) fn dvm_assign_worker( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &EventId, + worker_index: i32, + now: DateTime, +) -> Result { + let Some(mut job) = dvm_find_by_id(txn, databases, id)? else { + return Ok(false); + }; + if job.status != DvmJobStatus::Submitted { + return Ok(false); + } + let previous = job.clone(); + let now = encoding::micros(now); + job.worker_index = Some(worker_index); + job.status = DvmJobStatus::PickedUp; + job.picked_up_at = Some(now); + job.updated_at = now; + dvm_write(txn, databases, &job, Some(&previous))?; + Ok(true) +} + +/// `DvmJobRepository::update_status`: the `FieldUpdate` fields are applied by key presence. +pub(crate) fn dvm_update_status( + txn: &mut RwTxn<'_>, + databases: &Databases, + update: &DvmJobUpdate, + now: DateTime, +) -> Result, StorageError> { + let Some(mut job) = dvm_find_by_id(txn, databases, &update.id)? else { + return Ok(None); + }; + let previous = job.clone(); + let now = encoding::micros(now); + job.status = update.status; + job.updated_at = now; + if update.status.is_terminal() { + job.completed_at = Some(now); + } + match &update.result_event_id { + FieldUpdate::Absent => {} + FieldUpdate::Clear => job.result_event_id = None, + FieldUpdate::Set(id) => job.result_event_id = Some(*id), + } + match &update.error { + FieldUpdate::Absent => {} + FieldUpdate::Clear => job.error = None, + FieldUpdate::Set(error) => job.error = Some(error.clone()), + } + dvm_write(txn, databases, &job, Some(&previous))?; + Ok(Some(job)) +} + +/// `DvmJobRepository::find_pending_jobs`: submitted and picked-up jobs, oldest first. +pub(crate) fn dvm_find_pending_jobs( + txn: &RoTxn<'_>, + databases: &Databases, + limit: u64, + kinds: Option<&[Kind]>, +) -> Result, StorageError> { + // Each status range runs in creation order, so the merge only needs the first `limit` + // eligible rows of each: the backlog is not materialised for a small request. + let take = usize::try_from(limit).unwrap_or(usize::MAX); + if take == 0 { + return Ok(Vec::new()); + } + let mut jobs = Vec::new(); + for status in [DvmJobStatus::Submitted, DvmJobStatus::PickedUp] { + let prefix = text_part(status.as_str(), "dvm_jobs_by_status_created_at", "status")?; + let mut taken = 0; + for entry in databases + .dvm_jobs_by_status_created_at + .prefix_iter(txn, &prefix)? + { + let (key, _) = entry?; + let id = encoding::event_id_from(&key[key.len() - 32..])?; + let Some(job) = dvm_find_by_id(txn, databases, &id)? else { + continue; + }; + if kinds.is_some_and(|kinds| !kinds.is_empty() && !kinds.contains(&job.kind)) { + continue; + } + jobs.push(job); + taken += 1; + if taken >= take { + break; + } + } + } + jobs.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.id.cmp(&right.id)) + }); + jobs.truncate(take); + Ok(jobs) +} + +// --------------------------------------------------------------------------------------- +// Reports. +// --------------------------------------------------------------------------------------- + +fn report_key(id: i32) -> [u8; REPORT_ID_LEN] { + encoding::i32_key(id) +} + +fn report_event_key(report: &Report, event_id: &EventId) -> Vec { + let mut key = event_id.as_bytes().to_vec(); + key.extend_from_slice(&report_key(report.id)); + key +} + +fn report_created_key(report: &Report) -> Vec { + let mut key = stamp_key(report.created_at).to_vec(); + key.extend_from_slice(&report_key(report.id)); + key +} + +fn report_pubkey_key(report: &Report, pubkey: &Pubkey) -> Vec { + let mut key = pubkey.as_bytes().to_vec(); + key.extend_from_slice(&report_key(report.id)); + key +} + +fn report_load( + txn: &RoTxn<'_>, + databases: &Databases, + id: i32, +) -> Result, StorageError> { + let Some(value) = databases.reports.get(txn, &report_key(id))? else { + return Ok(None); + }; + rows::decode("reports", value).map(Some) +} + +/// Allocates the next `reports.id` from the `meta` sequence, which starts at 1. +fn next_report_id(txn: &mut RwTxn<'_>, databases: &Databases) -> Result { + let current: u64 = match databases.meta.get(txn, REPORT_SEQUENCE)? { + Some(value) => rows::decode("meta", value)?, + None => 0, + }; + let next = current + 1; + let value = rows::encode("meta", &next)?; + databases.meta.put(txn, REPORT_SEQUENCE, &value)?; + i32::try_from(next).map_err(|_| StorageError::ValueOutOfRange { + column: "reports.id", + value: next.to_string(), + }) +} + +fn report_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + report: &Report, +) -> Result<(), StorageError> { + let value = rows::encode("reports", report)?; + databases.reports.put(txn, &report_key(report.id), &value)?; + databases + .reports_by_event_id + .put(txn, &report_event_key(report, &report.event_id), &[])?; + if report.actionable { + databases + .reports_actionable_by_created_at + .put(txn, &report_created_key(report), &[])?; + if let Some(event_id) = &report.reported_event_id { + databases.reports_actionable_event.put( + txn, + &report_event_key(report, event_id), + &[], + )?; + } + if let Some(pubkey) = &report.reported_pubkey { + databases.reports_actionable_pubkey.put( + txn, + &report_pubkey_key(report, pubkey), + &[], + )?; + } + } + Ok(()) +} + +/// `ReportRepository::create`: inserts the row and returns it with its generated id. +pub(crate) fn report_create( + txn: &mut RwTxn<'_>, + databases: &Databases, + new: &NewReport, + now: DateTime, +) -> Result { + let report = Report { + id: next_report_id(txn, databases)?, + event_id: new.event_id, + reporter_pubkey: new.reporter_pubkey, + reported_pubkey: new.reported_pubkey, + reported_event_id: new.reported_event_id, + report_type: new.report_type, + weight: new.weight, + actionable: new.actionable, + created_at: encoding::micros(now), + }; + report_write(txn, databases, &report)?; + Ok(report) +} + +/// `ReportRepository::find_by_event_id`. +pub(crate) fn report_find_by_event_id( + txn: &RoTxn<'_>, + databases: &Databases, + event_id: &EventId, +) -> Result, StorageError> { + let mut reports = Vec::new(); + for entry in databases + .reports_by_event_id + .prefix_iter(txn, event_id.as_bytes())? + { + let (key, _) = entry?; + let id = encoding::i32_from_key(&key[32..])?; + if let Some(report) = report_load(txn, databases, id)? { + reports.push(report); + } + } + Ok(reports) +} + +/// `ReportRepository::find_actionable`: actionable rows, newest first. +pub(crate) fn report_find_actionable( + txn: &RoTxn<'_>, + databases: &Databases, + limit: u64, +) -> Result, StorageError> { + if limit == 0 { + return Ok(Vec::new()); + } + let mut reports = Vec::new(); + for entry in databases.reports_actionable_by_created_at.rev_iter(txn)? { + let (key, _) = entry?; + let id = encoding::i32_from_key(&key[8..])?; + if let Some(report) = report_load(txn, databases, id)? { + reports.push(report); + } + if reports.len() as u64 >= limit { + break; + } + } + Ok(reports) +} + +/// One actionable target, as the deduplicating walk keys it. +type ActionableKey = (Option<[u8; 32]>, Option<[u8; 32]>); + +/// `ReportRepository::find_actionable_targets`: every distinct target, deduplicated the way +/// the capture's `DISTINCT` does. +pub(crate) fn report_find_actionable_targets( + txn: &RoTxn<'_>, + databases: &Databases, +) -> Result, StorageError> { + let mut targets: BTreeSet = BTreeSet::new(); + for entry in databases.reports_actionable_by_created_at.iter(txn)? { + let (key, _) = entry?; + let id = encoding::i32_from_key(&key[8..])?; + if let Some(report) = report_load(txn, databases, id)? { + targets.insert(( + report.reported_pubkey.map(|pubkey| *pubkey.as_bytes()), + report.reported_event_id.map(|id| *id.as_bytes()), + )); + } + } + Ok(targets + .into_iter() + .map(|(pubkey, event_id)| ActionableTarget { + reported_pubkey: pubkey.map(Pubkey::from_byte_array), + reported_event_id: event_id.map(EventId::from_byte_array), + }) + .collect()) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use crate::repositories::ReportType; + + fn report() -> Report { + Report { + id: 7, + event_id: EventId::from_byte_array([1; 32]), + reporter_pubkey: Pubkey::from_byte_array([2; 32]), + reported_pubkey: Some(Pubkey::from_byte_array([3; 32])), + reported_event_id: Some(EventId::from_byte_array([4; 32])), + report_type: ReportType::Spam, + weight: 0.0, + actionable: true, + created_at: Utc::now(), + } + } + + #[test] + fn every_report_index_key_ends_with_the_id_width_the_codec_writes() { + // The NIP-56 lookups seek `reports_actionable_*` by the target's prefix and the walks + // read the id back from the tail, so the id's width is the one `report_key` writes: a + // widening has to change this assertion and the reads together, never one of them. + let report = report(); + assert_eq!(REPORT_ID_LEN, report_key(report.id).len()); + assert_eq!( + report_event_key(&report, &report.event_id).len(), + 32 + REPORT_ID_LEN + ); + assert_eq!( + report_pubkey_key(&report, &report.reporter_pubkey).len(), + 32 + REPORT_ID_LEN + ); + assert_eq!(report_created_key(&report).len(), 8 + REPORT_ID_LEN); + } +} diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs new file mode 100644 index 000000000..e9fc7b7d6 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -0,0 +1,1051 @@ +//! The LMDB backend: one environment, the 29 named databases, and the nine repository traits. +//! Mirrors `src/repositories/` through the seam the Postgres backend also implements, and +//! `docs/rewrite/decisions/0011-lmdb-backend.md` records why each choice below was made. +//! +//! Every heed call runs inside `tokio::task::spawn_blocking`, because `Env::write_txn` blocks +//! its thread on the single writer's mutex and a read transaction must not be held across an +//! await. A repository method is one whole write transaction, so it is atomic across every +//! database it touches. + +mod encoding; +mod events; +mod invoices; +mod metadata; +mod notifications; +mod rows; +mod users; + +use std::path::PathBuf; +use std::sync::Arc; + +use chrono::Utc; +use heed::types::Bytes; +use heed::{Database, Env, RoTxn, RwTxn}; +use nostream_config::Settings; +use nostream_core::{EventId, Filter, Kind, Pubkey}; + +use crate::backend::LmdbEnv; +use crate::error::StorageError; +use crate::repositories::{ + ActionableTarget, CreateInviteCodeOptions, DeliveryLogEntry, DeliveryLogFilters, DvmJob, + DvmJobRepository, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, + EventRetentionOptions, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, + InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, + NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, OutboxPayload, + QueryStream, ReconciliationItem, Report, ReportRepository, StorageFuture, User, UserPatch, + UserRepository, +}; + +/// The fixed map size every environment is opened with: `Env::resize` is `unsafe` and this +/// crate has one audited `unsafe` call, so the map is preallocated at 64 GiB and never grown. +/// +/// shortcut: the map is fixed at 64 GiB and never grown; upgrade when a deployment's data can +/// approach it, which needs `Env::resize` (a second `unsafe` call) or a refusal at open. +pub const LMDB_MAP_SIZE: usize = 64 << 30; + +/// The number of named databases the environment reserves, comfortably above the 29 it opens. +/// +/// shortcut: 29 databases open under a reserve of 64; upgrade when a tenth repository or a new +/// query path crosses that reserve, which moves this value and every store's reopen path. +pub const LMDB_MAX_DBS: u32 = 64; + +/// The number of concurrent read transactions the environment reserves. +pub const LMDB_MAX_READERS: u32 = 512; + +/// The capture's default page size (`DEFAULT_FILTER_LIMIT`). +pub(crate) const DEFAULT_FILTER_LIMIT: u64 = 500; + +/// The largest page a read materialises, however large a limit the wire asks for. +/// +/// `limits.client.subscription.maxLimit` is the relay's own bound, but it is applied only when +/// the deployment sets it, and a request reaches the store before some of those gates; this is +/// the backend's bound, so no page a read materialises exceeds it. A multi-filter read holds one +/// such page per filter, and a reconciliation over `ids` holds the request's own id list when +/// its criteria are exact — a prefix criterion is a byte range over the id field, so it takes +/// the bounded scan instead. 5 000 is exactly the shipped `maxLimit` +/// (`resources/default-settings.yaml`), so the clamp cannot shorten a page a configured relay +/// would have served. +pub(crate) const MAX_PAGE_KEYS: usize = 5_000; + +/// The retention batch size (`RETENTION_BATCH_SIZE`). +pub(crate) const RETENTION_BATCH_SIZE: usize = 1_000; + +/// How many rows one stream poll copies before it drops its read transaction. +pub(crate) const STREAM_BATCH_ITEMS: usize = 256; + +/// How many index keys one reconciliation poll evaluates before it drops its read transaction. +/// A filter that matches few of the keys it walks ends a batch on this count rather than on +/// [`STREAM_BATCH_ITEMS`], so the transaction never spans a whole range. +pub(crate) const STREAM_BATCH_KEYS: usize = 256; + +/// The settings gates the capture reads per query. The LMDB backend records only whether +/// NIP-50 is on: it has no search implementation (ADR 0011, "Oracle: two arms, one executor" — +/// entry 21, out of scope, not a permitted failure). +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(crate) struct Gates { + pub(crate) nip50: bool, + pub(crate) nip56: bool, +} + +/// The named databases of one environment. +#[derive(Debug, Clone, Copy)] +pub(crate) struct Databases { + pub(crate) events: Database, + pub(crate) events_by_created_at: Database, + pub(crate) events_by_author: Database, + pub(crate) events_deleted: Database, + pub(crate) events_expiring: Database, + pub(crate) replaceable: Database, + pub(crate) tags: Database, + pub(crate) users: Database, + pub(crate) invoices: Database, + pub(crate) invoices_pending: Database, + pub(crate) nip05: Database, + pub(crate) nip05_by_last_checked_at: Database, + pub(crate) invite_codes: Database, + pub(crate) invite_codes_by_created_at: Database, + pub(crate) invite_codes_by_expires_at: Database, + pub(crate) dvm_jobs: Database, + pub(crate) dvm_jobs_by_status_created_at: Database, + pub(crate) reports: Database, + pub(crate) reports_by_event_id: Database, + pub(crate) reports_actionable_by_created_at: Database, + pub(crate) reports_actionable_event: Database, + pub(crate) reports_actionable_pubkey: Database, + pub(crate) outbox: Database, + pub(crate) outbox_by_status_created_at: Database, + pub(crate) outbox_by_status_updated_at: Database, + pub(crate) notification_delivery_log: Database, + pub(crate) delivery_log_by_created_at: Database, + pub(crate) delivery_log_by_outbox: Database, + pub(crate) meta: Database, +} + +impl Databases { + /// Creates (or opens) every database inside the caller's write transaction. + fn create(env: &Env, txn: &mut RwTxn<'_>) -> Result { + Ok(Self { + events: env.create_database(txn, Some("events"))?, + events_by_created_at: env.create_database(txn, Some("events_by_created_at"))?, + events_by_author: env.create_database(txn, Some("events_by_author"))?, + events_deleted: env.create_database(txn, Some("events_deleted"))?, + events_expiring: env.create_database(txn, Some("events_expiring"))?, + replaceable: env.create_database(txn, Some("replaceable"))?, + tags: env.create_database(txn, Some("tags"))?, + users: env.create_database(txn, Some("users"))?, + invoices: env.create_database(txn, Some("invoices"))?, + invoices_pending: env.create_database(txn, Some("invoices_pending"))?, + nip05: env.create_database(txn, Some("nip05_verifications"))?, + nip05_by_last_checked_at: env.create_database(txn, Some("nip05_by_last_checked_at"))?, + invite_codes: env.create_database(txn, Some("invite_codes"))?, + invite_codes_by_created_at: env + .create_database(txn, Some("invite_codes_by_created_at"))?, + invite_codes_by_expires_at: env + .create_database(txn, Some("invite_codes_by_expires_at"))?, + dvm_jobs: env.create_database(txn, Some("dvm_jobs"))?, + dvm_jobs_by_status_created_at: env + .create_database(txn, Some("dvm_jobs_by_status_created_at"))?, + reports: env.create_database(txn, Some("reports"))?, + reports_by_event_id: env.create_database(txn, Some("reports_by_event_id"))?, + reports_actionable_by_created_at: env + .create_database(txn, Some("reports_actionable_by_created_at"))?, + reports_actionable_event: env.create_database(txn, Some("reports_actionable_event"))?, + reports_actionable_pubkey: env + .create_database(txn, Some("reports_actionable_pubkey"))?, + outbox: env.create_database(txn, Some("notification_outbox"))?, + outbox_by_status_created_at: env + .create_database(txn, Some("outbox_by_status_created_at"))?, + outbox_by_status_updated_at: env + .create_database(txn, Some("outbox_by_status_updated_at"))?, + notification_delivery_log: env + .create_database(txn, Some("notification_delivery_log"))?, + delivery_log_by_created_at: env + .create_database(txn, Some("delivery_log_by_created_at"))?, + delivery_log_by_outbox: env.create_database(txn, Some("delivery_log_by_outbox"))?, + meta: env.create_database(txn, Some("meta"))?, + }) + } +} + +/// How much of the fixed map one store has used (`Env::non_free_pages_size`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MapUsage { + /// The ceiling the store never grows past. + pub map_size: usize, + /// The bytes the pages in use occupy, as of the last committed transaction. + pub used_bytes: u64, +} + +/// An embedded LMDB event store with its metadata tail, built from the root a caller supplies. +#[derive(Debug, Clone)] +pub struct LmdbBackend { + env: Env, + databases: Databases, + settings: Option>, +} + +impl LmdbBackend { + /// Opens (creating if needed) the store under `/lmdb` and creates every database. + /// + /// The usage it reports is what the map's approach to [`LMDB_MAP_SIZE`] is read from: the + /// store never grows, so an operator sees it coming here rather than at the first failed + /// write. + pub async fn open(root: impl Into) -> Result { + let root = root.into(); + // The same rule `data_dir` answers, from the same constant. + let path = root.join(crate::backend::LMDB_DIR); + let opened = tokio::task::spawn_blocking(move || { + let environment = LmdbEnv::open(&path, LMDB_MAP_SIZE)?; + let handle = environment.env().clone(); + let mut transaction = handle.write_txn()?; + let databases = Databases::create(&handle, &mut transaction)?; + transaction.commit()?; + let used_bytes = handle.non_free_pages_size()?; + Ok::<_, StorageError>((environment, databases, used_bytes)) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)??; + let (environment, databases, used_bytes) = opened; + + tracing::info!( + path = %environment.path().display(), + map_size = LMDB_MAP_SIZE, + used_bytes, + used_percent = used_bytes as f64 / LMDB_MAP_SIZE as f64 * 100.0, + "the lmdb store is open" + ); + + Ok(Self { + env: environment.env().clone(), + databases, + settings: None, + }) + } + + /// How much of the fixed map the store has used, for an operator's tooling and the log + /// line [`open`](Self::open) writes. + /// + /// `Env::non_free_pages_size` opens a read transaction and walks every named database, so + /// it runs on the blocking pool (D7) rather than on the caller's runtime worker. + pub async fn map_usage(&self) -> Result { + let env = self.env.clone(); + tokio::task::spawn_blocking(move || { + Ok(MapUsage { + map_size: LMDB_MAP_SIZE, + used_bytes: env.non_free_pages_size()?, + }) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)? + } + + /// Supplies the `nip50`/`nip56` gates. Absent settings mean both features are off. + pub fn with_settings(mut self, settings: Arc) -> Self { + self.settings = Some(settings); + self + } + + /// The environment handle, for an operator's tooling. + pub fn env(&self) -> &Env { + &self.env + } + + pub(crate) fn gates(&self) -> Gates { + let Some(settings) = &self.settings else { + return Gates::default(); + }; + Gates { + nip50: settings + .nip50 + .as_ref() + .is_some_and(|nip50| nip50.enabled == Some(true)), + nip56: settings + .nip56 + .as_ref() + .is_some_and(|nip56| nip56.enabled && nip56.hide_actionable_reports), + } + } + + /// Runs one write transaction on the blocking pool; an error rolls it back. + pub(crate) async fn write(&self, work: F) -> Result + where + T: Send + 'static, + F: FnOnce(&mut RwTxn<'_>, &Databases) -> Result + Send + 'static, + { + let env = self.env.clone(); + let databases = self.databases; + tokio::task::spawn_blocking(move || { + let mut transaction = env.write_txn()?; + let value = work(&mut transaction, &databases)?; + transaction.commit()?; + Ok(value) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)? + } + + /// Runs one read transaction on the blocking pool. + pub(crate) async fn read(&self, work: F) -> Result + where + T: Send + 'static, + F: FnOnce(&RoTxn<'_>, &Databases) -> Result + Send + 'static, + { + let env = self.env.clone(); + let databases = self.databases; + tokio::task::spawn_blocking(move || { + let transaction = env.read_txn()?; + work(&transaction, &databases) + }) + .await + .map_err(|_| StorageError::LmdbTaskCancelled)? + } +} + +impl EventRepository for LmdbBackend { + fn create<'a>(&'a self, record: &'a EventRecord) -> StorageFuture<'a, u64> { + Box::pin(async move { + let record = record.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + events::create(txn, databases, std::slice::from_ref(&record), now) + }) + .await + }) + } + + fn create_many<'a>(&'a self, records: &'a [EventRecord]) -> StorageFuture<'a, u64> { + Box::pin(async move { + let records = records.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| events::create(txn, databases, &records, now)) + .await + }) + } + + fn upsert<'a>(&'a self, record: &'a EventRecord) -> StorageFuture<'a, u64> { + Box::pin(async move { + let record = record.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| events::upsert(txn, databases, &record, now)) + .await + }) + } + + fn upsert_many<'a>(&'a self, records: &'a [EventRecord]) -> StorageFuture<'a, u64> { + Box::pin(async move { + let records = records.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| events::upsert_many(txn, databases, &records, now)) + .await + }) + } + + fn find_by_filters<'a>( + &'a self, + filters: &'a [Filter], + ) -> StorageFuture<'a, QueryStream<'a, EventRecord>> { + Box::pin(async move { + let filters = filters.to_vec(); + let gates = self.gates(); + let now = Utc::now().timestamp(); + let page = self + .read(move |txn, databases| events::page(txn, databases, &filters, &gates, now)) + .await?; + let stream: QueryStream<'_, EventRecord> = Box::pin(events::ValuesStream::new( + self.env.clone(), + self.databases, + &page, + )); + Ok(stream) + }) + } + + fn count_by_filters<'a>(&'a self, filters: &'a [Filter]) -> StorageFuture<'a, u64> { + Box::pin(async move { + let filters = filters.to_vec(); + let gates = self.gates(); + let now = Utc::now().timestamp(); + self.read(move |txn, databases| events::count(txn, databases, &filters, &gates, now)) + .await + }) + } + + fn find_reconciliation_items<'a>( + &'a self, + filter: &'a Filter, + ) -> StorageFuture<'a, QueryStream<'a, ReconciliationItem>> { + Box::pin(async move { + let filter = filter.clone(); + let gates = self.gates(); + let now = Utc::now().timestamp(); + let stream: QueryStream<'_, ReconciliationItem> = + Box::pin(events::ReconciliationStream::new( + self.env.clone(), + self.databases, + &filter, + gates, + now, + )?); + Ok(stream) + }) + } + + fn delete_by_pubkey_and_ids<'a>( + &'a self, + pubkey: &'a Pubkey, + ids: &'a [EventId], + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let ids = ids.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + events::delete_by_pubkey_and_ids(txn, databases, &pubkey, &ids, now) + }) + .await + }) + } + + fn delete_by_pubkey_except_kinds<'a>( + &'a self, + pubkey: &'a Pubkey, + excluded_kinds: &'a [Kind], + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let kinds = excluded_kinds.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + events::delete_by_pubkey_except_kinds(txn, databases, &pubkey, &kinds, now) + }) + .await + }) + } + + fn has_active_request_to_vanish<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, bool> { + Box::pin(async move { + let pubkey = *pubkey; + self.read(move |txn, databases| { + events::has_active_request_to_vanish(txn, databases, &pubkey) + }) + .await + }) + } + + fn delete_expired_and_retained<'a>( + &'a self, + options: &'a EventRetentionOptions, + ) -> StorageFuture<'a, EventPurgeCounts> { + Box::pin(async move { + let options = options.clone(); + let now = Utc::now().timestamp(); + self.write(move |txn, databases| { + events::delete_expired_and_retained(txn, databases, &options, now) + }) + .await + }) + } +} + +impl InvoiceRepository for LmdbBackend { + fn find_by_id<'a>(&'a self, id: &'a str) -> StorageFuture<'a, Option> { + Box::pin(async move { + let id = id.to_owned(); + self.read(move |txn, databases| invoices::find_by_id(txn, databases, &id)) + .await + }) + } + + fn upsert<'a>(&'a self, invoice: &'a Invoice) -> StorageFuture<'a, u64> { + Box::pin(async move { + let invoice = invoice.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| invoices::upsert(txn, databases, &invoice, now)) + .await + }) + } + + fn update_status<'a>( + &'a self, + id: &'a str, + status: InvoiceStatus, + ) -> StorageFuture<'a, Option> { + Box::pin(async move { + let id = id.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + invoices::update_status(txn, databases, &id, status, now) + }) + .await + }) + } + + fn confirm_invoice<'a>( + &'a self, + invoice_id: &'a str, + amount_received: i64, + confirmed_at: chrono::DateTime, + ) -> StorageFuture<'a, ()> { + Box::pin(async move { + let invoice_id = invoice_id.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + invoices::confirm_invoice( + txn, + databases, + &invoice_id, + amount_received, + confirmed_at, + now, + ) + }) + .await + }) + } + + fn find_pending_invoices<'a>( + &'a self, + offset: u64, + limit: u64, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + self.read(move |txn, databases| { + invoices::find_pending_invoices(txn, databases, offset, limit) + }) + .await + }) + } +} + +impl UserRepository for LmdbBackend { + fn find_by_pubkey<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, Option> { + Box::pin(async move { + let pubkey = *pubkey; + self.read(move |txn, databases| users::find_by_pubkey(txn, databases, &pubkey)) + .await + }) + } + + fn upsert<'a>(&'a self, user: &'a UserPatch) -> StorageFuture<'a, u64> { + Box::pin(async move { + let user = user.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| users::upsert(txn, databases, &user, now)) + .await + }) + } + + fn get_balance_by_pubkey<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, i64> { + Box::pin(async move { + let pubkey = *pubkey; + self.read(move |txn, databases| users::get_balance_by_pubkey(txn, databases, &pubkey)) + .await + }) + } + + fn is_vanished<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, bool> { + Box::pin(async move { + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| users::is_vanished(txn, databases, &pubkey, now)) + .await + }) + } + + fn set_vanished<'a>(&'a self, pubkey: &'a Pubkey, vanished: bool) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + users::upsert_vanish_state(txn, databases, &pubkey, vanished, now) + }) + .await + }) + } + + fn admit_user<'a>( + &'a self, + pubkey: &'a Pubkey, + admitted_at: chrono::DateTime, + ) -> StorageFuture<'a, ()> { + Box::pin(async move { + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + users::admit_user(txn, databases, &pubkey, admitted_at, now) + }) + .await + }) + } + + fn revoke_admission<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| users::revoke_admission(txn, databases, &pubkey, now)) + .await + }) + } +} + +impl Nip05VerificationRepository for LmdbBackend { + fn find_by_pubkey<'a>( + &'a self, + pubkey: &'a Pubkey, + ) -> StorageFuture<'a, Option> { + Box::pin(async move { + let pubkey = *pubkey; + self.read(move |txn, databases| metadata::nip05_find_by_pubkey(txn, databases, &pubkey)) + .await + }) + } + + fn upsert<'a>(&'a self, verification: &'a Nip05Verification) -> StorageFuture<'a, u64> { + Box::pin(async move { + let verification = verification.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::nip05_upsert(txn, databases, &verification, now) + }) + .await + }) + } + + fn find_pending_verifications<'a>( + &'a self, + update_frequency_ms: i64, + max_failures: i32, + limit: u64, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let now = encoding::micros(Utc::now()); + self.read(move |txn, databases| { + metadata::nip05_find_pending( + txn, + databases, + update_frequency_ms, + max_failures, + limit, + now, + ) + }) + .await + }) + } + + fn delete_by_pubkey<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + self.write(move |txn, databases| { + metadata::nip05_delete_by_pubkey(txn, databases, &pubkey) + }) + .await + }) + } +} + +impl InviteCodeRepository for LmdbBackend { + fn create<'a>( + &'a self, + code: &'a str, + options: &'a CreateInviteCodeOptions, + ) -> StorageFuture<'a, InviteCode> { + Box::pin(async move { + let code = code.to_owned(); + let options = options.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::invite_create(txn, databases, &code, &options, now) + }) + .await + }) + } + + fn find_by_code<'a>(&'a self, code: &'a str) -> StorageFuture<'a, Option> { + Box::pin(async move { + let code = code.to_owned(); + self.read(move |txn, databases| metadata::invite_find_by_code(txn, databases, &code)) + .await + }) + } + + fn claim_code<'a>(&'a self, code: &'a str, pubkey: &'a Pubkey) -> StorageFuture<'a, bool> { + Box::pin(async move { + let code = code.to_owned(); + let pubkey = *pubkey; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::invite_claim_code(txn, databases, &code, &pubkey, now) + }) + .await + }) + } + + fn find_active_codes<'a>(&'a self, limit: u64) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let now = encoding::micros(Utc::now()); + self.read(move |txn, databases| { + metadata::invite_find_active_codes(txn, databases, limit, now) + }) + .await + }) + } + + fn delete_expired_codes<'a>(&'a self) -> StorageFuture<'a, u64> { + Box::pin(async move { + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| metadata::invite_delete_expired(txn, databases, now)) + .await + }) + } +} + +impl DvmJobRepository for LmdbBackend { + fn create<'a>( + &'a self, + id: &'a EventId, + requester: &'a Pubkey, + kind: Kind, + ) -> StorageFuture<'a, DvmJob> { + Box::pin(async move { + let id = *id; + let requester = *requester; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::dvm_create(txn, databases, &id, &requester, kind, now) + }) + .await + }) + } + + fn find_by_id<'a>(&'a self, id: &'a EventId) -> StorageFuture<'a, Option> { + Box::pin(async move { + let id = *id; + self.read(move |txn, databases| metadata::dvm_find_by_id(txn, databases, &id)) + .await + }) + } + + fn assign_worker<'a>(&'a self, id: &'a EventId, worker_index: i32) -> StorageFuture<'a, bool> { + Box::pin(async move { + let id = *id; + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::dvm_assign_worker(txn, databases, &id, worker_index, now) + }) + .await + }) + } + + fn update_status<'a>(&'a self, update: &'a DvmJobUpdate) -> StorageFuture<'a, Option> { + Box::pin(async move { + let update = update.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + metadata::dvm_update_status(txn, databases, &update, now) + }) + .await + }) + } + + fn find_pending_jobs<'a>( + &'a self, + limit: u64, + kinds: Option<&'a [Kind]>, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let kinds = kinds.map(<[Kind]>::to_vec); + self.read(move |txn, databases| { + metadata::dvm_find_pending_jobs(txn, databases, limit, kinds.as_deref()) + }) + .await + }) + } +} + +impl ReportRepository for LmdbBackend { + fn create<'a>(&'a self, report: &'a NewReport) -> StorageFuture<'a, Report> { + Box::pin(async move { + let report = report.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| metadata::report_create(txn, databases, &report, now)) + .await + }) + } + + fn create_many<'a>(&'a self, reports: &'a [NewReport]) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let reports = reports.to_vec(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + let mut created = Vec::with_capacity(reports.len()); + for report in &reports { + created.push(metadata::report_create(txn, databases, report, now)?); + } + Ok(created) + }) + .await + }) + } + + fn find_by_event_id<'a>(&'a self, event_id: &'a EventId) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let event_id = *event_id; + self.read(move |txn, databases| { + metadata::report_find_by_event_id(txn, databases, &event_id) + }) + .await + }) + } + + fn find_actionable<'a>(&'a self, limit: u64) -> StorageFuture<'a, Vec> { + Box::pin(async move { + self.read(move |txn, databases| metadata::report_find_actionable(txn, databases, limit)) + .await + }) + } + + fn find_actionable_targets<'a>(&'a self) -> StorageFuture<'a, Vec> { + Box::pin(async move { self.read(metadata::report_find_actionable_targets).await }) + } +} + +impl NotificationDeliveryLogRepository for LmdbBackend { + fn append<'a>(&'a self, entry: &'a NewDeliveryLogEntry) -> StorageFuture<'a, ()> { + Box::pin(async move { + let entry = entry.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| notifications::append(txn, databases, &entry, now)) + .await + }) + } + + fn find_recent<'a>( + &'a self, + limit: u64, + filters: &'a DeliveryLogFilters, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let filters = filters.clone(); + self.read(move |txn, databases| { + notifications::find_recent(txn, databases, limit, &filters) + }) + .await + }) + } + + fn find_successful_target_ids<'a>( + &'a self, + outbox_id: &'a str, + ) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let outbox_id = outbox_id.to_owned(); + self.read(move |txn, databases| { + notifications::find_successful_target_ids(txn, databases, &outbox_id) + }) + .await + }) + } + + fn delete_older_than<'a>(&'a self, cutoff: chrono::DateTime) -> StorageFuture<'a, u64> { + Box::pin(async move { + self.write(move |txn, databases| { + notifications::delete_older_than(txn, databases, cutoff) + }) + .await + }) + } +} + +impl NotificationOutboxRepository for LmdbBackend { + fn enqueue<'a>( + &'a self, + event_type: &'a str, + payload: &'a OutboxPayload, + ) -> StorageFuture<'a, OutboxMessage> { + Box::pin(async move { + let event_type = event_type.to_owned(); + let payload = payload.clone(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + notifications::enqueue(txn, databases, &event_type, &payload, now) + }) + .await + }) + } + + fn claim_batch<'a>(&'a self, limit: u64) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| notifications::claim_batch(txn, databases, limit, now)) + .await + }) + } + + fn mark_delivered<'a>(&'a self, id: &'a str) -> StorageFuture<'a, ()> { + Box::pin(async move { + let id = id.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + notifications::mark_delivered(txn, databases, &id, now) + }) + .await + }) + } + + fn mark_failed<'a>( + &'a self, + id: &'a str, + error: &'a str, + attempt_count: i32, + max_attempts: i32, + base_delay_ms: i64, + ) -> StorageFuture<'a, ()> { + Box::pin(async move { + let id = id.to_owned(); + let error = error.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + notifications::mark_failed( + txn, + databases, + &id, + &error, + attempt_count, + max_attempts, + base_delay_ms, + now, + ) + }) + .await + }) + } + + fn delete_terminal_older_than<'a>( + &'a self, + cutoff: chrono::DateTime, + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + self.write(move |txn, databases| { + notifications::delete_terminal_older_than(txn, databases, cutoff) + }) + .await + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + + #[tokio::test] + async fn the_backend_opens_and_creates_every_database() { + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let transaction = backend.env.read_txn().expect("a read transaction"); + for name in [ + "events", + "events_by_created_at", + "events_by_author", + "events_deleted", + "events_expiring", + "replaceable", + "tags", + "users", + "meta", + "delivery_log_by_outbox", + ] { + let database = backend + .env + .open_database::(&transaction, Some(name)) + .expect("the database opens"); + assert!(database.is_some(), "{name} exists"); + } + } + + #[tokio::test] + async fn the_store_reports_how_much_of_its_map_it_uses() { + // The map never grows (LMDB_MAP_SIZE), so the approach to it is what an operator has to + // see; before this the first signal was the first failed write. + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let usage = backend.map_usage().await.expect("the usage"); + assert_eq!(usage.map_size, LMDB_MAP_SIZE); + assert!(usage.used_bytes > 0, "the 29 databases occupy pages"); + assert!(usage.used_bytes <= LMDB_MAP_SIZE as u64); + } + + #[tokio::test] + async fn absent_settings_leave_both_gates_off() { + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let gates = backend.gates(); + assert!(!gates.nip50); + assert!(!gates.nip56); + } + + #[tokio::test] + async fn a_named_database_past_the_maximum_is_refused() { + // The environment reserves LMDB_MAX_DBS handles; the 29 the backend opens leave the + // rest free, and the 65th creation has to fail rather than silently succeed. + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let mut transaction = backend.env.write_txn().expect("a write transaction"); + let mut created = 0; + for index in 0..LMDB_MAX_DBS { + let name = format!("probe-{index}"); + if backend + .env + .create_database::(&mut transaction, Some(&name)) + .is_err() + { + break; + } + created += 1; + } + assert!(created < LMDB_MAX_DBS, "the handle limit is enforced"); + transaction.abort(); + } + + #[tokio::test] + async fn a_store_filled_past_its_map_size_reports_the_driver_error() { + // A small map is the only way to reach the boundary the fixed LMDB_MAP_SIZE stands + // in for: the environment never grows, so a full one is the driver's error (V27). + let directory = tempfile::tempdir().expect("tempdir"); + let environment = + LmdbEnv::open(directory.path().join("map"), 1 << 20).expect("a 1 MiB environment"); + let env = environment.env().clone(); + let mut transaction = env.write_txn().expect("a write transaction"); + let database = env + .create_database::(&mut transaction, Some("fill")) + .expect("a database"); + + let value = vec![0u8; 64 << 10]; + let mut written = 0u32; + loop { + match database.put(&mut transaction, &written.to_be_bytes(), &value) { + Ok(()) => written += 1, + Err(error) => { + let error = StorageError::from(error); + assert!( + matches!(error, StorageError::Lmdb(_)), + "a full map is the driver's error, got {error:?}" + ); + break; + } + } + } + assert!(written > 0, "the map held rows before it filled"); + transaction.abort(); + } +} diff --git a/crates/nostream-storage/src/lmdb/notifications.rs b/crates/nostream-storage/src/lmdb/notifications.rs new file mode 100644 index 000000000..3c73734c8 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/notifications.rs @@ -0,0 +1,648 @@ +//! `NotificationDeliveryLogRepository` and `NotificationOutboxRepository` against LMDB. +//! Mirrors `src/repositories/notification-delivery-log-repository.ts` and +//! `src/repositories/notification-outbox-repository.ts`. +//! +//! The outbox's `FOR UPDATE SKIP LOCKED` claim needs no port: LMDB has one writer, so a claim +//! cannot interleave with another claim, and it runs inside one write transaction. + +use std::collections::HashSet; + +use chrono::{DateTime, Duration, Utc}; +use heed::{RoTxn, RwTxn}; + +use super::Databases; +use super::encoding; +use super::encoding::{stamp_key, text_from, text_part}; +use super::rows; +use crate::error::StorageError; +use crate::repositories::{ + DeliveryLogEntry, DeliveryLogFilters, NOTIFICATION_OUTBOX_PROCESSING_LEASE_MS, + NewDeliveryLogEntry, OutboxMessage, OutboxPayload, OutboxStatus, +}; + +fn new_id() -> String { + uuid::Uuid::new_v4().to_string() +} + +/// Counts the outbox rows a claim loaded. Only the tests read the count, so a release build +/// drops it. +fn probe_load() { + #[cfg(test)] + probes::load(); +} + +/// The work counter the claim's cost test asserts on: a claim is bounded by the rows it +/// returns, never by the backlog behind them. +#[cfg(test)] +mod probes { + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Serialises the tests that count loads, so one cannot read another's numbers. + pub(super) static COUNTING: Mutex<()> = Mutex::new(()); + + static LOADS: AtomicUsize = AtomicUsize::new(0); + + pub(super) fn reset() { + LOADS.store(0, Ordering::Relaxed); + } + + pub(super) fn load() { + LOADS.fetch_add(1, Ordering::Relaxed); + } + + pub(super) fn loads() -> usize { + LOADS.load(Ordering::Relaxed) + } +} + +// --------------------------------------------------------------------------------------- +// The delivery log. +// --------------------------------------------------------------------------------------- + +fn delivery_created_key(entry: &DeliveryLogEntry) -> Vec { + let mut key = stamp_key(entry.created_at).to_vec(); + key.extend_from_slice(entry.id.as_bytes()); + key +} + +/// `outbox_id ‖ status ‖ id`, so one entry is addressable and two attempts do not share a key. +/// +/// The target is in the row, not the key: a target id may be longer than an LMDB key holds, +/// and the index only has to find the entry, whose row carries the id to report. +fn delivery_outbox_key(outbox_id: &str, entry: &DeliveryLogEntry) -> Result, StorageError> { + let mut key = text_part(outbox_id, "delivery_log_by_outbox", "outbox_id")?; + key.extend_from_slice(&text_part( + entry.status.as_str(), + "delivery_log_by_outbox", + "status", + )?); + key.extend_from_slice(entry.id.as_bytes()); + Ok(key) +} + +/// The entry id one `delivery_log_by_outbox` key ends with. +fn delivery_outbox_id(key: &[u8]) -> Result<&str, StorageError> { + let Some(start) = key.iter().rposition(|byte| *byte == 0) else { + return Err(StorageError::InvalidRow { + table: "delivery_log_by_outbox", + column: "key", + value: "a key without an entry id".to_owned(), + }); + }; + text_from(&key[start + 1..], "delivery_log_by_outbox") +} + +fn delivery_load( + txn: &RoTxn<'_>, + databases: &Databases, + id: &str, +) -> Result, StorageError> { + let Some(value) = databases + .notification_delivery_log + .get(txn, id.as_bytes())? + else { + return Ok(None); + }; + rows::decode("notification_delivery_log", value).map(Some) +} + +/// `append`: one delivery attempt, timestamped now. The column's foreign key is enforced +/// here, because the store has no server to enforce it. +pub(crate) fn append( + txn: &mut RwTxn<'_>, + databases: &Databases, + entry: &NewDeliveryLogEntry, + now: DateTime, +) -> Result<(), StorageError> { + if let Some(outbox_id) = &entry.outbox_id + && databases.outbox.get(txn, outbox_id.as_bytes())?.is_none() + { + return Err(StorageError::InvalidArgument { + message: "notification_delivery_log.outbox_id must name an outbox row", + }); + } + + let entry = DeliveryLogEntry { + id: new_id(), + outbox_id: entry.outbox_id.clone(), + event_type: entry.event_type.clone(), + target_id: entry.target_id.clone(), + target_type: entry.target_type, + status: entry.status, + attempt_number: entry.attempt_number, + error_snippet: entry.error_snippet.clone(), + created_at: encoding::micros(now), + }; + // The outbox key no longer carries the target, so the row's own text column is where the + // codec's refusal is met. + encoding::refuse_nul("notification_delivery_log", "target_id", &entry.target_id)?; + let value = rows::encode("notification_delivery_log", &entry)?; + databases + .notification_delivery_log + .put(txn, entry.id.as_bytes(), &value)?; + databases + .delivery_log_by_created_at + .put(txn, &delivery_created_key(&entry), &[])?; + if let Some(outbox_id) = &entry.outbox_id { + databases + .delivery_log_by_outbox + .put(txn, &delivery_outbox_key(outbox_id, &entry)?, &[])?; + } + Ok(()) +} + +/// `find_recent`: newest first, optionally filtered by status and event type. +pub(crate) fn find_recent( + txn: &RoTxn<'_>, + databases: &Databases, + limit: u64, + filters: &DeliveryLogFilters, +) -> Result, StorageError> { + if limit == 0 { + return Ok(Vec::new()); + } + let mut entries = Vec::new(); + for entry in databases.delivery_log_by_created_at.rev_iter(txn)? { + let (key, _) = entry?; + let id = text_from(&key[8..], "delivery_log_by_created_at")?; + let Some(entry) = delivery_load(txn, databases, id)? else { + continue; + }; + if filters.status.is_some_and(|status| status != entry.status) + || filters + .event_type + .as_ref() + .is_some_and(|event_type| *event_type != entry.event_type) + { + continue; + } + entries.push(entry); + if entries.len() as u64 >= limit { + break; + } + } + Ok(entries) +} + +/// `findSuccessfulTargetIds`: the distinct targets that succeeded for one message. +pub(crate) fn find_successful_target_ids( + txn: &RoTxn<'_>, + databases: &Databases, + outbox_id: &str, +) -> Result, StorageError> { + let mut prefix = text_part(outbox_id, "delivery_log_by_outbox", "outbox_id")?; + prefix.extend_from_slice(&text_part( + crate::repositories::NotificationDeliveryStatus::Success.as_str(), + "delivery_log_by_outbox", + "status", + )?); + + let mut seen = HashSet::new(); + let mut targets = Vec::new(); + for entry in databases.delivery_log_by_outbox.prefix_iter(txn, &prefix)? { + let (key, _) = entry?; + let id = delivery_outbox_id(key)?; + let Some(row) = delivery_load(txn, databases, id)? else { + continue; + }; + if seen.insert(row.target_id.clone()) { + targets.push(row.target_id); + } + } + Ok(targets) +} + +/// `deleteOlderThan`: prunes the log by creation time. +pub(crate) fn delete_older_than( + txn: &mut RwTxn<'_>, + databases: &Databases, + cutoff: DateTime, +) -> Result { + let cutoff = encoding::micros(cutoff).timestamp_micros(); + let mut expired = Vec::new(); + for entry in databases.delivery_log_by_created_at.iter(txn)? { + let (key, _) = entry?; + if encoding::i64_from_key(&key[..8])? >= cutoff { + break; + } + expired.push(text_from(&key[8..], "delivery_log_by_created_at")?.to_owned()); + } + + let mut affected = 0; + for id in expired { + let Some(entry) = delivery_load(txn, databases, &id)? else { + continue; + }; + databases + .notification_delivery_log + .delete(txn, id.as_bytes())?; + databases + .delivery_log_by_created_at + .delete(txn, &delivery_created_key(&entry))?; + if let Some(outbox_id) = &entry.outbox_id { + databases + .delivery_log_by_outbox + .delete(txn, &delivery_outbox_key(outbox_id, &entry)?)?; + } + affected += 1; + } + Ok(affected) +} + +// --------------------------------------------------------------------------------------- +// The outbox. +// --------------------------------------------------------------------------------------- + +fn outbox_created_key(message: &OutboxMessage) -> Result, StorageError> { + let mut key = text_part( + message.status.as_str(), + "outbox_by_status_created_at", + "status", + )?; + key.extend_from_slice(&stamp_key(message.created_at)); + key.extend_from_slice(message.id.as_bytes()); + Ok(key) +} + +fn outbox_updated_key(message: &OutboxMessage) -> Result, StorageError> { + let mut key = text_part( + message.status.as_str(), + "outbox_by_status_updated_at", + "status", + )?; + key.extend_from_slice(&stamp_key(message.updated_at)); + key.extend_from_slice(message.id.as_bytes()); + Ok(key) +} + +fn outbox_load( + txn: &RoTxn<'_>, + databases: &Databases, + id: &str, +) -> Result, StorageError> { + let Some(value) = databases.outbox.get(txn, id.as_bytes())? else { + return Ok(None); + }; + rows::decode("notification_outbox", value).map(Some) +} + +/// Writes a message and its two status indexes, removing the entries of the row it replaces. +fn outbox_write( + txn: &mut RwTxn<'_>, + databases: &Databases, + message: &OutboxMessage, + previous: Option<&OutboxMessage>, +) -> Result<(), StorageError> { + let value = rows::encode("notification_outbox", message)?; + databases.outbox.put(txn, message.id.as_bytes(), &value)?; + if let Some(previous) = previous { + databases + .outbox_by_status_created_at + .delete(txn, &outbox_created_key(previous)?)?; + databases + .outbox_by_status_updated_at + .delete(txn, &outbox_updated_key(previous)?)?; + } + databases + .outbox_by_status_created_at + .put(txn, &outbox_created_key(message)?, &[])?; + databases + .outbox_by_status_updated_at + .put(txn, &outbox_updated_key(message)?, &[])?; + Ok(()) +} + +/// `enqueue`: inserts a pending message and returns the stored row. +pub(crate) fn enqueue( + txn: &mut RwTxn<'_>, + databases: &Databases, + event_type: &str, + payload: &OutboxPayload, + now: DateTime, +) -> Result { + let now = encoding::micros(now); + let message = OutboxMessage { + id: new_id(), + event_type: event_type.to_owned(), + payload: payload.clone(), + status: OutboxStatus::Pending, + attempt_count: 0, + available_at: now, + last_error: None, + delivered_at: None, + created_at: now, + updated_at: now, + }; + outbox_write(txn, databases, &message, None)?; + Ok(message) +} + +/// `claimBatch`: pending rows that are due and processing rows whose lease expired, oldest +/// first, marked processing in the same write transaction. +pub(crate) fn claim_batch( + txn: &mut RwTxn<'_>, + databases: &Databases, + limit: u64, + now: DateTime, +) -> Result, StorageError> { + let now = encoding::micros(now); + let stale = now - Duration::milliseconds(NOTIFICATION_OUTBOX_PROCESSING_LEASE_MS); + + let mut candidates = Vec::new(); + // Each status range runs in creation order, so the merge only needs the first `limit` + // eligible rows of each: the global first `limit` cannot need more from either one. The + // backlog is never materialised: a small claim reads a small slice whatever it holds. + let take = usize::try_from(limit).unwrap_or(usize::MAX); + if take == 0 { + return Ok(Vec::new()); + } + for status in [OutboxStatus::Pending, OutboxStatus::Processing] { + let prefix = text_part(status.as_str(), "outbox_by_status_created_at", "status")?; + let mut taken = 0; + for entry in databases + .outbox_by_status_created_at + .prefix_iter(txn, &prefix)? + { + let (key, _) = entry?; + let id = text_from(&key[prefix.len() + 8..], "outbox_by_status_created_at")?; + let Some(message) = outbox_load(txn, databases, id)? else { + continue; + }; + probe_load(); + let due = match status { + OutboxStatus::Pending => message.available_at <= now, + OutboxStatus::Processing => message.updated_at < stale, + OutboxStatus::Delivered | OutboxStatus::Dead => false, + }; + if due { + candidates.push(message); + taken += 1; + if taken >= take { + break; + } + } + } + } + candidates.sort_by(|left, right| { + left.created_at + .cmp(&right.created_at) + .then_with(|| left.id.cmp(&right.id)) + }); + candidates.truncate(take); + + let mut claimed = Vec::with_capacity(candidates.len()); + for message in candidates { + let previous = message.clone(); + let mut message = message; + message.status = OutboxStatus::Processing; + message.updated_at = now; + outbox_write(txn, databases, &message, Some(&previous))?; + claimed.push(message); + } + Ok(claimed) +} + +/// `markDelivered`: delivered, timestamped, error cleared. +pub(crate) fn mark_delivered( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &str, + now: DateTime, +) -> Result<(), StorageError> { + let Some(mut message) = outbox_load(txn, databases, id)? else { + return Ok(()); + }; + let previous = message.clone(); + let now = encoding::micros(now); + message.status = OutboxStatus::Delivered; + message.delivered_at = Some(now); + message.last_error = None; + message.updated_at = now; + outbox_write(txn, databases, &message, Some(&previous)) +} + +/// `markFailed`: a retry with capped exponential backoff, or the dead transition. +#[allow(clippy::too_many_arguments)] +pub(crate) fn mark_failed( + txn: &mut RwTxn<'_>, + databases: &Databases, + id: &str, + error: &str, + attempt_count: i32, + max_attempts: i32, + base_delay_ms: i64, + now: DateTime, +) -> Result<(), StorageError> { + let Some(mut message) = outbox_load(txn, databases, id)? else { + return Ok(()); + }; + let previous = message.clone(); + let now = encoding::micros(now); + let dead = attempt_count >= max_attempts; + message.status = if dead { + OutboxStatus::Dead + } else { + OutboxStatus::Pending + }; + message.attempt_count = attempt_count; + message.available_at = if dead { + now + } else { + now + Duration::milliseconds(backoff_ms(base_delay_ms, attempt_count)) + }; + message.last_error = Some(truncate_bytes(error, 2_000)); + message.updated_at = now; + outbox_write(txn, databases, &message, Some(&previous)) +} + +/// `deleteTerminalOlderThan`: prunes delivered and dead messages, and nulls the outbox id of +/// every delivery-log entry that named one. +pub(crate) fn delete_terminal_older_than( + txn: &mut RwTxn<'_>, + databases: &Databases, + cutoff: DateTime, +) -> Result { + let cutoff = encoding::micros(cutoff).timestamp_micros(); + let mut terminal = Vec::new(); + for status in [OutboxStatus::Delivered, OutboxStatus::Dead] { + let prefix = text_part(status.as_str(), "outbox_by_status_updated_at", "status")?; + for entry in databases + .outbox_by_status_updated_at + .prefix_iter(txn, &prefix)? + { + let (key, _) = entry?; + if encoding::i64_from_key(&key[prefix.len()..prefix.len() + 8])? >= cutoff { + break; + } + terminal.push( + text_from(&key[prefix.len() + 8..], "outbox_by_status_updated_at")?.to_owned(), + ); + } + } + + let mut affected = 0; + for id in terminal { + let Some(message) = outbox_load(txn, databases, &id)? else { + continue; + }; + databases.outbox.delete(txn, id.as_bytes())?; + databases + .outbox_by_status_created_at + .delete(txn, &outbox_created_key(&message)?)?; + databases + .outbox_by_status_updated_at + .delete(txn, &outbox_updated_key(&message)?)?; + null_delivery_outbox(txn, databases, &id)?; + affected += 1; + } + Ok(affected) +} + +/// Sets `outbox_id` to `NULL` on every delivery-log row that named this outbox row. +fn null_delivery_outbox( + txn: &mut RwTxn<'_>, + databases: &Databases, + outbox_id: &str, +) -> Result<(), StorageError> { + let prefix = text_part(outbox_id, "delivery_log_by_outbox", "outbox_id")?; + let mut named = Vec::new(); + for entry in databases.delivery_log_by_outbox.prefix_iter(txn, &prefix)? { + let (key, _) = entry?; + named.push((key.to_vec(), delivery_outbox_id(key)?.to_owned())); + } + + for (index_key, id) in named { + let Some(mut row) = delivery_load(txn, databases, &id)? else { + continue; + }; + databases.delivery_log_by_outbox.delete(txn, &index_key)?; + row.outbox_id = None; + let value = rows::encode("notification_delivery_log", &row)?; + databases + .notification_delivery_log + .put(txn, row.id.as_bytes(), &value)?; + } + Ok(()) +} + +/// `min(60_000, max(0, base_delay_ms) * 2^max(0, attempt_count - 1))`, the same rule as +/// `postgres::notifications::backoff_ms`; kept local because that module is behind its own +/// feature and neither backend may depend on the other's. +fn backoff_ms(base_delay_ms: i64, attempt_count: i32) -> i64 { + let base = i128::from(base_delay_ms.max(0)); + let exponent = u32::try_from(attempt_count.saturating_sub(1).max(0)).unwrap_or(u32::MAX); + let factor = if exponent >= 62 { + i128::MAX + } else { + 1i128 << exponent + }; + i64::try_from(base.saturating_mul(factor).min(60_000)).unwrap_or(60_000) +} + +/// Truncates at a character boundary, keeping at most `limit` bytes. +fn truncate_bytes(value: &str, limit: usize) -> String { + if value.len() <= limit { + return value.to_owned(); + } + let mut end = limit; + while end > 0 && !value.is_char_boundary(end) { + end -= 1; + } + value[..end].to_owned() +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use crate::backend::LmdbEnv; + use crate::repositories::{NotificationDeliveryStatus, NotificationTargetType}; + use heed::Env; + + fn store() -> (tempfile::TempDir, Env, Databases) { + let directory = tempfile::tempdir().expect("a temporary directory"); + let environment = + LmdbEnv::open(directory.path().join("lmdb"), 8 << 20).expect("an environment"); + let env = environment.env().clone(); + let mut transaction = env.write_txn().expect("a write transaction"); + let databases = Databases::create(&env, &mut transaction).expect("the databases"); + transaction.commit().expect("the commit"); + (directory, env, databases) + } + + fn now() -> DateTime { + DateTime::::from_timestamp(1_700_000_000, 0).expect("an instant") + } + + /// A claim is bounded by the rows it returns: a small request over a large backlog reads + /// the head of each status range, not the backlog. + #[test] + fn a_small_claim_does_not_load_the_backlog() { + let _serial = probes::COUNTING + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let (_directory, env, databases) = store(); + let now = now(); + + let mut transaction = env.write_txn().expect("a write transaction"); + for step in 0..200 { + enqueue( + &mut transaction, + &databases, + "test", + &serde_json::json!({ "step": step }), + now, + ) + .expect("a message"); + } + transaction.commit().expect("the commit"); + + let mut transaction = env.write_txn().expect("a write transaction"); + probes::reset(); + let claimed = claim_batch(&mut transaction, &databases, 5, now).expect("the claim"); + let loads = probes::loads(); + transaction.commit().expect("the commit"); + + assert_eq!(claimed.len(), 5, "the claim is the request's size"); + assert!( + loads <= 10, + "a claim reads the head of each status range, not the backlog: {loads} rows loaded" + ); + } + + /// A target id may be longer than an LMDB key holds: the delivery index keys the entry, and + /// the row carries the target, so the success is recorded and reported. + #[test] + fn a_long_target_id_still_reaches_its_delivery_record() { + let (_directory, env, databases) = store(); + let now = now(); + let target = "t".repeat(430); + + let mut transaction = env.write_txn().expect("a write transaction"); + let message = enqueue( + &mut transaction, + &databases, + "test", + &serde_json::json!({}), + now, + ) + .expect("a message"); + let entry = NewDeliveryLogEntry { + outbox_id: Some(message.id.clone()), + event_type: "test".to_owned(), + target_id: target.clone(), + target_type: NotificationTargetType::Telegram, + status: NotificationDeliveryStatus::Success, + attempt_number: 1, + error_snippet: None, + }; + append(&mut transaction, &databases, &entry, now).expect("the log row"); + let targets = find_successful_target_ids(&transaction, &databases, &message.id) + .expect("the successful targets"); + transaction.commit().expect("the commit"); + + assert_eq!(targets, vec![target]); + } +} diff --git a/crates/nostream-storage/src/lmdb/rows.rs b/crates/nostream-storage/src/lmdb/rows.rs new file mode 100644 index 000000000..e40e48310 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/rows.rs @@ -0,0 +1,217 @@ +//! Per-row encode and decode for the LMDB store: one format byte, the event's fixed header, +//! then `serde_json` of the row the seam already declares. + +use chrono::{DateTime, Utc}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use super::encoding::{self, EVENT_HEADER_LEN, EventHeader, ROW_FORMAT_VERSION}; +use crate::error::StorageError; +use crate::repositories::{EventRecord, User}; + +/// One stored `users` row: the seam's [`User`] plus `tos_accepted_at`, which +/// `fromDBUser` omits from the read shape but a write still carries. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub(crate) struct StoredUser { + pub pubkey: nostream_core::Pubkey, + pub is_admitted: bool, + pub is_vanished: bool, + pub balance: i64, + pub tos_accepted_at: Option>, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl StoredUser { + /// A new row with the capture's column defaults. + pub(crate) fn new(pubkey: nostream_core::Pubkey, now: DateTime) -> Self { + Self { + pubkey, + is_admitted: false, + is_vanished: false, + balance: 0, + tos_accepted_at: None, + created_at: now, + updated_at: now, + } + } + + /// The read shape the seam returns (`fromDBUser` drops `tos_accepted_at`). + pub(crate) fn user(self) -> User { + User { + pubkey: self.pubkey, + is_admitted: self.is_admitted, + is_vanished: self.is_vanished, + balance: self.balance, + created_at: self.created_at, + updated_at: self.updated_at, + } + } +} + +/// Encodes one value: the format byte, then the JSON document. +pub(crate) fn encode(table: &'static str, row: &T) -> Result, StorageError> +where + T: Serialize, +{ + let mut value = Vec::from([ROW_FORMAT_VERSION]); + value.extend( + serde_json::to_vec(row).map_err(|error| StorageError::InvalidRow { + table, + column: "value", + value: error.to_string(), + })?, + ); + Ok(value) +} + +/// Decodes one value written by [`encode`]. +pub(crate) fn decode(table: &'static str, value: &[u8]) -> Result +where + T: DeserializeOwned, +{ + encoding::check_version(table, value)?; + serde_json::from_slice(&value[1..]).map_err(|error| StorageError::InvalidRow { + table, + column: "value", + value: error.to_string(), + }) +} + +/// Encodes one event value: version byte, fixed header, JSON of the whole record. +pub(crate) fn encode_event(record: &EventRecord) -> Result, StorageError> { + let kind = i32::try_from(record.event.kind.0).map_err(|_| StorageError::ValueOutOfRange { + column: "event_kind", + value: record.event.kind.0.to_string(), + })?; + let header = EventHeader { + deleted: record.deleted_at.is_some(), + created_at: record.event.created_at, + kind, + expires_at: record.expires_at, + pubkey: record.event.pubkey, + }; + let mut value = Vec::with_capacity(1 + EVENT_HEADER_LEN + 256); + value.push(ROW_FORMAT_VERSION); + value.extend_from_slice(&encoding::header_bytes(&header)); + value.extend( + serde_json::to_vec(record).map_err(|error| StorageError::InvalidRow { + table: "events", + column: "value", + value: error.to_string(), + })?, + ); + Ok(value) +} + +/// Decodes one event value. `row_id` is backend-defined and this backend writes none. +pub(crate) fn decode_event(value: &[u8]) -> Result { + #[cfg(test)] + probes::decode(); + let body = encoding::body("events", value)?; + let mut record: EventRecord = + serde_json::from_slice(body).map_err(|error| StorageError::InvalidRow { + table: "events", + column: "value", + value: error.to_string(), + })?; + record.row_id = None; + Ok(record) +} + +/// The body-decode counter the read tests assert on: every `serde_json` decode of an event body +/// passes through [`decode_event`], whatever reader asked for it. +/// +/// The count is per thread, so a test reads its own read's decodes and not a sibling's write +/// path; the blocking-pool callers never assert on it. +#[cfg(test)] +pub(crate) mod probes { + use std::cell::Cell; + + thread_local! { + static DECODES: Cell = const { Cell::new(0) }; + } + + pub(crate) fn reset() { + DECODES.with(|count| count.set(0)); + } + + pub(crate) fn decode() { + DECODES.with(|count| count.set(count.get() + 1)); + } + + pub(crate) fn decodes() -> usize { + DECODES.with(Cell::get) + } +} + +/// Decodes one event value's fixed header, without its JSON body. +pub(crate) fn event_header(value: &[u8]) -> Result { + encoding::event_header("events", value) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use nostream_core::Event; + + fn event() -> Event { + serde_json::from_value(serde_json::json!({ + "id": "0d684e8ec2431de586aa3cafbee2f6d308d19b28805e53deabcac3220e9136a5", + "pubkey": "f43c1f9bff677b8f27b602725ea0ad51af221344f69a6b352a74991a4479bac3", + "created_at": 1660407625i64, + "kind": 3, + "tags": [["p", "b34417513f66497d7b0e1a8406b6689ac32afb184027717e57d281ea19186315"]], + "content": "hello", + "sig": "ab".repeat(64) + })) + .expect("a wire event") + } + + #[test] + fn the_header_carries_the_live_state_without_decoding_the_body() { + let mut record = EventRecord::from_event(&event()).with_expires_at(2_000_000_000); + record.deleted_at = Some(DateTime::UNIX_EPOCH); + let value = encode_event(&record).expect("an encoded event"); + let header = event_header(&value).expect("a header"); + assert!(header.deleted); + assert_eq!(header.expires_at, Some(2_000_000_000)); + assert_eq!(header.created_at, 1_660_407_625); + assert_eq!(header.kind, 3); + assert_eq!(header.pubkey, record.event.pubkey); + } + + #[test] + fn a_value_without_a_body_is_an_invalid_row() { + let mut value = Vec::from([ROW_FORMAT_VERSION]); + value.extend_from_slice(&[0u8; EVENT_HEADER_LEN]); + let error = decode_event(&value).expect_err("no body"); + assert!(matches!(error, StorageError::InvalidRow { .. })); + } + + #[test] + fn a_tail_row_round_trips_behind_the_version_byte() { + let user = StoredUser::new(event().pubkey, DateTime::UNIX_EPOCH); + let value = encode("users", &user).expect("an encoded user"); + assert_eq!(value[0], ROW_FORMAT_VERSION); + let decoded: StoredUser = decode("users", &value).expect("a decoded user"); + assert_eq!(decoded.pubkey, user.pubkey); + assert!(!decoded.is_admitted); + } + + #[test] + fn a_bumped_version_is_refused_by_every_row_type() { + let user = StoredUser::new(event().pubkey, DateTime::UNIX_EPOCH); + let mut value = encode("users", &user).expect("an encoded user"); + value[0] = 2; + let error = decode::("users", &value).expect_err("version 2"); + match error { + StorageError::InvalidRow { value, .. } => { + assert_eq!(value, "unsupported format version 2"); + } + other => panic!("expected InvalidRow, got {other:?}"), + } + } +} diff --git a/crates/nostream-storage/src/lmdb/users.rs b/crates/nostream-storage/src/lmdb/users.rs new file mode 100644 index 000000000..b9eb2f6c5 --- /dev/null +++ b/crates/nostream-storage/src/lmdb/users.rs @@ -0,0 +1,230 @@ +//! `UserRepository` against LMDB. Mirrors `src/repositories/user-repository.ts`. +//! +//! A user row is one value in the `users` database, keyed by the 32-byte pubkey. Every write +//! is one whole write transaction, so `admit_user` and the balance credit are atomic against +//! every other database. + +use chrono::{DateTime, Utc}; +use heed::{RoTxn, RwTxn}; +use nostream_core::Pubkey; + +use super::Databases; +use super::encoding; +use super::events; +use super::rows::{self, StoredUser}; +use crate::error::StorageError; +use crate::repositories::{FieldUpdate, User, UserPatch}; + +/// The stored row of one pubkey. +pub(crate) fn load( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result, StorageError> { + let Some(value) = databases.users.get(txn, pubkey.as_bytes())? else { + return Ok(None); + }; + rows::decode("users", value).map(Some) +} + +/// Writes one user row, preserving nothing the caller did not supply. +pub(crate) fn store( + txn: &mut RwTxn<'_>, + databases: &Databases, + stored: &StoredUser, +) -> Result<(), StorageError> { + let value = rows::encode("users", stored)?; + databases.users.put(txn, stored.pubkey.as_bytes(), &value)?; + Ok(()) +} + +/// `findByPubkey`: the full row as the capture's `fromDBUser` maps it. +pub(crate) fn find_by_pubkey( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result, StorageError> { + Ok(load(txn, databases, pubkey)?.map(StoredUser::user)) +} + +/// `upsert`: the flags the patch carries and `updated_at`, plus `tos_accepted_at` only when the +/// patch carries it. A flag the patch leaves `Absent` is `false` on a new row and left alone on +/// an existing one, as the Postgres merge does. `balance` and `created_at` are never touched. +pub(crate) fn upsert( + txn: &mut RwTxn<'_>, + databases: &Databases, + patch: &UserPatch, + now: DateTime, +) -> Result { + let mut stored = match load(txn, databases, &patch.pubkey)? { + Some(stored) => stored, + None => StoredUser::new(patch.pubkey, encoding::micros(now)), + }; + if let Some(is_admitted) = patch.is_admitted.value() { + stored.is_admitted = *is_admitted; + } + if let Some(is_vanished) = patch.is_vanished.value() { + stored.is_vanished = *is_vanished; + } + stored.updated_at = encoding::micros(now); + match patch.tos_accepted_at { + FieldUpdate::Absent => {} + FieldUpdate::Clear => stored.tos_accepted_at = None, + FieldUpdate::Set(value) => stored.tos_accepted_at = Some(encoding::micros(value)), + } + store(txn, databases, &stored)?; + Ok(1) +} + +/// `getBalanceByPubkey`: the stored balance, `0` when no row exists. +pub(crate) fn get_balance_by_pubkey( + txn: &RoTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, +) -> Result { + Ok(load(txn, databases, pubkey)?.map_or(0, |stored| stored.balance)) +} + +/// `upsertVanishState`: inserts a minimal row or updates only the vanished flag. +pub(crate) fn upsert_vanish_state( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + vanished: bool, + now: DateTime, +) -> Result { + let mut stored = match load(txn, databases, pubkey)? { + Some(stored) => stored, + None => StoredUser::new(*pubkey, encoding::micros(now)), + }; + stored.is_vanished = vanished; + stored.updated_at = encoding::micros(now); + store(txn, databases, &stored)?; + Ok(1) +} + +/// The capture's `isVanished`: the stored flag, or one hydrated from a live vanish request. +pub(crate) fn is_vanished( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + now: DateTime, +) -> Result { + if let Some(stored) = load(txn, databases, pubkey)? { + return Ok(stored.is_vanished); + } + let vanished = events::has_active_request_to_vanish(txn, databases, pubkey)?; + upsert_vanish_state(txn, databases, pubkey, vanished, now)?; + Ok(vanished) +} + +/// `admitUser`: the schema function's upsert, which a write transaction makes atomic. +pub(crate) fn admit_user( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + admitted_at: DateTime, + now: DateTime, +) -> Result<(), StorageError> { + let mut stored = match load(txn, databases, pubkey)? { + Some(stored) => stored, + None => StoredUser::new(*pubkey, encoding::micros(now)), + }; + stored.is_admitted = true; + stored.tos_accepted_at = Some(encoding::micros(admitted_at)); + stored.updated_at = encoding::micros(now); + store(txn, databases, &stored) +} + +/// `revokeAdmission`: a targeted update so unrelated columns are never clobbered. +pub(crate) fn revoke_admission( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + now: DateTime, +) -> Result { + let Some(mut stored) = load(txn, databases, pubkey)? else { + return Ok(0); + }; + stored.is_admitted = false; + stored.updated_at = encoding::micros(now); + store(txn, databases, &stored)?; + Ok(1) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use super::*; + use crate::lmdb::LmdbBackend; + use crate::repositories::UserRepository; + + fn patch( + pubkey: Pubkey, + is_admitted: FieldUpdate, + is_vanished: FieldUpdate, + ) -> UserPatch { + UserPatch { + pubkey, + is_admitted, + is_vanished, + tos_accepted_at: FieldUpdate::Absent, + } + } + + /// A new row takes the capture's column defaults for the flags the patch does not carry. + #[tokio::test] + async fn a_new_row_writes_false_for_a_flag_the_patch_leaves_absent() { + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let pubkey = Pubkey::from_byte_array([1; 32]); + + backend + .upsert(&patch(pubkey, FieldUpdate::Absent, FieldUpdate::Absent)) + .await + .expect("the upsert"); + + let user = backend + .find_by_pubkey(&pubkey) + .await + .expect("the row") + .expect("a row"); + assert!(!user.is_admitted); + assert!(!user.is_vanished); + } + + /// An upsert that only needs the row to exist must not clear an admission or a vanish flag + /// that is already there, which is what the Postgres merge's omitted assignments do. + #[tokio::test] + async fn a_flag_the_patch_leaves_absent_keeps_the_stored_value() { + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + let pubkey = Pubkey::from_byte_array([2; 32]); + + backend + .upsert(&patch( + pubkey, + FieldUpdate::Set(true), + FieldUpdate::Set(true), + )) + .await + .expect("the upsert"); + backend + .upsert(&patch(pubkey, FieldUpdate::Absent, FieldUpdate::Absent)) + .await + .expect("the upsert"); + + let user = backend + .find_by_pubkey(&pubkey) + .await + .expect("the row") + .expect("a row"); + assert!(user.is_admitted); + assert!(user.is_vanished); + } +} diff --git a/crates/nostream-storage/src/records.rs b/crates/nostream-storage/src/records.rs index b6a8f8ad3..3019a551e 100644 --- a/crates/nostream-storage/src/records.rs +++ b/crates/nostream-storage/src/records.rs @@ -8,7 +8,7 @@ use std::net::IpAddr; use std::str::FromStr; use chrono::{DateTime, Utc}; -use nostream_core::{Event, Kind, Pubkey}; +use nostream_core::{Event, EventId, Kind, Pubkey}; use crate::error::StorageError; use crate::repositories::{ @@ -101,12 +101,23 @@ pub fn replacement_key(record: &EventRecord) -> Option<(Pubkey, u64, Replacement Some((record.event.pubkey, kind.0, key)) } -/// The merge predicate the replaceable upsert applies: the incoming row wins when it is newer, -/// or equal and the smaller id. +/// The merge predicate the replaceable upsert applies, on the two fields it reads: the incoming +/// row wins when it is newer, or equal and the smaller id. +/// +/// The fields are separate from the records because a backend may hold the stored row as a +/// header and a pointer rather than a whole record — LMDB reads `created_at` from the value's +/// fixed header and the id from the `replaceable` pointer, and decodes the body only once the +/// incoming row has won. +pub fn replaces_fields(candidate: (i64, EventId), current: (i64, EventId)) -> bool { + candidate.0 > current.0 || (candidate.0 == current.0 && candidate.1 < current.1) +} + +/// [`replaces_fields`] applied to the two records a backend holds in memory. pub fn replaces(candidate: &EventRecord, current: &EventRecord) -> bool { - candidate.event.created_at > current.event.created_at - || (candidate.event.created_at == current.event.created_at - && candidate.event.id < current.event.id) + replaces_fields( + (candidate.event.created_at, candidate.event.id), + (current.event.created_at, current.event.id), + ) } /// `records` with at most one row per conflict key, in first-seen order. diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index 01f738f01..1db5b06e9 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -21,10 +21,21 @@ use crate::error::StorageError; pub type StorageFuture<'a, T> = Pin> + Send + 'a>>; /// One row per item, in exactly the query's `ORDER BY`. The borrow of the store is held for -/// the stream's life, which is what both backends need: Postgres holds a portal and a -/// pooled connection; LMDB holds a read transaction. A channel would force a background -/// task and a buffer policy LMDB does not need, and a `Vec` would materialise a 10M-row -/// NIP-77 reconciliation. +/// the stream's life, which is what both backends need: Postgres holds a portal and a pooled +/// connection, and LMDB holds whatever its own query kept, opening a short read transaction +/// per batch (256 values) so a dropped stream pins no pages. +/// +/// An LMDB backend's two streams keep different things, and a caller plans memory from the +/// difference: `find_by_filters` collects its page first — at most each filter's limit, clamped +/// to the backend's bound, so at most `MAX_PAGE_KEYS × filters` ids in all — and copies each +/// batch's values out of that page, so its memory is the page; `find_reconciliation_items` +/// keeps only a cursor key and re-seeks it per batch, so its result is never materialised — +/// except for an `ids` filter naming exact ids, whose own list, bounded by the request, is read +/// as one batch and sorted before it is emitted. A prefix criterion is a byte range over the id +/// field rather than the request's list, so it takes the bounded scan. +/// Neither is a snapshot: a row written or deleted between two batches may appear or vanish. A +/// channel would force a background task and a buffer policy LMDB does not need, and a `Vec` +/// would materialise a 10M-row NIP-77 reconciliation. pub type QueryStream<'a, T> = Pin> + Send + 'a>>; /// The rows a retention pass deleted, expired and kept (`src/@types/repositories.ts` @@ -79,7 +90,7 @@ pub struct ReconciliationItem { } /// One user row (`src/@types/user.ts` `User`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct User { pub pubkey: Pubkey, pub is_admitted: bool, @@ -110,7 +121,8 @@ pub struct UserPatch { } /// `invoices.unit` (`src/@types/invoice.ts` `InvoiceUnit`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum InvoiceUnit { Msats, Sats, @@ -118,7 +130,8 @@ pub enum InvoiceUnit { } /// `invoices.status` (`src/@types/invoice.ts` `InvoiceStatus`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum InvoiceStatus { Pending, Completed, @@ -126,7 +139,7 @@ pub enum InvoiceStatus { } /// One payment invoice row (`src/@types/invoice.ts` `Invoice`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct Invoice { pub id: String, pub pubkey: Pubkey, @@ -144,7 +157,7 @@ pub struct Invoice { } /// One NIP-05 verification row (`src/@types/nip05.ts` `Nip05Verification`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct Nip05Verification { pub pubkey: Pubkey, pub nip05: String, @@ -168,7 +181,7 @@ pub struct CreateInviteCodeOptions { } /// One invite code (`src/@types/invite-code.ts` `InviteCode`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct InviteCode { pub code: String, pub created_by: Option, @@ -180,7 +193,8 @@ pub struct InviteCode { } /// `dvm_jobs.status` (`src/@types/dvm.ts` `DvmJobStatus`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] pub enum DvmJobStatus { Submitted, PickedUp, @@ -190,7 +204,7 @@ pub enum DvmJobStatus { } /// One DVM job row (`src/@types/dvm.ts` `DvmJob`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DvmJob { pub id: EventId, pub requester_pubkey: Pubkey, @@ -233,7 +247,8 @@ pub struct DvmJobUpdate { } /// `reports.report_type` (`src/@types/report.ts` `ReportType`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum ReportType { Nudity, Malware, @@ -258,7 +273,7 @@ pub struct NewReport { } /// One report row (`src/@types/report.ts` `Report`). -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct Report { pub id: i32, pub event_id: EventId, @@ -280,7 +295,8 @@ pub struct ActionableTarget { /// `notification_delivery_log.target_type` (`src/@types/operator-notifications.ts` /// `OperatorNotificationChannelType`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum NotificationTargetType { Http, Discord, @@ -290,7 +306,8 @@ pub enum NotificationTargetType { /// `notification_delivery_log.status` (`src/@types/operator-notifications.ts` /// `NotificationDeliveryStatus`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum NotificationDeliveryStatus { Success, Failed, @@ -310,7 +327,7 @@ pub struct NewDeliveryLogEntry { /// One stored delivery attempt (`src/@types/operator-notifications.ts` /// `NotificationDeliveryLogEntry`). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DeliveryLogEntry { pub id: String, pub outbox_id: Option, @@ -332,7 +349,8 @@ pub struct DeliveryLogFilters { /// `notification_outbox.status` (`src/@types/notification-outbox.ts` /// `NotificationOutboxStatus`). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum OutboxStatus { Pending, Processing, @@ -344,7 +362,7 @@ pub enum OutboxStatus { pub type OutboxPayload = serde_json::Value; /// One outbox message (`src/@types/notification-outbox.ts`). -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] pub struct OutboxMessage { pub id: String, pub event_type: String, diff --git a/crates/nostream-storage/tests/common/mod.rs b/crates/nostream-storage/tests/common/mod.rs index 1c0034750..ffba90522 100644 --- a/crates/nostream-storage/tests/common/mod.rs +++ b/crates/nostream-storage/tests/common/mod.rs @@ -1,17 +1,83 @@ //! Helpers shared by the storage integration tests. A directory module, so cargo does not //! build it as a target of its own. +//! +//! The executor below is the replay both parity targets share: it takes the nine repository +//! traits as `&dyn` references, so the same script drives Postgres and LMDB. It cannot reach +//! a serializable transaction or raw SQL, which is why the Postgres target keeps +//! `user.tos_accepted_at` and the `transaction` groups. #![allow(dead_code)] +use std::collections::HashMap; +use std::future::poll_fn; + +use chrono::{DateTime, Utc}; use nostream_config::Settings; -use serde_json::Value; +use serde_json::{Value, json}; + +use nostream_core::{Filter, Kind}; +use nostream_storage::repositories::{ + ActionableTarget, CreateInviteCodeOptions, DeliveryLogEntry, DeliveryLogFilters, DvmJob, + DvmJobRepository, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, + EventRetentionOptions, FieldUpdate, InviteCode, InviteCodeRepository, Invoice, + InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, + Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, + OutboxMessage, QueryStream, ReconciliationItem, Report, ReportRepository, User, UserPatch, + UserRepository, +}; + +/// The nine repositories one replay calls, as the traits rather than a concrete backend. +#[derive(Clone, Copy)] +pub struct Repos<'a> { + pub events: &'a dyn EventRepository, + pub invoices: &'a dyn InvoiceRepository, + pub users: &'a dyn UserRepository, + pub nip05: &'a dyn Nip05VerificationRepository, + pub invites: &'a dyn InviteCodeRepository, + pub dvm: &'a dyn DvmJobRepository, + pub reports: &'a dyn ReportRepository, + pub delivery_log: &'a dyn NotificationDeliveryLogRepository, + pub outbox: &'a dyn NotificationOutboxRepository, +} + +impl<'a> Repos<'a> { + /// The nine traits one backend implements. + pub fn all(backend: &'a T) -> Self + where + T: EventRepository + + InvoiceRepository + + UserRepository + + Nip05VerificationRepository + + InviteCodeRepository + + DvmJobRepository + + ReportRepository + + NotificationDeliveryLogRepository + + NotificationOutboxRepository, + { + Self { + events: backend, + invoices: backend, + users: backend, + nip05: backend, + invites: backend, + dvm: backend, + reports: backend, + delivery_log: backend, + outbox: backend, + } + } +} + +/// The capture's recording, compiled into the test binary. +pub fn fixture() -> Value { + serde_json::from_str(include_str!("../fixtures/repository-parity.json")) + .expect("tests/fixtures/repository-parity.json is valid JSON") +} /// The shipped defaults with a fixture's overrides applied — the settings both the capture /// and the replay load, so the gates under test are the fixture's. `PARITY_NIP50_LANGUAGE`, -/// when exported, replaces the fixture's language for every target that uses this helper -/// (both database targets), which is how the plan guard sees a deployment's language. -/// `PARITY_NIP50_LANGUAGE` overrides the fixture's `nip50.language`, so the NIP-50 plan -/// guard can be run against a deployed configuration (ADR 0003, revisit trigger 4). +/// when exported, replaces the fixture's language, which is how the plan guard sees a +/// deployment's language (ADR 0003, revisit trigger 4). pub fn settings_from_fixture(fixture: &Value) -> Settings { let raw = std::fs::read_to_string(concat!( env!("CARGO_MANIFEST_DIR"), @@ -37,3 +103,817 @@ pub fn settings_from_fixture(fixture: &Value) -> Settings { } serde_yaml::from_value(document).expect("the settings fit the typed schema") } + +// --------------------------------------------------------------------------------------- +// Encoders mirroring the generator. +// --------------------------------------------------------------------------------------- + +pub fn iso(value: &DateTime) -> Value { + Value::String(value.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) +} + +pub fn optional_iso(value: &Option>) -> Value { + match value { + Some(value) => iso(value), + None => Value::Null, + } +} + +pub fn parse_iso(value: &Value) -> DateTime { + DateTime::parse_from_rfc3339(value.as_str().expect("an ISO timestamp")) + .expect("a valid ISO timestamp") + .with_timezone(&Utc) +} + +pub fn text(value: &Value, key: &str) -> String { + value[key] + .as_str() + .unwrap_or_else(|| panic!("{key} is a string")) + .to_owned() +} + +pub fn integer(value: &Value, key: &str) -> i64 { + value[key] + .as_i64() + .unwrap_or_else(|| panic!("{key} is an integer")) +} + +pub fn optional_text(value: &Value, key: &str) -> Option { + value.get(key).and_then(Value::as_str).map(str::to_owned) +} + +pub fn text_of(value: &Value) -> String { + value.as_str().expect("a string").to_owned() +} + +pub fn event_row(record: &EventRecord) -> Value { + json!([ + record.event.id.to_string(), + record.event.created_at, + record.expires_at, + record.deleted_at.is_some(), + record.event_deduplication, + ]) +} + +pub fn event_record(value: &Value) -> EventRecord { + let event: nostream_core::Event = + serde_json::from_value(value["event"].clone()).expect("a wire event"); + let mut record = EventRecord::from_event(&event); + if let Some(address) = optional_text(value, "remote_address") { + record = record.with_remote_address(address.parse().expect("an IP address")); + } + if let Some(expires_at) = value.get("expires_at").and_then(Value::as_i64) { + record = record.with_expires_at(expires_at); + } + if let Some(deduplication) = value.get("deduplication") { + record = record.with_deduplication( + serde_json::from_value(deduplication.clone()).expect("deduplication values"), + ); + } + record +} + +pub fn invoice_view(invoice: &Invoice) -> Value { + json!({ + "id": invoice.id, + "pubkey": invoice.pubkey.to_string(), + "bolt11": invoice.bolt11, + "amount_requested": invoice.amount_requested, + "amount_paid": invoice.amount_paid, + "unit": invoice.unit.as_str(), + "status": invoice.status.as_str(), + "description": invoice.description, + "confirmed_at": optional_iso(&invoice.confirmed_at), + "expires_at": optional_iso(&invoice.expires_at), + "verify_url": invoice.verify_url, + }) +} + +pub fn optional_invoice_view(invoice: &Option) -> Value { + match invoice { + Some(invoice) => invoice_view(invoice), + None => Value::Null, + } +} + +pub fn user_view(user: &User) -> Value { + json!({ + "pubkey": user.pubkey.to_string(), + "is_admitted": user.is_admitted, + "is_vanished": user.is_vanished, + "balance": user.balance, + }) +} + +pub fn optional_user_view(user: &Option) -> Value { + match user { + Some(user) => user_view(user), + None => Value::Null, + } +} + +pub fn nip05_view(verification: &Nip05Verification) -> Value { + json!({ + "pubkey": verification.pubkey.to_string(), + "nip05": verification.nip05, + "domain": verification.domain, + "is_verified": verification.is_verified, + "last_verified_at": optional_iso(&verification.last_verified_at), + "last_checked_at": iso(&verification.last_checked_at), + "failure_count": verification.failure_count, + }) +} + +pub fn invite_view(invite: &InviteCode) -> Value { + json!({ + "code": invite.code, + "created_by": invite.created_by.map(|pubkey| pubkey.to_string()), + "claimed_by": invite.claimed_by.map(|pubkey| pubkey.to_string()), + "expires_at": optional_iso(&invite.expires_at), + "remaining_uses": invite.remaining_uses, + }) +} + +pub fn dvm_view(job: &DvmJob) -> Value { + json!({ + "id": job.id.to_string(), + "requester_pubkey": job.requester_pubkey.to_string(), + "kind": job.kind.0, + "worker_index": job.worker_index, + "status": job.status.as_str(), + "result_event_id": job.result_event_id.map(|id| id.to_string()), + "error": job.error, + }) +} + +pub fn optional_dvm_view(job: &Option) -> Value { + match job { + Some(job) => dvm_view(job), + None => Value::Null, + } +} + +pub fn report_view(report: &Report) -> Value { + json!({ + "id": report.id, + "event_id": report.event_id.to_string(), + "reporter_pubkey": report.reporter_pubkey.to_string(), + "reported_pubkey": report.reported_pubkey.map(|pubkey| pubkey.to_string()), + "reported_event_id": report.reported_event_id.map(|id| id.to_string()), + "report_type": report.report_type.as_str(), + "weight": report.weight, + "actionable": report.actionable, + }) +} + +pub fn delivery_view(entry: &DeliveryLogEntry) -> Value { + json!({ + "outbox_id": entry.outbox_id, + "event_type": entry.event_type, + "target_id": entry.target_id, + "target_type": entry.target_type.as_str(), + "status": entry.status.as_str(), + "attempt_number": entry.attempt_number, + "error_snippet": entry.error_snippet, + }) +} + +pub fn outbox_view(message: &OutboxMessage) -> Value { + json!({ + "event_type": message.event_type, + "payload": message.payload, + "status": message.status.as_str(), + "attempt_count": message.attempt_count, + "last_error": message.last_error, + }) +} + +pub fn sort_by_key(items: &mut [T], key: impl Fn(&T) -> String) { + items.sort_by_key(|item| key(item)); +} + +pub async fn collect( + mut stream: QueryStream<'_, T>, +) -> Result, nostream_storage::StorageError> { + let mut items = Vec::new(); + while let Some(item) = poll_fn(|context| stream.as_mut().poll_next(context)).await { + items.push(item?); + } + Ok(items) +} + +fn field_update(value: &Value) -> FieldUpdate { + match value["kind"].as_str().expect("a FieldUpdate kind") { + "absent" => FieldUpdate::Absent, + "clear" => FieldUpdate::Clear, + _ => FieldUpdate::Set(value["value"].clone()), + } +} + +fn parse_event_id(value: &Value) -> nostream_core::EventId { + value + .as_str() + .expect("an event id") + .parse() + .expect("a valid event id") +} + +fn parse_pubkey(value: &Value) -> nostream_core::Pubkey { + value + .as_str() + .expect("a pubkey") + .parse() + .expect("a valid pubkey") +} + +fn resolve_ref( + refs: &HashMap, + request: &Value, +) -> Result { + let event_type = request + .get("enqueue_event_type") + .or_else(|| request.get("outbox_id_from")) + .and_then(Value::as_str) + .expect("an enqueue reference"); + refs.get(event_type) + .cloned() + .ok_or(nostream_storage::StorageError::InvalidArgument { + message: "no enqueued row to address", + }) +} + +fn new_report(value: &Value) -> NewReport { + NewReport { + event_id: parse_event_id(&value["event_id"]), + reporter_pubkey: parse_pubkey(&value["reporter_pubkey"]), + reported_pubkey: optional_text(value, "reported_pubkey") + .map(|pubkey| pubkey.parse().expect("a pubkey")), + reported_event_id: optional_text(value, "reported_event_id") + .map(|id| id.parse().expect("an event id")), + report_type: value["report_type"] + .as_str() + .expect("a report type") + .parse() + .expect("a report type"), + weight: value["weight"].as_f64().expect("a weight") as f32, + actionable: value["actionable"].as_bool().expect("actionable"), + } +} + +/// One script entry, through the nine traits. +/// +/// The two arms this executor cannot run are `user.tos_accepted_at` (raw SQL, no seam form) +/// and `transaction` (no transaction type on the LMDB seam); the Postgres target handles the +/// former, and each target's replay loop handles the latter. +pub async fn run_op( + repos: Repos<'_>, + op: &Value, + refs: &mut HashMap, +) -> Result { + let name = op["op"].as_str().expect("an op name"); + let request = &op["request"]; + + match name { + "events.create" => { + let record = event_record(&request["record"]); + Ok(json!({ "count": repos.events.create(&record).await? })) + } + "events.create_many" => { + let records: Vec = request["records"] + .as_array() + .expect("records") + .iter() + .map(event_record) + .collect(); + Ok(json!({ "count": repos.events.create_many(&records).await? })) + } + "events.upsert" => { + let record = event_record(&request["record"]); + Ok(json!({ "count": repos.events.upsert(&record).await? })) + } + "events.upsert_many" => { + let records: Vec = request["records"] + .as_array() + .expect("records") + .iter() + .map(event_record) + .collect(); + Ok(json!({ "count": repos.events.upsert_many(&records).await? })) + } + "events.find_by_filters" => { + let filters: Vec = request["filters"] + .as_array() + .expect("filters") + .iter() + .map(|filter| serde_json::from_value(filter.clone()).expect("a filter")) + .collect(); + if filters.is_empty() { + let failed = repos.events.find_by_filters(&[]).await.is_err(); + return Ok(json!({ "error": failed })); + } + let rows = collect(repos.events.find_by_filters(&filters).await?).await?; + Ok(json!({ "rows": rows.iter().map(event_row).collect::>() })) + } + "events.count_by_filters" => { + let filters: Vec = request["filters"] + .as_array() + .expect("filters") + .iter() + .map(|filter| serde_json::from_value(filter.clone()).expect("a filter")) + .collect(); + Ok(json!({ "count": repos.events.count_by_filters(&filters).await? })) + } + "events.find_reconciliation_items" => { + let filter: Filter = + serde_json::from_value(request["filter"].clone()).expect("a filter"); + let items = collect(repos.events.find_reconciliation_items(&filter).await?).await?; + Ok(json!({ + "items": items.iter().map(|ReconciliationItem { created_at, event_id }| { + json!([created_at, event_id.to_string()]) + }).collect::>() + })) + } + "events.delete_by_pubkey_and_ids" => { + let pubkey = parse_pubkey(&request["pubkey"]); + let ids: Vec = request["ids"] + .as_array() + .expect("ids") + .iter() + .map(parse_event_id) + .collect(); + Ok(json!({ "count": repos.events.delete_by_pubkey_and_ids(&pubkey, &ids).await? })) + } + "events.delete_by_pubkey_except_kinds" => { + let pubkey = parse_pubkey(&request["pubkey"]); + let kinds: Vec = request["kinds"] + .as_array() + .expect("kinds") + .iter() + .map(|kind| Kind(kind.as_u64().expect("a kind"))) + .collect(); + Ok( + json!({ "count": repos.events.delete_by_pubkey_except_kinds(&pubkey, &kinds).await? }), + ) + } + "events.has_active_request_to_vanish" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "value": repos.events.has_active_request_to_vanish(&pubkey).await? })) + } + "events.delete_expired_and_retained" => { + let options = EventRetentionOptions { + max_days: request["options"].get("max_days").and_then(Value::as_i64), + kind_whitelist: request["options"] + .get("kind_whitelist") + .and_then(Value::as_array) + .map(|selectors| { + selectors + .iter() + .map(|selector| match selector { + Value::Array(range) => nostream_core::KindSelector::Range([ + Kind(range[0].as_u64().expect("a range start")), + Kind(range[1].as_u64().expect("a range end")), + ]), + other => nostream_core::KindSelector::Single(Kind( + other.as_u64().expect("a kind"), + )), + }) + .collect() + }), + pubkey_whitelist: request["options"] + .get("pubkey_whitelist") + .and_then(Value::as_array) + .map(|pubkeys| { + pubkeys + .iter() + .map(|pubkey| nostream_core::SettingsPubkey(text_of(pubkey))) + .collect() + }), + }; + let EventPurgeCounts { + deleted, + expired, + retained, + } = repos.events.delete_expired_and_retained(&options).await?; + Ok(json!({ "deleted": deleted, "expired": expired, "retained": retained })) + } + "user.find_by_pubkey" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "user": optional_user_view(&repos.users.find_by_pubkey(&pubkey).await?) })) + } + "user.upsert" => { + let patch = &request["patch"]; + let tos_accepted_at = match field_update(&patch["tos_accepted_at"]) { + FieldUpdate::Absent => FieldUpdate::Absent, + FieldUpdate::Clear => FieldUpdate::Clear, + FieldUpdate::Set(value) => FieldUpdate::Set(parse_iso(&value)), + }; + let user = UserPatch { + pubkey: parse_pubkey(&patch["pubkey"]), + is_admitted: FieldUpdate::Set(patch["is_admitted"].as_bool().expect("is_admitted")), + is_vanished: FieldUpdate::Set(patch["is_vanished"].as_bool().expect("is_vanished")), + tos_accepted_at, + }; + Ok(json!({ "count": repos.users.upsert(&user).await? })) + } + "user.get_balance_by_pubkey" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "balance": repos.users.get_balance_by_pubkey(&pubkey).await? })) + } + "user.is_vanished" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "value": repos.users.is_vanished(&pubkey).await? })) + } + "user.set_vanished" => { + let pubkey = parse_pubkey(&request["pubkey"]); + let vanished = request["vanished"].as_bool().expect("vanished"); + Ok(json!({ "count": repos.users.set_vanished(&pubkey, vanished).await? })) + } + "user.admit_user" => { + let pubkey = parse_pubkey(&request["pubkey"]); + repos + .users + .admit_user(&pubkey, parse_iso(&request["admitted_at"])) + .await?; + Ok(json!({ "ok": true })) + } + "user.tos_accepted_at" => Err(nostream_storage::StorageError::NotImplemented { + module: "tests::common::user_tos_accepted_at", + }), + "user.revoke_admission" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "count": repos.users.revoke_admission(&pubkey).await? })) + } + "invoice.find_by_id" => { + let id = text(request, "id"); + Ok(json!({ "invoice": optional_invoice_view(&repos.invoices.find_by_id(&id).await?) })) + } + "invoice.upsert" => { + let value = &request["invoice"]; + let invoice = Invoice { + id: text(value, "id"), + pubkey: parse_pubkey(&value["pubkey"]), + bolt11: text(value, "bolt11"), + amount_requested: integer(value, "amount_requested"), + amount_paid: value.get("amount_paid").and_then(Value::as_i64), + unit: value["unit"] + .as_str() + .expect("a unit") + .parse() + .expect("a unit"), + status: value["status"] + .as_str() + .expect("a status") + .parse() + .expect("a status"), + description: text(value, "description"), + confirmed_at: value + .get("confirmed_at") + .filter(|value| !value.is_null()) + .map(parse_iso), + expires_at: value + .get("expires_at") + .filter(|value| !value.is_null()) + .map(parse_iso), + updated_at: parse_iso(&value["created_at"]), + created_at: parse_iso(&value["created_at"]), + verify_url: optional_text(value, "verify_url"), + }; + Ok(json!({ "count": repos.invoices.upsert(&invoice).await? })) + } + "invoice.update_status" => { + let id = text(request, "id"); + let status: InvoiceStatus = request["status"] + .as_str() + .expect("a status") + .parse() + .expect("a status"); + Ok(json!({ + "invoice": optional_invoice_view(&repos.invoices.update_status(&id, status).await?) + })) + } + "invoice.confirm_invoice" => { + let id = text(request, "invoice_id"); + repos + .invoices + .confirm_invoice( + &id, + integer(request, "amount_received"), + parse_iso(&request["confirmed_at"]), + ) + .await?; + Ok(json!({ "ok": true })) + } + "invoice.find_pending_invoices" => { + let offset = integer(request, "offset") as u64; + let limit = integer(request, "limit") as u64; + let invoices = repos.invoices.find_pending_invoices(offset, limit).await?; + Ok(json!({ + "invoices": invoices.iter().map(invoice_view).collect::>() + })) + } + "nip05.find_by_pubkey" => { + let pubkey = parse_pubkey(&request["pubkey"]); + let verification = repos.nip05.find_by_pubkey(&pubkey).await?; + Ok(json!({ + "verification": verification.as_ref().map(nip05_view).unwrap_or(Value::Null) + })) + } + "nip05.upsert" => { + let value = &request["verification"]; + let verification = Nip05Verification { + pubkey: parse_pubkey(&value["pubkey"]), + nip05: text(value, "nip05"), + domain: text(value, "domain"), + is_verified: value["is_verified"].as_bool().expect("is_verified"), + last_verified_at: value + .get("last_verified_at") + .filter(|value| !value.is_null()) + .map(parse_iso), + last_checked_at: parse_iso(&value["last_checked_at"]), + failure_count: integer(value, "failure_count") as i32, + created_at: Utc::now(), + updated_at: Utc::now(), + }; + Ok(json!({ "count": repos.nip05.upsert(&verification).await? })) + } + "nip05.find_pending_verifications" => { + let verifications = repos + .nip05 + .find_pending_verifications( + integer(request, "update_frequency_ms"), + integer(request, "max_failures") as i32, + integer(request, "limit") as u64, + ) + .await?; + Ok(json!({ + "verifications": verifications.iter().map(nip05_view).collect::>() + })) + } + "nip05.delete_by_pubkey" => { + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "count": repos.nip05.delete_by_pubkey(&pubkey).await? })) + } + "invite.create" => { + let options = CreateInviteCodeOptions { + expires_at: request["options"] + .get("expires_at") + .filter(|value| !value.is_null()) + .map(parse_iso), + remaining_uses: match field_update( + request["options"] + .get("remaining_uses") + .unwrap_or(&json!({"kind": "absent"})), + ) { + FieldUpdate::Absent => FieldUpdate::Absent, + FieldUpdate::Clear => FieldUpdate::Clear, + FieldUpdate::Set(value) => { + FieldUpdate::Set(value.as_i64().expect("remaining uses") as i32) + } + }, + created_by: optional_text(&request["options"], "created_by") + .map(|pubkey| pubkey.parse().expect("a pubkey")), + }; + let code = text(request, "code"); + let invite = repos.invites.create(&code, &options).await?; + Ok(json!({ "invite": invite_view(&invite) })) + } + "invite.find_by_code" => { + let code = text(request, "code"); + let invite = repos.invites.find_by_code(&code).await?; + Ok(json!({ "invite": invite.as_ref().map(invite_view).unwrap_or(Value::Null) })) + } + "invite.claim_code" => { + let code = text(request, "code"); + let pubkey = parse_pubkey(&request["pubkey"]); + Ok(json!({ "value": repos.invites.claim_code(&code, &pubkey).await? })) + } + "invite.find_active_codes" => { + let mut invites = repos + .invites + .find_active_codes(integer(request, "limit") as u64) + .await?; + sort_by_key(&mut invites, |invite| invite.code.clone()); + Ok(json!({ "invites": invites.iter().map(invite_view).collect::>() })) + } + "invite.delete_expired_codes" => { + Ok(json!({ "count": repos.invites.delete_expired_codes().await? })) + } + "dvm.create" => { + let id = parse_event_id(&request["id"]); + let requester = parse_pubkey(&request["requester"]); + let kind = Kind(request["kind"].as_u64().expect("a kind")); + let job = repos.dvm.create(&id, &requester, kind).await?; + Ok(json!({ "job": dvm_view(&job) })) + } + "dvm.find_by_id" => { + let id = parse_event_id(&request["id"]); + Ok(json!({ "job": optional_dvm_view(&repos.dvm.find_by_id(&id).await?) })) + } + "dvm.assign_worker" => { + let id = parse_event_id(&request["id"]); + let worker_index = integer(request, "worker_index") as i32; + Ok(json!({ "value": repos.dvm.assign_worker(&id, worker_index).await? })) + } + "dvm.update_status" => { + let update = &request["update"]; + let dvm_update = DvmJobUpdate { + id: parse_event_id(&update["id"]), + status: update["status"] + .as_str() + .expect("a status") + .parse() + .expect("a status"), + result_event_id: match field_update(&update["result_event_id"]) { + FieldUpdate::Absent => FieldUpdate::Absent, + FieldUpdate::Clear => FieldUpdate::Clear, + FieldUpdate::Set(value) => FieldUpdate::Set(parse_event_id(&value)), + }, + error: match field_update(&update["error"]) { + FieldUpdate::Absent => FieldUpdate::Absent, + FieldUpdate::Clear => FieldUpdate::Clear, + FieldUpdate::Set(value) => { + FieldUpdate::Set(value.as_str().expect("an error").to_owned()) + } + }, + }; + Ok(json!({ "job": optional_dvm_view(&repos.dvm.update_status(&dvm_update).await?) })) + } + "dvm.find_pending_jobs" => { + let kinds: Option> = + request.get("kinds").and_then(Value::as_array).map(|kinds| { + kinds + .iter() + .map(|kind| Kind(kind.as_u64().expect("a kind"))) + .collect() + }); + let mut jobs = repos + .dvm + .find_pending_jobs(integer(request, "limit") as u64, kinds.as_deref()) + .await?; + sort_by_key(&mut jobs, |job| job.id.to_string()); + Ok(json!({ "jobs": jobs.iter().map(dvm_view).collect::>() })) + } + "report.create" => { + let report = new_report(&request["report"]); + let created = repos.reports.create(&report).await?; + Ok(json!({ "report": report_view(&created) })) + } + "report.create_many" => { + let reports: Vec = request["reports"] + .as_array() + .expect("reports") + .iter() + .map(new_report) + .collect(); + let mut created = repos.reports.create_many(&reports).await?; + sort_by_key(&mut created, |report| report.id.to_string()); + Ok(json!({ "reports": created.iter().map(report_view).collect::>() })) + } + "report.find_by_event_id" => { + let event_id = parse_event_id(&request["event_id"]); + let mut reports = repos.reports.find_by_event_id(&event_id).await?; + sort_by_key(&mut reports, |report| report.id.to_string()); + Ok(json!({ "reports": reports.iter().map(report_view).collect::>() })) + } + "report.find_actionable" => { + let mut reports = repos + .reports + .find_actionable(integer(request, "limit") as u64) + .await?; + sort_by_key(&mut reports, |report| report.id.to_string()); + Ok(json!({ "reports": reports.iter().map(report_view).collect::>() })) + } + "report.find_actionable_targets" => { + let mut targets: Vec = + repos.reports.find_actionable_targets().await?; + sort_by_key(&mut targets, |target| { + format!( + "{}/{}", + target + .reported_pubkey + .map(|pubkey| pubkey.to_string()) + .unwrap_or_else(|| "null".to_owned()), + target + .reported_event_id + .map(|id| id.to_string()) + .unwrap_or_else(|| "null".to_owned()) + ) + }); + Ok(json!({ + "targets": targets.iter().map(|target| json!({ + "reported_pubkey": target.reported_pubkey.map(|pubkey| pubkey.to_string()), + "reported_event_id": target + .reported_event_id + .map(|id| id.to_string()), + })).collect::>() + })) + } + "delivery_log.append" => { + let entry = &request["entry"]; + let outbox_id = match entry.get("outbox_id_from") { + Some(_) => Some(resolve_ref(refs, entry)?), + None => optional_text(entry, "outbox_id"), + }; + let new_entry = NewDeliveryLogEntry { + outbox_id, + event_type: text(entry, "event_type"), + target_id: text(entry, "target_id"), + target_type: entry["target_type"] + .as_str() + .expect("a target type") + .parse() + .expect("a target type"), + status: entry["status"] + .as_str() + .expect("a status") + .parse() + .expect("a status"), + attempt_number: integer(entry, "attempt_number") as i32, + error_snippet: optional_text(entry, "error_snippet"), + }; + repos.delivery_log.append(&new_entry).await?; + Ok(json!({ "ok": true })) + } + "delivery_log.find_recent" => { + let filters = DeliveryLogFilters { + status: request["filters"] + .get("status") + .and_then(Value::as_str) + .map(|status| status.parse().expect("a status")), + event_type: request["filters"] + .get("event_type") + .and_then(Value::as_str) + .map(str::to_owned), + }; + let mut entries = repos + .delivery_log + .find_recent(integer(request, "limit") as u64, &filters) + .await?; + sort_by_key(&mut entries, |entry| { + format!("{}/{:02}", entry.target_id, entry.attempt_number) + }); + Ok(json!({ "entries": entries.iter().map(delivery_view).collect::>() })) + } + "delivery_log.find_successful_target_ids" => { + let outbox_id = match request.get("outbox_id_from") { + Some(_) => resolve_ref(refs, request)?, + None => text(request, "outbox_id"), + }; + let mut ids = repos + .delivery_log + .find_successful_target_ids(&outbox_id) + .await?; + ids.sort(); + Ok(json!({ "target_ids": ids })) + } + "delivery_log.delete_older_than" => { + let cutoff = parse_iso(&request["cutoff"]); + Ok(json!({ "count": repos.delivery_log.delete_older_than(cutoff).await? })) + } + "outbox.enqueue" => { + let event_type = text(request, "event_type"); + let message = repos + .outbox + .enqueue(&event_type, &request["payload"]) + .await?; + refs.insert(message.event_type.clone(), message.id.clone()); + Ok(json!({ "message": outbox_view(&message) })) + } + "outbox.claim_batch" => { + let mut messages = repos + .outbox + .claim_batch(integer(request, "limit") as u64) + .await?; + sort_by_key(&mut messages, |message| message.event_type.clone()); + Ok(json!({ "messages": messages.iter().map(outbox_view).collect::>() })) + } + "outbox.mark_delivered" => { + let id = resolve_ref(refs, request)?; + repos.outbox.mark_delivered(&id).await?; + Ok(json!({ "ok": true })) + } + "outbox.mark_failed" => { + let id = resolve_ref(refs, request)?; + repos + .outbox + .mark_failed( + &id, + &text(request, "error"), + integer(request, "attempt_count") as i32, + integer(request, "max_attempts") as i32, + integer(request, "base_delay_ms"), + ) + .await?; + Ok(json!({ "ok": true })) + } + "outbox.delete_terminal_older_than" => { + let cutoff = parse_iso(&request["cutoff"]); + Ok(json!({ "count": repos.outbox.delete_terminal_older_than(cutoff).await? })) + } + "transaction" => Err(nostream_storage::StorageError::NotImplemented { + module: "tests::common::transaction", + }), + other => panic!("unknown op {other}"), + } +} diff --git a/crates/nostream-storage/tests/parity.rs b/crates/nostream-storage/tests/parity.rs index 1d2d517b5..805364a6c 100644 --- a/crates/nostream-storage/tests/parity.rs +++ b/crates/nostream-storage/tests/parity.rs @@ -4,6 +4,10 @@ //! Run with `--ignored` and `PARITY_ADMIN_URL` set; `scripts/storage-parity.sh` does both. //! The test creates the database if absent and runs the migrator, so this target and //! `migrations.rs` have no ordering dependency. +//! +//! The 51 method arms live in `common`, shared with the LMDB arm. What stays here is what +//! only this backend can reach: raw SQL on `users.tos_accepted_at` and the serializable +//! transaction groups. #![cfg(feature = "postgres")] // This integration-test target may unwrap: a failed assertion here is a failed test, @@ -12,35 +16,20 @@ #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use std::collections::HashMap; -use std::future::poll_fn; use std::sync::Arc; -use chrono::{DateTime, Utc}; use serde_json::{Value, json}; use sqlx::Executor; use sqlx::postgres::PgPool; -use nostream_core::{Filter, Kind}; -use nostream_storage::postgres::{PostgresBackend, PostgresTransaction}; -mod common; +use common::{Repos, fixture, settings_from_fixture}; +use nostream_storage::postgres::PostgresBackend; +use nostream_storage::repositories::EventRepository; -use nostream_storage::repositories::{ - ActionableTarget, CreateInviteCodeOptions, DeliveryLogEntry, DeliveryLogFilters, DvmJob, - DvmJobRepository, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, - EventRetentionOptions, FieldUpdate, InviteCode, InviteCodeRepository, Invoice, - InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, - Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, - OutboxMessage, QueryStream, ReconciliationItem, Report, ReportRepository, User, UserPatch, - UserRepository, -}; +mod common; const DATABASE: &str = "nostream_parity_rust"; -fn fixture() -> Value { - serde_json::from_str(include_str!("fixtures/repository-parity.json")) - .expect("tests/fixtures/repository-parity.json is valid JSON") -} - fn admin_url() -> String { std::env::var("PARITY_ADMIN_URL").expect( "PARITY_ADMIN_URL must name an administrative database, e.g. \ @@ -63,246 +52,40 @@ fn with_database(url: &str, database: &str) -> String { } } -fn iso(value: &DateTime) -> Value { - Value::String(value.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) -} - -fn optional_iso(value: &Option>) -> Value { - match value { - Some(value) => iso(value), - None => Value::Null, - } -} - -fn parse_iso(value: &Value) -> DateTime { - DateTime::parse_from_rfc3339(value.as_str().expect("an ISO timestamp")) - .expect("a valid ISO timestamp") - .with_timezone(&Utc) -} - -fn text(value: &Value, key: &str) -> String { - value[key] - .as_str() - .unwrap_or_else(|| panic!("{key} is a string")) - .to_owned() -} - -fn integer(value: &Value, key: &str) -> i64 { - value[key] - .as_i64() - .unwrap_or_else(|| panic!("{key} is an integer")) -} - -fn optional_text(value: &Value, key: &str) -> Option { - value.get(key).and_then(Value::as_str).map(str::to_owned) -} - -// --------------------------------------------------------------------------------------- -// Encoders mirroring the generator. -// --------------------------------------------------------------------------------------- - -fn event_row(record: &EventRecord) -> Value { - json!([ - record.event.id.to_string(), - record.event.created_at, - record.expires_at, - record.deleted_at.is_some(), - record.event_deduplication, - ]) -} - -fn event_record(value: &Value) -> EventRecord { - let event: nostream_core::Event = - serde_json::from_value(value["event"].clone()).expect("a wire event"); - let mut record = EventRecord::from_event(&event); - if let Some(address) = optional_text(value, "remote_address") { - record = record.with_remote_address(address.parse().expect("an IP address")); - } - if let Some(expires_at) = value.get("expires_at").and_then(Value::as_i64) { - record = record.with_expires_at(expires_at); - } - if let Some(deduplication) = value.get("deduplication") { - record = record.with_deduplication( - serde_json::from_value(deduplication.clone()).expect("deduplication values"), - ); - } - record -} - -fn invoice_view(invoice: &Invoice) -> Value { - json!({ - "id": invoice.id, - "pubkey": invoice.pubkey.to_string(), - "bolt11": invoice.bolt11, - "amount_requested": invoice.amount_requested, - "amount_paid": invoice.amount_paid, - "unit": invoice.unit.as_str(), - "status": invoice.status.as_str(), - "description": invoice.description, - "confirmed_at": optional_iso(&invoice.confirmed_at), - "expires_at": optional_iso(&invoice.expires_at), - "verify_url": invoice.verify_url, - }) -} - -fn optional_invoice_view(invoice: &Option) -> Value { - match invoice { - Some(invoice) => invoice_view(invoice), - None => Value::Null, - } -} - -fn user_view(user: &User) -> Value { - json!({ - "pubkey": user.pubkey.to_string(), - "is_admitted": user.is_admitted, - "is_vanished": user.is_vanished, - "balance": user.balance, - }) -} - -fn optional_user_view(user: &Option) -> Value { - match user { - Some(user) => user_view(user), - None => Value::Null, +/// One script entry: the two probes this backend owns, then the shared executor. +async fn run_op( + backend: &PostgresBackend, + repos: Repos<'_>, + op: &Value, + refs: &mut HashMap, +) -> Result { + if op["op"].as_str() == Some("user.tos_accepted_at") { + return user_tos_accepted_at(backend, &op["request"]).await; } + common::run_op(repos, op, refs).await } -fn nip05_view(verification: &Nip05Verification) -> Value { - json!({ - "pubkey": verification.pubkey.to_string(), - "nip05": verification.nip05, - "domain": verification.domain, - "is_verified": verification.is_verified, - "last_verified_at": optional_iso(&verification.last_verified_at), - "last_checked_at": iso(&verification.last_checked_at), - "failure_count": verification.failure_count, - }) -} - -fn invite_view(invite: &InviteCode) -> Value { - json!({ - "code": invite.code, - "created_by": invite.created_by.map(|pubkey| pubkey.to_string()), - "claimed_by": invite.claimed_by.map(|pubkey| pubkey.to_string()), - "expires_at": optional_iso(&invite.expires_at), - "remaining_uses": invite.remaining_uses, - }) -} - -fn dvm_view(job: &DvmJob) -> Value { - json!({ - "id": job.id.to_string(), - "requester_pubkey": job.requester_pubkey.to_string(), - "kind": job.kind.0, - "worker_index": job.worker_index, - "status": job.status.as_str(), - "result_event_id": job.result_event_id.map(|id| id.to_string()), - "error": job.error, - }) -} - -fn optional_dvm_view(job: &Option) -> Value { - match job { - Some(job) => dvm_view(job), +/// `users.tos_accepted_at` has no seam form (`fromDBUser` omits the column), so this probe is +/// raw SQL against the master pool and stays in the Postgres arm. +async fn user_tos_accepted_at( + backend: &PostgresBackend, + request: &Value, +) -> Result { + let pubkey = parse_pubkey(&request["pubkey"]); + let row = sqlx::query("SELECT tos_accepted_at FROM users WHERE pubkey = $1") + .bind(pubkey.as_bytes().to_vec()) + .fetch_optional(backend.master()) + .await?; + let value = match row { None => Value::Null, - } -} - -fn report_view(report: &Report) -> Value { - json!({ - "id": report.id, - "event_id": report.event_id.to_string(), - "reporter_pubkey": report.reporter_pubkey.to_string(), - "reported_pubkey": report.reported_pubkey.map(|pubkey| pubkey.to_string()), - "reported_event_id": report.reported_event_id.map(|id| id.to_string()), - "report_type": report.report_type.as_str(), - "weight": report.weight, - "actionable": report.actionable, - }) -} - -fn delivery_view(entry: &DeliveryLogEntry) -> Value { - json!({ - "outbox_id": entry.outbox_id, - "event_type": entry.event_type, - "target_id": entry.target_id, - "target_type": entry.target_type.as_str(), - "status": entry.status.as_str(), - "attempt_number": entry.attempt_number, - "error_snippet": entry.error_snippet, - }) -} - -fn outbox_view(message: &OutboxMessage) -> Value { - json!({ - "event_type": message.event_type, - "payload": message.payload, - "status": message.status.as_str(), - "attempt_count": message.attempt_count, - "last_error": message.last_error, - }) -} - -fn sort_by_key(items: &mut [T], key: impl Fn(&T) -> String) { - items.sort_by_key(|item| key(item)); -} - -async fn collect( - mut stream: QueryStream<'_, T>, -) -> Result, nostream_storage::StorageError> { - let mut items = Vec::new(); - while let Some(item) = poll_fn(|context| stream.as_mut().poll_next(context)).await { - items.push(item?); - } - Ok(items) -} - -/// The three repositories a serializable transaction carries; every other op uses the -/// backend's pools. -#[derive(Clone, Copy)] -enum Client<'a> { - Backend(&'a PostgresBackend), - Transaction(&'a PostgresTransaction), -} - -impl<'a> Client<'a> { - fn users(&self) -> &'a dyn UserRepository { - match self { - Self::Backend(backend) => *backend, - Self::Transaction(transaction) => *transaction, + Some(row) => { + let value: Option = sqlx::Row::try_get(&row, "tos_accepted_at")?; + value + .map(|value| common::iso(&value.and_utc())) + .unwrap_or(Value::Null) } - } - - fn invoices(&self) -> &'a dyn InvoiceRepository { - match self { - Self::Backend(backend) => *backend, - Self::Transaction(transaction) => *transaction, - } - } - - fn outbox(&self) -> &'a dyn NotificationOutboxRepository { - match self { - Self::Backend(backend) => *backend, - Self::Transaction(transaction) => *transaction, - } - } -} - -fn field_update(value: &Value) -> FieldUpdate { - match value["kind"].as_str().expect("a FieldUpdate kind") { - "absent" => FieldUpdate::Absent, - "clear" => FieldUpdate::Clear, - _ => FieldUpdate::Set(value["value"].clone()), - } -} - -fn parse_event_id(value: &Value) -> nostream_core::EventId { - value - .as_str() - .expect("an event id") - .parse() - .expect("a valid event id") + }; + Ok(json!({ "value": value })) } fn parse_pubkey(value: &Value) -> nostream_core::Pubkey { @@ -313,639 +96,8 @@ fn parse_pubkey(value: &Value) -> nostream_core::Pubkey { .expect("a valid pubkey") } -async fn run_op( - backend: &PostgresBackend, - client: Client<'_>, - op: &Value, - refs: &mut HashMap, -) -> Result { - let name = op["op"].as_str().expect("an op name"); - let request = &op["request"]; - - match name { - "events.create" => { - let record = event_record(&request["record"]); - Ok(json!({ "count": EventRepository::create(backend, &record).await? })) - } - "events.create_many" => { - let records: Vec = request["records"] - .as_array() - .expect("records") - .iter() - .map(event_record) - .collect(); - Ok(json!({ "count": EventRepository::create_many(backend, &records).await? })) - } - "events.upsert" => { - let record = event_record(&request["record"]); - Ok(json!({ "count": EventRepository::upsert(backend, &record).await? })) - } - "events.upsert_many" => { - let records: Vec = request["records"] - .as_array() - .expect("records") - .iter() - .map(event_record) - .collect(); - Ok(json!({ "count": EventRepository::upsert_many(backend, &records).await? })) - } - "events.find_by_filters" => { - let filters: Vec = request["filters"] - .as_array() - .expect("filters") - .iter() - .map(|filter| serde_json::from_value(filter.clone()).expect("a filter")) - .collect(); - if filters.is_empty() { - let failed = EventRepository::find_by_filters(backend, &[]) - .await - .is_err(); - return Ok(json!({ "error": failed })); - } - let rows = collect(EventRepository::find_by_filters(backend, &filters).await?).await?; - Ok(json!({ "rows": rows.iter().map(event_row).collect::>() })) - } - "events.count_by_filters" => { - let filters: Vec = request["filters"] - .as_array() - .expect("filters") - .iter() - .map(|filter| serde_json::from_value(filter.clone()).expect("a filter")) - .collect(); - Ok(json!({ "count": EventRepository::count_by_filters(backend, &filters).await? })) - } - "events.find_reconciliation_items" => { - let filter: Filter = - serde_json::from_value(request["filter"].clone()).expect("a filter"); - let items = - collect(EventRepository::find_reconciliation_items(backend, &filter).await?) - .await?; - Ok(json!({ - "items": items.iter().map(|ReconciliationItem { created_at, event_id }| { - json!([created_at, event_id.to_string()]) - }).collect::>() - })) - } - "events.delete_by_pubkey_and_ids" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let ids: Vec = request["ids"] - .as_array() - .expect("ids") - .iter() - .map(parse_event_id) - .collect(); - Ok( - json!({ "count": EventRepository::delete_by_pubkey_and_ids(backend, &pubkey, &ids).await? }), - ) - } - "events.delete_by_pubkey_except_kinds" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let kinds: Vec = request["kinds"] - .as_array() - .expect("kinds") - .iter() - .map(|kind| Kind(kind.as_u64().expect("a kind"))) - .collect(); - Ok( - json!({ "count": EventRepository::delete_by_pubkey_except_kinds(backend, &pubkey, &kinds).await? }), - ) - } - "events.has_active_request_to_vanish" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok( - json!({ "value": EventRepository::has_active_request_to_vanish(backend, &pubkey).await? }), - ) - } - "events.delete_expired_and_retained" => { - let options = EventRetentionOptions { - max_days: request["options"].get("max_days").and_then(Value::as_i64), - kind_whitelist: request["options"] - .get("kind_whitelist") - .and_then(Value::as_array) - .map(|selectors| { - selectors - .iter() - .map(|selector| match selector { - Value::Array(range) => nostream_core::KindSelector::Range([ - Kind(range[0].as_u64().expect("a range start")), - Kind(range[1].as_u64().expect("a range end")), - ]), - other => nostream_core::KindSelector::Single(Kind( - other.as_u64().expect("a kind"), - )), - }) - .collect() - }), - pubkey_whitelist: request["options"] - .get("pubkey_whitelist") - .and_then(Value::as_array) - .map(|pubkeys| { - pubkeys - .iter() - .map(|pubkey| nostream_core::SettingsPubkey(text_of(pubkey))) - .collect() - }), - }; - let EventPurgeCounts { - deleted, - expired, - retained, - } = EventRepository::delete_expired_and_retained(backend, &options).await?; - Ok(json!({ "deleted": deleted, "expired": expired, "retained": retained })) - } - "user.find_by_pubkey" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok( - json!({ "user": optional_user_view(&client.users().find_by_pubkey(&pubkey).await?) }), - ) - } - "user.upsert" => { - let patch = &request["patch"]; - let tos_accepted_at = match field_update(&patch["tos_accepted_at"]) { - FieldUpdate::Absent => FieldUpdate::Absent, - FieldUpdate::Clear => FieldUpdate::Clear, - FieldUpdate::Set(value) => FieldUpdate::Set(parse_iso(&value)), - }; - let user = UserPatch { - pubkey: parse_pubkey(&patch["pubkey"]), - is_admitted: FieldUpdate::Set(patch["is_admitted"].as_bool().expect("is_admitted")), - is_vanished: FieldUpdate::Set(patch["is_vanished"].as_bool().expect("is_vanished")), - tos_accepted_at, - }; - Ok(json!({ "count": client.users().upsert(&user).await? })) - } - "user.get_balance_by_pubkey" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok(json!({ "balance": client.users().get_balance_by_pubkey(&pubkey).await? })) - } - "user.is_vanished" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok(json!({ "value": client.users().is_vanished(&pubkey).await? })) - } - "user.set_vanished" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let vanished = request["vanished"].as_bool().expect("vanished"); - Ok(json!({ "count": client.users().set_vanished(&pubkey, vanished).await? })) - } - "user.admit_user" => { - let pubkey = parse_pubkey(&request["pubkey"]); - client - .users() - .admit_user(&pubkey, parse_iso(&request["admitted_at"])) - .await?; - Ok(json!({ "ok": true })) - } - "user.tos_accepted_at" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let row = sqlx::query("SELECT tos_accepted_at FROM users WHERE pubkey = $1") - .bind(pubkey.as_bytes().to_vec()) - .fetch_optional(backend.master()) - .await?; - let value = match row { - None => Value::Null, - Some(row) => { - let value: Option = - sqlx::Row::try_get(&row, "tos_accepted_at")?; - value - .map(|value| iso(&value.and_utc())) - .unwrap_or(Value::Null) - } - }; - Ok(json!({ "value": value })) - } - "user.revoke_admission" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok(json!({ "count": client.users().revoke_admission(&pubkey).await? })) - } - "invoice.find_by_id" => { - let id = text(request, "id"); - Ok( - json!({ "invoice": optional_invoice_view(&client.invoices().find_by_id(&id).await?) }), - ) - } - "invoice.upsert" => { - let value = &request["invoice"]; - let invoice = Invoice { - id: text(value, "id"), - pubkey: parse_pubkey(&value["pubkey"]), - bolt11: text(value, "bolt11"), - amount_requested: integer(value, "amount_requested"), - amount_paid: value.get("amount_paid").and_then(Value::as_i64), - unit: value["unit"] - .as_str() - .expect("a unit") - .parse() - .expect("a unit"), - status: value["status"] - .as_str() - .expect("a status") - .parse() - .expect("a status"), - description: text(value, "description"), - confirmed_at: value - .get("confirmed_at") - .filter(|value| !value.is_null()) - .map(parse_iso), - expires_at: value - .get("expires_at") - .filter(|value| !value.is_null()) - .map(parse_iso), - updated_at: parse_iso(&value["created_at"]), - created_at: parse_iso(&value["created_at"]), - verify_url: optional_text(value, "verify_url"), - }; - Ok(json!({ "count": client.invoices().upsert(&invoice).await? })) - } - "invoice.update_status" => { - let id = text(request, "id"); - let status: InvoiceStatus = request["status"] - .as_str() - .expect("a status") - .parse() - .expect("a status"); - Ok(json!({ - "invoice": optional_invoice_view( - &client.invoices().update_status(&id, status).await? - ) - })) - } - "invoice.confirm_invoice" => { - let id = text(request, "invoice_id"); - client - .invoices() - .confirm_invoice( - &id, - integer(request, "amount_received"), - parse_iso(&request["confirmed_at"]), - ) - .await?; - Ok(json!({ "ok": true })) - } - "invoice.find_pending_invoices" => { - let offset = integer(request, "offset") as u64; - let limit = integer(request, "limit") as u64; - let invoices = client - .invoices() - .find_pending_invoices(offset, limit) - .await?; - Ok(json!({ - "invoices": invoices.iter().map(invoice_view).collect::>() - })) - } - "nip05.find_by_pubkey" => { - let pubkey = parse_pubkey(&request["pubkey"]); - let verification = - Nip05VerificationRepository::find_by_pubkey(backend, &pubkey).await?; - Ok(json!({ - "verification": verification.as_ref().map(nip05_view).unwrap_or(Value::Null) - })) - } - "nip05.upsert" => { - let value = &request["verification"]; - let verification = Nip05Verification { - pubkey: parse_pubkey(&value["pubkey"]), - nip05: text(value, "nip05"), - domain: text(value, "domain"), - is_verified: value["is_verified"].as_bool().expect("is_verified"), - last_verified_at: value - .get("last_verified_at") - .filter(|value| !value.is_null()) - .map(parse_iso), - last_checked_at: parse_iso(&value["last_checked_at"]), - failure_count: integer(value, "failure_count") as i32, - created_at: Utc::now(), - updated_at: Utc::now(), - }; - Ok( - json!({ "count": Nip05VerificationRepository::upsert(backend, &verification).await? }), - ) - } - "nip05.find_pending_verifications" => { - let verifications = backend - .find_pending_verifications( - integer(request, "update_frequency_ms"), - integer(request, "max_failures") as i32, - integer(request, "limit") as u64, - ) - .await?; - Ok(json!({ - "verifications": verifications.iter().map(nip05_view).collect::>() - })) - } - "nip05.delete_by_pubkey" => { - let pubkey = parse_pubkey(&request["pubkey"]); - Ok( - json!({ "count": Nip05VerificationRepository::delete_by_pubkey(backend, &pubkey).await? }), - ) - } - "invite.create" => { - let options = CreateInviteCodeOptions { - expires_at: request["options"] - .get("expires_at") - .filter(|value| !value.is_null()) - .map(parse_iso), - remaining_uses: match field_update( - request["options"] - .get("remaining_uses") - .unwrap_or(&json!({"kind": "absent"})), - ) { - FieldUpdate::Absent => FieldUpdate::Absent, - FieldUpdate::Clear => FieldUpdate::Clear, - FieldUpdate::Set(value) => { - FieldUpdate::Set(value.as_i64().expect("remaining uses") as i32) - } - }, - created_by: optional_text(&request["options"], "created_by") - .map(|pubkey| pubkey.parse().expect("a pubkey")), - }; - let code = text(request, "code"); - let invite = InviteCodeRepository::create(backend, &code, &options).await?; - Ok(json!({ "invite": invite_view(&invite) })) - } - "invite.find_by_code" => { - let code = text(request, "code"); - let invite = InviteCodeRepository::find_by_code(backend, &code).await?; - Ok(json!({ "invite": invite.as_ref().map(invite_view).unwrap_or(Value::Null) })) - } - "invite.claim_code" => { - let code = text(request, "code"); - let pubkey = parse_pubkey(&request["pubkey"]); - Ok(json!({ "value": InviteCodeRepository::claim_code(backend, &code, &pubkey).await? })) - } - "invite.find_active_codes" => { - let mut invites = - InviteCodeRepository::find_active_codes(backend, integer(request, "limit") as u64) - .await?; - sort_by_key(&mut invites, |invite| invite.code.clone()); - Ok(json!({ "invites": invites.iter().map(invite_view).collect::>() })) - } - "invite.delete_expired_codes" => { - Ok(json!({ "count": InviteCodeRepository::delete_expired_codes(backend).await? })) - } - "dvm.create" => { - let id = parse_event_id(&request["id"]); - let requester = parse_pubkey(&request["requester"]); - let kind = Kind(request["kind"].as_u64().expect("a kind")); - let job = DvmJobRepository::create(backend, &id, &requester, kind).await?; - Ok(json!({ "job": dvm_view(&job) })) - } - "dvm.find_by_id" => { - let id = parse_event_id(&request["id"]); - Ok( - json!({ "job": optional_dvm_view(&DvmJobRepository::find_by_id(backend, &id).await?) }), - ) - } - "dvm.assign_worker" => { - let id = parse_event_id(&request["id"]); - let worker_index = integer(request, "worker_index") as i32; - Ok( - json!({ "value": DvmJobRepository::assign_worker(backend, &id, worker_index).await? }), - ) - } - "dvm.update_status" => { - let update = &request["update"]; - let dvm_update = DvmJobUpdate { - id: parse_event_id(&update["id"]), - status: update["status"] - .as_str() - .expect("a status") - .parse() - .expect("a status"), - result_event_id: match field_update(&update["result_event_id"]) { - FieldUpdate::Absent => FieldUpdate::Absent, - FieldUpdate::Clear => FieldUpdate::Clear, - FieldUpdate::Set(value) => FieldUpdate::Set(parse_event_id(&value)), - }, - error: match field_update(&update["error"]) { - FieldUpdate::Absent => FieldUpdate::Absent, - FieldUpdate::Clear => FieldUpdate::Clear, - FieldUpdate::Set(value) => { - FieldUpdate::Set(value.as_str().expect("an error").to_owned()) - } - }, - }; - Ok( - json!({ "job": optional_dvm_view(&DvmJobRepository::update_status(backend, &dvm_update).await?) }), - ) - } - "dvm.find_pending_jobs" => { - let kinds: Option> = - request.get("kinds").and_then(Value::as_array).map(|kinds| { - kinds - .iter() - .map(|kind| Kind(kind.as_u64().expect("a kind"))) - .collect() - }); - let mut jobs = DvmJobRepository::find_pending_jobs( - backend, - integer(request, "limit") as u64, - kinds.as_deref(), - ) - .await?; - sort_by_key(&mut jobs, |job| job.id.to_string()); - Ok(json!({ "jobs": jobs.iter().map(dvm_view).collect::>() })) - } - "report.create" => { - let report = new_report(&request["report"]); - let created = ReportRepository::create(backend, &report).await?; - Ok(json!({ "report": report_view(&created) })) - } - "report.create_many" => { - let reports: Vec = request["reports"] - .as_array() - .expect("reports") - .iter() - .map(new_report) - .collect(); - let mut created = ReportRepository::create_many(backend, &reports).await?; - sort_by_key(&mut created, |report| report.id.to_string()); - Ok(json!({ "reports": created.iter().map(report_view).collect::>() })) - } - "report.find_by_event_id" => { - let event_id = parse_event_id(&request["event_id"]); - let mut reports = ReportRepository::find_by_event_id(backend, &event_id).await?; - sort_by_key(&mut reports, |report| report.id.to_string()); - Ok(json!({ "reports": reports.iter().map(report_view).collect::>() })) - } - "report.find_actionable" => { - let mut reports = - ReportRepository::find_actionable(backend, integer(request, "limit") as u64) - .await?; - sort_by_key(&mut reports, |report| report.id.to_string()); - Ok(json!({ "reports": reports.iter().map(report_view).collect::>() })) - } - "report.find_actionable_targets" => { - let mut targets: Vec = - ReportRepository::find_actionable_targets(backend).await?; - sort_by_key(&mut targets, |target| { - format!( - "{}/{}", - target - .reported_pubkey - .map(|pubkey| pubkey.to_string()) - .unwrap_or_else(|| "null".to_owned()), - target - .reported_event_id - .map(|id| id.to_string()) - .unwrap_or_else(|| "null".to_owned()) - ) - }); - Ok(json!({ - "targets": targets.iter().map(|target| json!({ - "reported_pubkey": target.reported_pubkey.map(|pubkey| pubkey.to_string()), - "reported_event_id": target - .reported_event_id - .map(|id| id.to_string()), - })).collect::>() - })) - } - "delivery_log.append" => { - let entry = &request["entry"]; - let outbox_id = match entry.get("outbox_id_from") { - Some(_) => Some(resolve_ref(refs, entry)?), - None => optional_text(entry, "outbox_id"), - }; - let new_entry = NewDeliveryLogEntry { - outbox_id, - event_type: text(entry, "event_type"), - target_id: text(entry, "target_id"), - target_type: entry["target_type"] - .as_str() - .expect("a target type") - .parse() - .expect("a target type"), - status: entry["status"] - .as_str() - .expect("a status") - .parse() - .expect("a status"), - attempt_number: integer(entry, "attempt_number") as i32, - error_snippet: optional_text(entry, "error_snippet"), - }; - NotificationDeliveryLogRepository::append(backend, &new_entry).await?; - Ok(json!({ "ok": true })) - } - "delivery_log.find_recent" => { - let filters = DeliveryLogFilters { - status: request["filters"] - .get("status") - .and_then(Value::as_str) - .map(|status| status.parse().expect("a status")), - event_type: request["filters"] - .get("event_type") - .and_then(Value::as_str) - .map(str::to_owned), - }; - let mut entries = NotificationDeliveryLogRepository::find_recent( - backend, - integer(request, "limit") as u64, - &filters, - ) - .await?; - sort_by_key(&mut entries, |entry| { - format!("{}/{:02}", entry.target_id, entry.attempt_number) - }); - Ok(json!({ "entries": entries.iter().map(delivery_view).collect::>() })) - } - "delivery_log.find_successful_target_ids" => { - let outbox_id = match request.get("outbox_id_from") { - Some(_) => resolve_ref(refs, request)?, - None => text(request, "outbox_id"), - }; - let mut ids = - NotificationDeliveryLogRepository::find_successful_target_ids(backend, &outbox_id) - .await?; - ids.sort(); - Ok(json!({ "target_ids": ids })) - } - "delivery_log.delete_older_than" => { - let cutoff = parse_iso(&request["cutoff"]); - Ok( - json!({ "count": NotificationDeliveryLogRepository::delete_older_than(backend, cutoff).await? }), - ) - } - "outbox.enqueue" => { - let event_type = text(request, "event_type"); - let message = client - .outbox() - .enqueue(&event_type, &request["payload"]) - .await?; - refs.insert(message.event_type.clone(), message.id.clone()); - Ok(json!({ "message": outbox_view(&message) })) - } - "outbox.claim_batch" => { - let mut messages = client - .outbox() - .claim_batch(integer(request, "limit") as u64) - .await?; - sort_by_key(&mut messages, |message| message.event_type.clone()); - Ok(json!({ "messages": messages.iter().map(outbox_view).collect::>() })) - } - "outbox.mark_delivered" => { - let id = resolve_ref(refs, request)?; - client.outbox().mark_delivered(&id).await?; - Ok(json!({ "ok": true })) - } - "outbox.mark_failed" => { - let id = resolve_ref(refs, request)?; - client - .outbox() - .mark_failed( - &id, - &text(request, "error"), - integer(request, "attempt_count") as i32, - integer(request, "max_attempts") as i32, - integer(request, "base_delay_ms"), - ) - .await?; - Ok(json!({ "ok": true })) - } - "outbox.delete_terminal_older_than" => { - let cutoff = parse_iso(&request["cutoff"]); - Ok(json!({ "count": client.outbox().delete_terminal_older_than(cutoff).await? })) - } - other => panic!("unknown op {other}"), - } -} - -fn resolve_ref( - refs: &HashMap, - request: &Value, -) -> Result { - let event_type = request - .get("enqueue_event_type") - .or_else(|| request.get("outbox_id_from")) - .and_then(Value::as_str) - .expect("an enqueue reference"); - refs.get(event_type) - .cloned() - .ok_or(nostream_storage::StorageError::InvalidArgument { - message: "no enqueued row to address", - }) -} - -fn text_of(value: &Value) -> String { - value.as_str().expect("a string").to_owned() -} - -fn new_report(value: &Value) -> NewReport { - NewReport { - event_id: parse_event_id(&value["event_id"]), - reporter_pubkey: parse_pubkey(&value["reporter_pubkey"]), - reported_pubkey: optional_text(value, "reported_pubkey") - .map(|pubkey| pubkey.parse().expect("a pubkey")), - reported_event_id: optional_text(value, "reported_event_id") - .map(|id| id.parse().expect("an event id")), - report_type: value["report_type"] - .as_str() - .expect("a report type") - .parse() - .expect("a report type"), - weight: value["weight"].as_f64().expect("a weight") as f32, - actionable: value["actionable"].as_bool().expect("actionable"), - } -} - +/// The capture's four serializable transaction groups. Each nested call runs against the +/// transaction for the three traits it carries and against the backend for the rest. async fn run_transaction( backend: &PostgresBackend, op: &Value, @@ -961,7 +113,13 @@ async fn run_transaction( // A fresh map per nested op loses it, and `mark_delivered` then has no row // to address. for (index, nested) in ops.iter().enumerate() { - match run_op(backend, Client::Transaction(&transaction), nested, refs).await { + let repos = Repos { + invoices: &transaction, + users: &transaction, + outbox: &transaction, + ..Repos::all(backend) + }; + match run_op(backend, repos, nested, refs).await { Ok(result) => results.push(result), Err(_) => { failed_at = Some(index); @@ -1035,14 +193,14 @@ async fn replay(admin_url: &str, database: &str) -> Vec { let backend = PostgresBackend::connect(&with_database(admin_url, database)) .await .expect("the parity database is reachable") - .with_settings(Arc::new(common::settings_from_fixture(&fixture))); + .with_settings(Arc::new(settings_from_fixture(&fixture))); backend.migrate().await.expect("the schema migrates"); - let corpus: Vec = fixture["corpus"] + let corpus: Vec = fixture["corpus"] .as_array() .expect("corpus") .iter() - .map(event_record) + .map(common::event_record) .collect(); EventRepository::create_many(&backend, &corpus) .await @@ -1060,7 +218,7 @@ async fn replay(admin_url: &str, database: &str) -> Vec { let actual = if name == "transaction" { run_transaction(&backend, op, &mut refs).await } else { - run_op(&backend, Client::Backend(&backend), op, &mut refs) + run_op(&backend, Repos::all(&backend), op, &mut refs) .await .unwrap_or_else(|error| panic!("op {index} ({name}) failed: {error}")) }; diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs new file mode 100644 index 000000000..5cc20f545 --- /dev/null +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -0,0 +1,1381 @@ +//! Behavioural parity for the embedded backend: replays `tests/fixtures/repository-parity.json` +//! through `LmdbBackend` and compares every result with the TypeScript implementation's +//! recording, then covers the cases the replay cannot reach. +//! +//! No database and no network: the store is opened on a `tempfile::TempDir`. Every entry the +//! seam can run is replayed, and the list of the ones it cannot is asserted entry by entry, so +//! an exclusion can only grow by editing `EXCLUDED`, which the diff shows. + +#![cfg(feature = "lmdb")] +// This integration-test target may unwrap: a failed assertion here is a failed test, +// which is the point. The allowance names this file rather than exempting test targets +// workspace-wide, so a new test file still has to ask. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use chrono::{DateTime, Duration, Utc}; +use serde_json::{Value, json}; + +use common::{Repos, fixture, settings_from_fixture}; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; +use nostream_storage::LmdbBackend; +use nostream_storage::StorageError; +use nostream_storage::repositories::{ + CreateInviteCodeOptions, DeliveryLogFilters, DvmJobStatus, EventRecord, EventRepository, + EventRetentionOptions, FieldUpdate, InviteCodeRepository, Invoice, InvoiceRepository, + InvoiceStatus, InvoiceUnit, NewDeliveryLogEntry, NewReport, Nip05Verification, + NotificationDeliveryLogRepository, NotificationDeliveryStatus, NotificationOutboxRepository, + NotificationTargetType, ReconciliationItem, ReportType, UserRepository, +}; + +mod common; + +/// The entries this arm cannot replay, fixed by index: 21 is `search` (NIP-50 is out of scope), +/// 48 and 50 are `user.tos_accepted_at` (raw SQL, no seam form), and 112 and 115 are the two +/// `transaction` groups the capture rolled back — a seam with no rollback cannot reproduce them. +/// The two groups it committed (111, 117) are replayed: their nested ops in order leave exactly +/// the state the commit left, which is why entries after 111 observe its rows. +// shortcut: five fixture entries are excluded by index and the count is a constant; upgrade +// when the capture gains an entry, which fails the count assertion and forces this list to be +// re-derived rather than quietly skipped. +const EXCLUDED: [usize; 5] = [21, 48, 50, 112, 115]; + +/// The entries that are replayed; `EXCLUDED.len() + REPLAYED` is the fixture's entry count. +const REPLAYED: usize = 113; + +/// The exclusion list as the marker line prints it. +const EXCLUDED_MARKER: &str = "21,48,50,112,115"; + +/// Replays a transaction group the capture committed. The group's comparison is the aggregate +/// `{failed: false, results: [...]}`, and running its nested ops in order through the seam +/// produces exactly that shape and leaves the state the commit left. A group the capture rolled +/// back needs rollback, which this seam has no form for, and is listed in `EXCLUDED` instead. +async fn run_committed_transaction( + backend: &LmdbBackend, + op: &Value, + refs: &mut HashMap, +) -> Value { + let mut results = Vec::new(); + for nested in op["request"]["ops"].as_array().expect("transaction ops") { + let name = nested["op"].as_str().expect("an op name"); + let result = common::run_op(Repos::all(backend), nested, refs) + .await + .unwrap_or_else(|error| panic!("a committed transaction's {name} failed: {error}")); + results.push(result); + } + json!({ "failed": false, "results": results }) +} + +/// One store on a temporary directory, with the fixture's settings gates. +async fn open_store() -> (tempfile::TempDir, LmdbBackend) { + let fixture = fixture(); + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens") + .with_settings(Arc::new(settings_from_fixture(&fixture))); + (directory, backend) +} + +/// A signed event of the given kind and creation time, by the key `secret`. +fn event(secret: u8, kind: u64, created_at: i64) -> EventRecord { + let mut event = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(kind), + tags: Vec::new(), + content: format!("{secret}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut event).expect("the event signs"); + EventRecord::from_event(&event) +} + +fn one_entry(outbox_id: Option) -> NewDeliveryLogEntry { + NewDeliveryLogEntry { + outbox_id, + event_type: "note.created".to_owned(), + target_id: "https://example.test/hook".to_owned(), + target_type: NotificationTargetType::Http, + status: NotificationDeliveryStatus::Success, + attempt_number: 1, + error_snippet: None, + } +} + +/// `8-4-4-4-12`, all lowercase hex, version 4, variant 1. +fn assert_uuid_v4(value: &str) { + let groups: Vec<&str> = value.split('-').collect(); + let lengths: Vec = groups.iter().map(|group| group.len()).collect(); + assert_eq!( + lengths, + vec![8, 4, 4, 4, 12], + "{value} is hyphenated 8-4-4-4-12" + ); + assert!( + groups.iter().all(|group| group + .chars() + .all(|character| character.is_ascii_digit() || matches!(character, 'a'..='f'))), + "{value} is lowercase hex" + ); + assert_eq!(groups[2].as_bytes()[0], b'4', "{value} is version 4"); + assert!( + matches!(groups[3].as_bytes()[0], b'8' | b'9' | b'a' | b'b'), + "{value} carries the RFC 4122 variant" + ); +} + +#[tokio::test] +async fn the_typescript_script_replays_with_an_empty_diff() { + let fixture = fixture(); + let script = fixture["script"].as_array().expect("script"); + assert_eq!( + script.len(), + REPLAYED + EXCLUDED.len(), + "the fixture's entry count is the one the exclusion list was written against" + ); + for (index, name) in [ + (21, "events.find_by_filters"), + (48, "user.tos_accepted_at"), + (50, "user.tos_accepted_at"), + (112, "transaction"), + (115, "transaction"), + ] { + assert_eq!( + script[index]["op"].as_str(), + Some(name), + "entry {index} is not what the exclusion list says it is" + ); + } + // Each entry is excluded exactly when the seam has no form for it: a transaction group + // the capture rolled back, the two raw-SQL probes, and the one filter carrying `search`. + for (index, op) in script.iter().enumerate() { + let name = op["op"].as_str().expect("an op name"); + let excluded_kind = match name { + "transaction" => op["result"]["compare"]["failed"] != Value::Bool(false), + "user.tos_accepted_at" => true, + _ => op["request"]["filters"] + .as_array() + .is_some_and(|filters| filters.iter().any(|filter| filter.get("search").is_some())), + }; + assert_eq!( + EXCLUDED.contains(&index), + excluded_kind, + "entry {index} ({name}) is excluded exactly when the seam cannot run it" + ); + } + + let (_directory, backend) = open_store().await; + let corpus: Vec = fixture["corpus"] + .as_array() + .expect("corpus") + .iter() + .map(common::event_record) + .collect(); + backend + .create_many(&corpus) + .await + .expect("the corpus inserts"); + + let mut refs = HashMap::new(); + let mut replayed = 0; + for (index, op) in script.iter().enumerate() { + if EXCLUDED.contains(&index) { + continue; + } + let name = op["op"].as_str().expect("an op name"); + let actual = if name == "transaction" { + run_committed_transaction(&backend, op, &mut refs).await + } else { + common::run_op(Repos::all(&backend), op, &mut refs) + .await + .unwrap_or_else(|error| panic!("op {index} ({name}) failed: {error}")) + }; + assert_eq!( + actual, op["result"]["compare"], + "op {index} ({name}) differs from the TypeScript capture" + ); + replayed += 1; + } + assert_eq!(replayed, REPLAYED); + println!("LMDB_PARITY_COMPLETE replayed={replayed} excluded={EXCLUDED_MARKER}"); +} + +#[tokio::test] +async fn a_soft_deleted_event_is_readable_by_filter_and_excluded_from_count() { + let (_directory, backend) = open_store().await; + let record = event(1, 1, 1_700_000_000); + EventRepository::create(&backend, &record) + .await + .expect("the event stores"); + + let filter = Filter { + ids: Some(vec![record.event.id.to_string()]), + ..Filter::default() + }; + let live = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(live.len(), 1); + assert!(live[0].deleted_at.is_none()); + + assert_eq!( + backend + .delete_by_pubkey_and_ids(&record.event.pubkey, &[record.event.id]) + .await + .unwrap(), + 1 + ); + + let after = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + after.len(), + 1, + "find_by_filters does not apply the live predicate (V14)" + ); + assert!( + after[0].deleted_at.is_some(), + "the row carries its deletion time" + ); + assert_eq!( + backend + .count_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + 0, + "count_by_filters applies the live predicate (V13)" + ); + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + assert!(items.is_empty(), "reconciliation lists live rows only"); +} + +#[tokio::test] +async fn a_filter_result_larger_than_one_batch_arrives_whole_and_in_order() { + let (_directory, backend) = open_store().await; + // 500 is above STREAM_BATCH_ITEMS (256), so the stream copies at least two batches. + let records: Vec = (0..500) + .map(|index| event((index % 200 + 1) as u8, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }; + let seen = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(seen.len(), 500, "the whole page arrives"); + assert!( + seen.windows(2) + .all(|pair| pair[0].event.created_at < pair[1].event.created_at), + "a filter without a limit orders by created_at ascending (V6)" + ); +} + +#[tokio::test] +async fn admit_user_creates_an_admitted_row() { + let (_directory, backend) = open_store().await; + let pubkey = Pubkey::from_byte_array([7; 32]); + assert!(backend.find_by_pubkey(&pubkey).await.unwrap().is_none()); + + let admitted_at = Utc::now(); + backend.admit_user(&pubkey, admitted_at).await.unwrap(); + + let user = backend + .find_by_pubkey(&pubkey) + .await + .unwrap() + .expect("the call creates the row"); + assert!(user.is_admitted); + assert_eq!(user.balance, 0); +} + +#[tokio::test] +async fn confirm_invoice_scales_by_unit_and_rejects_an_overflow() { + let (_directory, backend) = open_store().await; + let now = Utc::now(); + + // sats: the credit is `amount_received × 1000` (V18). + let sats_payee = Pubkey::from_byte_array([9; 32]); + backend.admit_user(&sats_payee, now).await.unwrap(); + let sats_invoice = Invoice { + id: "invoice-sats".to_owned(), + pubkey: sats_payee, + bolt11: "lnbc1".to_owned(), + amount_requested: 1_000, + amount_paid: None, + unit: InvoiceUnit::Sats, + status: InvoiceStatus::Pending, + description: "test".to_owned(), + confirmed_at: None, + expires_at: None, + updated_at: now, + created_at: now, + verify_url: None, + }; + InvoiceRepository::upsert(&backend, &sats_invoice) + .await + .unwrap(); + backend + .confirm_invoice("invoice-sats", 1_000, now) + .await + .unwrap(); + + let confirmed = backend.find_by_id("invoice-sats").await.unwrap().unwrap(); + assert_eq!(confirmed.amount_paid, Some(1_000)); + assert!(confirmed.confirmed_at.is_some()); + assert_eq!( + backend.get_balance_by_pubkey(&sats_payee).await.unwrap(), + 1_000_000 + ); + + // btc: `amount_received × 100_000_000 × 1000`. + let btc_payee = Pubkey::from_byte_array([10; 32]); + backend.admit_user(&btc_payee, now).await.unwrap(); + let btc_invoice = Invoice { + id: "invoice-btc".to_owned(), + pubkey: btc_payee, + unit: InvoiceUnit::Btc, + ..sats_invoice.clone() + }; + InvoiceRepository::upsert(&backend, &btc_invoice) + .await + .unwrap(); + backend + .confirm_invoice("invoice-btc", 1, now) + .await + .unwrap(); + assert_eq!( + backend.get_balance_by_pubkey(&btc_payee).await.unwrap(), + 100_000_000_000 + ); + + // An `int8` overflow is an error, not a wrap, and nothing is written (V19). + let overflowing = Invoice { + id: "invoice-overflow".to_owned(), + ..btc_invoice.clone() + }; + InvoiceRepository::upsert(&backend, &overflowing) + .await + .unwrap(); + let before = backend.get_balance_by_pubkey(&btc_payee).await.unwrap(); + let error = backend + .confirm_invoice("invoice-overflow", i64::MAX, now) + .await + .unwrap_err(); + assert!( + matches!( + error, + StorageError::ValueOutOfRange { + column: "users.balance", + .. + } + ), + "an amount that cannot be scaled is ValueOutOfRange, got {error:?}" + ); + assert_eq!( + backend.get_balance_by_pubkey(&btc_payee).await.unwrap(), + before, + "the failed confirmation credits nothing" + ); + assert!( + backend + .find_by_id("invoice-overflow") + .await + .unwrap() + .unwrap() + .confirmed_at + .is_none(), + "the failed confirmation leaves the invoice unconfirmed" + ); +} + +#[tokio::test] +async fn a_nanosecond_instant_reads_back_at_microsecond_precision() { + let (_directory, backend) = open_store().await; + let now = Utc::now(); + let payee = Pubkey::from_byte_array([11; 32]); + backend.admit_user(&payee, now).await.unwrap(); + // Three digits past the microsecond, which no `timestamp` column can hold (V38). + let instant = DateTime::::from_timestamp_nanos(1_700_000_000_123_456_789); + let invoice = Invoice { + id: "invoice-micros".to_owned(), + pubkey: payee, + bolt11: "lnbc1".to_owned(), + amount_requested: 1, + amount_paid: None, + unit: InvoiceUnit::Sats, + status: InvoiceStatus::Pending, + description: "test".to_owned(), + confirmed_at: None, + expires_at: Some(instant), + updated_at: now, + created_at: instant, + verify_url: None, + }; + InvoiceRepository::upsert(&backend, &invoice).await.unwrap(); + + let stored = backend.find_by_id("invoice-micros").await.unwrap().unwrap(); + assert_eq!( + stored.created_at.timestamp_subsec_nanos(), + 123_456_000, + "the insert truncates the caller's created_at (V38)" + ); + assert_eq!( + stored + .expires_at + .expect("the expiry") + .timestamp_subsec_nanos(), + 123_456_000, + "and its expires_at" + ); + + backend + .confirm_invoice("invoice-micros", 1, instant) + .await + .unwrap(); + + let confirmed_at = backend + .find_by_id("invoice-micros") + .await + .unwrap() + .unwrap() + .confirmed_at + .expect("the confirmation is stamped"); + assert_eq!( + confirmed_at.timestamp_micros(), + 1_700_000_000_123_456, + "a nanosecond instant reads back at microsecond precision (V38)" + ); + assert_eq!(confirmed_at.timestamp_subsec_nanos(), 123_456_000); + + // The NIP-05 row keeps the caller's last check and verification at the same precision. + use nostream_storage::repositories::Nip05VerificationRepository; + + let watched = Pubkey::from_byte_array([12; 32]); + Nip05VerificationRepository::upsert( + &backend, + &Nip05Verification { + pubkey: watched, + nip05: "micros@example.test".to_owned(), + domain: "example.test".to_owned(), + is_verified: true, + last_verified_at: Some(instant), + last_checked_at: instant, + failure_count: 0, + created_at: instant, + updated_at: instant, + }, + ) + .await + .unwrap(); + let verification = Nip05VerificationRepository::find_by_pubkey(&backend, &watched) + .await + .unwrap() + .unwrap(); + assert_eq!( + verification.last_checked_at.timestamp_subsec_nanos(), + 123_456_000, + "the NIP-05 row truncates last_checked_at" + ); + assert_eq!( + verification + .last_verified_at + .expect("the verification") + .timestamp_subsec_nanos(), + 123_456_000, + "and last_verified_at" + ); +} + +#[tokio::test] +async fn a_delivery_log_entry_must_name_an_existing_outbox_row() { + let (_directory, backend) = open_store().await; + let unknown = one_entry(Some("00000000-0000-4000-8000-000000000000".to_owned())); + match backend.append(&unknown).await { + Err(StorageError::InvalidArgument { .. }) => {} + other => panic!("an unknown outbox id is rejected, got {other:?}"), + } + assert!( + backend + .find_recent(10, &DeliveryLogFilters::default()) + .await + .unwrap() + .is_empty(), + "the rejected entry is not stored" + ); + + let message = backend + .enqueue("note.created", &json!({ "id": "note-1" })) + .await + .unwrap(); + backend.append(&one_entry(Some(message.id))).await.unwrap(); + assert_eq!( + backend + .find_recent(10, &DeliveryLogFilters::default()) + .await + .unwrap() + .len(), + 1 + ); +} + +#[tokio::test] +async fn a_target_id_a_text_column_cannot_hold_is_refused_rather_than_truncated() { + let (_directory, backend) = open_store().await; + let message = backend + .enqueue("note.created", &json!({ "id": "note-3" })) + .await + .unwrap(); + let mut entry = one_entry(Some(message.id.clone())); + entry.target_id = "https://example.test/\0hook".to_owned(); + + // `notification_delivery_log.target_id` is a `text` column, and PostgreSQL cannot store + // `0x00` in one, so the capture could never have kept this row; `append` refuses it through + // `encoding::refuse_nul` rather than writing a value the other backend cannot hold. + match backend.append(&entry).await { + Err(StorageError::InvalidRow { column, .. }) => { + assert_eq!(column, "target_id"); + } + other => panic!("a NUL target id is refused, got {other:?}"), + } + assert!( + backend + .find_successful_target_ids(&message.id) + .await + .unwrap() + .is_empty(), + "the refused entry stored nothing to read back" + ); +} + +#[tokio::test] +async fn deleting_an_outbox_row_nulls_its_delivery_log_entries() { + let (_directory, backend) = open_store().await; + let message = backend + .enqueue("note.created", &json!({ "id": "note-2" })) + .await + .unwrap(); + backend + .append(&one_entry(Some(message.id.clone()))) + .await + .unwrap(); + backend.mark_delivered(&message.id).await.unwrap(); + + let cutoff: DateTime = Utc::now() + Duration::seconds(60); + assert_eq!( + backend.delete_terminal_older_than(cutoff).await.unwrap(), + 1, + "the delivered row is deleted" + ); + + let entries = backend + .find_recent(10, &DeliveryLogFilters::default()) + .await + .unwrap(); + assert_eq!(entries.len(), 1, "the delivery log keeps its entry"); + assert!( + entries[0].outbox_id.is_none(), + "the deleted outbox row leaves no dangling id (V40)" + ); +} + +#[tokio::test] +async fn an_invite_code_rejects_a_negative_remaining_uses() { + let (_directory, backend) = open_store().await; + let negative = CreateInviteCodeOptions { + remaining_uses: FieldUpdate::Set(-5), + ..CreateInviteCodeOptions::default() + }; + match InviteCodeRepository::create(&backend, "negative", &negative).await { + Err(StorageError::InvalidArgument { .. }) => {} + other => panic!("a negative remaining_uses is rejected, got {other:?}"), + } + + let zero = CreateInviteCodeOptions { + remaining_uses: FieldUpdate::Set(0), + ..CreateInviteCodeOptions::default() + }; + let invite = InviteCodeRepository::create(&backend, "zero", &zero) + .await + .unwrap(); + assert_eq!(invite.remaining_uses, Some(0)); +} + +#[tokio::test] +async fn two_enqueues_return_distinct_uuids() { + let (_directory, backend) = open_store().await; + let first = backend + .enqueue("note.created", &json!({ "id": "a" })) + .await + .unwrap(); + let second = backend + .enqueue("note.created", &json!({ "id": "b" })) + .await + .unwrap(); + + assert_ne!(first.id, second.id); + assert_uuid_v4(&first.id); + assert_uuid_v4(&second.id); +} + +/// Five events of one author, created at 100, 200, … 500 seconds. +async fn five_events_of_one_author(backend: &LmdbBackend) -> Vec { + let records: Vec = (1..=5).map(|step| event(5, 1, step * 100)).collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + records +} + +/// The `created_at` values a page returned, newest first. +fn created_ats(records: &[EventRecord]) -> Vec { + records + .iter() + .map(|record| record.event.created_at) + .collect() +} + +/// A signed event of one author, by the key `secret`, carrying `tags`. +fn tagged_event(secret: u8, created_at: i64, tags: Vec>) -> EventRecord { + let mut wire = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(1), + tags, + content: format!("{secret}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut wire).expect("the event signs"); + EventRecord::from_event(&wire) +} + +#[tokio::test] +async fn a_limited_ids_filter_returns_the_newest_events() { + let (_directory, backend) = open_store().await; + let records = five_events_of_one_author(&backend).await; + let filter = Filter { + ids: Some( + records + .iter() + .map(|record| record.event.id.to_string()) + .collect(), + ), + limit: Some(3), + ..Filter::default() + }; + + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + created_ats(&page), + vec![500, 400, 300], + "a limit orders the page newest first whatever index answered it (V6)" + ); + assert_eq!( + backend + .count_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + 3, + "the count applies the same limit and order (V6)" + ); +} + +#[tokio::test] +async fn a_limited_author_filter_returns_the_newest_events() { + let (_directory, backend) = open_store().await; + let records = five_events_of_one_author(&backend).await; + let filter = Filter { + authors: Some(vec![records[0].event.pubkey.to_string()]), + limit: Some(3), + ..Filter::default() + }; + + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + created_ats(&page), + vec![500, 400, 300], + "a limit orders the page newest first whatever index answered it (V6)" + ); +} + +/// The page and count of one filter: `(rows, count)`. +async fn page_and_count(backend: &LmdbBackend, filter: &Filter) -> (usize, u64) { + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + let count = backend + .count_by_filters(std::slice::from_ref(filter)) + .await + .unwrap(); + (page.len(), count) +} + +#[tokio::test] +async fn an_id_prefix_of_either_parity_selects_the_event_it_prefixes() { + let (_directory, backend) = open_store().await; + let record = event(4, 1, 1_700_000_000); + backend + .create_many(std::slice::from_ref(&record)) + .await + .expect("the event stores"); + let id = record.event.id.to_string(); + + // The capture's `substring(event_id from 1 for n/2)` takes an even prefix's whole bytes, + // and an odd one's final nibble is a range; both select the row (R12). + for length in [8, 9, 16] { + let filter = Filter { + ids: Some(vec![id[..length].to_owned()]), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (1, 1), + "an id prefix of {length} characters names the row" + ); + } + + let mut other = id[..8].to_owned(); + let digit = other.pop().expect("the prefix is not empty"); + other.push(if digit == '0' { '1' } else { '0' }); + let filter = Filter { + ids: Some(vec![other]), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (0, 0), + "the prefix range stops at the prefix's own bytes" + ); +} + +#[tokio::test] +async fn a_prefix_author_with_kinds_selects_the_event() { + let (_directory, backend) = open_store().await; + let record = event(12, 1, 1_700_000_000); + backend + .create_many(std::slice::from_ref(&record)) + .await + .expect("the event stores"); + let author = record.event.pubkey.to_string(); + + // The author index's key is `pubkey ‖ kind ‖ created_at ‖ event_id`, so the kind is at the + // field's own offset whatever the author criterion covers (R12). + for length in [8, 9, 16, 63, 64] { + let filter = Filter { + authors: Some(vec![author[..length].to_owned()]), + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (1, 1), + "an author prefix of {length} characters with the row's kind names it" + ); + + let filter = Filter { + authors: Some(vec![author[..length].to_owned()]), + kinds: Some(vec![Kind(2)]), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (0, 0), + "another kind excludes it at the same offset" + ); + } +} + +#[tokio::test] +async fn a_repeated_kind_counts_its_row_once() { + let (_directory, backend) = open_store().await; + let note = event(13, 1, 1_700_000_000); + let highest = event(14, i32::MAX as u64, 1_700_000_001); + backend + .create_many(&[note.clone(), highest.clone()]) + .await + .expect("the events store"); + + // A duplicated criterion, and two kinds that the key codec maps to the same `i32`, name one + // index range; a count that visited it per criterion would count the row twice (V6, UNION). + for (record, kinds) in [ + (¬e, vec![Kind(1), Kind(1)]), + ( + &highest, + vec![Kind(i32::MAX as u64), Kind(i32::MAX as u64 + 1)], + ), + ] { + let filter = Filter { + authors: Some(vec![record.event.pubkey.to_string()]), + kinds: Some(kinds), + ..Filter::default() + }; + assert_eq!( + page_and_count(&backend, &filter).await, + (1, 1), + "the row is one row however its kind is repeated" + ); + } +} + +#[tokio::test] +async fn a_reconciliation_lists_every_live_row_across_batches() { + let (_directory, backend) = open_store().await; + let records: Vec = (0..500) + .map(|index| event((index % 200 + 1) as u8, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(1), + ..Filter::default() + }; + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + assert_eq!( + items.len(), + 500, + "every live row arrives, not one per batch fewer (R20)" + ); + assert!( + items + .windows(2) + .all(|pair| (pair[0].created_at, pair[0].event_id) + < (pair[1].created_at, pair[1].event_id)), + "reconciliation is oldest first and ignores the filter's limit" + ); +} + +#[tokio::test] +async fn a_reconciliation_over_ids_lists_exactly_those_rows_oldest_first() { + let (_directory, backend) = open_store().await; + let records: Vec = (0..300) + .map(|index| event(41, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + ids: Some(vec![ + records[250].event.id.to_string(), + records[12].event.id.to_string(), + records[199].event.id.to_string(), + ]), + ..Filter::default() + }; + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + + assert_eq!( + items, + vec![ + ReconciliationItem { + created_at: 1_700_000_012, + event_id: records[12].event.id, + }, + ReconciliationItem { + created_at: 1_700_000_199, + event_id: records[199].event.id, + }, + ReconciliationItem { + created_at: 1_700_000_250, + event_id: records[250].event.id, + }, + ], + "the request's ids, oldest first, whatever order the ids arrive in" + ); +} + +#[tokio::test] +async fn a_reconciliation_over_an_id_prefix_lists_every_row_the_prefix_selects() { + // A prefix criterion is a byte range over the id field rather than the request's own list, + // so it is answered by the bounded created-at scan; `""` is that range over every row. + let (_directory, backend) = open_store().await; + let records: Vec = (0..400) + .map(|index| event(42, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + ids: Some(vec![String::new()]), + ..Filter::default() + }; + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + assert_eq!( + items.len(), + 400, + "every row the criterion selects arrives across the batches" + ); + assert!( + items + .windows(2) + .all(|pair| (pair[0].created_at, pair[0].event_id) + < (pair[1].created_at, pair[1].event_id)), + "and oldest first" + ); + + let prefix = records[7].event.id.to_string()[..2].to_owned(); + let filter = Filter { + ids: Some(vec![prefix.clone()]), + ..Filter::default() + }; + let items = common::collect(backend.find_reconciliation_items(&filter).await.unwrap()) + .await + .unwrap(); + let expected: Vec = records + .iter() + .filter(|record| record.event.id.to_string().starts_with(&prefix)) + .map(|record| ReconciliationItem { + created_at: record.event.created_at, + event_id: record.event.id, + }) + .collect(); + assert!( + !expected.is_empty(), + "the prefix selects the row it came from" + ); + assert_eq!( + items, expected, + "the prefix selects the same rows, oldest first" + ); +} + +#[tokio::test] +async fn an_empty_tag_value_criterion_matches_nothing() { + let (_directory, backend) = open_store().await; + let record = tagged_event(3, 1_700_000_000, vec![vec!["e".to_owned(), String::new()]]); + EventRepository::create(&backend, &record) + .await + .expect("the event stores"); + + let mut filter = Filter { + ids: Some(vec![record.event.id.to_string()]), + limit: Some(10), + ..Filter::default() + }; + let reachable = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(reachable.len(), 1, "the row is reachable by its id"); + + filter.tags.insert("#e".to_owned(), vec![String::new()]); + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert!( + page.is_empty(), + "event_tags holds no empty value, so no row can match one (V42)" + ); +} + +#[tokio::test] +async fn an_empty_criterion_family_matches_nothing() { + let (_directory, backend) = open_store().await; + let record = event(6, 1, 1_700_000_000); + EventRepository::create(&backend, &record) + .await + .expect("the event stores"); + + for filter in [ + Filter { + ids: Some(Vec::new()), + ..Filter::default() + }, + Filter { + authors: Some(Vec::new()), + ..Filter::default() + }, + Filter { + tags: BTreeMap::from([("#e".to_owned(), Vec::new())]), + ..Filter::default() + }, + ] { + let page = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert!( + page.is_empty(), + "a present-but-empty family matches nothing, not the next path (V45): {filter:?}" + ); + assert_eq!( + backend + .count_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + 0, + "the count of an empty family is zero (V45): {filter:?}" + ); + } +} + +#[tokio::test] +async fn a_delivery_log_entry_keeps_its_index_entry_when_a_sibling_is_pruned() { + let (_directory, backend) = open_store().await; + let message = backend + .enqueue("note.created", &json!({ "id": "note-3" })) + .await + .unwrap(); + let entry = one_entry(Some(message.id.clone())); + backend.append(&entry).await.unwrap(); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + backend.append(&entry).await.unwrap(); + + let entries = backend + .find_recent(10, &DeliveryLogFilters::default()) + .await + .unwrap(); + assert_eq!(entries.len(), 2, "two attempts, one target"); + let cutoff = entries[1].created_at + Duration::milliseconds(1); + assert_eq!( + backend.delete_older_than(cutoff).await.unwrap(), + 1, + "only the older attempt is pruned" + ); + assert_eq!( + backend + .find_successful_target_ids(&message.id) + .await + .unwrap(), + vec![entry.target_id.clone()], + "the surviving attempt keeps its index entry (V40)" + ); +} + +#[tokio::test] +async fn a_retention_pass_classifies_a_row_once_and_spares_the_whitelist() { + let (_directory, backend) = open_store().await; + let now = Utc::now(); + let long_ago = 1_700_000_000; + let expired_at = now.timestamp() - 1; + + // Deleted and expired at once: the capture counts it as deleted, and its precedence is + // what makes the total one row rather than two (R13, V8). + let deleted_and_expired = event(21, 1, long_ago).with_expires_at(expired_at); + let expired_only = event(22, 1, long_ago).with_expires_at(expired_at); + let retained_only = event(23, 1, long_ago); + // Every whitelist keeps kind 62, so this row is never a candidate. + let vanish = event(24, 62, long_ago); + backend + .create_many(&[ + deleted_and_expired.clone(), + expired_only, + retained_only, + vanish.clone(), + ]) + .await + .expect("the events store"); + assert_eq!( + backend + .delete_by_pubkey_and_ids( + &deleted_and_expired.event.pubkey, + &[deleted_and_expired.event.id] + ) + .await + .unwrap(), + 1 + ); + + let skipped = backend + .delete_expired_and_retained(&EventRetentionOptions { + max_days: Some(0), + kind_whitelist: None, + pubkey_whitelist: None, + }) + .await + .unwrap(); + assert_eq!( + (skipped.deleted, skipped.expired, skipped.retained), + (0, 0, 0), + "max_days 0 touches nothing (V8)" + ); + + let counts = backend + .delete_expired_and_retained(&EventRetentionOptions { + max_days: Some(1), + kind_whitelist: None, + pubkey_whitelist: None, + }) + .await + .unwrap(); + assert_eq!( + (counts.deleted, counts.expired, counts.retained), + (1, 1, 1), + "each qualifying row is classified once, in the capture's precedence" + ); + + let filter = Filter { + ids: Some(vec![vanish.event.id.to_string()]), + ..Filter::default() + }; + let survivors = common::collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + survivors.len(), + 1, + "a kind-62 row survives an empty configured whitelist (V8)" + ); +} + +#[tokio::test] +async fn a_retention_pass_takes_at_most_one_batch_of_candidates() { + let (_directory, backend) = open_store().await; + let records: Vec = (0..1_001) + .map(|index| event((index % 200 + 1) as u8, 1, 1_700_000_000 + index)) + .collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let options = EventRetentionOptions { + max_days: Some(1), + kind_whitelist: None, + pubkey_whitelist: None, + }; + let first = backend.delete_expired_and_retained(&options).await.unwrap(); + assert_eq!( + (first.deleted, first.expired, first.retained), + (0, 0, 1_000), + "one pass takes RETENTION_BATCH_SIZE candidates (V7)" + ); + let second = backend.delete_expired_and_retained(&options).await.unwrap(); + assert_eq!( + (second.deleted, second.expired, second.retained), + (0, 0, 1), + "the next pass takes the row the batch ceiling left behind" + ); +} + +/// A `LIMIT 0` returns no rows, in every finder whose walk arms its bound after the first +/// match: Postgres answers an empty list, so a bound of zero must not return one. +#[tokio::test] +async fn a_zero_limit_returns_no_rows() { + use nostream_storage::repositories::{Nip05VerificationRepository, ReportRepository}; + + let (_directory, backend) = open_store().await; + let now = Utc::now(); + + let watched = Pubkey::from_byte_array([21; 32]); + Nip05VerificationRepository::upsert( + &backend, + &Nip05Verification { + pubkey: watched, + nip05: "pending@example.test".to_owned(), + domain: "example.test".to_owned(), + is_verified: false, + last_verified_at: None, + last_checked_at: now - Duration::hours(2), + failure_count: 0, + created_at: now - Duration::hours(2), + updated_at: now - Duration::hours(2), + }, + ) + .await + .unwrap(); + assert!( + backend + .find_pending_verifications(0, 3, 1) + .await + .unwrap() + .len() + == 1, + "the row the zero limit must not return" + ); + assert!( + backend + .find_pending_verifications(0, 3, 0) + .await + .unwrap() + .is_empty(), + "a zero limit returns no pending verification" + ); + + InviteCodeRepository::create( + &backend, + "invite-zero", + &CreateInviteCodeOptions { + expires_at: Some(now + Duration::hours(1)), + remaining_uses: FieldUpdate::Set(3), + created_by: None, + }, + ) + .await + .unwrap(); + assert_eq!(backend.find_active_codes(1).await.unwrap().len(), 1); + assert!( + backend.find_active_codes(0).await.unwrap().is_empty(), + "a zero limit returns no invite code" + ); + + ReportRepository::create( + &backend, + &NewReport { + event_id: EventId::from_byte_array([22; 32]), + reporter_pubkey: Pubkey::from_byte_array([23; 32]), + reported_pubkey: Some(Pubkey::from_byte_array([24; 32])), + reported_event_id: None, + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + }, + ) + .await + .unwrap(); + assert_eq!(backend.find_actionable(1).await.unwrap().len(), 1); + assert!( + backend.find_actionable(0).await.unwrap().is_empty(), + "a zero limit returns no report" + ); + + backend.append(&one_entry(None)).await.unwrap(); + let filters = DeliveryLogFilters::default(); + assert_eq!(backend.find_recent(1, &filters).await.unwrap().len(), 1); + assert!( + backend.find_recent(0, &filters).await.unwrap().is_empty(), + "a zero limit returns no delivery-log entry" + ); +} + +/// `dvm_jobs.id` is the primary key: creating a row that already exists is refused instead of +/// overwriting the worker and result a later state wrote. +#[tokio::test] +async fn creating_a_job_that_exists_is_refused() { + use nostream_storage::repositories::DvmJobRepository; + + let (_directory, backend) = open_store().await; + let id = EventId::from_byte_array([31; 32]); + let requester = Pubkey::from_byte_array([32; 32]); + + DvmJobRepository::create(&backend, &id, &requester, Kind(5_000)) + .await + .expect("the job"); + assert!(backend.assign_worker(&id, 3).await.unwrap()); + + let error = DvmJobRepository::create(&backend, &id, &requester, Kind(5_000)) + .await + .expect_err("a second create of the same id"); + assert!( + matches!(error, StorageError::InvalidArgument { .. }), + "the primary key refuses the row, got {error:?}" + ); + + let stored = DvmJobRepository::find_by_id(&backend, &id) + .await + .unwrap() + .expect("the job"); + assert_eq!(stored.status, DvmJobStatus::PickedUp); + assert_eq!( + stored.worker_index, + Some(3), + "the refused create changed nothing" + ); +} + +/// A non-empty `search` with the NIP-50 gate on is refused by every query method, the +/// reconciliation stream included: this backend cannot search, and returning a broader filter's +/// rows would answer a question it did not ask. +#[tokio::test] +async fn reconciliation_refuses_a_search_the_gate_enables() { + let (_directory, backend) = open_store().await; + let filter = Filter { + search: Some("nostr".to_owned()), + ..Filter::default() + }; + + let error = backend + .find_reconciliation_items(&filter) + .await + .err() + .expect("the refusal"); + assert!( + matches!( + error, + StorageError::NotImplemented { + module: "storage::lmdb::nip50_search" + } + ), + "a search the gate enables is NotImplemented, got {error:?}" + ); +} diff --git a/crates/nostream-storage/tests/storage.rs b/crates/nostream-storage/tests/storage.rs index f0e790900..5f425bbe3 100644 --- a/crates/nostream-storage/tests/storage.rs +++ b/crates/nostream-storage/tests/storage.rs @@ -1,31 +1,40 @@ -//! The LMDB environment slice, the trait surface, and the backend's dyn compatibility. +//! The backend seam: the enum, the data directory, and the dyn-compatible trait surface. // This integration-test target may unwrap: a failed assertion here is a failed test, // which is the point. The allowance names this file rather than exempting test targets // workspace-wide, so a new test file still has to ask. #![allow(clippy::unwrap_used, clippy::expect_used)] -use std::path::Path; +use std::path::{Path, PathBuf}; -use nostream_storage::{Backend, LmdbEnv}; +use nostream_core::EventStoreBackend; +use nostream_storage::data_dir; -#[tokio::test] -async fn the_lmdb_environment_round_trips_a_value() { - let directory = tempfile::tempdir().expect("tempdir"); - let env = LmdbEnv::open(directory.path(), 1024 * 1024).expect("the environment opens"); - - env.put("greeting", "hello").expect("put"); - assert_eq!(env.get("greeting").expect("get"), Some("hello".to_owned())); - assert_eq!(env.get("absent").expect("get"), None); - assert_eq!(env.path(), Path::new(directory.path())); +#[test] +fn the_backend_parses_the_settings_spelling() { + assert_eq!( + EventStoreBackend::parse("postgres"), + Some(EventStoreBackend::Postgres) + ); + assert_eq!( + EventStoreBackend::parse("lmdb"), + Some(EventStoreBackend::Lmdb) + ); + assert_eq!(EventStoreBackend::parse("mongodb"), None); + assert_eq!(EventStoreBackend::parse("LMDB"), None); + assert_eq!(EventStoreBackend::Lmdb.to_string(), "lmdb"); } #[test] -fn the_backend_parses_the_settings_spelling() { - assert_eq!(Backend::parse("postgres"), Some(Backend::Postgres)); - assert_eq!(Backend::parse("lmdb"), Some(Backend::Lmdb)); - assert_eq!(Backend::parse("mongodb"), None); - assert_eq!(Backend::Lmdb.to_string(), "lmdb"); +fn the_lmdb_backend_keeps_a_data_directory_of_its_own() { + assert_eq!( + data_dir(Path::new("/data"), EventStoreBackend::Lmdb), + Some(PathBuf::from("/data/lmdb")) + ); + assert_eq!( + data_dir(Path::new("/data"), EventStoreBackend::Postgres), + None + ); } #[tokio::test] @@ -411,3 +420,34 @@ async fn the_postgres_backend_coerces_to_all_nine_trait_objects() { let _: &dyn NotificationDeliveryLogRepository = &backend; let _: &dyn NotificationOutboxRepository = &backend; } + +/// The embedded backend coerces into the same nine forms, from a store it opens itself. +#[cfg(feature = "lmdb")] +#[tokio::test] +async fn the_lmdb_backend_coerces_to_all_nine_trait_objects() { + use nostream_storage::LmdbBackend; + use nostream_storage::repositories::{ + DvmJobRepository, EventRepository, InviteCodeRepository, InvoiceRepository, + Nip05VerificationRepository, NotificationDeliveryLogRepository, + NotificationOutboxRepository, ReportRepository, UserRepository, + }; + + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens"); + assert!( + data_dir(directory.path(), EventStoreBackend::Lmdb) + .expect("a data directory") + .is_dir() + ); + let _: &dyn EventRepository = &backend; + let _: &dyn InvoiceRepository = &backend; + let _: &dyn UserRepository = &backend; + let _: &dyn Nip05VerificationRepository = &backend; + let _: &dyn InviteCodeRepository = &backend; + let _: &dyn DvmJobRepository = &backend; + let _: &dyn ReportRepository = &backend; + let _: &dyn NotificationDeliveryLogRepository = &backend; + let _: &dyn NotificationOutboxRepository = &backend; +} diff --git a/docs/rewrite/decisions/0011-lmdb-backend.md b/docs/rewrite/decisions/0011-lmdb-backend.md new file mode 100644 index 000000000..c8f34cf4e --- /dev/null +++ b/docs/rewrite/decisions/0011-lmdb-backend.md @@ -0,0 +1,369 @@ +# ADR 0011: The LMDB event store behind the existing seam (job 024) + +- **Status:** accepted (job 024, Phase 2) +- **Deciders:** the rewrite team, on the evidence in job 024's recon and spec +- **Date:** 2026-11-06 + +## Context + +`nostream-storage` shipped one backend. `src/backend.rs` selected between `postgres` and +`lmdb` and then answered `Phase 1 stub` from `data_dir`, so the second backend the rewrite +exists to provide had a name, a settings value and no code. This job adds it: the event +store, the metadata tail, and the nine repository traits implemented over heed 0.20.5, +with the same integration test the Postgres backend passes replayed against it. + +`spec.md` records ten decisions (D1–D10) with their reasons and their measurements. This +document keeps the parts a later reader needs to judge the code: where the state lives, how +a value is shaped, what the seam does not offer, which schema it implements, and what the oracle +therefore measures. + +## The backend is a feature of the crate, not a crate + +`lmdb = ["dep:heed", "dep:uuid", "dep:tracing"]`, default on beside `postgres` (D1). heed was +already a hard dependency, the seams' own ADR says the `--no-default-features` build is what +this follow-up uses (ADR 0003, "Backend coupling"), and a second crate would repeat five +manifest fields plus `[lints] workspace = true` to isolate code that shares every type in +`nostream-storage`. The cost is that `--no-default-features` now links no driver at all — +which is also the point: `cargo check -p nostream-storage --no-default-features --all-targets` +must compile, and it does. + +`nostream_storage::Backend` is deleted (D8) and everything uses +`nostream_core::EventStoreBackend`. Two enums with identical variants and spellings were a +drift waiting to happen, and neither had a consumer: `data_dir` now returns +`Option` — `Some(root/lmdb)` for LMDB, `None` for Postgres — and the LMDB backend +opens `/lmdb` itself. + +## Metadata lives in LMDB databases of its own + +The eight tail tables are 22 named databases in the same environment (D2): one primary +database per table keyed by its primary key, plus one per query path (`invoices_pending`, +`nip05_by_last_checked_at`, `invite_codes_by_created_at`/`_by_expires_at`, +`dvm_jobs_by_status_created_at`, four `reports_*`, four `outbox_*`/`delivery_log_*`), and a +`meta` database for the report sequence. + +**The reason is one write transaction spanning events and tail.** NIP-56's hidden-content +predicate is a `NOT EXISTS` over `reports` inside the events query +(`crates/nostream-storage/src/postgres/sql.rs:291-297`). One environment makes that two +point lookups — `reports_actionable_event` by event id and `reports_actionable_pubkey` by +author — inside the same read transaction that already holds the event row, instead of a +second store read per candidate. An embedded SQL store would have needed a second +dependency and could still tear against the LMDB file. + +The price is recorded with the decision: 29 named databases maintained by hand, `DISTINCT` +and `OFFSET` become cursor walks (`BTreeSet` for `find_actionable_targets`, a skip count for +`find_pending_invoices`), and the columns' `CHECK` and foreign-key constraints move into +Rust (§ Constraints move into Rust, below). + +Rows are versioned: one format byte (`ROW_FORMAT_VERSION = 1`), then `serde_json`, the byte +per database rather than per table. ADR 0003 trigger 6 asked for a version marker before a +second backend persisted `EventRecord`, and this is it. + +## An event value is a 55-byte header, then JSON + +`events` holds `1 + 54 = 55` bytes of header, then `serde_json` of `EventRecord` (D5): + +|bytes|field| +|---|---| +|1|format version (`1`)| +|1|flags: bit 0 is `deleted_at.is_some()`, the soft-delete bit (`FLAG_DELETED`)| +|8|`created_at`, big-endian i64| +|4|`kind`, big-endian i32| +|8|`expires_at`, big-endian i64, `i64::MIN` for NULL| +|32|author pubkey| +|1|reserved, zero| + +The live predicate and the NIP-56 lookup read that header without decoding the body, so +`count_by_filters` and reconciliation copy 55 bytes per candidate instead of an event. +The flags byte is header byte 0 — the first byte after the format byte — and it already +carries the soft-delete bit, so a second flag lands there behind `FLAG_DELETED` rather than +in the reserved byte. The reserved byte is the header's last (byte 53 of the 54-byte header), +kept so the header stays the length R7, V35 and AC8 pin; a flag that needs its own bit goes +in the flags byte, and the reserved byte is there for a field that has to be a byte wide. + +`row_id` stays `None` (D5): it is the Postgres row's `uuid` and nothing reads it through +this backend. Text components of a composite key are NUL-terminated except the final one +(`text_part`), so `tag_key` is `name NUL value NUL created_at ‖ event_id`. A value carrying +`0x00` is refused as `InvalidRow` by `refuse_nul` rather than splitting the key, and +PostgreSQL's `text` cannot hold that byte either, so both backends reject the same value. A +row's own text column meets the same check: the outbox key stopped carrying the target, so +`append` refuses `notification_delivery_log.target_id` through `refuse_nul` rather than +writing a value the capture could not have stored. The three component codecs a key reader +needs — `text_part`, `text_from` and `stamp_key` — have one home, `lmdb/encoding.rs`, beside +the key builders; the tail modules call them rather than keeping copies whose rules could +drift apart. + +LMDB bounds a key at 511 bytes (`MDB_MAXKEYSIZE`, which this environment does not raise), and +a `tags` key carries the tag value verbatim. **That is the one place this backend refuses a +row Postgres would keep**: a tag value of 469 bytes or more (with a one-byte tag name) pushes +the key past the bound, so the event's write fails and the relay refuses the event, where the +capture stores it. It is a recorded divergence, not a silent one — the write returns +`StorageError::Lmdb` — and it is not fixed here: the repairs (hashing the value into the key and +looking the row up twice, or a larger `max_key_size` at the open) change the index the benchmark +job will measure. On the read side a criterion too long to name a stored key selects nothing, +rather than failing the whole request. + +The `replaceable` pointer is the second variable-length key, and it adds no second divergence. +Its components are length-prefixed — `pubkey ‖ kind ‖ (tag ‖ len ‖ bytes)*`, a string element +written as its own bytes and any other JSON value as its `serde_json` text behind a different +tag — so JSON escaping cannot inflate a value into a refusal: a 235-byte quote-only `d` value +is 235 key bytes, not 474. The same value costs 39 + len bytes here and 43 + len in the `tags` +key, so the tag index refuses first (469 bytes against the pointer's 473), and a value past +the pointer's own bound is refused as `InvalidRow` naming `event_deduplication` before LMDB +sees the key. PostgreSQL keeps either row; when such a value has to be stored, the same repair +the tag bound needs applies. + +Soft delete is a flag **and** the row's value, never an index removal (D4): `find_by_filters` +does not apply the live predicate (`sql.rs:119-140` against `:262-267`), so the row must stay +in every read index while `count_by_filters` and reconciliation exclude it. The +`events_deleted` database is the retention scan's path, not the query's. + +## The seam has no transaction type (D3) + +`RwTxn` borrows the `Env` (heed 0.20.5 `src/txn.rs:119-122`) and a `spawn_blocking` closure +must be `'static`, so a seam-level transaction would need a leaked `&'static Env` plus a +mutex, and would then hold LMDB's **single** writer across every `await` between `begin()` +and `commit()`. That is a worse failure mode than the one it fixes: one stuck caller stops +every write in the process. + +What replaces it: a repository method is one whole write transaction, so every write the +seam performs is atomic across every database it touches. What it leaves **unmeasured** is +atomicity *across repository calls* — a caller that needs two writes to succeed or fail +together has no form for it here. `PostgresTransaction` and `begin()` are untouched. + +This resolves nothing about ADR 0003 trigger 2 (a second backend needs a transaction +abstraction) except the shape a future one must take: a held LMDB transaction cannot satisfy +it, so the abstraction has to be closure-shaped — `with_write_txn(|tx| …)` — with the +single writer never held across an `await`. The trigger stays open. + +## Blocking and streams (D7) + +Every heed call — `write_txn`, `read_txn`, `create_database`, iteration and `commit` — runs +inside `tokio::task::spawn_blocking`, because `Env::write_txn` blocks the calling thread on +the writer's mutex and a read transaction must not be held across an `await`. `LmdbBackend` +therefore has two helpers, `write` and `read`, that own their transaction and commit or drop +it inside the closure; no `RwTxn` or `RoTxn` leaves one. + +A stream is a key page, not a held transaction, and the two query methods keep different +things — which is what a caller planning memory needs to know: + +- `find_by_filters` collects its union page inside one blocking closure, so the stream **holds + the page**: at most each filter's limit, clamped to `MAX_PAGE_KEYS = 5 000` ids. Each poll + opens a read transaction, copies up to `STREAM_BATCH_ITEMS = 256` values from that page and + drops it; nothing more is read from the index, and the page is the only thing resident. +- `find_reconciliation_items` **holds a cursor key**. Each poll opens a read transaction, + resumes exclusively after the last key the previous batch evaluated, and drops it. A batch + ends at `STREAM_BATCH_ITEMS` items or `STREAM_BATCH_KEYS = 256` evaluated keys, whichever + comes first, so a filter that matches few of the keys it walks cannot make one poll span a + whole range — the cursor resumes exactly where the key cap stopped. + It scans the filter's own index when that index narrows the filter to a single range that + runs in `created_at` order (`{authors: [a], kinds: [k]}`, or one exact tag value). An `ids` + filter whose criteria are all exact ids is read from its own ranges as one batch and sorted, + because that list is the request's own bound and an `events` key runs in id order; a prefix + criterion is a byte range over the id field (requirement 12), where `""` is the whole + database, so it is not the request's list and takes the bounded scan below. Any other filter + scans `events_by_created_at`, because only that range's order is the oldest-first order the + seam promises. Reading a candidate costs its header, plus its decoded record only for the tag + families the key has not already answered. + +**A stream is therefore not a snapshot** (D7): a row written or deleted between two batches +may appear or vanish, where Postgres' portal holds one snapshot. No comparable behaviour +exists to compare, so the divergence is recorded for the benchmark job, and ADR 0003 trigger +1 ("a long-lived LMDB read transaction hurts writes") is answered by construction: no +transaction outlives a poll, and a dropped stream pins no pages. + +`Env::resize` is `unsafe` and this crate has one audited `unsafe` call, so the map is +preallocated at `LMDB_MAP_SIZE = 64 GiB` and never grown (D6). The same single +`#[allow(unsafe_code)]` open sets `LMDB_MAX_DBS = 64` (the 29 named databases, comfortably +inside) and `LMDB_MAX_READERS = 512`. + +Because the map never grows, its approach has to be visible before the first failed write: +`LmdbBackend::open` logs the map size, the bytes the pages in use occupy +(`Env::non_free_pages_size`) and that as a percentage, and `LmdbBackend::map_usage` reports the +same numbers to an operator's tooling. + +## What a read holds, and which index it reads + +The resident set of a read is the page, never the range it was chosen from, and the size of +that page is the backend's to bound: the effective limit is the filter's own or the capture's +default of 500, clamped to `MAX_PAGE_KEYS = 5 000` — the shipped +`limits.client.subscription.maxLimit` — so a wire `limit` cannot choose an allocation, whether +or not a deployment sets `maxLimit`. A filter's branch holds at most that many rows — the page +itself, kept as a heap rooted at the row it would drop first, so an offer is one comparison +against that root and the page is sorted once, when it is returned — and that page is one +accumulator across every range the plan seeks: the top `limit` is global to them. A range whose +key order *is* the page's order stops at the page's own worst kept row — the heap's root, not +the first row the range happened to offer — keeping every row at that timestamp so a tie breaks +as the capture's does. + +One read keeps a second thing resident, and it is a count's rather than a page's: a `tags` count +whose criteria can select one row twice holds an id per row its matcher accepts, and that set is +bounded by `MAX_PAGE_KEYS` — the same 5 000. Below the cap the walk reads no body; at the cap it +stops inserting and decides each further key with a bounded-memory check instead: a repeated key +reads the row's body and asks whether an earlier key of the walk already answered for it, an +earlier criterion that matches one of its tags or, within one `#g` prefix range, a smaller value +the prefix matches. The total is therefore the same at every corpus size, and the set is a +constant 270 352 B once full (measured live: 5 000 ids at a `HashSet` capacity of 7 168 slots, +against 1 081 360 B for the same 20 000-row corpus with the set unbounded). The price of the +cap is a body decode per repeated key past it; an index order that visits each row once is what +would remove that too. + +The index is chosen from the filter, and which ranges can stop early follows from the key: + +| filter | index | stops when full | +|---|---|---| +| `ids` | `events`, a point or prefix range per criterion | never: the key is the id, so the range runs in id order | +| `authors` + `kinds` | `events_by_author`, one range per whole-pubkey criterion and kind, and one whole-tail range per prefix criterion | when every criterion is a whole pubkey, so a range is one `pubkey ‖ kind ‖ created_at` run; a prefix range spans every kind | +| `authors` alone | `events_by_created_at` | yes — see below | +| a `#` + letter criterion | `tags`, one range per criterion | for an exact value; a `#g` prefix spans values, so its range runs whole | +| anything else | `events_by_created_at` | yes | + +Every hex criterion's bounds are widened to the field they name — 32 bytes for an id or a +pubkey — before anything the key carries after it is appended. A prefix is therefore compared +against its own field's bytes and never against the kind, the timestamp or the id that follow, +whether the path is `events` or `events_by_author`. That is why an author prefix with `kinds` +seeks no kind at byte 32: its upper bound runs to the end of the pubkey field, a kind appended +after that bound is compared where no key writes one, and every kind's range would then contain +every other kind's keys — a count visiting each row once per kind. `author_kind_ranges` +(`events.rs`) is where the split is composed, a whole pubkey to one range per kind and a prefix +to one range over its whole tail, and it bounds the composition too: a whole-pubkey × kind +product past `MAX_PAGE_KEYS` composes no range at all and the filter is answered from +`events_by_created_at` instead. `hex_ranges` (`encoding.rs`) is the one place the widening rule +lives. A criterion another already covers, and a repeated kind, seek one range, so a count that +walks ranges whole visits those keys once. + +The `authors`-alone row is the capture's own shape rather than the faster index: the author +index orders by kind before `created_at`, so a page ordered by time cannot stop inside it, and +the same query in Postgres walks the created-at index and filters (ADR 0003 records 5 508 rows +discarded to return 500 for the kind-only form). The cost is that a quiet author's page reads +further into the created-at index than the author's own timeline would; the page still stops at +its limit, and the choice belongs to the benchmark job once it can choose with statistics +rather than a rule. + +Two consequences of the same bound: + +- `count_by_filters` has no page, so it materialises no result set: each filter counts the rows + it visits and skips one the count already holds — an earlier filter's page when that filter + had a limit, or a predicate match when it had none — which is the capture's `UNION`. An + unlimited filter walks its ranges whole, which is what counting every match costs, and a + `tags` walk that can select one row twice also holds that row's id, up to `MAX_PAGE_KEYS` + ids (see above). The slot is filled before the cross-filter check, so the set also holds rows + an earlier filter already counts; the same cap bounds that residue. +- A `tags` range is built from the family's own criteria, so a key inside it has already proved + that family: the event's body is decoded only for the filter's *other* tag families, and a + filter with one family decodes none. The id a key ends with answers the rest of the walk, so a + row two criteria — or two values of a `#g` prefix — both select is counted at the first key + the walk reaches and skipped at the rest without a body decode, while the set has room: the + ranges run in the criteria's order, so that key is not necessarily the row's smallest matching + one. Once the set is full the walk reads the body of a repeated key instead and asks whether + an earlier key of the walk already answered for it — an earlier criterion, or a smaller value + of the same prefix — so the count stays exact and its resident set stays a constant. Only those + two shapes hold the set, whose population is one id per row its matcher accepts: a single exact + criterion names one key per row. The tags index has no order that visits each row once, which + is why the cap trades a slot past it for a body decode. + +## Constraints move into Rust + +The store has no server to enforce the columns' `CHECK` and foreign-key constraints, so the +two a caller can violate are enforced on write (R17): a negative +`CreateInviteCodeOptions::remaining_uses` is `InvalidArgument`, and a delivery-log entry +naming an unknown outbox row is `InvalidArgument` while deleting an outbox row walks +`delivery_log_by_outbox` and sets `outbox_id` to NULL. The `status`, `unit`, `report_type` +and `target_type` spellings need no code: they are typed enums parsed by `column_enum!`, so +an invalid spelling cannot reach the writer. + +Every `DateTime` is truncated to microseconds before encoding, because every column it +lands in is a PostgreSQL `timestamp`; a call takes one `Utc::now()` and uses it for every row +it writes, standing in for the statements' `CURRENT_TIMESTAMP`. + +`admit_user` and `confirm_invoice` are Rust in one write transaction each, following the +schema functions they replace. `confirm_invoice` scales by `sats × 1000`, `msats × 1`, +`btc × 100_000_000 × 1000` with checked arithmetic, and answers `ValueOutOfRange` where the +`int8` would overflow. `reports.id` is allocated from `meta` starting at 1; every other +generated id is a UUID v4 in canonical text form, which is why `uuid` (v4) is the job's only +new dependency. + +## The schema contract is 37 migrations (D9) + +The store implements the contract both schema oracles pin: 37 migrations +(`crates/nostream-storage/migrations/`, the count `crates/nostream-storage/tests/migrations.rs:25` +asserts as `EXPECTED_MIGRATIONS`) and the captured catalogue's own `subject.migration_count: 37` +(`crates/nostream-storage/tests/fixtures/schema.json`). No table, column or index outside that +contract is added here, and the same tree is what the Postgres arm is migrated from. + +**The seam therefore stays 9 of the capture's 10 repositories.** The tenth is +`user_subscriptions`: it has a TypeScript repository +(`src/repositories/user-subscription-repository.ts`) and no Rust trait, and the capture holds no +entry that exercises it, so it belongs to the job that adds that repository's slice. +`events.first_seen` is inside the contract and carries no index — the sqlx tree declares the +column (`crates/nostream-storage/migrations/20220524153400_create_events_table.sql:10`) and +indexes it nowhere — so this backend stores it in the row's JSON body, sets it at create and +preserves it across a replace, and seeks nothing by it. + +The two knex migrations the sqlx tree lacks are closed elsewhere, each by the job that first needs +it: + +- `migrations/20261016_120000_create_user_subscriptions_table.js` creates `user_subscriptions` + and adds three nullable `invoices` columns (`plan_id`, `period_start`, `period_end`) — closed by + the job that adds the capture's tenth repository. +- `migrations/20261025_120000_add_moderation_stats_first_seen_indexes.js` adds two partial + `events` indexes on `first_seen` (`events_live_first_seen_pubkey_idx`, + `events_live_first_seen_remote_address_idx`, built outside a transaction) and changes no column + this store encodes — closed by the job that ports the moderation-stats query. + +Both consequences — nine repositories of ten, and a `first_seen` stored but never indexed — are +stated in `lib.rs`'s "does not own" paragraph, which is where a reader of the crate meets them +first. + +## Oracle: two arms, one executor + +The fixture is the capture's recording and both backends are compared against the same +`result.compare` (D10). The 51 method arms moved out of `tests/parity.rs` into +`tests/common/`, where the executor takes a `Repos<'a>` of nine `&dyn` repository +references, so one script drives both backends; what stays Postgres-only is what only +Postgres can reach: raw SQL on `users.tos_accepted_at` and the serializable `transaction` +groups. + +`tests/parity_lmdb.rs` opens a store on a `tempfile::TempDir`, inserts the fixture's +483-event corpus and replays **113 of 118** entries. The five it does not replay: + +|index|op|why the seam cannot run it| +|---|---|---| +|21|`events.find_by_filters` with `search`|NIP-50 is out of scope; the backend answers `NotImplemented`| +|48, 50|`user.tos_accepted_at`|raw SQL against `backend.master()`, no seam form| +|112, 115|`transaction`|the capture **rolled these back**; reproducing a rollback needs a transaction type this job deliberately does not add| + +The other two `transaction` groups, 111 and 117, **committed** in the capture and are +replayed: running their nested ops in order through the seam leaves exactly the state the +commit left, and their comparison — the aggregate `{failed: false, results: [...]}` — is what +those calls return. That is also the evidence for the count rather than a preference: entry +116, `outbox.claim_batch`, expects the outbox row that group 111 committed, so excluding 111 +cannot pass. What the transaction decision leaves unmeasured is rollback: the arm proves the +committed groups' effects, not that two calls can fail together. + +The target asserts every exclusion against the fixture — an entry is excluded exactly when +its op is one of the five above — so the list cannot grow without the diff showing it. + +## Consequences + +- One environment, one write transaction, one error taxonomy; no second dependency, and no + tear between two files. +- 29 named databases and their key encodings are maintained by hand; `DISTINCT`/`OFFSET` + became cursor walks; two `CHECK`/foreign-key constraints became Rust code. +- Postgres remains the reference: every query path is written against `sql.rs`'s predicates + and orderings, and the shared executor is what proves it. +- Retention's batch composition can differ from Postgres' planner-chosen order when more than + 1000 rows qualify; both return 1000 rows classified by the same precedence (Assumption 10 + in `spec.md`). The corpus is 483 events, so no current entry reaches it. +- A page is bounded by `MAX_PAGE_KEYS` and a read holds a page rather than a range; a count + materialises no result set, and the ids it holds for a tag walk that can select a row twice + are bounded by the same `MAX_PAGE_KEYS` — 270 352 B of resident set once full, however large + the corpus, with a body decode per repeated key past the cap. So a client cannot choose how + much a request materialises. + The price is that a `#g`-prefix page walks its whole (time-bounded) range, an author page + without kinds reads the created-at index, and so does an author × kind filter whose + composition passes `MAX_PAGE_KEYS` ranges; an `ids` read whose criteria are exact is bounded + by the request's own list, and a prefix criterion takes the bounded created-at scan. +- Two JSON values PostgreSQL's `jsonb` considers equal but that serialize differently + (`1` against `1.0`) would be distinct deduplication pointers here; the repository builds + the value in one shape and the relay passes a single string, so today's callers cannot + reach it (Assumption 11). diff --git a/scripts/storage-parity.sh b/scripts/storage-parity.sh index f8021cbbb..3e8291e42 100755 --- a/scripts/storage-parity.sh +++ b/scripts/storage-parity.sh @@ -285,6 +285,10 @@ echo "== behavioural parity" cargo test -p nostream-storage --test parity -- --ignored --test-threads=1 \ || fail "the parity target failed" +echo "== lmdb behavioural parity (no database)" +./scripts/rust.sh cargo test -p nostream-storage --test parity_lmdb -- --nocapture \ + || fail "the lmdb parity target failed" + echo "== tls parity (the same replay over sslmode=require)" # The test skips itself when the variable is absent, so the run has to prove it executed: # a renamed or quietly skipped TLS leg is the failure mode this check closes.