From 677501522cd1b1c2489c6e27186839d48f5a0354 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 00:55:43 -0500 Subject: [PATCH 01/36] feat(core): advertise the audited NIP set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SUPPORTED_NIPS becomes the Phase 0 audit's 33 must-have rows plus NIP-67, ascending and free of duplicates, with the array length as the second guard. Nothing serves the list yet: the workspace has no HTTP listener and no NIP-11 document (docs/rewrite/nip-parity.md §10, D1). --- crates/nostream-core/src/constants.rs | 13 ++++++- crates/nostream-core/tests/supported_nips.rs | 41 ++++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) create mode 100644 crates/nostream-core/tests/supported_nips.rs diff --git a/crates/nostream-core/src/constants.rs b/crates/nostream-core/src/constants.rs index d7c934ad9..69081ba6f 100644 --- a/crates/nostream-core/src/constants.rs +++ b/crates/nostream-core/src/constants.rs @@ -7,8 +7,17 @@ pub const ALL_RELAYS: &str = "ALL_RELAYS"; /// the negentropy storage seam's timestamp bound. pub const MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991; -/// The NIP-11 `supported_nips` advertisement (`src/constants/base.ts` `SUPPORTED_NIPS` shape). -pub const SUPPORTED_NIPS: [u16; 4] = [11, 42, 45, 50]; +/// The NIP-11 `supported_nips` advertisement: the Phase 0 audit's `must-have` set +/// (`docs/rewrite/nip-parity.md` §2, 33 rows) plus NIP-67, whose hint this tree implements +/// (§10, decision D2). Ascending, no duplicates. +/// +/// Nothing serves this yet — the workspace has no HTTP listener and no NIP-11 document — so +/// the list is a decision record rather than an answer on the wire (D1). The array length is +/// the second guard: an id added without the type changing does not compile. +pub const SUPPORTED_NIPS: [u16; 34] = [ + 1, 2, 3, 4, 5, 9, 11, 12, 13, 14, 16, 17, 20, 22, 25, 28, 33, 40, 42, 43, 44, 45, 50, 56, 59, + 62, 65, 66, 67, 70, 77, 89, 90, 98, +]; /// The payment processors the settings surface names (`src/constants/base.ts` `PaymentsProcessors`). pub const PAYMENTS_PROCESSORS: [&str; 6] = diff --git a/crates/nostream-core/tests/supported_nips.rs b/crates/nostream-core/tests/supported_nips.rs new file mode 100644 index 000000000..c04b72e34 --- /dev/null +++ b/crates/nostream-core/tests/supported_nips.rs @@ -0,0 +1,41 @@ +//! R2: the advertisement is the audited set. +//! +//! `SUPPORTED_NIPS` is the Phase 0 audit's `must-have` list plus NIP-67 +//! (`docs/rewrite/nip-parity.md` §2, §10). The literal below is written out again rather than +//! compared against the constant, so the tests fail on a missing id, an extra id, a wrong id a +//! reordering or a duplicate — and on nothing else. +//! +//! Two limits are worth stating, because they are what these tests do **not** prove: +//! +//! - **No client is told this list.** Nothing in the workspace serves a NIP-11 document or +//! listens on HTTP (`docs/rewrite/nip-parity.md` §10, decision D1), so this is a test of the +//! decision record and not of a response. +//! - **No behaviour behind an id.** A NIP in the list is claimed, not shown, here; the +//! behaviour is the per-NIP table's and the harness map's business (`§10`). + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +use nostream_core::constants::SUPPORTED_NIPS; + +#[test] +fn the_advertisement_is_the_audited_set() { + assert_eq!( + SUPPORTED_NIPS, + [ + 1, 2, 3, 4, 5, 9, 11, 12, 13, 14, 16, 17, 20, 22, 25, 28, 33, 40, 42, 43, 44, 45, 50, + 56, 59, 62, 65, 66, 67, 70, 77, 89, 90, 98, + ] + ); +} + +#[test] +fn the_advertisement_is_ascending_and_free_of_duplicates() { + for pair in SUPPORTED_NIPS.windows(2) { + assert!( + pair[0] < pair[1], + "the advertisement must ascend strictly, found {} then {}", + pair[0], + pair[1] + ); + } +} From 52a7c1325742f9322155fbb2228ed6db3af47886 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 00:55:43 -0500 Subject: [PATCH 02/36] feat(core): carry NIP-67's EOSE hints EoseMessage gains an optional third element, an array of hint strings. Hints are emitted from exactly two causes (D3): the read guard withheld a matching event (auth), or a single-filter subscription carried a limit and the cap decided the answer (finish/more, decided by asking storage for limit + 1). A no-limit, no-trim read keeps the two-element frame, which is what the recorded sessions require. --- crates/nostream-core/src/messages.rs | 43 +++++++- crates/nostream-core/tests/wire_codec.rs | 27 ++++- .../nostream-relay/src/handlers/subscribe.rs | 100 ++++++++++++++++-- crates/nostream-relay/tests/connection.rs | 8 +- crates/nostream-relay/tests/handlers.rs | 79 ++++++++++++++ 5 files changed, 240 insertions(+), 17 deletions(-) diff --git a/crates/nostream-core/src/messages.rs b/crates/nostream-core/src/messages.rs index 1940c3e71..6722a9dac 100644 --- a/crates/nostream-core/src/messages.rs +++ b/crates/nostream-core/src/messages.rs @@ -105,13 +105,31 @@ pub struct ClosedMessage { pub params: Option, } -/// `["EOSE", ]` (`src/@types/messages.ts` `EndOfStoredEventsNotice`, -/// NIP-15). +/// `["EOSE", , ?]` (`src/@types/messages.ts` `EndOfStoredEventsNotice`, +/// NIP-15; the optional third element is NIP-67). +/// +/// The third element is **an array of hint strings** — `67.md` §"Specification": *"A relay that +/// implements this NIP MAY append a third element to the `EOSE` message: an array of hint +/// strings"*. Issue cameri/nostream#718 proposed `{"completeness": "limit"|"full"}` instead; that +/// object is **rejected** here, because the registry defines an array, an object is a different +/// message, and a client parsing one would not parse the other. +/// +/// The hints stay [`String`] rather than an enum: `67.md` says clients MUST ignore unknown hint +/// values, and nothing in this tree decodes an `EOSE`, so no code path could reject one. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct EoseMessage { pub subscription_id: SubscriptionId, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hints: Vec, } +/// `67.md` §"Specification": the relay withheld stored events that `AUTH` could release. +pub const EOSE_HINT_AUTH: &str = "auth"; +/// `67.md` §"Specification": every stored event matching the filters has been sent. +pub const EOSE_HINT_FINISH: &str = "finish"; +/// `67.md` §"Specification": the relay holds more matching stored events than it sent. +pub const EOSE_HINT_MORE: &str = "more"; + /// `["EVENT", , ]` (`src/@types/messages.ts` `OutgoingEventMessage`). /// /// The event is shared: the hub hands one `Arc` to every connection and each matching @@ -219,7 +237,7 @@ impl RelayMessage { &["e(&message.subscription_id), "e(&message.message)], ), }, - Self::Eose(message) => frame("EOSE", &["e(&message.subscription_id)]), + Self::Eose(message) => eose_frame(message), Self::Event(message) => event_frame(message), Self::Count(message) => frame( "COUNT", @@ -294,6 +312,25 @@ fn event_frame(message: &OutgoingEventMessage) -> String { } } +/// `["EOSE", ]`, or `["EOSE", , ["auth","finish"]]` when the relay has hints to give +/// (`67.md` §"Specification"). The hints keep the order the builder pushed them in, which is +/// why the caller puts `auth` before a completeness hint. +fn eose_frame(message: &EoseMessage) -> String { + if message.hints.is_empty() { + return frame("EOSE", &["e(&message.subscription_id)]); + } + + let hints: Vec = message.hints.iter().map(|hint| quote(hint)).collect(); + + frame( + "EOSE", + &[ + "e(&message.subscription_id), + &format!("[{}]", hints.join(",")), + ], + ) +} + fn frame(verb: &str, parts: &[&str]) -> String { let mut text = format!("[{}", quote(verb)); diff --git a/crates/nostream-core/tests/wire_codec.rs b/crates/nostream-core/tests/wire_codec.rs index 040e9a04c..2d926428a 100644 --- a/crates/nostream-core/tests/wire_codec.rs +++ b/crates/nostream-core/tests/wire_codec.rs @@ -93,13 +93,38 @@ fn event_frame_is_the_verb_the_id_and_the_event_in_wire_order() { fn eose_frame_is_the_verb_and_the_subscription() { assert_eq!( RelayMessage::Eose(EoseMessage { - subscription_id: "subscriptionId".to_owned() + subscription_id: "subscriptionId".to_owned(), + hints: Vec::new(), }) .to_text(), r#"["EOSE","subscriptionId"]"# ); } +/// NIP-67's third element: an array of hint strings, in the order the builder pushed them. +#[test] +fn eose_frame_carries_the_hints_as_a_third_element() { + assert_eq!( + RelayMessage::Eose(EoseMessage { + subscription_id: "subscriptionId".to_owned(), + hints: vec![ + nostream_core::messages::EOSE_HINT_AUTH.to_owned(), + nostream_core::messages::EOSE_HINT_FINISH.to_owned(), + ], + }) + .to_text(), + r#"["EOSE","subscriptionId",["auth","finish"]]"# + ); + assert_eq!( + RelayMessage::Eose(EoseMessage { + subscription_id: "subscriptionId".to_owned(), + hints: vec![nostream_core::messages::EOSE_HINT_MORE.to_owned()], + }) + .to_text(), + r#"["EOSE","subscriptionId",["more"]]"# + ); +} + #[test] fn ok_frame_has_four_elements_without_params_and_five_with_them() { let plain = RelayMessage::Ok(CommandResult { diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index 6f24122f4..11fefa9cd 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -22,7 +22,10 @@ use std::sync::Arc; use futures_util::StreamExt; use nostream_core::expiration; use nostream_core::filter::{count_filter_values, normalise_subscription}; -use nostream_core::messages::{EoseMessage, OutgoingEventMessage, RelayMessage, SubscribeMessage}; +use nostream_core::messages::{ + EOSE_HINT_AUTH, EOSE_HINT_FINISH, EOSE_HINT_MORE, EoseMessage, OutgoingEventMessage, + RelayMessage, SubscribeMessage, +}; use nostream_core::nip42; use nostream_core::{Filter, Kind}; @@ -238,6 +241,12 @@ async fn mint_invite_code( /// /// The read is already in flight when this runs: it is registered before the invite mint, so /// a `CLOSE` that landed during the mint has set its cancellation. +/// +/// The `EOSE` may carry NIP-67's third element (`67.md` §"Specification"), and only from the +/// two causes that make the two-element form misleading: the read guard withheld a matching +/// event (`auth`), or a single-filter subscription's `limit` decided the answer +/// (`finish`/`more`). A read with neither cause keeps `["EOSE", ]`, which is what makes +/// every recorded session's EOSE byte-identical to the capture's. async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &HandlerContext) { let subscription_id = read.subscription_id().to_owned(); @@ -248,7 +257,38 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha return; } - let stream = match context.stores.events.find_by_filters(filters).await { + // NIP-67 can only be answered exactly for a single filter carrying a `limit`: a + // multi-filter union is deduplicated after each filter's own limit, so "did this + // subscription's cap cut the result" has no cheap exact answer there — and `more` is + // optional in the NIP for exactly that reason. + let hint_eligible = filters.len() == 1 && filters[0].limit.is_some(); + let limit = if hint_eligible { + filters[0].limit + } else { + None + }; + + // The probe: ask storage for one more row than the client's cap, so the `(limit + 1)`th + // matching row is the one that says `more` without being sent. `u64::MAX` saturates, which + // admits nothing beyond the maximum and so cannot hide a row. The backends already honour + // `limit`; the in-memory double does not, and the loop below drops the extra row either way. + let storage_filters: Vec = match limit { + Some(limit) => filters + .iter() + .map(|filter| Filter { + limit: Some(limit.saturating_add(1)), + ..filter.clone() + }) + .collect(), + None => filters.to_vec(), + }; + + let stream = match context + .stores + .events + .find_by_filters(&storage_filters) + .await + { Ok(stream) => stream, Err(error) => { tracing::error!(error = %error, "unable to stream events"); @@ -263,6 +303,13 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha let matches_any = |event: &nostream_core::Event| filters.iter().any(|filter| filter.matches(event)); + // `trimmed`: the guard withheld a matching, unexpired event this read would otherwise have + // delivered. `more`: the cap cut the result. `delivered`: matching rows inside the cap, in + // stream order — a deleted, expired or non-matching row is not one of them. + let mut trimmed = false; + let mut more = false; + let mut matched = 0u64; + loop { if *read.cancellation().borrow() { context.connection.end_read(read); @@ -316,14 +363,29 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha let event = record.event; + if expiration::is_expired_event(&event, now) || !matches_any(&event) { + continue; + } + + // The `(limit + 1)`th matching row is beyond the cap, so nothing after it can + // change the answer: it sets `more` and ends the read undelivered. + if let Some(limit) = limit + && matched >= limit + { + more = true; + break; + } + + matched += 1; + // The guard is the crate's one definition of "may this socket read this // event"; it is asked per event, as the capture's // `createReadAuthorizationGuard` is, so an AUTH that lands mid-read applies to - // the rest of the stream. - if expiration::is_expired_event(&event, now) - || !context.read_authorization_guard().allows(&event) - || !matches_any(&event) - { + // the rest of the stream. A withheld row still occupied a place inside the + // cap, which is why the counter moves before the guard is asked. + if !context.read_authorization_guard().allows(&event) { + trimmed = true; + continue; } @@ -343,9 +405,27 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha context.connection.end_read(read); - context - .sink - .send(&RelayMessage::Eose(EoseMessage { subscription_id })); + // `auth` first, then the completeness hint. `finish` is never sent beside `auth`: `67.md` + // defines `finish` as "every stored event matching the subscription's filters" has been + // sent, and a withheld event makes that false even though the NIP's example shows the pair. + let mut hints = Vec::new(); + + if trimmed { + hints.push(EOSE_HINT_AUTH.to_owned()); + } + + if hint_eligible { + if more { + hints.push(EOSE_HINT_MORE.to_owned()); + } else if !trimmed { + hints.push(EOSE_HINT_FINISH.to_owned()); + } + } + + context.sink.send(&RelayMessage::Eose(EoseMessage { + subscription_id, + hints, + })); } /// `canSubscribe` (`:258-301`): six reasons, the capture's strings — including diff --git a/crates/nostream-relay/tests/connection.rs b/crates/nostream-relay/tests/connection.rs index 1654e8f66..ad0893f6b 100644 --- a/crates/nostream-relay/tests/connection.rs +++ b/crates/nostream-relay/tests/connection.rs @@ -589,16 +589,18 @@ async fn restricted_read_with(authenticate_mid_read: bool) -> Vec { #[tokio::test] async fn a_restricted_read_is_judged_against_the_authenticated_set_at_each_event() { - // No AUTH: the public record arrives and the restricted one is withheld. + // No AUTH: the public record arrives and the restricted one is withheld, which is what + // NIP-67's `auth` hint reports (`67.md` §"Specification"). let withheld = restricted_read_with(false).await; assert_eq!(withheld.len(), 2, "{withheld:?}"); assert!(withheld[0].starts_with(r#"["EVENT","restricted","#)); assert!(withheld[0].contains("public")); - assert_eq!(withheld[1], r#"["EOSE","restricted"]"#); + assert_eq!(withheld[1], r#"["EOSE","restricted",["auth"]]"#); // AUTH between the two records: the second one is delivered, which a guard built once per - // read could not do. + // read could not do. Nothing was withheld, and the read carries no limit, so the `EOSE` + // keeps its two-element form. let authenticated = restricted_read_with(true).await; assert_eq!(authenticated.len(), 3, "{authenticated:?}"); diff --git a/crates/nostream-relay/tests/handlers.rs b/crates/nostream-relay/tests/handlers.rs index 6de0c50b1..084394007 100644 --- a/crates/nostream-relay/tests/handlers.rs +++ b/crates/nostream-relay/tests/handlers.rs @@ -1032,6 +1032,85 @@ async fn a_restricted_read_delivers_only_to_the_author_or_a_p_tagged_reader() { assert_eq!(frames[1], r#"["EOSE","mine"]"#); } +// --------------------------------------------------------------------------------------- +// NIP-67 EOSE hints +// --------------------------------------------------------------------------------------- + +/// `count` stored notes by the client's key, so a `limit` can sit above or below the count. +async fn seed_notes(fixture: &Fixture, count: usize) { + for index in 0..count { + fixture + .seed_event(signed_event( + &CLIENT_SECRET, + 1, + vec![], + &format!("note {index}"), + NOW_SECS, + )) + .await; + } +} + +/// Every frame but the last is an `EVENT` for the given subscription. +fn events_then_eose(frames: &[String]) -> bool { + let (last, events) = frames.split_last().expect("an EOSE at least"); + + events + .iter() + .all(|frame| frame.starts_with(r#"["EVENT","s1","#)) + && last.starts_with(r#"["EOSE","s1""#) +} + +/// Reading without a limit is the case the client's legacy heuristic is exact for, so the relay +/// adds nothing (`67.md` §"Specification": the third element is optional, and its absence is not +/// a claim). This is also the shape the sixteen recorded sessions pin. +#[tokio::test] +async fn eose_hints_are_absent_without_a_limit() { + let fixture = Fixture::new(settings()); + seed_notes(&fixture, 2).await; + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r#"["REQ","s1",{"kinds":[1]}]"#).await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 3, "{frames:?}"); + assert!(events_then_eose(&frames), "{frames:?}"); + assert_eq!(frames[2], r#"["EOSE","s1"]"#); +} + +/// The cap was not reached: every matching stored event has been sent. +#[tokio::test] +async fn eose_hints_finish_when_the_limit_was_not_reached() { + let fixture = Fixture::new(settings()); + seed_notes(&fixture, 2).await; + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r#"["REQ","s1",{"kinds":[1],"limit":3}]"#).await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 3, "{frames:?}"); + assert!(events_then_eose(&frames), "{frames:?}"); + assert_eq!(frames[2], r#"["EOSE","s1",["finish"]]"#); +} + +/// The cap was reached: the probe row exists, is not sent, and ends the read. +#[tokio::test] +async fn eose_hints_more_when_the_limit_was_reached() { + let fixture = Fixture::new(settings()); + seed_notes(&fixture, 3).await; + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r#"["REQ","s1",{"kinds":[1],"limit":2}]"#).await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 3, "exactly the limit's events: {frames:?}"); + assert!(events_then_eose(&frames), "{frames:?}"); + assert_eq!(frames[2], r#"["EOSE","s1",["more"]]"#); +} + #[tokio::test] async fn a_close_aborts_an_in_flight_read_without_a_frame() { let stored = signed_event(&CLIENT_SECRET, 1, vec![], "stored", NOW_SECS); From 8372a5b13cd03656c40734a3b7c2586f3b054491 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 00:55:43 -0500 Subject: [PATCH 03/36] feat(core): adopt NIP-44's extended length A plaintext of 65 536 bytes or more is written behind the six-byte [0x00, 0x00][u32] prefix, and the format's ceiling is the u32 that carries it: 4 294 967 295 bytes. The payload and data ceilings are derived from it at compile time through the same const fn the encoder uses. The length refusals are reachable as a function of a length alone, so a test can assert the 5 726 623 156-character base64 ceiling it cannot allocate. --- crates/nostream-core/src/nip44.rs | 221 ++++++++++++++++++--- crates/nostream-core/tests/nip44_parity.rs | 36 +++- 2 files changed, 225 insertions(+), 32 deletions(-) diff --git a/crates/nostream-core/src/nip44.rs b/crates/nostream-core/src/nip44.rs index 42049868f..f462dbe17 100644 --- a/crates/nostream-core/src/nip44.rs +++ b/crates/nostream-core/src/nip44.rs @@ -2,10 +2,22 @@ //! //! `base64(0x02 || nonce[32] || ciphertext || mac[32])`, where the ChaCha20 key, its 12-byte //! nonce and the HMAC-SHA256 key are HKDF-expanded from the conversation key and the -//! per-message nonce, and the plaintext is padded with a two-byte big-endian length prefix. +//! per-message nonce, and the plaintext is padded behind a length prefix: two big-endian bytes +//! below 65 536, the six-byte `[0x00, 0x00][u32 big-endian]` form at or above it (`44.md` +//! §"Encryption" step 4). The format's ceiling is that `u32`, so a plaintext may be +//! 4 294 967 295 bytes. //! //! The nonce is an argument: the capture generates a random one inside `nip44Encrypt`, and //! randomness is the relay's to supply (spec D13 for the clock, the same reasoning here). +//! +//! The format's ceiling is not the relay's. A payload that large cannot arrive over a socket: +//! `network.maxPayloadSize` (524288 in the shipped settings, +//! `resources/default-settings.yaml:209`) bounds both the WebSocket message and the frame the +//! relay reads (`crates/nostream-relay/src/transport.rs:49-76`), and an absent or zero value +//! falls back to 131072. Only a non-socket caller, such as `nostream-cli import`, can hand this +//! module something bigger. Taking the format's bound rather than inventing a smaller one is +//! deliberate: `44.md` §"Implementation considerations" asks an implementation for its own +//! maximum, and the transport already is this relay's. use base64::Engine; use base64::alphabet; @@ -23,20 +35,48 @@ use crate::event::Pubkey; type HmacSha256 = Hmac; const MIN_PLAINTEXT_SIZE: usize = 1; -const MAX_PLAINTEXT_SIZE: usize = 65_535; + +/// The format's ceiling, `u32::MAX` (`44.md` §"Details"). +const MAX_PLAINTEXT_SIZE: usize = 4_294_967_295; + +/// The plaintext length at which the prefix widens to the six-byte form. +const EXTENDED_PREFIX_THRESHOLD: usize = 65_536; + +/// The width of the two prefix forms. +const SHORT_PREFIX_LENGTH: usize = 2; +const EXTENDED_PREFIX_LENGTH: usize = 6; /// The NIP-44 version byte this module writes and accepts. const VERSION: u8 = 2; -/// The smallest and largest base64 payload lengths the capture accepts -/// (`src/utils/nip44.ts:120-122`). +/// The smallest base64 payload length the capture accepts (`src/utils/nip44.ts:120-122`). const MIN_PAYLOAD_LENGTH: usize = 132; -const MAX_PAYLOAD_LENGTH: usize = 87_472; -/// The smallest and largest decoded payload lengths the capture accepts -/// (`src/utils/nip44.ts:125-127`). +/// The largest base64 payload length: `MAX_DATA_LENGTH` bytes in whole base64 groups. Public +/// beside [`payload_length_refusal`] so a test can name the ceiling it cannot allocate. +pub const MAX_PAYLOAD_LENGTH: usize = base64_length(MAX_DATA_LENGTH); + +/// The smallest decoded payload length the capture accepts (`src/utils/nip44.ts:125-127`). const MIN_DATA_LENGTH: usize = 99; -const MAX_DATA_LENGTH: usize = 65_603; + +/// The largest decoded payload length: version, nonce, prefixed padded plaintext and MAC at +/// [`MAX_PLAINTEXT_SIZE`]. +const MAX_DATA_LENGTH: usize = + 1 + 32 + prefix_length(MAX_PLAINTEXT_SIZE) + calc_padded_len(MAX_PLAINTEXT_SIZE) + 32; + +/// The prefix width a plaintext of `unpadded_len` bytes is written with. +const fn prefix_length(unpadded_len: usize) -> usize { + if unpadded_len < EXTENDED_PREFIX_THRESHOLD { + SHORT_PREFIX_LENGTH + } else { + EXTENDED_PREFIX_LENGTH + } +} + +/// The base64 length of `data_len` bytes, rounded up to a whole four-character group. +const fn base64_length(data_len: usize) -> usize { + data_len.div_ceil(3) * 4 +} /// The capture decodes with Node's lenient `Buffer.from(payload, 'base64')`, which ignores /// non-canonical trailing bits (`src/utils/nip44.ts:124`). The strict `STANDARD` engine @@ -96,8 +136,8 @@ pub fn decrypt(payload: &str, conversation_key: &[u8; 32]) -> Result MAX_PAYLOAD_LENGTH { - return Err(invalid("invalid payload size")); + if let Some(refusal) = payload_length_refusal(payload.len()) { + return Err(invalid(refusal)); } // The same engine `validate_payload` uses: the capture decodes both with Node's lenient @@ -147,8 +187,8 @@ pub fn validate_payload(payload: &str) -> Option { return Some("unsupported encryption version".to_owned()); } - if payload.len() < MIN_PAYLOAD_LENGTH || payload.len() > MAX_PAYLOAD_LENGTH { - return Some("invalid payload size".to_owned()); + if let Some(refusal) = payload_length_refusal(payload.len()) { + return Some(refusal); } if !payload.len().is_multiple_of(4) || !is_base64(payload) { @@ -174,8 +214,9 @@ pub fn validate_payload(payload: &str) -> Option { } /// The length a plaintext of `unpadded_len` bytes is padded to (`calcPaddedLen`, -/// `src/utils/nip44.ts:63-71`). -fn calc_padded_len(unpadded_len: usize) -> usize { +/// `src/utils/nip44.ts:63-71`). `const` so the payload and data ceilings are derived from +/// [`MAX_PLAINTEXT_SIZE`] by the same arithmetic the encoder uses. +const fn calc_padded_len(unpadded_len: usize) -> usize { if unpadded_len <= 32 { return 32; } @@ -190,35 +231,86 @@ fn calc_padded_len(unpadded_len: usize) -> usize { chunk * ((unpadded_len - 1) / chunk + 1) } -/// `u16` big-endian length prefix, the plaintext, then zero padding (`pad`, -/// `src/utils/nip44.ts:72-82`). +/// The payload-length half of [`decrypt`] and [`validate_payload`], as a function of the length +/// alone: a caller that asks about a length it cannot allocate gets the same refusal the +/// decoder would give. +/// +/// shortcut: public where nothing outside this module and the parity test calls it, because the +/// format's ceiling is 5 726 623 156 base64 characters and no test can build one; fold it back +/// to a private helper when a non-test caller needs the refusal, or when `MAX_PAYLOAD_LENGTH` +/// stops being the assertion the test names. +pub fn payload_length_refusal(length: usize) -> Option { + (!(MIN_PAYLOAD_LENGTH..=MAX_PAYLOAD_LENGTH).contains(&length)) + .then(|| "invalid payload size".to_owned()) +} + +/// The plaintext-length half of [`pad`], as a function of the length alone — so the ceiling is +/// reachable by a test that cannot allocate it. +fn check_plaintext_length(length: usize) -> Result<(), CoreError> { + if !(MIN_PLAINTEXT_SIZE..=MAX_PLAINTEXT_SIZE).contains(&length) { + return Err(invalid("invalid plaintext length")); + } + + Ok(()) +} + +/// The length prefix, the plaintext, then zero padding (`pad`, `src/utils/nip44.ts:72-82`). +/// The prefix is the two-byte `u16` form below 65 536 bytes and the six-byte +/// `[0x00, 0x00][u32 big-endian]` form at or above it (`44.md` §"Encryption" step 4). fn pad(plaintext: &str) -> Result, CoreError> { let unpadded = plaintext.as_bytes(); + check_plaintext_length(unpadded.len())?; - if unpadded.len() < MIN_PLAINTEXT_SIZE || unpadded.len() > MAX_PLAINTEXT_SIZE { - return Err(invalid("invalid plaintext length")); + let prefix = prefix_length(unpadded.len()); + let padded_len = prefix + calc_padded_len(unpadded.len()); + let mut padded = Vec::with_capacity(padded_len); + + if prefix == SHORT_PREFIX_LENGTH { + padded.extend_from_slice(&(unpadded.len() as u16).to_be_bytes()); + } else { + padded.extend_from_slice(&[0x00, 0x00]); + padded.extend_from_slice(&(unpadded.len() as u32).to_be_bytes()); } - let mut padded = Vec::with_capacity(2 + calc_padded_len(unpadded.len())); - padded.extend_from_slice(&(unpadded.len() as u16).to_be_bytes()); padded.extend_from_slice(unpadded); - padded.resize(2 + calc_padded_len(unpadded.len()), 0); + padded.resize(padded_len, 0); Ok(padded) } -/// Reads the length prefix back and checks the padding (`unpad`, -/// `src/utils/nip44.ts:84-91`). +/// Reads the length prefix back and checks the padding (`unpad`, `src/utils/nip44.ts:84-91`). fn unpad(padded: &[u8]) -> Result { - let prefix = padded.get(..2).ok_or_else(|| invalid("invalid padding"))?; - let unpadded_len = usize::from(u16::from_be_bytes([prefix[0], prefix[1]])); + let marker = padded + .get(..SHORT_PREFIX_LENGTH) + .ok_or_else(|| invalid("invalid padding"))?; + let marker = u16::from_be_bytes([marker[0], marker[1]]); + + let (prefix, unpadded_len) = if marker == 0 { + let extended = padded + .get(..EXTENDED_PREFIX_LENGTH) + .ok_or_else(|| invalid("invalid padding"))?; + let length = u32::from_be_bytes( + extended[SHORT_PREFIX_LENGTH..] + .try_into() + .map_err(|_| invalid("invalid padding"))?, + ); + + (EXTENDED_PREFIX_LENGTH, length as usize) + } else { + (SHORT_PREFIX_LENGTH, usize::from(marker)) + }; - if unpadded_len == 0 || padded.len() != 2 + calc_padded_len(unpadded_len) { + // An extended prefix can only describe a length the short form cannot, so a smaller one + // under it is padding that no `pad` wrote. + if unpadded_len < MIN_PLAINTEXT_SIZE + || (prefix == EXTENDED_PREFIX_LENGTH && unpadded_len < EXTENDED_PREFIX_THRESHOLD) + || padded.len() != prefix + calc_padded_len(unpadded_len) + { return Err(invalid("invalid padding")); } let unpadded = padded - .get(2..2 + unpadded_len) + .get(prefix..prefix + unpadded_len) .ok_or_else(|| invalid("invalid padding"))?; String::from_utf8(unpadded.to_vec()).map_err(|_| invalid("invalid padding")) @@ -359,4 +451,79 @@ mod tests { Some("invalid payload size".to_owned()) ); } + + #[test] + fn the_ceilings_are_derived_from_the_formats_plaintext_ceiling() { + assert_eq!(MAX_DATA_LENGTH, 4_294_967_367); + assert_eq!(MAX_PAYLOAD_LENGTH, 5_726_623_156); + assert_eq!(MAX_PAYLOAD_LENGTH, base64_length(MAX_DATA_LENGTH)); + } + + #[test] + fn plaintext_lengths_are_checked_at_the_formats_bounds() { + for (length, allowed) in [ + (0, false), + (MIN_PLAINTEXT_SIZE, true), + (65_535, true), + (65_536, true), + (MAX_PLAINTEXT_SIZE, true), + (MAX_PLAINTEXT_SIZE + 1, false), + ] { + assert_eq!( + check_plaintext_length(length).is_ok(), + allowed, + "length {length}" + ); + } + } + + #[test] + fn payload_lengths_are_checked_at_the_derived_bounds() { + for (length, refused) in [ + (MIN_PAYLOAD_LENGTH - 1, true), + (MIN_PAYLOAD_LENGTH, false), + (MAX_PAYLOAD_LENGTH, false), + (MAX_PAYLOAD_LENGTH + 1, true), + ] { + assert_eq!( + payload_length_refusal(length), + refused.then(|| "invalid payload size".to_owned()), + "length {length}" + ); + } + } + + #[test] + fn the_prefix_widens_at_65_536_bytes() { + let short = pad(&"x".repeat(65_535)).unwrap(); + + assert_eq!(&short[..2], &[0xff, 0xff]); + assert_eq!(short.len(), 2 + calc_padded_len(65_535)); + assert_eq!(unpad(&short).unwrap(), "x".repeat(65_535)); + + let extended = pad(&"x".repeat(65_536)).unwrap(); + + assert_eq!(&extended[..6], &[0x00, 0x00, 0x00, 0x01, 0x00, 0x00]); + assert_eq!(extended.len(), 6 + calc_padded_len(65_536)); + assert_eq!(unpad(&extended).unwrap(), "x".repeat(65_536)); + } + + #[test] + fn unpad_rejects_an_extended_prefix_below_the_threshold() { + // `[0x00, 0x00][u32 = 65 535]` followed by a consistent padding: the length is one the + // short form carries, so no `pad` wrote this prefix. + let mut padded = vec![0x00, 0x00, 0x00, 0x00, 0xff, 0xff]; + padded.resize(EXTENDED_PREFIX_LENGTH + calc_padded_len(65_535), 0); + + assert_eq!(unpad(&padded).unwrap_err().to_string(), "invalid padding"); + } + + #[test] + fn unpad_rejects_a_length_the_padding_does_not_cover() { + // `[0x00, 0x00][u32 = 100 000]` with 65 536 bytes of padding behind it. + let mut padded = vec![0x00, 0x00, 0x00, 0x01, 0x86, 0xa0]; + padded.resize(EXTENDED_PREFIX_LENGTH + calc_padded_len(65_536), 0); + + assert_eq!(unpad(&padded).unwrap_err().to_string(), "invalid padding"); + } } diff --git a/crates/nostream-core/tests/nip44_parity.rs b/crates/nostream-core/tests/nip44_parity.rs index 4bed622f7..431bcbbe7 100644 --- a/crates/nostream-core/tests/nip44_parity.rs +++ b/crates/nostream-core/tests/nip44_parity.rs @@ -8,6 +8,13 @@ //! random one passes a fixed nonce instead — the "valid base64 string" and "different //! ciphertexts" cases are exactly that re-expression. The ten padding cases observe the //! padded length through the payload size, as the capture does. +//! +//! One case **diverges from the capture by design**: `44.md` at the audit cutoff carries a +//! `u32` plaintext length with the six-byte `[0x00, 0x00][u32]` prefix, so the capture's +//! "throws for plaintext exceeding 65535 bytes" case becomes the round trip at 65 536 bytes +//! (`accepts_a_plaintext_of_65536_bytes_with_the_extended_prefix`), and the payload ceiling +//! moves to the one derived from the format's. `docs/rewrite/nip-parity.md` §9 item 5 and +//! §10 record it. #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] @@ -15,7 +22,10 @@ use base64::Engine; use base64::engine::general_purpose::STANDARD; use nostream_core::Pubkey; use nostream_core::crypto::public_key; -use nostream_core::nip44::{decrypt, encrypt, get_conversation_key, validate_payload}; +use nostream_core::nip44::{ + MAX_PAYLOAD_LENGTH, decrypt, encrypt, get_conversation_key, payload_length_refusal, + validate_payload, +}; const KNOWN_CONVERSATION_KEY: &str = "c41c775356fd92eadc63ff5a0dc1da211b268cbea22316767095b2871ea1412d"; @@ -131,12 +141,25 @@ fn throws_for_empty_plaintext() { assert_eq!(error.to_string(), "invalid plaintext length"); } +/// The format's extended length: a plaintext of 65 536 bytes is written behind the six-byte +/// prefix and round-trips. Against the pre-`733a047` format the capture implements, `encrypt` +/// answers `invalid plaintext length` here. +/// +/// The prefix itself is inside the encrypted region, so it is not readable in the payload; its +/// width is what the decoded length pins (a two-byte prefix would make this 65 603 bytes, not +/// 65 607), and `decrypt` succeeding is the proof that `unpad` read `[0x00, 0x00][u32]`: read as +/// a `u16`, those bytes are a length of zero. The prefix bytes as written are asserted where +/// they are visible, in `nip44`'s own `the_prefix_widens_at_65_536_bytes`. #[test] -fn throws_for_plaintext_exceeding_65535_bytes() { +fn accepts_a_plaintext_of_65536_bytes_with_the_extended_prefix() { let key = conversation_key(1, 2); - let error = encrypt(&"x".repeat(65_536), &key, &nonce(1)).unwrap_err(); + let plaintext = "x".repeat(65_536); + let payload = encrypt(&plaintext, &key, &nonce(1)).unwrap(); + let decoded = STANDARD.decode(&payload).unwrap(); - assert_eq!(error.to_string(), "invalid plaintext length"); + // version, nonce, the six-byte prefix, the padded plaintext, the MAC. + assert_eq!(decoded.len(), 1 + 32 + 6 + 65_536 + 32); + assert_eq!(decrypt(&payload, &key).unwrap(), plaintext); } // --- nip44Decrypt (7) -------------------------------------------------------------------- @@ -237,12 +260,15 @@ fn returns_error_string_for_payload_that_is_too_short() { ); } +/// One byte past the base64 ceiling the format derives. The ceiling itself is 5 726 623 156 +/// characters, so the refusal is asked of the length helper rather than of a string that size. #[test] fn returns_error_string_for_payload_that_is_too_long() { assert_eq!( - validate_payload(&"A".repeat(87_473)), + payload_length_refusal(MAX_PAYLOAD_LENGTH + 1), Some("invalid payload size".to_owned()) ); + assert_eq!(payload_length_refusal(MAX_PAYLOAD_LENGTH), None); } #[test] From fb0a6112753fb7588ea47c97da5280ef395461dc Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 00:55:43 -0500 Subject: [PATCH 04/36] feat(relay): delete addressable events named by a NIP-09 a tag An a tag's :: resolves to the author's live addressable row when the row is not newer than the request, and its id joins the e-tag ids in one delete_by_pubkey_and_ids. The pubkey must equal the request's author, and the storage call re-checks it independently. --- .../nostream-relay/src/strategies/delete.rs | 99 ++++++++++++++++++- crates/nostream-relay/tests/strategies.rs | 88 +++++++++++++++++ 2 files changed, 183 insertions(+), 4 deletions(-) diff --git a/crates/nostream-relay/src/strategies/delete.rs b/crates/nostream-relay/src/strategies/delete.rs index af6604dab..8f19f3a04 100644 --- a/crates/nostream-relay/src/strategies/delete.rs +++ b/crates/nostream-relay/src/strategies/delete.rs @@ -1,15 +1,18 @@ //! The `delete` event strategy. Mirrors `src/handlers/event-strategies/delete-event-strategy.ts`. //! //! A NIP-09 request: every `e` tag whose value parses as an event id is soft-deleted — exact -//! 32 bytes of lowercase hex, which is what `EventId`'s own parser accepts — **before** the -//! request itself is stored, so a client that re-reads after the `OK` sees the deletions +//! 32 bytes of lowercase hex, which is what `EventId`'s own parser accepts — and every `a` tag +//! naming one of the author's addressable events resolves to that event's id (`09.md` +//! §"Deletion Request" covers both the event id and the addressable `a` form). The deletions run +//! **before** the request itself is stored, so a client that re-reads after the `OK` sees them //! already applied. Then the common create-or-dedupe. use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use nostream_core::{Event, EventId}; +use futures_util::StreamExt; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey}; use crate::error::RelayError; use crate::handlers::HandlerContext; @@ -26,13 +29,19 @@ impl EventStrategy for DeleteStrategy { context: &'a HandlerContext, ) -> Pin> + Send + 'a>> { Box::pin(async move { - let ids: Vec = event + let mut ids: Vec = event .tags .iter() .filter(|tag| tag.len() >= 2 && tag[0] == "e") .filter_map(|tag| tag[1].parse::().ok()) .collect(); + for address in addresses(event) { + if let Some(id) = resolve(&address, event, context).await? { + ids.push(id); + } + } + if !ids.is_empty() { context .stores @@ -59,3 +68,85 @@ impl EventStrategy for DeleteStrategy { }) } } + +/// One `a` tag's address, kept only when the request may name it. +struct Address { + kind: Kind, + /// The `d` identifier, `""` for a tag that omits it — the key the addressable strategy + /// stores a row without a `d` tag under. + d: String, +} + +/// The addresses a deletion request may act on: `::`, split into at most three +/// parts so a `d` may hold colons of its own. +/// +/// `09.md`'s `a` branch is about addressable events, and only those carry a `d` in their +/// address, so a regular replaceable kind named by an `a` tag is ignored. An address naming +/// another author is not this request's to delete: the pubkey must be the request's own. The +/// storage call re-checks the author as a second, independent guard. +fn addresses(event: &Event) -> Vec
{ + event + .tags + .iter() + .filter(|tag| tag.len() >= 2 && tag[0] == "a") + .filter_map(|tag| { + let mut parts = tag[1].splitn(3, ':'); + let kind = Kind(parts.next()?.parse::().ok()?); + let pubkey = parts.next()?.parse::().ok()?; + let d = parts.next().unwrap_or_default().to_owned(); + + (kind.is_parameterized_replaceable() && pubkey == event.pubkey) + .then_some(Address { kind, d }) + }) + .collect() +} + +/// The addressable row an address names, if it is still live and no newer than the request. +/// +/// Addressable kinds are stored one live row per `(pubkey, kind, d)` (`parameterized_replaceable` +/// makes the first `d` tag the deduplication key), so the author's rows of that kind are read and +/// the one whose first `d` tag equals the address's is kept. `09.md` asks for *"all versions … +/// up to the `created_at` timestamp of the deletion request"*, so a row newer than the request +/// survives. +async fn resolve( + address: &Address, + request: &Event, + context: &HandlerContext, +) -> Result, RelayError> { + let filters = [Filter { + kinds: Some(vec![address.kind]), + authors: Some(vec![request.pubkey.to_string()]), + ..Default::default() + }]; + + let mut rows = context.stores.events.find_by_filters(&filters).await?; + + while let Some(row) = rows.next().await { + let row = row?; + + if row.deleted_at.is_some() { + continue; + } + + let stored = &row.event; + + if d_identifier(stored) != address.d || stored.created_at > request.created_at { + continue; + } + + return Ok(Some(stored.id)); + } + + Ok(None) +} + +/// The value of an event's first `d` tag, `""` when it carries none — the same key the +/// addressable strategy deduplicates on. +fn d_identifier(event: &Event) -> &str { + event + .tags + .iter() + .find(|tag| tag.len() >= 2 && tag[0] == "d") + .map(|tag| tag[1].as_str()) + .unwrap_or_default() +} diff --git a/crates/nostream-relay/tests/strategies.rs b/crates/nostream-relay/tests/strategies.rs index 0fbb605e0..9705ded94 100644 --- a/crates/nostream-relay/tests/strategies.rs +++ b/crates/nostream-relay/tests/strategies.rs @@ -442,6 +442,94 @@ async fn delete_returns_an_error_when_the_create_fails() { assert_eq!(frames(&mut sink), Vec::::new()); } +// --- NIP-09 `a` tags (09.md §"Deletion Request") ----------------------------------------- + +/// A kind-30023 row carrying `d`, under [`other_event_id`] so it cannot be confused with the +/// request. +fn addressable_row(d: Option<&str>, created_at: i64) -> Event { + Event { + id: other_event_id(), + created_at, + ..event( + 30_023, + d.map(|value| vec![tag(&["d", value])]).unwrap_or_default(), + "a long-form post", + ) + } +} + +/// A kind-5 request naming `30023::`. +fn address_delete_event(address: &str) -> Event { + event(5, vec![tag(&["a", address])], "") +} + +#[tokio::test] +async fn delete_soft_deletes_an_addressable_event_named_by_an_a_tag() { + let fixture = Fixture::new(settings()); + let target = addressable_row(Some("post-1"), NOW_SECS); + fixture.seed_event(target.clone()).await; + + let (context, mut sink) = socket(&fixture); + let event = address_delete_event(&format!("30023:{}:post-1", author())); + + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!(fixture.events.deletes(), vec![(author(), vec![target.id])]); + assert_eq!(fixture.events.live_events(), vec![event.clone()]); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + +#[tokio::test] +async fn delete_ignores_an_a_tag_naming_another_authors_address() { + let fixture = Fixture::new(settings()); + let target = addressable_row(Some("post-1"), NOW_SECS); + fixture.seed_event(target.clone()).await; + + let (context, mut sink) = socket(&fixture); + // The address names `author()`, but the request is signed by somebody else. + let mut event = address_delete_event(&format!("30023:{}:post-1", author())); + event.pubkey = Pubkey::from_byte_array([0x44; 32]); + + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!(fixture.events.deletes(), Vec::new()); + assert!(fixture.events.live_events().contains(&target)); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + +#[tokio::test] +async fn delete_leaves_an_addressable_event_newer_than_the_request() { + let fixture = Fixture::new(settings()); + let target = addressable_row(Some("post-1"), NOW_SECS + 10); + fixture.seed_event(target.clone()).await; + + let (context, mut sink) = socket(&fixture); + let event = address_delete_event(&format!("30023:{}:post-1", author())); + + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!(fixture.events.deletes(), Vec::new()); + assert!(fixture.events.live_events().contains(&target)); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + +#[tokio::test] +async fn delete_matches_an_empty_d_identifier() { + let fixture = Fixture::new(settings()); + // A row with no `d` tag is the one stored under the address's empty `d`. + let target = addressable_row(None, NOW_SECS); + fixture.seed_event(target.clone()).await; + + let (context, mut sink) = socket(&fixture); + let event = address_delete_event(&format!("30023:{}:", author())); + + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!(fixture.events.deletes(), vec![(author(), vec![target.id])]); + assert_eq!(fixture.events.live_events(), vec![event.clone()]); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + // --------------------------------------------------------------------------- // dvm_job_request — `dvm-job-request-event-strategy.spec.ts` // --------------------------------------------------------------------------- From 7903ef4bf1b80ecc06aa6710f114bff87e5d069c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 00:55:43 -0500 Subject: [PATCH 05/36] test(relay): pin the six owed NIP vectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Socket tests for NIP-14, 22, 28, 50, 65 and 89 — the must-have rows the audit found live and named by no test. NIP-50's ranking stays with the Postgres-backed parity target, and the test records why. --- crates/nostream-relay/tests/nip_vectors.rs | 368 +++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 crates/nostream-relay/tests/nip_vectors.rs diff --git a/crates/nostream-relay/tests/nip_vectors.rs b/crates/nostream-relay/tests/nip_vectors.rs new file mode 100644 index 000000000..9648beb52 --- /dev/null +++ b/crates/nostream-relay/tests/nip_vectors.rs @@ -0,0 +1,368 @@ +//! The vectors the audit found owed: one socket test per NIP that was live in the capture and +//! named by no test (`docs/rewrite/nip-parity.md` §9, R9). +//! +//! Five of these are passive accept-and-serve behaviours — the relay stores the NIP's event shape +//! and hands it back unchanged, which is the same shape the capture's own NIP-89 integration +//! feature takes (*"passive accept-and-serve"*). NIP-50 is a read-side behaviour with a runnable +//! in-memory oracle: `Filter::matches` implements the capture's search matcher. What no test here +//! reaches is NIP-50's **ranking**, which lives in the SQL backend's `ts_rank` and needs +//! Postgres, so it stays with the `#[ignore]`d `parity` target (`docs/rewrite/nip-parity.md` +//! §10). +//! +//! Each test proves the relay accepts and returns the NIP's shape. None proves the NIP's +//! client-side tag rules are enforced: kind 1111 stays on the default arm, and 31989/31990 are +//! addressable only because they fall in 30000-39999. + +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod common; + +use common::{CLIENT_SECRET, NOW_SECS, Server, fast_config, relay_keys, signed_event}; +use nostream_config::Settings; +use nostream_core::Event; +use nostream_relay::testing::{Fixture, nip50_settings, settings}; + +/// A relay over `document` with the test relay key: without one, every publish is answered +/// `error: unable to process event` (the capture's missing-`SECRET` path). +async fn start(document: Settings) -> Server { + Server::start_with(Fixture::new(document), relay_keys(), fast_config()).await +} + +/// The shipped `nip50` document, whose `enabled` is unset and so off. +fn nip50_off() -> Settings { + let mut settings = settings(); + settings.nip50 = Some(nip50_settings()); + + settings +} + +/// The same document with NIP-50 on. +fn nip50_on() -> Settings { + let mut settings = settings(); + settings.nip50 = Some(nostream_config::settings::Nip50Settings { + enabled: Some(true), + ..nip50_settings() + }); + + settings +} + +/// The event frame a client sends for one signed event. +fn event_frame(event: &Event) -> String { + format!( + r#"["EVENT",{}]"#, + serde_json::to_string(event).expect("event json") + ) +} + +/// The `OK` frame an accepted event's publish is answered with. +fn accepted(event: &Event) -> String { + format!(r#"["OK","{}",true,""]"#, event.id) +} + +/// Publishes each event, asserting the relay accepted it. +async fn publish(server: &Server, events: &[&Event]) { + let (mut client, _challenge) = server.connect().await; + + for event in events { + client.send(&event_frame(event)).await; + + assert_eq!(client.expect_frame().await, accepted(event)); + } +} + +/// Reads `filters` on a fresh subscription and returns its frames, `EOSE` last. +async fn read(server: &Server, subscription_id: &str, filters: &str) -> Vec { + let (mut client, _challenge) = server.connect().await; + + client + .send(&format!(r#"["REQ","{subscription_id}",{filters}]"#)) + .await; + + client.read_until_eose().await +} + +// --------------------------------------------------------------------------------------- +// NIP-14 — Subject tag +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn nip14_a_kind_1_event_returns_its_subject_tag() { + let server = start(settings()).await; + let event = signed_event( + &CLIENT_SECRET, + 1, + vec![vec!["subject", "A test subject"]], + "hello", + NOW_SECS, + ); + + publish(&server, &[&event]).await; + + let frames = read(&server, "s1", r#"{"kinds":[1]}"#).await; + + assert_eq!(frames.len(), 2, "{frames:?}"); + assert!(frames[0].contains(r#""kind":1"#), "{}", frames[0]); + assert!( + frames[0].contains(r#"["subject","A test subject"]"#), + "{}", + frames[0] + ); +} + +// --------------------------------------------------------------------------------------- +// NIP-22 — Comments (kind 1111) +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn nip22_a_kind_1111_comment_is_stored_and_served_unchanged() { + let server = start(settings()).await; + let root = signed_event(&CLIENT_SECRET, 1, vec![], "the root note", NOW_SECS); + let root_id = root.id.to_string(); + let root_author = root.pubkey.to_string(); + // The NIP's shape: the root scope in uppercase (`K`/`E`/`P`) and the parent scope in + // lowercase (`k`/`e`/`p`). Here the comment replies to the root, so both scopes name it. + let comment = signed_event( + &CLIENT_SECRET, + 1111, + vec![ + vec!["K", "1"], + vec!["E", root_id.as_str()], + vec!["P", root_author.as_str()], + vec!["k", "1"], + vec!["e", root_id.as_str()], + vec!["p", root_author.as_str()], + ], + "a comment", + NOW_SECS, + ); + + publish(&server, &[&root, &comment]).await; + + let frames = read(&server, "s1", r#"{"kinds":[1111]}"#).await; + + assert_eq!(frames.len(), 2, "{frames:?}"); + let expected = [ + r#"["K","1"]"#.to_owned(), + format!(r#"["E","{root_id}"]"#), + format!(r#"["P","{root_author}"]"#), + r#"["k","1"]"#.to_owned(), + format!(r#"["e","{root_id}"]"#), + format!(r#"["p","{root_author}"]"#), + ]; + + for tag in expected { + assert!(frames[0].contains(&tag), "{tag} missing from {}", frames[0]); + } +} + +// --------------------------------------------------------------------------------------- +// NIP-28 — Public chat channels (kinds 40, 41, 42) +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn nip28_channel_metadata_is_replaceable_and_channel_events_are_not() { + let server = start(settings()).await; + let create = signed_event( + &CLIENT_SECRET, + 40, + vec![vec!["name", "the channel"]], + "", + NOW_SECS, + ); + let channel_id = create.id.to_string(); + let metadata = signed_event( + &CLIENT_SECRET, + 41, + vec![vec!["e", channel_id.as_str()], vec!["name", "first name"]], + "", + NOW_SECS, + ); + let message = signed_event( + &CLIENT_SECRET, + 42, + vec![vec!["e", channel_id.as_str()]], + "hello channel", + NOW_SECS, + ); + // A newer kind 41 for the same author replaces the first. + let newer_metadata = signed_event( + &CLIENT_SECRET, + 41, + vec![vec!["e", channel_id.as_str()], vec!["name", "second name"]], + "", + NOW_SECS + 1, + ); + + publish(&server, &[&create, &metadata, &message, &newer_metadata]).await; + + let frames = read(&server, "s1", r#"{"kinds":[40,41,42]}"#).await; + + assert_eq!(frames.len(), 4, "three events and the EOSE: {frames:?}"); + assert!( + frames[..3] + .iter() + .any(|frame| frame.contains(&format!(r#""id":"{channel_id}""#))) + ); + assert!( + frames[..3] + .iter() + .any(|frame| frame.contains(&format!(r#""id":"{}""#, message.id))) + ); + assert!( + frames[..3] + .iter() + .any(|frame| frame.contains(&format!(r#""id":"{}""#, newer_metadata.id))), + "the newer kind 41 replaced the first: {frames:?}" + ); + assert!( + !frames[..3] + .iter() + .any(|frame| frame.contains(&format!(r#""id":"{}""#, metadata.id))), + "the replaced kind 41 is not served: {frames:?}" + ); +} + +// --------------------------------------------------------------------------------------- +// NIP-50 — Search +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn nip50_a_search_filter_is_served_only_when_enabled() { + let matching = signed_event( + &CLIENT_SECRET, + 1, + vec![], + "a note about nostalgia", + NOW_SECS, + ); + let other = signed_event( + &CLIENT_SECRET, + 1, + vec![], + "a note about something else", + NOW_SECS, + ); + + // The shipped document has NIP-50 off, and the search criterion is dropped rather than + // refused: the read is the plain kind-1 read. + let off = start(nip50_off()).await; + publish(&off, &[&matching, &other]).await; + + let frames = read(&off, "s1", r#"{"search":"nostalgia","kinds":[1]}"#).await; + + assert_eq!(frames.len(), 3, "both events with search off: {frames:?}"); + + // With NIP-50 on, the matcher decides. + let on = start(nip50_on()).await; + publish(&on, &[&matching, &other]).await; + + let frames = read(&on, "s1", r#"{"search":"nostalgia","kinds":[1]}"#).await; + + assert_eq!(frames.len(), 2, "only the matching event: {frames:?}"); + assert!( + frames[0].contains("nostalgia"), + "the served event is the matching one: {}", + frames[0] + ); + assert_eq!(frames[1], r#"["EOSE","s1"]"#); +} + +// --------------------------------------------------------------------------------------- +// NIP-65 — Relay lists (kind 10002) +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn nip65_a_relay_list_is_stored_and_replaces_the_previous_one() { + let server = start(settings()).await; + let first = signed_event( + &CLIENT_SECRET, + 10_002, + vec![ + vec!["r", "wss://relay.one.example"], + vec!["r", "wss://relay.two.example"], + vec!["r", "wss://relay.three.example"], + vec!["r", "wss://relay.four.example"], + ], + "", + NOW_SECS, + ); + let second = signed_event( + &CLIENT_SECRET, + 10_002, + vec![vec!["r", "wss://relay.five.example"]], + "", + NOW_SECS + 1, + ); + + publish(&server, &[&first, &second]).await; + + let frames = read(&server, "s1", r#"{"kinds":[10002]}"#).await; + + assert_eq!(frames.len(), 2, "one relay list and the EOSE: {frames:?}"); + assert!(frames[0].contains(&format!(r#""id":"{}""#, second.id))); + assert!(!frames[0].contains(&format!(r#""id":"{}""#, first.id))); + + // NIP-65 makes every `r` value a URL, and the event schema is what refuses a bad one, before + // any handler sees the event. + let refused = signed_event( + &CLIENT_SECRET, + 10_002, + vec![vec!["r", "not a url"]], + "", + NOW_SECS + 2, + ); + let (mut client, _challenge) = server.connect().await; + + client.send(&event_frame(&refused)).await; + + assert_eq!( + client.expect_frame().await, + r#"["NOTICE","invalid: Invalid relay URL"]"# + ); +} + +// --------------------------------------------------------------------------------------- +// NIP-89 — Recommended application handlers (kinds 31989, 31990) +// --------------------------------------------------------------------------------------- + +#[tokio::test] +async fn nip89_recommendation_and_handler_events_are_stored_and_served() { + let server = start(settings()).await; + let handler = signed_event( + &CLIENT_SECRET, + 31_990, + vec![ + vec!["d", "example-app"], + vec!["k", "1"], + vec!["web", "https://example.com/app"], + ], + "", + NOW_SECS, + ); + let address = format!("31990:{}:example-app", handler.pubkey); + let recommendation = signed_event( + &CLIENT_SECRET, + 31_989, + vec![vec!["d", "1"], vec!["a", address.as_str()]], + "", + NOW_SECS, + ); + // A second recommendation under the same `d` replaces the first, because 31989 is addressable. + let newer_recommendation = signed_event( + &CLIENT_SECRET, + 31_989, + vec![vec!["d", "1"], vec!["a", address.as_str()], vec!["k", "1"]], + "", + NOW_SECS + 1, + ); + + publish(&server, &[&handler, &recommendation, &newer_recommendation]).await; + + let frames = read(&server, "s1", r#"{"kinds":[31989,31990]}"#).await; + + assert_eq!(frames.len(), 3, "both kinds and the EOSE: {frames:?}"); + assert!(frames[0].contains(&format!(r#""id":"{}""#, handler.id))); + assert!(frames[1].contains(&format!(r#""id":"{}""#, newer_recommendation.id))); + assert!(!frames[1].contains(&format!(r#""id":"{}""#, recommendation.id))); +} From 0f265169442ae5de0a15025359d7020a9184fd6d Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 00:55:43 -0500 Subject: [PATCH 06/36] test(relay): record the nip-generic-events harness session One session publishing and reading back the NIP-14, 22, 28, 65 and 89 shapes, recorded from the capture tree at 242a3001. Its seven stored read-backs carry the capture's row key order, so they declare the fixture's existing JSON comparison; the declared-tolerance count stays at one and the not-byte-compared ledger lists the new steps. --- .../tests/capture/gen-wire-sessions.mjs | 83 +++ .../tests/fixtures/wire-sessions.json | 583 ++++++++++++++++++ crates/nostream-relay/tests/wire_parity.rs | 21 +- 3 files changed, 685 insertions(+), 2 deletions(-) diff --git a/crates/nostream-relay/tests/capture/gen-wire-sessions.mjs b/crates/nostream-relay/tests/capture/gen-wire-sessions.mjs index eece0dcbf..a89b9e6eb 100644 --- a/crates/nostream-relay/tests/capture/gen-wire-sessions.mjs +++ b/crates/nostream-relay/tests/capture/gen-wire-sessions.mjs @@ -995,6 +995,89 @@ const runAll = async () => { }, }) + + // 17. NIP-14/22/28/65/89: the event shapes the generic store-and-serve path carries, published + // and then read back. No filter carries a `limit` and no kind is restricted, so no step can + // depend on a NIP-67 hint; each kind is published once, so nothing depends on the doubles' + // missing replaceability either. The read-back frames are compared as JSON values, not bytes, + // because a stored row's key order is the capture's and every other EVENT frame in the fixture + // is the other path's. + await runSession({ + id: 'nip-generic-events', + settings: settingsFor({ limits: noConnectionLimit(baseLimits()) }), + script: async ({ send }) => { + const subject = makeEvent({ + kind: 1, + content: 'nip14', + tags: [['subject', 'A test subject']], + }) + const rootId = 'aa'.repeat(32) + const rootAuthor = 'bb'.repeat(32) + const comment = makeEvent({ + kind: 1111, + content: 'nip22', + tags: [ + ['K', '1'], + ['E', rootId], + ['P', rootAuthor], + ['k', '1'], + ['e', rootId], + ['p', rootAuthor], + ], + }) + const channel = makeEvent({ + kind: 40, + content: 'nip28', + tags: [['name', 'the channel']], + }) + const channelMessage = makeEvent({ + kind: 42, + content: 'nip28 message', + tags: [['e', channel.id]], + }) + const relayList = makeEvent({ + kind: 10002, + content: '', + tags: [['r', 'wss://relay.one.example']], + }) + const handler = makeEvent({ + kind: 31990, + content: '', + tags: [ + ['d', 'example-app'], + ['k', '1'], + ['web', 'https://example.com/app'], + ], + }) + const recommendation = makeEvent({ + kind: 31989, + content: '', + tags: [['d', '1'], ['a', `31990:${CLIENT_PUBKEY}:example-app`]], + }) + + for (const event of [subject, comment, channel, channelMessage, relayList, handler, recommendation]) { + await send(['EVENT', event], { expect: 1 }) + } + + // One filter per kind, so no ordering between rows is asserted, and no filter carries a + // `limit`, so the EOSE keeps its two-element form on both sides. + for (const [id, kind] of [ + ['v1', 1], + ['v2', 1111], + ['v3', 40], + ['v4', 42], + ['v5', 10002], + ['v6', 31990], + ['v7', 31989], + ]) { + await send(['REQ', id, { kinds: [kind] }], { + expect: 2, + compare: 'json', + why: 'a REQ result carries the capture’s stored-row field order; the values are compared, the key order is the capture’s — job 025', + }) + } + }, + }) } // The stream restarts here: module loading draws nothing this fixture wants to replay, and the diff --git a/crates/nostream-relay/tests/fixtures/wire-sessions.json b/crates/nostream-relay/tests/fixtures/wire-sessions.json index 678543bba..f2bb91064 100644 --- a/crates/nostream-relay/tests/fixtures/wire-sessions.json +++ b/crates/nostream-relay/tests/fixtures/wire-sessions.json @@ -7475,6 +7475,589 @@ "challenge": "836083c53e97aeba32c1e6c0836090361eb77a252ad8a18b562b463195d50a14", "invite_code": null } + }, + { + "id": "nip-generic-events", + "challenge": "YEyYCe_MOiyZ-Yp4GLqr_M5GMBN3vb4rAHks19zbx6o", + "clock": { + "start_ms": 1700000000000 + }, + "seed": 20261023, + "settings": { + "info": { + "relay_url": "wss://nostream.your-domain.com", + "name": "nostream.your-domain.com", + "description": "A nostr relay written in Typescript.", + "banner": "https://nostream.your-domain.com/banner.png", + "icon": "https://nostream.your-domain.com/icon.png", + "pubkey": "replace-with-your-pubkey-in-hex", + "contact": "mailto:operator@your-domain.com", + "terms_of_service": "https://nostream.your-domain.com/terms" + }, + "payments": { + "enabled": false, + "processor": "zebedee", + "feeSchedules": { + "admission": [ + { + "enabled": false, + "description": "Admission fee charged per public key in msats (1000 msats = 1 satoshi)", + "amount": 1000000, + "whitelists": { + "pubkeys": [ + "replace-with-your-pubkey-in-hex" + ], + "event_kinds": [ + 9735 + ] + } + } + ] + }, + "invoiceExpirySeconds": 86400 + }, + "paymentsProcessors": { + "zebedee": { + "baseURL": "https://api.zebedee.io/", + "callbackBaseURL": "https://nostream.your-domain.com/callbacks/zebedee", + "ipWhitelist": [ + "3.225.112.64", + "::ffff:3.225.112.64" + ] + }, + "lnbits": { + "baseURL": "https://lnbits.your-domain.com/", + "callbackBaseURL": "https://nostream.your-domain.com/callbacks/lnbits" + }, + "lnurl": { + "invoiceURL": "https://getalby.com/lnurlp/your-username" + }, + "nodeless": { + "baseURL": "https://nodeless.io", + "storeId": "your-nodeless-io-store-id" + }, + "opennode": { + "baseURL": "api.opennode.com", + "callbackBaseURL": "https://nostream.your-domain.com/callbacks/opennode" + }, + "nwc": { + "invoiceExpirySeconds": 900, + "replyTimeoutMs": 10000 + } + }, + "nip05": { + "mode": "disabled", + "verifyExpiration": 604800000, + "verifyUpdateFrequency": 86400000, + "maxConsecutiveFailures": 20, + "domainWhitelist": [], + "domainBlacklist": [] + }, + "nip42": { + "authRequired": false, + "sessionExpirySeconds": 0, + "restrictedReads": { + "enabled": false, + "kinds": [ + 4, + 1059 + ] + } + }, + "nip43": { + "enabled": false, + "inviteCodeExpirySeconds": 600, + "defaultMaxUses": 1, + "allowInviteRequests": false, + "inviteRequestWhitelist": [] + }, + "nip45": { + "enabled": true + }, + "nip50": { + "enabled": false, + "language": "simple", + "maxQueryLength": 256 + }, + "nip66": { + "enabled": false, + "probeIntervalSeconds": 3600, + "timeouts": { + "dnsMs": 10000, + "tlsMs": 10000, + "wsRttMs": 10000, + "nip11Ms": 10000 + }, + "targets": [], + "dnsCacheTtlSeconds": 300 + }, + "wot": { + "enabled": false, + "seedPubkey": "", + "minimumFollowers": 1, + "maxDepth": 2, + "refreshIntervalHours": 24 + }, + "nip56": { + "enabled": false, + "trustedModerators": [], + "hideActionableReports": false + }, + "network": { + "maxPayloadSize": 524288, + "trustedProxies": [ + "127.0.0.1", + "::ffff:127.0.0.1", + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + }, + "workers": { + "count": 0 + }, + "eventStore": { + "backend": "postgres" + }, + "mirroring": { + "static": [] + }, + "dvm": { + "workers": [] + }, + "limits": { + "rateLimiter": { + "strategy": "ewma" + }, + "invoice": { + "rateLimits": [ + { + "period": 60000, + "rate": 12 + }, + { + "period": 3600000, + "rate": 30 + } + ], + "ipWhitelist": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + }, + "invite": { + "rateLimits": [ + { + "description": "5 invite requests per hour per pubkey", + "period": 3600000, + "rate": 5 + } + ] + }, + "admissionCheck": { + "rateLimits": [ + { + "description": "30 admission checks/min or 1 check every 2 seconds", + "period": 60000, + "rate": 30 + } + ], + "ipWhitelist": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + }, + "admin": { + "rateLimits": [ + { + "description": "30 admin requests/min", + "period": 60000, + "rate": 30 + } + ], + "loginRateLimits": [ + { + "description": "10 login attempts per 15 minutes", + "period": 900000, + "rate": 10 + } + ], + "ipWhitelist": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + }, + "connection": { + "rateLimits": [ + { + "period": 1000, + "rate": 12 + }, + { + "period": 60000, + "rate": 48 + } + ], + "ipWhitelist": [ + "127.0.0.1", + "::1", + "::ffff:127.0.0.1" + ] + }, + "event": { + "retention": { + "maxDays": -1, + "kind": { + "whitelist": [ + 62 + ] + }, + "pubkey": { + "whitelist": [] + } + }, + "eventId": { + "minLeadingZeroBits": 0 + }, + "pow": { + "enabled": false, + "floorBits": 0, + "ceilingBits": 24, + "targetEventsPerSecond": 50, + "periodMs": 60000, + "wotThresholds": [] + }, + "kind": { + "whitelist": [], + "blacklist": [] + }, + "pubkey": { + "minBalance": 0, + "minLeadingZeroBits": 0, + "whitelist": [], + "blacklist": [] + }, + "createdAt": { + "maxPositiveDelta": 900, + "maxNegativeDelta": 0 + }, + "content": [ + { + "description": "64 KB for event kind ranges 0-10 and 40-49", + "maxLength": 102400, + "kinds": [ + [ + 0, + 10 + ], + [ + 40, + 49 + ] + ] + }, + { + "description": "96 KB for event kind ranges 11-39 and 50-max", + "maxLength": 102400, + "kinds": [ + [ + 11, + 39 + ], + [ + 50, + 9007199254740991 + ] + ] + } + ], + "rateLimits": [ + { + "description": "6 events/min for event kinds 0, 3, 40 and 41", + "kinds": [ + 0, + 3, + 40, + 41 + ], + "period": 60000, + "rate": 6 + }, + { + "description": "12 events/min for event kinds 1, 2, 4 and 42", + "kinds": [ + 1, + 2, + 4, + 42 + ], + "period": 60000, + "rate": 12 + }, + { + "description": "30 events/min for event kind ranges 5-7 and 43-49", + "kinds": [ + [ + 5, + 7 + ], + [ + 43, + 49 + ] + ], + "period": 60000, + "rate": 30 + }, + { + "description": "24 events/min for replaceable events and parameterized replaceable events", + "kinds": [ + [ + 10000, + 19999 + ], + [ + 30000, + 39999 + ] + ], + "period": 60000, + "rate": 24 + }, + { + "description": "60 events/min for Marmot group events (kind 445)", + "kinds": [ + 445 + ], + "period": 60000, + "rate": 60 + }, + { + "description": "60 events/min for ephemeral events", + "kinds": [ + [ + 20000, + 29999 + ] + ], + "period": 60000, + "rate": 60 + }, + { + "description": "720 events/hour for all events", + "period": 3600000, + "rate": 720 + } + ], + "whitelists": { + "pubkeys": [], + "ipAddresses": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + } + }, + "client": { + "subscription": { + "maxSubscriptions": 10, + "maxFilters": 10, + "maxFilterValues": 2500, + "maxSubscriptionIdLength": 256, + "maxLimit": 5000, + "minPrefixLength": 4 + } + }, + "message": { + "rateLimits": [ + { + "description": "240 raw messages/min", + "period": 60000, + "rate": 240 + } + ], + "ipWhitelist": [ + "::1", + "10.10.10.1", + "::ffff:10.10.10.1" + ] + } + }, + "admin": { + "enabled": false, + "sessionTtlSeconds": 86400, + "nip98": { + "enabled": false, + "allowedPubkeys": [], + "maxSkewSeconds": 60 + }, + "notifications": { + "enabled": false, + "targets": [], + "events": { + "admission.invoice.created": true, + "admission.invoice.paid": true, + "admission.invoice.failed": true, + "settings.changed": true, + "relay.restarted": false + }, + "retry": { + "maxAttempts": 5, + "baseDelayMs": 1000 + }, + "deliveryLogRetentionDays": 30 + } + } + }, + "steps": [ + { + "connect": true, + "expect": [ + "[\"AUTH\",\"{{challenge}}\"]" + ] + }, + { + "send": { + "text": "[\"EVENT\",{\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"kind\":1,\"created_at\":1700000000,\"content\":\"nip14\",\"tags\":[[\"subject\",\"A test subject\"]],\"id\":\"5a817569bf079d278e77109178a56f0b339c238c9324fbd831023012818d23ee\",\"sig\":\"b4adf24b065467544b47c62a06680348374ae0ab48c6173d90caaedb4c285c9f510d3279dfb358b7e39b6726e03862fd719837fadbe8d11141c8350bdeb946cd\"}]" + }, + "expect": [ + "[\"OK\",\"5a817569bf079d278e77109178a56f0b339c238c9324fbd831023012818d23ee\",true,\"\"]" + ] + }, + { + "send": { + "text": "[\"EVENT\",{\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"kind\":1111,\"created_at\":1700000000,\"content\":\"nip22\",\"tags\":[[\"K\",\"1\"],[\"E\",\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"],[\"P\",\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"],[\"k\",\"1\"],[\"e\",\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"],[\"p\",\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"]],\"id\":\"46b5cacbf77960162bf0d7b66cc8efcc6ad248f34fbdc80ed97052b9034b105c\",\"sig\":\"65f1ed07bb3f9f88c5a35d2000978be63b3dd659b67dbfb04fb189ce4631b32d495c954ca88a9e55f06d6f37b9c542adbf11799ff1c47f150e363eaa730ca814\"}]" + }, + "expect": [ + "[\"OK\",\"46b5cacbf77960162bf0d7b66cc8efcc6ad248f34fbdc80ed97052b9034b105c\",true,\"\"]" + ] + }, + { + "send": { + "text": "[\"EVENT\",{\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"kind\":40,\"created_at\":1700000000,\"content\":\"nip28\",\"tags\":[[\"name\",\"the channel\"]],\"id\":\"a568281922bd58433eed9a8db1912d17c0b887631b53ce80410600a20b728d1c\",\"sig\":\"f84015cc91d82f766a23d93f9910edbc9d5b7607741b27840172afc272a15ea645ee952939a5e02466037d7fac6caa4ee6e9f0ecd0baec53fea5362bc39602e1\"}]" + }, + "expect": [ + "[\"OK\",\"a568281922bd58433eed9a8db1912d17c0b887631b53ce80410600a20b728d1c\",true,\"\"]" + ] + }, + { + "send": { + "text": "[\"EVENT\",{\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"kind\":42,\"created_at\":1700000000,\"content\":\"nip28 message\",\"tags\":[[\"e\",\"a568281922bd58433eed9a8db1912d17c0b887631b53ce80410600a20b728d1c\"]],\"id\":\"e11eae30ca8f4bee275bdd09c1de41cb12f20dc0c95c4a367d7b9156d55753bd\",\"sig\":\"72cb4822fc40a8fe9e9939f0576dd6a895659ca3e297421387dfe3dce446d7114fdd03d6346948f072620d6dc65f6b6490c617aff16c2dc981b77e516126fca5\"}]" + }, + "expect": [ + "[\"OK\",\"e11eae30ca8f4bee275bdd09c1de41cb12f20dc0c95c4a367d7b9156d55753bd\",true,\"\"]" + ] + }, + { + "send": { + "text": "[\"EVENT\",{\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"kind\":10002,\"created_at\":1700000000,\"content\":\"\",\"tags\":[[\"r\",\"wss://relay.one.example\"]],\"id\":\"bbcfe396e0bbac69cadc7966685623455e0ab3937f20eded80fdc81a6187250d\",\"sig\":\"3e60d23d6157b29f3ec2e323c34105e7da17dfd2048659d56b76e63dc7fe6d6005067ee82ab8f88c754107d87fb07a671aa0f4fa2124f0b095e1d05c481ae717\"}]" + }, + "expect": [ + "[\"OK\",\"bbcfe396e0bbac69cadc7966685623455e0ab3937f20eded80fdc81a6187250d\",true,\"\"]" + ] + }, + { + "send": { + "text": "[\"EVENT\",{\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"kind\":31990,\"created_at\":1700000000,\"content\":\"\",\"tags\":[[\"d\",\"example-app\"],[\"k\",\"1\"],[\"web\",\"https://example.com/app\"]],\"id\":\"13d1dd1821b2a617592c03b5062e4245f8c72667cdfce9d3167677255f8a37b3\",\"sig\":\"23a3123048ed4f67b0aa43642f0878564ba22452b38136e94126f87b5a77b4bc8217d0a490b7381634720b7daf9d734accaa36f83c14156fa1cf6ff1aa3f3c73\"}]" + }, + "expect": [ + "[\"OK\",\"13d1dd1821b2a617592c03b5062e4245f8c72667cdfce9d3167677255f8a37b3\",true,\"\"]" + ] + }, + { + "send": { + "text": "[\"EVENT\",{\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"kind\":31989,\"created_at\":1700000000,\"content\":\"\",\"tags\":[[\"d\",\"1\"],[\"a\",\"31990:8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271:example-app\"]],\"id\":\"1e9a2f8d62ef969cb708b8f6ffd526cb9c2ca6741fcc55f3fc163113212b7020\",\"sig\":\"43b6e3b1d28b4e053d91921d13173674dc0934cee6a283c9c2d080d67e911d01f446fb4e3d1a60d840f9abd8e4cc69c68e62876408604cccbdf207b27bb8e042\"}]" + }, + "expect": [ + "[\"OK\",\"1e9a2f8d62ef969cb708b8f6ffd526cb9c2ca6741fcc55f3fc163113212b7020\",true,\"\"]" + ] + }, + { + "send": { + "text": "[\"REQ\",\"v1\",{\"kinds\":[1]}]" + }, + "compare": "json", + "compare_why": "a REQ result carries the capture’s stored-row field order; the values are compared, the key order is the capture’s — job 025", + "expect": [ + "[\"EVENT\",\"v1\",{\"id\":\"5a817569bf079d278e77109178a56f0b339c238c9324fbd831023012818d23ee\",\"kind\":1,\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"created_at\":1700000000,\"content\":\"nip14\",\"tags\":[[\"subject\",\"A test subject\"]],\"sig\":\"b4adf24b065467544b47c62a06680348374ae0ab48c6173d90caaedb4c285c9f510d3279dfb358b7e39b6726e03862fd719837fadbe8d11141c8350bdeb946cd\"}]", + "[\"EOSE\",\"v1\"]" + ] + }, + { + "send": { + "text": "[\"REQ\",\"v2\",{\"kinds\":[1111]}]" + }, + "compare": "json", + "compare_why": "a REQ result carries the capture’s stored-row field order; the values are compared, the key order is the capture’s — job 025", + "expect": [ + "[\"EVENT\",\"v2\",{\"id\":\"46b5cacbf77960162bf0d7b66cc8efcc6ad248f34fbdc80ed97052b9034b105c\",\"kind\":1111,\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"created_at\":1700000000,\"content\":\"nip22\",\"tags\":[[\"K\",\"1\"],[\"E\",\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"],[\"P\",\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"],[\"k\",\"1\"],[\"e\",\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"],[\"p\",\"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\"]],\"sig\":\"65f1ed07bb3f9f88c5a35d2000978be63b3dd659b67dbfb04fb189ce4631b32d495c954ca88a9e55f06d6f37b9c542adbf11799ff1c47f150e363eaa730ca814\"}]", + "[\"EOSE\",\"v2\"]" + ] + }, + { + "send": { + "text": "[\"REQ\",\"v3\",{\"kinds\":[40]}]" + }, + "compare": "json", + "compare_why": "a REQ result carries the capture’s stored-row field order; the values are compared, the key order is the capture’s — job 025", + "expect": [ + "[\"EVENT\",\"v3\",{\"id\":\"a568281922bd58433eed9a8db1912d17c0b887631b53ce80410600a20b728d1c\",\"kind\":40,\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"created_at\":1700000000,\"content\":\"nip28\",\"tags\":[[\"name\",\"the channel\"]],\"sig\":\"f84015cc91d82f766a23d93f9910edbc9d5b7607741b27840172afc272a15ea645ee952939a5e02466037d7fac6caa4ee6e9f0ecd0baec53fea5362bc39602e1\"}]", + "[\"EOSE\",\"v3\"]" + ] + }, + { + "send": { + "text": "[\"REQ\",\"v4\",{\"kinds\":[42]}]" + }, + "compare": "json", + "compare_why": "a REQ result carries the capture’s stored-row field order; the values are compared, the key order is the capture’s — job 025", + "expect": [ + "[\"EVENT\",\"v4\",{\"id\":\"e11eae30ca8f4bee275bdd09c1de41cb12f20dc0c95c4a367d7b9156d55753bd\",\"kind\":42,\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"created_at\":1700000000,\"content\":\"nip28 message\",\"tags\":[[\"e\",\"a568281922bd58433eed9a8db1912d17c0b887631b53ce80410600a20b728d1c\"]],\"sig\":\"72cb4822fc40a8fe9e9939f0576dd6a895659ca3e297421387dfe3dce446d7114fdd03d6346948f072620d6dc65f6b6490c617aff16c2dc981b77e516126fca5\"}]", + "[\"EOSE\",\"v4\"]" + ] + }, + { + "send": { + "text": "[\"REQ\",\"v5\",{\"kinds\":[10002]}]" + }, + "compare": "json", + "compare_why": "a REQ result carries the capture’s stored-row field order; the values are compared, the key order is the capture’s — job 025", + "expect": [ + "[\"EVENT\",\"v5\",{\"id\":\"bbcfe396e0bbac69cadc7966685623455e0ab3937f20eded80fdc81a6187250d\",\"kind\":10002,\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"created_at\":1700000000,\"content\":\"\",\"tags\":[[\"r\",\"wss://relay.one.example\"]],\"sig\":\"3e60d23d6157b29f3ec2e323c34105e7da17dfd2048659d56b76e63dc7fe6d6005067ee82ab8f88c754107d87fb07a671aa0f4fa2124f0b095e1d05c481ae717\"}]", + "[\"EOSE\",\"v5\"]" + ] + }, + { + "send": { + "text": "[\"REQ\",\"v6\",{\"kinds\":[31990]}]" + }, + "compare": "json", + "compare_why": "a REQ result carries the capture’s stored-row field order; the values are compared, the key order is the capture’s — job 025", + "expect": [ + "[\"EVENT\",\"v6\",{\"id\":\"13d1dd1821b2a617592c03b5062e4245f8c72667cdfce9d3167677255f8a37b3\",\"kind\":31990,\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"created_at\":1700000000,\"content\":\"\",\"tags\":[[\"d\",\"example-app\"],[\"k\",\"1\"],[\"web\",\"https://example.com/app\"]],\"sig\":\"23a3123048ed4f67b0aa43642f0878564ba22452b38136e94126f87b5a77b4bc8217d0a490b7381634720b7daf9d734accaa36f83c14156fa1cf6ff1aa3f3c73\"}]", + "[\"EOSE\",\"v6\"]" + ] + }, + { + "send": { + "text": "[\"REQ\",\"v7\",{\"kinds\":[31989]}]" + }, + "compare": "json", + "compare_why": "a REQ result carries the capture’s stored-row field order; the values are compared, the key order is the capture’s — job 025", + "expect": [ + "[\"EVENT\",\"v7\",{\"id\":\"1e9a2f8d62ef969cb708b8f6ffd526cb9c2ca6741fcc55f3fc163113212b7020\",\"kind\":31989,\"pubkey\":\"8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271\",\"created_at\":1700000000,\"content\":\"\",\"tags\":[[\"d\",\"1\"],[\"a\",\"31990:8f92e601a0dd56fb250621d9c12e3249e94ab82bb58e9698b6e50400554ef271:example-app\"]],\"sig\":\"43b6e3b1d28b4e053d91921d13173674dc0934cee6a283c9c2d080d67e911d01f446fb4e3d1a60d840f9abd8e4cc69c68e62876408604cccbdf207b27bb8e042\"}]", + "[\"EOSE\",\"v7\"]" + ] + } + ], + "relay_secret": "683c29961bc5247d177a5019710ee3b776fb07ef636d834b848b4f726ace3f51", + "random": { + "challenge": "604c9809efcc3a2c99f98a7818baabfcce46301377bdbe2b00792cd7dcdbc7aa", + "invite_code": null + } } ] } diff --git a/crates/nostream-relay/tests/wire_parity.rs b/crates/nostream-relay/tests/wire_parity.rs index ed9a699b8..3657b0391 100644 --- a/crates/nostream-relay/tests/wire_parity.rs +++ b/crates/nostream-relay/tests/wire_parity.rs @@ -963,6 +963,7 @@ async fn every_recorded_session_replays_frame_for_frame() { "count-disabled", "nip42-expiry", "event-rejections", + "nip-generic-events", ] { assert!( ids.contains(&required), @@ -1046,8 +1047,14 @@ async fn every_declared_tolerance_is_one_of_the_two() { /// capture's three event-building paths disagree about key order) and a session whose frames /// the capture does not record (AC29). Weakening the fixture by another step or session fails /// here, and each exception still compares everything but its declared difference. +/// +/// `nip-generic-events`' seven read steps are stored-row read-backs, which carry the capture's +/// own key order (`id, kind, pubkey, created_at, content, tags, sig`) where every other EVENT +/// frame in the fixture is the broadcast path's — the same declared difference +/// `measured-session:13` carries, one per kind because one filter per kind is what keeps the row +/// order out of the comparison. #[test] -fn the_not_byte_compared_steps_are_exactly_the_declared_two_and_one_session() { +fn the_not_byte_compared_steps_are_exactly_the_declared_ones() { let fixture = fixture(); let mut json_steps = Vec::new(); @@ -1074,7 +1081,17 @@ fn the_not_byte_compared_steps_are_exactly_the_declared_two_and_one_session() { assert_eq!( json_steps, - ["measured-session:13", "invite-mint:2"], + [ + "measured-session:13", + "invite-mint:2", + "nip-generic-events:8", + "nip-generic-events:9", + "nip-generic-events:10", + "nip-generic-events:11", + "nip-generic-events:12", + "nip-generic-events:13", + "nip-generic-events:14", + ], "the steps compared as JSON values" ); From 2e7f3a85d6d50fc1d5c7c1ef2c2ea6f1d6c1fa3a Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 00:55:43 -0500 Subject: [PATCH 07/36] docs(rewrite): record Phase 2's advertisement and harness map MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §9's twelve items gain a Status line — five done in this job, seven deferred with their reason and where they now live — and the new §10 states D1, D2 and D3, the 34-id advertisement and one harness row per advertised id. --- .changeset/nip-currency.md | 9 +++ docs/rewrite/nip-parity.md | 120 ++++++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 .changeset/nip-currency.md diff --git a/.changeset/nip-currency.md b/.changeset/nip-currency.md new file mode 100644 index 000000000..5f6e99058 --- /dev/null +++ b/.changeset/nip-currency.md @@ -0,0 +1,9 @@ +--- +--- + +feat(core): the audited NIP advertisement, NIP-67 EOSE hints, NIP-44's extended length and NIP-09 `a` tags + +No published package changes: the work is Rust-only (`crates/`), and no `src/` or `test/` +file moves. `SUPPORTED_NIPS` becomes the Phase 0 audit's 34-id set; the relay's `EOSE` +may carry NIP-67's hint array; NIP-44 gains the six-byte length prefix and the format's +2³²−1 ceiling; and a NIP-09 `a` tag now deletes the author's addressable event. diff --git a/docs/rewrite/nip-parity.md b/docs/rewrite/nip-parity.md index 4ca4d95f4..3bd094bce 100644 --- a/docs/rewrite/nip-parity.md +++ b/docs/rewrite/nip-parity.md @@ -769,7 +769,9 @@ keep. ## 9. Open items for Phase 2 This job files no issues: the dispositions above are the deliverable, and Phase 2 -turns them into work. The items, in the order they affect the rewrite: +turns them into work. The items, in the order they affect the rewrite. Each carries a +**Status:** line once Phase 2 has an answer for it; a status that drifts from the code +is a defect, and §10 is the same record in table form. 1. **The advertisement set needs a decision, not a copy.** Nine live features are unadvertised (NIP-05, 56, 59, 62, 66, 70, 89, 90, 98), five advertised numbers have @@ -779,14 +781,17 @@ turns them into work. The items, in the order they affect the rewrite: behaviour, decide 15 and 22 in item 2, name the nine — but it should be one list that matches what the relay does, and `README.md:69-95` should stop contradicting `package.json:7` in both directions. + - **Status:** done in this job — decision **D2** fixes the list at 34 ids in `crates/nostream-core/src/constants.rs`, pinned by `crates/nostream-core/tests/supported_nips.rs`; NIP-15 is out and NIP-67 is in (§10). Nothing serves the list yet (**D1**). 2. **NIP-15 and NIP-22 are number collisions.** `README.md:79` and `:82` name the old NIPs. Either the relay drops those numbers from `supported_nips` (NIP-15 has no relay duty; NIP-22's kind 1111 is client-side) or it implements the current texts. Implementing NIP-22's tag rules is the larger of the two. + - **Status:** done in this job — 15 is dropped and 22 stays with the current text, whose event shape `nip_vectors::nip22_a_kind_1111_comment_is_stored_and_served_unchanged` pins (§10). 3. **NIP-42 / NIP-67: the `auth` hint.** One change closes both: emit the third `EOSE` element with `"auth"` where restricted kinds were trimmed (and `"finish"` / `"more"` where the limit was reached), and add 67 to `supported_nips`. This is the `opportunistic` disposition on 67 and the narrow gap in §4's NIP-42. + - **Status:** done in this job — the third `EOSE` element, carrying `auth` for a trimmed read and `finish`/`more` for a capped one, emitted only from **D3**'s two causes so the recorded sessions keep their two-element frames. `messages::{EOSE_HINT_AUTH,EOSE_HINT_FINISH,EOSE_HINT_MORE}`, `handlers::eose_hints_*`, `connection::a_restricted_read_is_judged_against_the_authenticated_set_at_each_event`; 67 is in the advertised set (**D2**). 4. **NIP-43 / NIP-86: the invite flow.** `43.md` now delegates claims to NIP-86's `createclaim`. The rewrite either exposes that method (which also covers the NIP-86 disposition) or keeps the removed kind-28935 `REQ` path and says so. If it @@ -794,14 +799,17 @@ turns them into work. The items, in the order they affect the rewrite: `Authorization` on every method (`86.md:255-259`) — and must stay behind `admin.nip98.allowedPubkeys` (`src/handlers/request-handlers/admin-auth-middleware.ts:41-100`); as a public route it would mint invite codes for anyone. + - **Status:** deferred — `createclaim` needs NIP-86's HTTP surface, and this tree has no HTTP listener at all (**D1**); the kind-28935 `REQ` path stays as it is. The item lives on #865's checklist. 5. **NIP-44's extended length.** Adopt the 6-byte prefix and the raised cap, or the relay keeps rejecting valid large gift wraps at admission. This is a prerequisite for NIP-17 parity, not a separate feature. The 4 GiB figure is the format's ceiling, not the relay's: `network.maxPayloadSize` (524288, `resources/default-settings.yaml:161`) still caps what arrives, so the raise is a format change, not a bigger buffer. + - **Status:** done in this job — the six-byte `[0x00, 0x00][u32]` prefix and the format's 2³²−1 plaintext ceiling, with `MAX_PAYLOAD_LENGTH` (5 726 623 156) and `MAX_DATA_LENGTH` (4 294 967 367) derived from it. `nip44_parity::accepts_a_plaintext_of_65536_bytes_with_the_extended_prefix`, `nip44::tests::the_ceilings_are_derived_from_the_formats_plaintext_ceiling`. The relay's own bound stays `network.maxPayloadSize`. 6. **NIP-62's deletion.** Replace the soft delete for vanish requests with a real delete, and remove gift wraps that p-tag the vanished pubkey. + - **Status:** deferred — a hard delete across both storage backends plus a gift-wrap `p`-tag sweep; both backends still soft-delete, and the work is its own job. 7. **NIP-78's AUTH guard — two halves, two decisions.** The serving half: set `nip42.restrictedReads` to `enabled: true` with `kinds: [78, 30078]`, or take the broader `[4, 1059, 78, 30078]` — which a relay already running `enabled: true` with @@ -830,6 +838,7 @@ turns them into work. The items, in the order they affect the rewrite: refusals — admission (`isUserAdmitted`, `:125`), NIP-05 (`:132`), content limits, PoW and lists (`canAcceptEvent`, `:97`), rate limits (`:86`) and vanish blocks (`:118`) — are policy, not AUTH, and a port carries them unchanged. + - **Status:** deferred — kinds 78 and 30078 appear nowhere in `crates`, and the owner rule plus the kindless COUNT stay open; the item lives on #865's checklist. 8. **NIP-01's prefix matching.** Keep it or move to exact 64-character values as the current text requires; whether `min_prefix` stays advertised is item 11's decision, per key. Keeping it is not free: the prefix arms are `substring(...)` predicates @@ -838,13 +847,16 @@ turns them into work. The items, in the order they affect the rewrite: `events`. The rewrite can keep the extension cheaply by making a prefix a range seek in its key layout; otherwise the honest choice is exact values. Either way the decision should be written down, because today it is an undocumented extension. + - **Status:** deferred — prefix matching is kept exactly as the capture has it (a 4..64-character hex prefix) and `min_prefix` is not advertised; the decision belongs with NIP-11's field set, item 11. 9. **NIP-45's HyperLogLog response** is optional under the cutoff text. Take it when `COUNT` is next touched, not before — and set the exact-versus-approximate policy at the same time, because HLL still visits every matching event to fill its registers; its leverage is the deterministic `offset` and cross-query caching, not a cheaper single pass. + - **Status:** deferred — optional under the cutoff text, and `COUNT` still answers the plain `{"count":N}`; the item belongs to the next `COUNT` change. 10. **NIP-29 and NIP-96 are out of scope** until the rewrite takes on relay groups or a file-storage service; neither exists in nostream today. + - **Status:** deferred — relay groups and file storage are outside the rewrite's scope, as the audit recorded. 11. **NIP-11's field set.** Four keys in `limitation` are not in `11.md` today (`max_filters` and `min_prefix` were removed, `max_filter_values` and `search_supported` were never defined) and two top-level keys never were @@ -860,8 +872,114 @@ turns them into work. The items, in the order they affect the rewrite: the four untested keys are `min_prefix`, `supported_nip_extensions`, `supported_mips` and `max_event_tags` (§1). NIP-78's two kinds have no test either, and item 7 carries that pair. + - **Status:** deferred — **D1** defers the HTTP surface and the document itself, and the field set is decided with the document; the four untested keys stay untested until then. 12. **NIP-09 `a`-tag deletion.** `DeleteEventStrategy` accepts only `e` tags (`src/handlers/event-strategies/delete-event-strategy.ts:22-31`); a kind-5 event whose tags are `a`-only deletes nothing. The rewrite either implements addressable-event deletion or records the omission deliberately — but §5 and the §7 row now say so, and Phase 2 must not inherit the silence. + - **Status:** done in this job — an `a` tag resolves to the author's addressable row, subject to the author-equality rule and the request's `created_at`, and its id joins the `e`-tag ids in one `delete_by_pubkey_and_ids`; `strategies::delete_soft_deletes_an_addressable_event_named_by_an_a_tag` and three more (§10). + +## 10. Phase 2: the advertisement and the harness map + +This section is Phase 2's record, set against §2's dated audit of the capture. It states +**two deltas** from §2 rather than rewriting a measured table: **67 is in** the +advertised set (`67.md` §"Relay advertisement", and the hint §4 asks for) and **15 is +out** (it is not `must-have`; the number now means Marketplace, §3). §2 and §3 keep the +capture's wording. + +**The decisions, by id.** + +- **D1 — NIP-11's HTTP surface is not the advertisement's job.** `SUPPORTED_NIPS` is + fixed and the wire gap recorded; nothing listens on HTTP and no NIP-11 document is + built. The surface is a dependency decision (no HTTP server crate anywhere in the + workspace), a second bind in `nostreamd`, and a document assembled from `info`, + `limits`, `payments` and `peering`. §9 item 11 carries it. +- **D2 — the advertised set is the audit's 33 `must-have` rows plus NIP-67: 34 ids.** + 04, 14 and 28 stay as behaviour the generic path satisfies, which costs nothing and + avoids a silent parity change; the tombstones 12/16/20/33 stay because nostream + advertises them and the behaviour their text names is implemented; 15 is out; 98 + stays because the list is what the audit says the relay must carry, and this job + records the gap rather than narrowing the claim. The array length in the type is the + second guard: an id added without the type changing does not compile. +- **D3 — the third `EOSE` element appears only when the two-element form would + mislead.** A hint is emitted from exactly two causes: the read guard withheld a + matching event (`auth`), or a single-filter subscription carried a `limit` and the cap + decided the answer (`finish`/`more`). A no-limit, no-trim read keeps + `["EOSE", ]`, which is what the sixteen sessions recorded from the capture require + of the harness. `finish` is not sent beside `auth`: `67.md` defines it as "the relay + has sent every stored event matching the subscription's filters", and a withheld event + makes that false. + +**The advertisement.** + +| where | value | +| --- | --- | +| `crates/nostream-core/src/constants.rs` | `pub const SUPPORTED_NIPS: [u16; 34]`, ascending and free of duplicates | +| `crates/nostream-core/tests/supported_nips.rs` | `the_advertisement_is_the_audited_set`, `the_advertisement_is_ascending_and_free_of_duplicates` | +| served to a client? | nothing: no HTTP listener and no NIP-11 document (**D1**) | + +The 34 ids: 01, 02, 03, 04, 05, 09, 11, 12, 13, 14, 16, 17, 20, 22, 25, 28, 33, 40, +42, 43, 44, 45, 50, 56, 59, 62, 65, 66, 67, 70, 77, 89, 90, 98. + +**The harness map**, one row per advertised id. The harness column is one of four forms: + +- `session ` — the capture recorded this behaviour's own frames, in + `crates/nostream-relay/tests/fixtures/wire-sessions.json`, replayed byte for byte by + `crates/nostream-relay/tests/wire_parity.rs`; +- `divergent: ::` — this relay deliberately does it differently from the + capture, so **no session may record it**; a Rust test pins it instead; +- `vectors-only: ::` — a Rust test pins the behaviour and the capture has + no session for it; +- `no-wire-surface: ` — the NIP's duty is not a client-visible frame. + +`` is the test file's stem. Every target lives in that crate's `tests/` +directory; the two marked *(nostream-core)* are the behaviours whose pinning test is in +`crates/nostream-core/tests/` because the behaviour itself is the core crate's (the +filter matcher, the event schema, the NIP-44 format and the reconciliation engine), and +no relay-side test reaches them. + +| # | implementation (Rust) | harness | +| --- | --- | --- | +| 01 | `core/{event,filter,messages}.rs`; `relay/handlers/*` | `session measured-session`, `live-dispatch`, `req-close-abort` | +| 02 | `relay/strategies/contact_list.rs` | `vectors-only: strategies::contact_list_upserts_the_event` | +| 03 | `core/nip03.rs`; `relay/strategies/timestamp.rs` | `vectors-only: strategies::timestamp_stores_and_broadcasts_a_valid_proof` | +| 04 | `core/event.rs`; `relay/strategies/default.rs`; `core/nip42.rs` | `vectors-only: handlers::a_restricted_read_delivers_only_to_the_author_or_a_p_tagged_reader` | +| 05 | `core/nip05.rs`; `relay/handlers/event.rs`; `services/workers/maintenance.rs` | `vectors-only: handlers::nip05_gates_publication` | +| 09 | `relay/strategies/delete.rs` | `divergent: strategies::delete_soft_deletes_an_addressable_event_named_by_an_a_tag` | +| 11 | none: the constant is the record | `no-wire-surface: no HTTP listener and no NIP-11 document (D1)` | +| 12 | `core/filter.rs` generic `#` criteria | `vectors-only: event_parity::nip12_d_filter_returns_true_if_d_filter_contains_an_empty_tag_value_in_the_event` *(nostream-core)* | +| 13 | `core/pow.rs`, `core/adaptive_pow.rs`; `relay/handlers/event.rs` | `session event-rejections` | +| 14 | the generic path (`relay/strategies/default.rs`) | `session nip-generic-events` | +| 16 | `core/event.rs`; `relay/strategies/{replaceable,ephemeral}.rs` | `vectors-only: strategies::replaceable_upserts_the_event` | +| 17 | `relay/strategies/gift_wrap.rs`; `core/nip44.rs` | `session event-rejections`, step 3 | +| 20 | `core/messages.rs` `OK`/`CLOSED` encoders | `session measured-session` | +| 22 | the generic path | `session nip-generic-events` | +| 25 | `core/validation.rs` | `vectors-only: wire_codec::strict_event_schema_checks_the_refinements` *(nostream-core)* | +| 28 | the generic path; kind 41 replaceable via `core/event.rs` | `session nip-generic-events` | +| 33 | `core/event.rs`; `relay/strategies/parameterized_replaceable.rs` | `vectors-only: strategies::parameterized_replaceable_uses_the_first_d_tag_value` | +| 40 | `core/expiration.rs`; `relay/handlers/event.rs` | `session event-rejections` | +| 42 | `core/nip42.rs`; `relay/handlers/{auth,subscribe}.rs` | `session req-auth-required`, `nip42-expiry` | +| 43 | `core/nip43.rs`; `relay/strategies/{join,leave,invite}_request.rs` | `session invite-mint` | +| 44 | `core/nip44.rs` | `divergent: nip44_parity::produces_the_canonical_payload_from_the_nip44_spec_test_vector` *(nostream-core)* | +| 45 | `relay/handlers/count.rs`; `core/messages.rs` | `session count-disabled`, `measured-session` | +| 50 | `core/filter.rs`; `storage/postgres/sql.rs` | `vectors-only: nip_vectors::nip50_a_search_filter_is_served_only_when_enabled` | +| 56 | `core/nip56.rs`; `relay/strategies/report.rs`; `relay/hidden_content.rs` | `session hidden-report` | +| 59 | `relay/strategies/gift_wrap.rs`; `core/event.rs` | `session event-rejections`, step 3 | +| 62 | `relay/strategies/vanish.rs` | `vectors-only: strategies::vanish_soft_deletes_the_other_events_and_stores_the_request` | +| 65 | `core/validation.rs`; `core/event.rs` | `session nip-generic-events` | +| 66 | `core/nip66.rs`; `services/monitor.rs` | `no-wire-surface: the probes and the publisher are #865 checklist stubs` | +| 67 | `core/messages.rs`; `relay/handlers/subscribe.rs` | `divergent: handlers::eose_hints_more_when_the_limit_was_reached` | +| 70 | `core/event.rs`; `relay/handlers/event.rs` | `session event-rejections` | +| 77 | `core/negentropy.rs` | `vectors-only: negentropy_parity::answers_that_message_exactly_like_the_reference` *(nostream-core; the `NEG-*` handlers are #865's)* | +| 89 | the generic path | `session nip-generic-events` | +| 90 | `core/event.rs`; `relay/strategies/dvm_job_request.rs`; `services/workers/dvm_orchestrator.rs` | `vectors-only: strategies::dvm_job_request_creates_the_event` | +| 98 | `core/nip98.rs` (stub) | `no-wire-surface: header verification needs the admin HTTP surface (D1, #865)` | + +Two limits of the map, stated so a later reader does not take it for more than it is. +**NIP-50's ranking** is not in it: `ts_rank` lives in the SQL backend and needs Postgres, +so ranking stays with the `#[ignore]`d `parity` target while the vector above pins the +matcher's accept-and-drop behaviour. **The census is a different oracle**: it maps 658 +TypeScript unit cases to Rust tests and is untouched here, because a NIP with a vector +and no TypeScript case has nothing to map, and hand-adding one fails `ts_case_parity` by +design. From 4a0a95760ad34fe6736d2119be9a0f7e23679ed8 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 00:57:35 -0500 Subject: [PATCH 08/36] docs(relay): mark the harness session's deliberate one-kind-per-read limit A comment only: the recorded fixture is unchanged, because the generator's output does not depend on it. --- crates/nostream-relay/tests/capture/gen-wire-sessions.mjs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/nostream-relay/tests/capture/gen-wire-sessions.mjs b/crates/nostream-relay/tests/capture/gen-wire-sessions.mjs index a89b9e6eb..51e9d0e56 100644 --- a/crates/nostream-relay/tests/capture/gen-wire-sessions.mjs +++ b/crates/nostream-relay/tests/capture/gen-wire-sessions.mjs @@ -1059,8 +1059,9 @@ const runAll = async () => { await send(['EVENT', event], { expect: 1 }) } - // One filter per kind, so no ordering between rows is asserted, and no filter carries a - // `limit`, so the EOSE keeps its two-element form on both sides. + // shortcut: one filter per kind, so no ordering between stored rows is asserted; upgrade + // when a backend's row order is what the replay must pin. No filter carries a `limit`, so + // the EOSE keeps its two-element form on both sides. for (const [id, kind] of [ ['v1', 1], ['v2', 1111], From 02d935cab677f9513b95ec9fb998046d0c1d060f Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 01:59:44 -0500 Subject: [PATCH 09/36] fix(storage): resolve and delete a NIP-09 address in one operation --- crates/nostream-relay/src/unavailable.rs | 10 ++ crates/nostream-services/src/testing.rs | 10 ++ crates/nostream-services/src/unavailable.rs | 10 ++ crates/nostream-storage/src/lib.rs | 11 ++ crates/nostream-storage/src/lmdb/events.rs | 68 ++++++++- crates/nostream-storage/src/lmdb/mod.rs | 26 ++++ .../nostream-storage/src/postgres/events.rs | 36 +++++ .../src/postgres/repository.rs | 12 ++ crates/nostream-storage/src/repositories.rs | 19 +++ .../nostream-storage/tests/address_delete.rs | 142 ++++++++++++++++++ crates/nostream-storage/tests/storage.rs | 10 ++ 11 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 crates/nostream-storage/tests/address_delete.rs diff --git a/crates/nostream-relay/src/unavailable.rs b/crates/nostream-relay/src/unavailable.rs index 0c23895e6..11a25f5ed 100644 --- a/crates/nostream-relay/src/unavailable.rs +++ b/crates/nostream-relay/src/unavailable.rs @@ -113,6 +113,16 @@ impl EventRepository for UnavailableEventRepository { Box::pin(async { Err(unavailable()) }) } + fn delete_by_address<'a>( + &'a self, + _pubkey: &'a Pubkey, + _kind: Kind, + _identifier: &'a str, + _not_after: i64, + ) -> StorageFuture<'a, u64> { + Box::pin(async { Err(unavailable()) }) + } + fn delete_by_pubkey_except_kinds<'a>( &'a self, _pubkey: &'a Pubkey, diff --git a/crates/nostream-services/src/testing.rs b/crates/nostream-services/src/testing.rs index e43605347..fc08e90ca 100644 --- a/crates/nostream-services/src/testing.rs +++ b/crates/nostream-services/src/testing.rs @@ -573,6 +573,16 @@ impl EventRepository for InMemoryEventRepository { }) } + fn delete_by_address<'a>( + &'a self, + _pubkey: &'a Pubkey, + _kind: Kind, + _identifier: &'a str, + _not_after: i64, + ) -> StorageFuture<'a, u64> { + Box::pin(async { Ok(0) }) + } + fn delete_by_pubkey_except_kinds<'a>( &'a self, _pubkey: &'a Pubkey, diff --git a/crates/nostream-services/src/unavailable.rs b/crates/nostream-services/src/unavailable.rs index 0977e33f2..610696446 100644 --- a/crates/nostream-services/src/unavailable.rs +++ b/crates/nostream-services/src/unavailable.rs @@ -100,6 +100,16 @@ impl EventRepository for UnavailableEventRepository { Box::pin(async { Err(unavailable()) }) } + fn delete_by_address<'a>( + &'a self, + _pubkey: &'a nostream_core::Pubkey, + _kind: nostream_core::Kind, + _identifier: &'a str, + _not_after: i64, + ) -> StoreFuture<'a, u64> { + Box::pin(async { Err(unavailable()) }) + } + fn delete_by_pubkey_except_kinds<'a>( &'a self, _pubkey: &'a nostream_core::Pubkey, diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index 8bf3ae02c..225335bb5 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -43,3 +43,14 @@ pub use error::StorageError; pub use lmdb::{LmdbBackend, MapUsage}; #[cfg(feature = "postgres")] pub use postgres::{PostgresBackend, PostgresTransaction}; + +/// The most rows one filter's page can hold on any shipped backend, whatever `limit` it +/// carries. +/// +/// A caller that must know whether a page was truncated compares the rows a stream yielded +/// against `min(limit, MAX_PAGE_ROWS)`: a stream that yields that many has a full page and says +/// nothing about what lies beyond it. LMDB clamps every filter to this bound +/// (`lmdb::MAX_PAGE_KEYS`, which is this constant); Postgres honours the filter's own `limit`, +/// and its default (`DEFAULT_FILTER_LIMIT`) is smaller. A new backend that clamps lower must +/// lower this constant, and the LMDB clamp's own test fails if the two drift apart. +pub const MAX_PAGE_ROWS: u64 = 5_000; diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index f5fe51d25..e2aa58a1c 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -17,7 +17,7 @@ use chrono::{DateTime, Utc}; use futures_core::Stream; use heed::types::Bytes; use heed::{Database, Env, RoTxn, RwTxn}; -use nostream_core::{EventId, Filter, Kind, KindSelector, Pubkey}; +use nostream_core::{Event, EventId, Filter, Kind, KindSelector, Pubkey}; use tokio::task::JoinHandle; use super::encoding::{self, EventHeader}; @@ -351,6 +351,64 @@ pub(crate) fn delete_by_pubkey_and_ids( Ok(affected) } +/// `delete_by_address`: the author's live rows of one addressable kind whose first `d` tag is +/// `identifier` and whose `created_at` is no later than `not_after`, soft-deleted. +/// +/// The walk is the author-and-kind prefix of `events_by_author` — `pubkey ‖ kind ‖ created_at ‖ +/// id` — so it is one index range rather than a filter page, and no page bound applies: a page +/// is what would put the address outside the search. The NIP-56 gate is a read's, not a +/// delete's, so a reported row is reachable here. +pub(crate) fn delete_by_address( + txn: &mut RwTxn<'_>, + databases: &Databases, + pubkey: &Pubkey, + kind: Kind, + identifier: &str, + not_after: i64, + now: DateTime, +) -> Result { + let kind_number = i32::try_from(kind.0).unwrap_or(i32::MAX); + let mut prefix = Vec::with_capacity(36); + prefix.extend_from_slice(pubkey.as_bytes()); + prefix.extend_from_slice(&encoding::i32_key(kind_number)); + + let mut keys = Vec::new(); + for entry in databases.events_by_author.prefix_iter(txn, &prefix)? { + let (key, _) = entry?; + keys.push(key.to_vec()); + } + + let mut affected = 0; + for key in keys { + let header = author_key_header(&key)?; + let Some(value) = databases.events.get(txn, header.event_id.as_bytes())? else { + continue; + }; + let record = rows::decode_event(value)?; + if record.deleted_at.is_some() + || record.event.created_at > not_after + || kind_of(&record)? == REQUEST_TO_VANISH + || d_identifier(&record.event) != identifier + { + continue; + } + soft_delete(txn, databases, &record, now)?; + affected += 1; + } + Ok(affected) +} + +/// The value of an event's first `d` tag, `""` when it carries none — the key the addressable +/// strategy deduplicates on (`parameterized_replaceable.rs:29-35`). +fn d_identifier(event: &Event) -> &str { + event + .tags + .iter() + .find(|tag| tag.len() >= 2 && tag[0] == "d") + .map(|tag| tag[1].as_str()) + .unwrap_or_default() +} + /// `delete_by_pubkey_except_kinds`: every other live row of one author, soft-deleted. pub(crate) fn delete_by_pubkey_except_kinds( txn: &mut RwTxn<'_>, @@ -2378,6 +2436,14 @@ mod tests { "the page the backend materialises is its own bound, not the wire's" ); assert_eq!(page[0].0, MAX_PAGE_KEYS as i64 + 1, "and it is the newest"); + + // The relay decides whether a page was full from `MAX_PAGE_ROWS`, so the two constants + // are the same number and cannot drift apart unnoticed. + assert_eq!( + MAX_PAGE_KEYS as u64, + crate::MAX_PAGE_ROWS, + "the relay's page bound is the backend's" + ); } #[test] diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index e9fc7b7d6..d20215e73 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -422,6 +422,32 @@ impl EventRepository for LmdbBackend { }) } + fn delete_by_address<'a>( + &'a self, + pubkey: &'a Pubkey, + kind: Kind, + identifier: &'a str, + not_after: i64, + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + let pubkey = *pubkey; + let identifier = identifier.to_owned(); + let now = encoding::micros(Utc::now()); + self.write(move |txn, databases| { + events::delete_by_address( + txn, + databases, + &pubkey, + kind, + &identifier, + not_after, + now, + ) + }) + .await + }) + } + fn has_active_request_to_vanish<'a>(&'a self, pubkey: &'a Pubkey) -> StorageFuture<'a, bool> { Box::pin(async move { let pubkey = *pubkey; diff --git a/crates/nostream-storage/src/postgres/events.rs b/crates/nostream-storage/src/postgres/events.rs index 80f5c18b6..142f713c4 100644 --- a/crates/nostream-storage/src/postgres/events.rs +++ b/crates/nostream-storage/src/postgres/events.rs @@ -237,6 +237,42 @@ where .rows_affected()) } +/// `delete_by_address`: one statement that resolves and deletes, so a replacement landing +/// between a read and a delete cannot survive it. +/// +/// The address is `(event_pubkey, event_kind, event_deduplication->>0)` — the key the +/// addressable strategy upserts on — and `event_created_at <= not_after` is `09.md`'s "all +/// versions up to the `created_at` timestamp of the deletion request". Kind 62 is excluded, as +/// in [`delete_by_pubkey_and_ids`]. +pub(crate) async fn delete_by_address<'e, E>( + executor: E, + pubkey: &Pubkey, + kind: Kind, + identifier: &str, + not_after: i64, +) -> Result +where + E: Executor<'e, Database = Postgres>, +{ + let mut args = Args::default(); + let pubkey_slot = args.push(Arg::Bytes(pubkey.as_bytes().to_vec())); + let kind_slot = args.push(Arg::Int(i32::try_from(kind.0).unwrap_or(i32::MAX))); + let identifier_slot = args.push(Arg::Text(identifier.to_owned())); + let not_after_slot = args.push(Arg::BigInt(not_after)); + let sql = format!( + "UPDATE events SET deleted_at = now() WHERE event_pubkey = {pubkey_slot} \ + AND event_kind = {kind_slot} AND event_deduplication->>0 = {identifier_slot} \ + AND event_created_at <= {not_after_slot} AND event_kind <> {REQUEST_TO_VANISH} \ + AND deleted_at IS NULL" + ); + + Ok(EventQuery { sql, args } + .into_query() + .execute(executor) + .await? + .rows_affected()) +} + /// `delete_by_pubkey_except_kinds`: a soft delete of every other live kind. pub(crate) async fn delete_by_pubkey_except_kinds<'e, E>( executor: E, diff --git a/crates/nostream-storage/src/postgres/repository.rs b/crates/nostream-storage/src/postgres/repository.rs index 16064b98e..e2989bad8 100644 --- a/crates/nostream-storage/src/postgres/repository.rs +++ b/crates/nostream-storage/src/postgres/repository.rs @@ -69,6 +69,18 @@ impl EventRepository for PostgresBackend { Box::pin(async move { events::delete_by_pubkey_and_ids(&self.master, pubkey, ids).await }) } + fn delete_by_address<'a>( + &'a self, + pubkey: &'a Pubkey, + kind: Kind, + identifier: &'a str, + not_after: i64, + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + events::delete_by_address(&self.master, pubkey, kind, identifier, not_after).await + }) + } + fn delete_by_pubkey_except_kinds<'a>( &'a self, pubkey: &'a Pubkey, diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index 1db5b06e9..858417598 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -431,6 +431,25 @@ pub trait EventRepository: Send + Sync { ids: &'a [EventId], ) -> StorageFuture<'a, u64>; + /// Soft-deletes the author's live addressable rows that one address names — the `d` + /// identifier and the kinds 30000..=39999, with `created_at` no later than `not_after` — + /// returning the count. Kind 62 is never touched, as in + /// [`delete_by_pubkey_and_ids`](Self::delete_by_pubkey_and_ids). + /// + /// This exists as its own operation rather than as a read followed by + /// `delete_by_pubkey_and_ids`, for two reasons a caller cannot work around: a read is + /// bounded by the backend's page, so a narrow address must not be found by scanning a page; + /// and a read and a delete are two operations, so a replacement landing between them + /// survives the delete the client was told had happened. Both are properties of the store, + /// so the store performs the resolution and the deletion as one. + fn delete_by_address<'a>( + &'a self, + pubkey: &'a Pubkey, + kind: Kind, + identifier: &'a str, + not_after: i64, + ) -> StorageFuture<'a, u64>; + /// Soft-deletes every event of one pubkey except the given kinds. fn delete_by_pubkey_except_kinds<'a>( &'a self, diff --git a/crates/nostream-storage/tests/address_delete.rs b/crates/nostream-storage/tests/address_delete.rs new file mode 100644 index 000000000..e5e995fed --- /dev/null +++ b/crates/nostream-storage/tests/address_delete.rs @@ -0,0 +1,142 @@ +//! `delete_by_address`: the NIP-09 `a`-tag delete, over the real store. +//! +//! The seam resolves the address and soft-deletes it in one operation, walking the author's kind +//! index rather than a read page, so neither the page bound nor the gap between a read and a +//! delete can leave a named row alive. + +#![cfg(feature = "lmdb")] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod common; + +use std::sync::Arc; + +use common::{Repos, collect, fixture, settings_from_fixture}; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; +use nostream_storage::LmdbBackend; +use nostream_storage::repositories::{EventRecord, EventRepository, NewReport, ReportType}; + +/// A store on a temporary directory with the fixture's settings gates. +async fn open_store() -> (tempfile::TempDir, LmdbBackend) { + let fixture = fixture(); + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens") + .with_settings(Arc::new(settings_from_fixture(&fixture))); + + (directory, backend) +} + +/// A kind-30023 row under the address `30023::`, by the key `secret`. +fn addressable(secret: u8, d: &str, created_at: i64) -> EventRecord { + let mut event = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(30_023), + tags: vec![vec!["d".to_owned(), d.to_owned()]], + content: format!("{secret}-{d}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut event).expect("the event signs"); + EventRecord::from_event(&event) +} + +/// 501 addressable rows of one author, so the target sits past the backend's page bound: a +/// read-then-delete seam could not reach it. +#[tokio::test] +async fn delete_by_address_reaches_a_row_past_the_page_bound() { + let (_directory, backend) = open_store().await; + let mut target = None; + + for index in 0..501u64 { + let record = addressable(7, &format!("post-{index}"), 1_700_000_000 + index as i64); + backend.create(&record).await.expect("the store creates"); + + if index == 500 { + target = Some(record); + } + } + + let target = target.expect("the five hundred and first row"); + let filter = Filter { + authors: Some(vec![target.event.pubkey.to_string()]), + ..Default::default() + }; + + // A streamed read is what a read-then-delete seam would have used, and it stops at the + // backend's page, so the five hundred and first row is not in it. + let served = collect( + backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .expect("a read"), + ) + .await + .expect("a read drains") + .len(); + assert!( + served < 501, + "the read page is bounded ({served} of 501), which is why the delete may not go through it" + ); + + let deleted = backend + .delete_by_address(&target.event.pubkey, Kind(30_023), "post-500", i64::MAX) + .await + .expect("the store deletes"); + + assert_eq!(deleted, 1); + assert_eq!( + backend + .count_by_filters(std::slice::from_ref(&filter)) + .await + .expect("a read after the delete"), + 500, + "the row named by the address is the one gone" + ); +} + +/// A row NIP-56 withholds from reads is still a row the author's deletion request names: the +/// report gate lives on the read path, and the delete walks the author's index instead. +#[tokio::test] +async fn delete_by_address_reaches_a_reported_row() { + let (_directory, backend) = open_store().await; + let target = addressable(7, "post-1", 1_700_000_000); + backend.create(&target).await.expect("the store creates"); + + let repos = Repos::all(&backend); + repos + .reports + .create(&NewReport { + event_id: EventId::from_byte_array([9; 32]), + reporter_pubkey: Pubkey::from_byte_array([8; 32]), + reported_pubkey: Some(target.event.pubkey), + reported_event_id: Some(target.event.id), + report_type: ReportType::Spam, + weight: 1.0, + actionable: true, + }) + .await + .expect("the report stores"); + + let filter = Filter { + authors: Some(vec![target.event.pubkey.to_string()]), + ..Default::default() + }; + assert_eq!( + backend + .count_by_filters(std::slice::from_ref(&filter)) + .await + .expect("a read"), + 0, + "the read path withholds the reported row" + ); + + let deleted = backend + .delete_by_address(&target.event.pubkey, Kind(30_023), "post-1", i64::MAX) + .await + .expect("the store deletes"); + + assert_eq!(deleted, 1, "the delete reaches the row the read cannot"); +} diff --git a/crates/nostream-storage/tests/storage.rs b/crates/nostream-storage/tests/storage.rs index 5f425bbe3..9a0620905 100644 --- a/crates/nostream-storage/tests/storage.rs +++ b/crates/nostream-storage/tests/storage.rs @@ -105,6 +105,16 @@ async fn the_repository_traits_are_dyn_compatible() { unsupported("noop::events::delete_by_pubkey_and_ids") } + fn delete_by_address<'a>( + &'a self, + _pubkey: &'a Pubkey, + _kind: Kind, + _identifier: &'a str, + _not_after: i64, + ) -> StorageFuture<'a, u64> { + unsupported("noop::events::delete_by_address") + } + fn delete_by_pubkey_except_kinds<'a>( &'a self, _pubkey: &'a Pubkey, From e56161ff4bebab8462a2c999bf1166c45413ba9d Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 01:59:44 -0500 Subject: [PATCH 10/36] fix(relay): delete NIP-09 a-tag addresses through the storage seam --- .../nostream-relay/src/strategies/delete.rs | 93 +++++------------- crates/nostream-relay/src/testing.rs | 97 ++++++++++++++++++- crates/nostream-relay/tests/strategies.rs | 67 ++++++++++++- 3 files changed, 182 insertions(+), 75 deletions(-) diff --git a/crates/nostream-relay/src/strategies/delete.rs b/crates/nostream-relay/src/strategies/delete.rs index 8f19f3a04..4b565ab9f 100644 --- a/crates/nostream-relay/src/strategies/delete.rs +++ b/crates/nostream-relay/src/strategies/delete.rs @@ -1,18 +1,25 @@ //! The `delete` event strategy. Mirrors `src/handlers/event-strategies/delete-event-strategy.ts`. //! -//! A NIP-09 request: every `e` tag whose value parses as an event id is soft-deleted — exact -//! 32 bytes of lowercase hex, which is what `EventId`'s own parser accepts — and every `a` tag -//! naming one of the author's addressable events resolves to that event's id (`09.md` -//! §"Deletion Request" covers both the event id and the addressable `a` form). The deletions run -//! **before** the request itself is stored, so a client that re-reads after the `OK` sees them -//! already applied. Then the common create-or-dedupe. +//! A NIP-09 request names its targets two ways, and each is one storage operation: +//! +//! - an `e` tag whose value parses as an event id — exact 32 bytes of lowercase hex, which is +//! what `EventId`'s own parser accepts — goes to `delete_by_pubkey_and_ids`; +//! - an `a` tag naming `::` goes to `delete_by_address`, which resolves and +//! deletes in the store (see [`EventRepository::delete_by_address`]). +//! +//! The second is deliberately *not* "read the row, then delete its id": a read is bounded by +//! the backend's page, so an address could sit outside it, and a read and a delete are two +//! operations, so a replacement landing between them would survive the delete the client was +//! told had happened. One operation per address is what makes the `OK` true. +//! +//! The deletions run **before** the request itself is stored, so a client that re-reads after +//! the `OK` sees them already applied. Then the common create-or-dedupe. use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use futures_util::StreamExt; -use nostream_core::{Event, EventId, Filter, Kind, Pubkey}; +use nostream_core::{Event, EventId, Kind, Pubkey}; use crate::error::RelayError; use crate::handlers::HandlerContext; @@ -29,19 +36,13 @@ impl EventStrategy for DeleteStrategy { context: &'a HandlerContext, ) -> Pin> + Send + 'a>> { Box::pin(async move { - let mut ids: Vec = event + let ids: Vec = event .tags .iter() .filter(|tag| tag.len() >= 2 && tag[0] == "e") .filter_map(|tag| tag[1].parse::().ok()) .collect(); - for address in addresses(event) { - if let Some(id) = resolve(&address, event, context).await? { - ids.push(id); - } - } - if !ids.is_empty() { context .stores @@ -50,6 +51,14 @@ impl EventStrategy for DeleteStrategy { .await?; } + for address in addresses(event) { + context + .stores + .events + .delete_by_address(&event.pubkey, address.kind, &address.d, event.created_at) + .await?; + } + let record = event_record(event, context); let count = context.stores.events.create(&record).await?; @@ -82,8 +91,8 @@ struct Address { /// /// `09.md`'s `a` branch is about addressable events, and only those carry a `d` in their /// address, so a regular replaceable kind named by an `a` tag is ignored. An address naming -/// another author is not this request's to delete: the pubkey must be the request's own. The -/// storage call re-checks the author as a second, independent guard. +/// another author is not this request's to delete: the pubkey must be the request's own, and +/// the store's delete re-checks the author as a second, independent guard. fn addresses(event: &Event) -> Vec
{ event .tags @@ -100,53 +109,3 @@ fn addresses(event: &Event) -> Vec
{ }) .collect() } - -/// The addressable row an address names, if it is still live and no newer than the request. -/// -/// Addressable kinds are stored one live row per `(pubkey, kind, d)` (`parameterized_replaceable` -/// makes the first `d` tag the deduplication key), so the author's rows of that kind are read and -/// the one whose first `d` tag equals the address's is kept. `09.md` asks for *"all versions … -/// up to the `created_at` timestamp of the deletion request"*, so a row newer than the request -/// survives. -async fn resolve( - address: &Address, - request: &Event, - context: &HandlerContext, -) -> Result, RelayError> { - let filters = [Filter { - kinds: Some(vec![address.kind]), - authors: Some(vec![request.pubkey.to_string()]), - ..Default::default() - }]; - - let mut rows = context.stores.events.find_by_filters(&filters).await?; - - while let Some(row) = rows.next().await { - let row = row?; - - if row.deleted_at.is_some() { - continue; - } - - let stored = &row.event; - - if d_identifier(stored) != address.d || stored.created_at > request.created_at { - continue; - } - - return Ok(Some(stored.id)); - } - - Ok(None) -} - -/// The value of an event's first `d` tag, `""` when it carries none — the same key the -/// addressable strategy deduplicates on. -fn d_identifier(event: &Event) -> &str { - event - .tags - .iter() - .find(|tag| tag.len() >= 2 && tag[0] == "d") - .map(|tag| tag[1].as_str()) - .unwrap_or_default() -} diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index 9914b3938..2e48c6e7c 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -22,6 +22,7 @@ use nostream_config::settings::{ }; use nostream_core::clock::Clock; use nostream_core::{Event, EventId, Filter, Kind, Pubkey}; +use nostream_storage::MAX_PAGE_ROWS; use nostream_storage::StorageError; use nostream_storage::repositories::{ ActionableTarget, CreateInviteCodeOptions, DvmJob, DvmJobStatus, EventPurgeCounts, EventRecord, @@ -153,9 +154,11 @@ pub struct InMemoryEventRepository { fail_count: AtomicBool, fail_queries: AtomicBool, fail_delete: AtomicBool, + fail_reads: AtomicBool, upsert_failure: Mutex>, upserts: Mutex>>, deletes: Mutex)>>, + address_deletes: Mutex>, delete_except_kinds_calls: Mutex)>>, /// When set, `find_by_filters` holds its stream until a permit is released, which is how /// a test gets a read in flight long enough to `CLOSE` it. @@ -204,6 +207,12 @@ impl InMemoryEventRepository { self.fail_queries.store(true, Ordering::SeqCst); } + /// Makes a read yield one error and end, so a handler's error path can be reached. The + /// stream carries no rows first, which is the case a `finish` hint would misreport. + pub fn fail_reads(&self, fail: bool) { + self.fail_reads.store(fail, Ordering::SeqCst); + } + /// Makes every read wait for a permit. Grant one with `Semaphore::add_permits` to let a /// read through. pub fn gate_reads(&self) -> Arc { @@ -267,6 +276,14 @@ impl InMemoryEventRepository { .clone() } + /// The `(pubkey, kind, d, not_after)` each `delete_by_address` was called with. + pub fn address_deletes(&self) -> Vec<(Pubkey, Kind, String, i64)> { + self.address_deletes + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() + } + /// The `(pubkey, excluded kinds)` each `delete_by_pubkey_except_kinds` was called with. pub fn delete_except_kinds_calls(&self) -> Vec<(Pubkey, Vec)> { self.delete_except_kinds_calls @@ -308,6 +325,17 @@ impl InMemoryEventRepository { } } +/// The value of an event's first `d` tag, `""` when it carries none — the key the addressable +/// strategy deduplicates on. +fn d_identifier(event: &Event) -> &str { + event + .tags + .iter() + .find(|tag| tag.len() >= 2 && tag[0] == "d") + .map(|tag| tag[1].as_str()) + .unwrap_or_default() +} + impl EventRepository for InMemoryEventRepository { fn create<'a>(&'a self, record: &'a EventRecord) -> StorageFuture<'a, u64> { Box::pin(async move { @@ -415,19 +443,47 @@ impl EventRepository for InMemoryEventRepository { .unwrap_or_else(|poisoned| poisoned.into_inner()) .clone(); let per_item = self.gate_per_item.load(std::sync::atomic::Ordering::SeqCst); + let failing = self.fail_reads.load(Ordering::SeqCst); Box::pin(async move { + if failing { + let failed: Vec> = + vec![Err(StorageError::InvalidArgument { + message: "in-memory double: the read failed", + })]; + + return Ok(Box::pin(stream::iter(failed)) as QueryStream<'a, EventRecord>); + } + let records = self .records .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) .clone(); - let matched: Vec> = records - .into_iter() - .filter(|record| filters.iter().any(|filter| filter.matches(&record.event))) - .map(Ok) - .collect(); + // A page, as the real backends answer one: each filter's own `limit`, never more + // than the backend bound, in insertion order, the union of the filters with no row + // twice. The double used to answer every match, which is what a handler that + // counted pages could not be tested against. + let mut matched: Vec> = Vec::new(); + let mut seen: Vec = Vec::new(); + + for filter in filters { + let bound = filter.limit.unwrap_or(u64::MAX).min(MAX_PAGE_ROWS); + let mut taken = 0u64; + + for record in &records { + if taken >= bound { + break; + } + + if filter.matches(&record.event) && !seen.contains(&record.event.id) { + seen.push(record.event.id); + matched.push(Ok(record.clone())); + taken += 1; + } + } + } let Some(gate) = gate else { return Ok(Box::pin(stream::iter(matched)) as QueryStream<'a, EventRecord>); @@ -534,6 +590,37 @@ impl EventRepository for InMemoryEventRepository { }) } + /// The addressable row one `(pubkey, kind, d)` names, soft-deleted when it is no newer than + /// `not_after` — one operation, so there is no window between a lookup and a delete. + fn delete_by_address<'a>( + &'a self, + pubkey: &'a Pubkey, + kind: Kind, + identifier: &'a str, + not_after: i64, + ) -> StorageFuture<'a, u64> { + Box::pin(async move { + self.address_deletes + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push((*pubkey, kind, identifier.to_owned(), not_after)); + + if self.fail_delete.load(Ordering::SeqCst) { + return Err(StorageError::InvalidArgument { + message: "in-memory double: the delete failed", + }); + } + + Ok(self.soft_delete(|record| { + record.event.pubkey == *pubkey + && record.event.kind == kind + && record.event.created_at <= not_after + && record.event.kind != Kind::REQUEST_TO_VANISH + && d_identifier(&record.event) == identifier + })) + }) + } + /// The same soft delete over every live row of one author but the excluded kinds. fn delete_by_pubkey_except_kinds<'a>( &'a self, diff --git a/crates/nostream-relay/tests/strategies.rs b/crates/nostream-relay/tests/strategies.rs index 9705ded94..10e485f02 100644 --- a/crates/nostream-relay/tests/strategies.rs +++ b/crates/nostream-relay/tests/strategies.rs @@ -463,6 +463,11 @@ fn address_delete_event(address: &str) -> Event { event(5, vec![tag(&["a", address])], "") } +/// The one address delete a request naming `post-1` makes, in the order the seam takes it. +fn one_address_delete(d: &str, not_after: i64) -> Vec<(Pubkey, Kind, String, i64)> { + vec![(author(), Kind(30_023), d.to_owned(), not_after)] +} + #[tokio::test] async fn delete_soft_deletes_an_addressable_event_named_by_an_a_tag() { let fixture = Fixture::new(settings()); @@ -474,7 +479,54 @@ async fn delete_soft_deletes_an_addressable_event_named_by_an_a_tag() { DeleteStrategy.execute(&event, &context).await.unwrap(); - assert_eq!(fixture.events.deletes(), vec![(author(), vec![target.id])]); + assert_eq!( + fixture.events.address_deletes(), + one_address_delete("post-1", NOW_SECS) + ); + assert_eq!(fixture.events.live_events(), vec![event.clone()]); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + +/// The lookup and the deletion are one storage operation, so there is no window between them for +/// a replacement to land in: the seam is asked once, by address, and no id list is resolved. +#[tokio::test] +async fn delete_resolves_and_deletes_an_address_in_one_operation() { + let fixture = Fixture::new(settings()); + let target = addressable_row(Some("post-1"), NOW_SECS); + fixture.seed_event(target.clone()).await; + + let (context, mut sink) = socket(&fixture); + let event = address_delete_event(&format!("30023:{}:post-1", author())); + + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!( + fixture.events.deletes(), + Vec::new(), + "the a-tag path never builds an id list" + ); + assert_eq!( + fixture.events.address_deletes(), + one_address_delete("post-1", NOW_SECS) + ); + assert_eq!(fixture.events.live_events(), vec![event.clone()]); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + +/// The delete does not consult a read, which is what makes it reach a row NIP-56 hides from +/// reads: a failing stream changes nothing about the address path. +#[tokio::test] +async fn delete_deletes_by_address_without_a_read() { + let fixture = Fixture::new(settings()); + let target = addressable_row(Some("post-1"), NOW_SECS); + fixture.seed_event(target.clone()).await; + fixture.events.fail_reads(true); + + let (context, mut sink) = socket(&fixture); + let event = address_delete_event(&format!("30023:{}:post-1", author())); + + DeleteStrategy.execute(&event, &context).await.unwrap(); + assert_eq!(fixture.events.live_events(), vec![event.clone()]); assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); } @@ -492,6 +544,7 @@ async fn delete_ignores_an_a_tag_naming_another_authors_address() { DeleteStrategy.execute(&event, &context).await.unwrap(); + assert_eq!(fixture.events.address_deletes(), Vec::new()); assert_eq!(fixture.events.deletes(), Vec::new()); assert!(fixture.events.live_events().contains(&target)); assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); @@ -508,7 +561,12 @@ async fn delete_leaves_an_addressable_event_newer_than_the_request() { DeleteStrategy.execute(&event, &context).await.unwrap(); - assert_eq!(fixture.events.deletes(), Vec::new()); + // The bound travels with the delete, which is what makes it a compare-and-delete rather than + // a list of ids chosen earlier. + assert_eq!( + fixture.events.address_deletes(), + one_address_delete("post-1", NOW_SECS) + ); assert!(fixture.events.live_events().contains(&target)); assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); } @@ -525,7 +583,10 @@ async fn delete_matches_an_empty_d_identifier() { DeleteStrategy.execute(&event, &context).await.unwrap(); - assert_eq!(fixture.events.deletes(), vec![(author(), vec![target.id])]); + assert_eq!( + fixture.events.address_deletes(), + one_address_delete("", NOW_SECS) + ); assert_eq!(fixture.events.live_events(), vec![event.clone()]); assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); } From 09b1f13cdfce603917b8465d55fa842595a40336 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 01:59:44 -0500 Subject: [PATCH 11/36] fix(relay): claim finish only on a clean full-free page with no further match --- .../nostream-relay/src/handlers/subscribe.rs | 56 +++++++--- crates/nostream-relay/tests/handlers.rs | 101 +++++++++++++++++- 2 files changed, 142 insertions(+), 15 deletions(-) diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index 11fefa9cd..de8b8309c 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -28,6 +28,7 @@ use nostream_core::messages::{ }; use nostream_core::nip42; use nostream_core::{Filter, Kind}; +use nostream_storage::MAX_PAGE_ROWS; use crate::connection::InFlightRead; use crate::error::RelayError; @@ -268,15 +269,16 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha None }; - // The probe: ask storage for one more row than the client's cap, so the `(limit + 1)`th - // matching row is the one that says `more` without being sent. `u64::MAX` saturates, which + // The probe: ask storage for one more row than the client's cap, so a `(limit + 1)`th + // **matching** row is what says `more` without being sent. `u64::MAX` saturates, which // admits nothing beyond the maximum and so cannot hide a row. The backends already honour // `limit`; the in-memory double does not, and the loop below drops the extra row either way. - let storage_filters: Vec = match limit { - Some(limit) => filters + let probe = limit.map(|limit| limit.saturating_add(1)); + let storage_filters: Vec = match probe { + Some(probe) => filters .iter() .map(|filter| Filter { - limit: Some(limit.saturating_add(1)), + limit: Some(probe), ..filter.clone() }) .collect(), @@ -303,12 +305,15 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha let matches_any = |event: &nostream_core::Event| filters.iter().any(|filter| filter.matches(event)); - // `trimmed`: the guard withheld a matching, unexpired event this read would otherwise have - // delivered. `more`: the cap cut the result. `delivered`: matching rows inside the cap, in - // stream order — a deleted, expired or non-matching row is not one of them. + // `rows`: every row the page yielded, skipped ones included — a full page is what makes the + // backend's remaining extent unknown. `matched`: rows that were a match inside the client's + // window, which is what `limit` bounds. `trimmed`: the guard withheld a match. `more`: a + // match beyond the window was found. `failed`: the stream errored. + let mut rows = 0u64; + let mut matched = 0u64; let mut trimmed = false; let mut more = false; - let mut matched = 0u64; + let mut failed = false; loop { if *read.cancellation().borrow() { @@ -357,10 +362,13 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha match record { Ok(record) => { + // Every row the page yielded counts towards whether the page was full, whether + // or not it turns out to be a match. + rows += 1; + if record.deleted_at.is_some() { continue; } - let event = record.event; if expiration::is_expired_event(&event, now) || !matches_any(&event) { @@ -398,6 +406,8 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha } Err(error) => { tracing::error!(error = %error, "error streaming events"); + failed = true; + break; } } @@ -405,9 +415,27 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha context.connection.end_read(read); - // `auth` first, then the completeness hint. `finish` is never sent beside `auth`: `67.md` - // defines `finish` as "every stored event matching the subscription's filters" has been - // sent, and a withheld event makes that false even though the NIP's example shows the pair. + // The rule, in one place, so the next reader does not have to re-derive it. `finish` claims + // that every stored event matching the subscription's filters has been sent (`67.md` + // §"Specification"), and this read can fail to establish that three ways, so `finish` needs + // all three: + // + // 1. the stream ended **cleanly** — after a storage error, how far the read got is unknown; + // 2. the page was **not full** — a page that yielded `min(probe, MAX_PAGE_ROWS)` rows says + // nothing about the rows beyond it, whether the bound came from the client's `limit` or + // from the backend's own page cap, and a skipped row spends a page slot just as a match + // does (which is why `rows`, not `matched`, decides this); + // 3. **no further match** was found — `more` stayed false, which is only meaningful once 1 + // and 2 hold, because the loop keeps asking past deleted, expired, withheld and + // non-matching rows until it sees a match, the genuine end, or the page cap. + // + // Where any of the three cannot be established the relay sends no completeness hint: an + // absent hint is honest and a wrong one is not. `more` needs one further **match** beyond + // the delivered window, `auth` is unaffected, and `finish` is never sent beside `auth` + // because a withheld event falsifies it. + let page_full = probe.is_some_and(|probe| rows >= probe.min(MAX_PAGE_ROWS)); + let finish = !failed && !more && !page_full; + let mut hints = Vec::new(); if trimmed { @@ -417,7 +445,7 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha if hint_eligible { if more { hints.push(EOSE_HINT_MORE.to_owned()); - } else if !trimmed { + } else if finish && !trimmed { hints.push(EOSE_HINT_FINISH.to_owned()); } } diff --git a/crates/nostream-relay/tests/handlers.rs b/crates/nostream-relay/tests/handlers.rs index 084394007..f37a50994 100644 --- a/crates/nostream-relay/tests/handlers.rs +++ b/crates/nostream-relay/tests/handlers.rs @@ -32,7 +32,7 @@ use nostream_relay::connection::Connection; use nostream_relay::relay_key::RelayKeySource; use nostream_relay::sink::FrameSink; use nostream_relay::testing::{Fixture, nip42_settings, nip43_settings, nip45_settings, settings}; -use nostream_storage::repositories::User; +use nostream_storage::repositories::{EventRepository, User}; /// The relay's public key for [`RELAY_SECRET`]. fn relay_pubkey() -> nostream_core::Pubkey { @@ -1111,6 +1111,105 @@ async fn eose_hints_more_when_the_limit_was_reached() { assert_eq!(frames[2], r#"["EOSE","s1",["more"]]"#); } +/// A skipped row spends a page slot just as a match does, so a page that came back full of skips +/// cannot say whether the row after it was a match. Deleted rows sort ahead of live ones here, +/// which is the shape the review measured against LMDB: two deleted rows and one live one fill a +/// three-row probe for a `limit` of 2. +#[tokio::test] +async fn eose_hints_are_absent_when_the_page_was_full_of_skips() { + let fixture = Fixture::new(settings()); + + for index in 0..2 { + let deleted = signed_event( + &CLIENT_SECRET, + 1, + vec![], + &format!("gone {index}"), + NOW_SECS, + ); + fixture.seed_event(deleted.clone()).await; + fixture + .events + .delete_by_pubkey_and_ids(&deleted.pubkey, &[deleted.id]) + .await + .expect("the double deletes"); + } + + seed_notes(&fixture, 2).await; + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r#"["REQ","s1",{"kinds":[1],"limit":2}]"#).await; + let frames = client.read_until_eose().await; + + assert_eq!( + frames.len(), + 2, + "two deleted rows cost the page a slot each, so one live row is left: {frames:?}" + ); + assert!(events_then_eose(&frames), "{frames:?}"); + assert_eq!(frames[1], r#"["EOSE","s1"]"#); +} + +/// The backend caps the page below the client's `limit`, so the probe came back full and the rows +/// beyond it are unknown: exactly 5000 sent is not a claim that nothing was left. +#[tokio::test] +async fn eose_hints_are_absent_when_the_backend_page_cap_was_reached() { + let fixture = Fixture::new(settings()); + seed_notes(&fixture, 5_001).await; + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client + .send(r#"["REQ","s1",{"kinds":[1],"limit":5000}]"#) + .await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 5_001, "{frames:?}"); + assert!(events_then_eose(&frames), "{frames:?}"); + assert_eq!(frames[5_000], r#"["EOSE","s1"]"#); +} + +/// A storage error leaves the read's extent unknown, so no hint is added after it. +#[tokio::test] +async fn eose_hints_are_absent_when_the_stream_failed() { + let fixture = Fixture::new(settings()); + seed_notes(&fixture, 1).await; + fixture.events.fail_reads(true); + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r#"["REQ","s1",{"kinds":[1],"limit":5}]"#).await; + let frames = client.read_until_eose().await; + + assert_eq!(frames, vec![r#"["EOSE","s1"]"#.to_owned()], "{frames:?}"); +} + +/// `auth` and the completeness hint compose: a withheld match falsifies `finish` even where the +/// page was neither full nor capped, and `auth` is what is sent. +#[tokio::test] +async fn eose_hints_send_auth_without_finish_when_a_match_was_withheld() { + let mut settings = settings(); + settings.nip42 = Some(nip42_settings()); + let fixture = Fixture::new(settings); + + let stranger_secret = [0x77; 32]; + let withheld = signed_event(&stranger_secret, 4, vec![], "ciphertext", NOW_SECS); + fixture.seed_event(withheld).await; + seed_notes(&fixture, 1).await; + + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client + .send(r#"["REQ","s1",{"kinds":[1,4],"limit":2}]"#) + .await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 2, "{frames:?}"); + assert_eq!(frames[1], r#"["EOSE","s1",["auth"]]"#); +} + #[tokio::test] async fn a_close_aborts_an_in_flight_read_without_a_frame() { let stored = signed_event(&CLIENT_SECRET, 1, vec![], "stored", NOW_SECS); From fc42a48e1c0788b46d33ef0be502a7666185a685 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 02:01:52 -0500 Subject: [PATCH 12/36] test(relay): cover both spellings of an empty NIP-09 d identifier --- crates/nostream-relay/tests/strategies.rs | 47 +++++++++++++++-------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/crates/nostream-relay/tests/strategies.rs b/crates/nostream-relay/tests/strategies.rs index 10e485f02..24ce4622b 100644 --- a/crates/nostream-relay/tests/strategies.rs +++ b/crates/nostream-relay/tests/strategies.rs @@ -573,22 +573,37 @@ async fn delete_leaves_an_addressable_event_newer_than_the_request() { #[tokio::test] async fn delete_matches_an_empty_d_identifier() { - let fixture = Fixture::new(settings()); - // A row with no `d` tag is the one stored under the address's empty `d`. - let target = addressable_row(None, NOW_SECS); - fixture.seed_event(target.clone()).await; - - let (context, mut sink) = socket(&fixture); - let event = address_delete_event(&format!("30023:{}:", author())); - - DeleteStrategy.execute(&event, &context).await.unwrap(); - - assert_eq!( - fixture.events.address_deletes(), - one_address_delete("", NOW_SECS) - ); - assert_eq!(fixture.events.live_events(), vec![event.clone()]); - assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); + // A row with no `d` tag is stored under the address's empty `d`, and an address may spell + // that `d` as an empty third part or leave the third part off entirely. + for address in [ + format!("30023:{}:", author()), + format!("30023:{}", author()), + ] { + let fixture = Fixture::new(settings()); + let target = addressable_row(None, NOW_SECS); + fixture.seed_event(target.clone()).await; + + let (context, mut sink) = socket(&fixture); + let event = address_delete_event(&address); + + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!( + fixture.events.address_deletes(), + one_address_delete("", NOW_SECS), + "{address}" + ); + assert_eq!( + fixture.events.live_events(), + vec![event.clone()], + "{address}" + ); + assert_eq!( + frames(&mut sink), + vec![ok_frame(&event, true, "")], + "{address}" + ); + } } // --------------------------------------------------------------------------- From e81019d3147e85dcc369b073c1e15004112fce71 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 02:03:05 -0500 Subject: [PATCH 13/36] refactor(relay): hold finish's proof in one binding and leave more its own branch --- crates/nostream-relay/src/handlers/subscribe.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index de8b8309c..7105c1b45 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -425,16 +425,18 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // nothing about the rows beyond it, whether the bound came from the client's `limit` or // from the backend's own page cap, and a skipped row spends a page slot just as a match // does (which is why `rows`, not `matched`, decides this); - // 3. **no further match** was found — `more` stayed false, which is only meaningful once 1 - // and 2 hold, because the loop keeps asking past deleted, expired, withheld and - // non-matching rows until it sees a match, the genuine end, or the page cap. + // 3. **no further match** was found — the `more` branch below is where this one lives: a + // further match ends the read with `more` set, so the code that can reach the hint is the + // code where none was found. It is only meaningful once 1 and 2 hold, because the loop + // keeps asking past deleted, expired, withheld and non-matching rows until it sees a + // match, the genuine end, or the page cap. // // Where any of the three cannot be established the relay sends no completeness hint: an // absent hint is honest and a wrong one is not. `more` needs one further **match** beyond // the delivered window, `auth` is unaffected, and `finish` is never sent beside `auth` // because a withheld event falsifies it. let page_full = probe.is_some_and(|probe| rows >= probe.min(MAX_PAGE_ROWS)); - let finish = !failed && !more && !page_full; + let proven = !failed && !page_full; let mut hints = Vec::new(); @@ -445,7 +447,7 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha if hint_eligible { if more { hints.push(EOSE_HINT_MORE.to_owned()); - } else if finish && !trimmed { + } else if proven && !trimmed { hints.push(EOSE_HINT_FINISH.to_owned()); } } From e732898f43784b092b4af7efe021d1cc1f17cb6d Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 02:16:01 -0500 Subject: [PATCH 14/36] docs(storage): mark the kind-range check as the caller's --- crates/nostream-storage/src/repositories.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index 858417598..c3d13ae2d 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -442,6 +442,10 @@ pub trait EventRepository: Send + Sync { /// and a read and a delete are two operations, so a replacement landing between them /// survives the delete the client was told had happened. Both are properties of the store, /// so the store performs the resolution and the deletion as one. + /// + /// `shortcut: the addressable kind range is the caller's to check, not the store's; upgrade + /// when a caller other than the NIP-09 `a`-tag strategy passes a kind`, because the store + /// takes the kind it is given exactly as `delete_by_pubkey_and_ids` takes the ids it is given. fn delete_by_address<'a>( &'a self, pubkey: &'a Pubkey, From 1a83905dc3ac34acd7c50afec1e253e1f766756e Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 02:50:17 -0500 Subject: [PATCH 15/36] fix(relay): charge an overlapping row against the filter's own page limit --- crates/nostream-relay/src/testing.rs | 80 +++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 8 deletions(-) diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index 2e48c6e7c..c91cd38e5 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -461,10 +461,12 @@ impl EventRepository for InMemoryEventRepository { .unwrap_or_else(|poisoned| poisoned.into_inner()) .clone(); - // A page, as the real backends answer one: each filter's own `limit`, never more - // than the backend bound, in insertion order, the union of the filters with no row - // twice. The double used to answer every match, which is what a handler that - // counted pages could not be tested against. + // A page, as the real backends answer one: each filter takes its own `limit` in + // insertion order, never more than the backend bound, and the union is deduplicated + // afterwards — so a row another filter already returned still spends a slot of this + // filter's limit, exactly as it does in both backends. The double used to answer + // every match, and then to not charge an overlap against the limit, which is how a + // handler test stops meaning anything. let mut matched: Vec> = Vec::new(); let mut seen: Vec = Vec::new(); @@ -477,11 +479,18 @@ impl EventRepository for InMemoryEventRepository { break; } - if filter.matches(&record.event) && !seen.contains(&record.event.id) { - seen.push(record.event.id); - matched.push(Ok(record.clone())); - taken += 1; + if !filter.matches(&record.event) { + continue; } + + taken += 1; + + if seen.contains(&record.event.id) { + continue; + } + + seen.push(record.event.id); + matched.push(Ok(record.clone())); } } @@ -1686,6 +1695,7 @@ mod tests { #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] use super::*; + use nostream_core::Signature; fn claimant(byte: u8) -> Pubkey { Pubkey::from_byte_array([byte; 32]) @@ -1699,6 +1709,60 @@ mod tests { } } + #[tokio::test] + async fn a_row_another_filter_returned_still_spends_this_filters_limit() { + // Both backends take each filter's page first and deduplicate the union afterwards, so a + // row the first filter already returned is still one of the second filter's `limit` + // rows. The double used to skip it without charging the slot, which handed a handler + // more rows than either backend would (`double=["old","new"]` against `lmdb=["new"]` + // for two overlapping filters each capped at one). + let events = InMemoryEventRepository::new(); + let author = Pubkey::from_byte_array([7; 32]); + + for (index, content) in ["old", "new"].into_iter().enumerate() { + let event = Event { + id: EventId::from_byte_array([index as u8; 32]), + pubkey: author, + created_at: index as i64 + 1, + kind: Kind(1), + tags: Vec::new(), + content: content.to_owned(), + sig: Signature::from_byte_array([0; 64]), + }; + events + .create(&EventRecord::from_event(&event)) + .await + .unwrap(); + } + + let filters = [ + Filter { + kinds: Some(vec![Kind(1)]), + limit: Some(1), + ..Filter::default() + }, + Filter { + authors: Some(vec![author.to_string()]), + limit: Some(1), + ..Filter::default() + }, + ]; + + let stream = events.find_by_filters(&filters).await.unwrap(); + futures_util::pin_mut!(stream); + + let mut rows = Vec::new(); + while let Some(record) = futures_util::StreamExt::next(&mut stream).await { + rows.push(record.unwrap()); + } + + assert_eq!( + rows.len(), + 1, + "one row, because the second filter's first match was the first filter's row" + ); + } + #[tokio::test] async fn a_shared_invite_code_admits_a_second_pubkey_until_its_uses_run_out() { // The capture's `claimCode` conditions on `code`, the remaining uses and the expiry From b242e430148d970e538faff648a7293a2dc0ddf3 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 02:50:17 -0500 Subject: [PATCH 16/36] test(storage): pin the created_at window inside delete_by_address --- .../nostream-storage/tests/address_delete.rs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/nostream-storage/tests/address_delete.rs b/crates/nostream-storage/tests/address_delete.rs index e5e995fed..e278863c6 100644 --- a/crates/nostream-storage/tests/address_delete.rs +++ b/crates/nostream-storage/tests/address_delete.rs @@ -140,3 +140,47 @@ async fn delete_by_address_reaches_a_reported_row() { assert_eq!(deleted, 1, "the delete reaches the row the read cannot"); } + +/// The request's own `created_at` is the authorisation window, so the comparison is inside the +/// delete: a row newer than the request survives, and one no newer than it goes. Both rows sit +/// under the same author and kind, one address apart, so only the bound decides. +#[tokio::test] +async fn delete_by_address_keeps_a_row_newer_than_the_request() { + let (_directory, backend) = open_store().await; + let author = addressable(7, "kept", 1_010); + let at_the_bound = addressable(7, "gone", 1_000); + backend.create(&author).await.expect("the store creates"); + backend + .create(&at_the_bound) + .await + .expect("the store creates"); + + assert_eq!( + backend + .delete_by_address(&author.event.pubkey, Kind(30_023), "kept", 1_000) + .await + .expect("the store deletes"), + 0, + "a replacement newer than the request is not its to delete" + ); + assert_eq!( + backend + .delete_by_address(&at_the_bound.event.pubkey, Kind(30_023), "gone", 1_000) + .await + .expect("the store deletes"), + 1, + "a row at exactly the request's created_at is deleted" + ); + + assert_eq!( + backend + .count_by_filters(&[Filter { + authors: Some(vec![author.event.pubkey.to_string()]), + ..Default::default() + }]) + .await + .expect("a read"), + 1, + "only the newer row is left" + ); +} From 31251427b66c8ce87c5091ac0171ebb548adfcd8 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 02:50:17 -0500 Subject: [PATCH 17/36] docs(rewrite): describe the address-walk delete and scope the page bound --- crates/nostream-storage/src/lib.rs | 20 +++++++++++++------- docs/rewrite/nip-parity.md | 14 ++++++++++++-- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index 225335bb5..b3df47e89 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -44,13 +44,19 @@ pub use lmdb::{LmdbBackend, MapUsage}; #[cfg(feature = "postgres")] pub use postgres::{PostgresBackend, PostgresTransaction}; -/// The most rows one filter's page can hold on any shipped backend, whatever `limit` it -/// carries. +/// The largest page the relay assumes any shipped backend can return for one filter, whatever +/// `limit` it carries. /// /// A caller that must know whether a page was truncated compares the rows a stream yielded -/// against `min(limit, MAX_PAGE_ROWS)`: a stream that yields that many has a full page and says -/// nothing about what lies beyond it. LMDB clamps every filter to this bound -/// (`lmdb::MAX_PAGE_KEYS`, which is this constant); Postgres honours the filter's own `limit`, -/// and its default (`DEFAULT_FILTER_LIMIT`) is smaller. A new backend that clamps lower must -/// lower this constant, and the LMDB clamp's own test fails if the two drift apart. +/// against `min(limit, MAX_PAGE_ROWS)`: a stream that yields that many has a page whose end the +/// relay cannot see, and must say nothing about what lies beyond it. +/// +/// The two backends reach the bound differently. LMDB clamps every filter to it in code +/// (`lmdb::MAX_PAGE_KEYS`, which is this constant, and the clamp's own test fails if the two +/// drift apart). Postgres does **not** clamp by constant: it honours the filter's own `limit`, +/// with `DEFAULT_FILTER_LIMIT = 500` when there is none. The bound reaches it through the wire +/// instead — `limits.client.subscription.maxLimit` ships at this number and a `REQ` above it is +/// refused (`subscribe.rs:525`) — so the two agree only while that setting does. Raising it +/// above this constant leaves Postgres able to serve more than the relay will believe, which +/// suppresses `finish` where it could have been proved and never claims it where it should not. pub const MAX_PAGE_ROWS: u64 = 5_000; diff --git a/docs/rewrite/nip-parity.md b/docs/rewrite/nip-parity.md index 3bd094bce..63fbb83a2 100644 --- a/docs/rewrite/nip-parity.md +++ b/docs/rewrite/nip-parity.md @@ -878,7 +878,17 @@ is a defect, and §10 is the same record in table form. whose tags are `a`-only deletes nothing. The rewrite either implements addressable-event deletion or records the omission deliberately — but §5 and the §7 row now say so, and Phase 2 must not inherit the silence. - - **Status:** done in this job — an `a` tag resolves to the author's addressable row, subject to the author-equality rule and the request's `created_at`, and its id joins the `e`-tag ids in one `delete_by_pubkey_and_ids`; `strategies::delete_soft_deletes_an_addressable_event_named_by_an_a_tag` and three more (§10). + - **Status:** done in this job — an `a` tag is handed to `EventRepository::delete_by_address`, + which resolves the address and soft-deletes it in **one** storage operation: LMDB walks the + author-and-kind prefix of `events_by_author` (one index range, no read page, so a narrow + address is never found by scanning a page that sorts it out of view) and Postgres runs one + `UPDATE … WHERE`; the request's `created_at` is compared inside that transaction, so a + replacement cannot land between a lookup and the delete, and the author-equality and + addressable-kind rules still gate the tag before it is read. Records with an actionable + NIP-56 report are reachable too, because the delete no longer goes through the read path + that withholds them. `strategies::delete_resolves_and_deletes_an_address_in_one_operation`, + `delete_by_address_reaches_a_row_past_the_page_bound`, + `delete_by_address_reaches_a_reported_row` and four more (§10). ## 10. Phase 2: the advertisement and the harness map @@ -946,7 +956,7 @@ no relay-side test reaches them. | 03 | `core/nip03.rs`; `relay/strategies/timestamp.rs` | `vectors-only: strategies::timestamp_stores_and_broadcasts_a_valid_proof` | | 04 | `core/event.rs`; `relay/strategies/default.rs`; `core/nip42.rs` | `vectors-only: handlers::a_restricted_read_delivers_only_to_the_author_or_a_p_tagged_reader` | | 05 | `core/nip05.rs`; `relay/handlers/event.rs`; `services/workers/maintenance.rs` | `vectors-only: handlers::nip05_gates_publication` | -| 09 | `relay/strategies/delete.rs` | `divergent: strategies::delete_soft_deletes_an_addressable_event_named_by_an_a_tag` | +| 09 | `relay/strategies/delete.rs` | `divergent: strategies::delete_resolves_and_deletes_an_address_in_one_operation` | | 11 | none: the constant is the record | `no-wire-surface: no HTTP listener and no NIP-11 document (D1)` | | 12 | `core/filter.rs` generic `#` criteria | `vectors-only: event_parity::nip12_d_filter_returns_true_if_d_filter_contains_an_empty_tag_value_in_the_event` *(nostream-core)* | | 13 | `core/pow.rs`, `core/adaptive_pow.rs`; `relay/handlers/event.rs` | `session event-rejections` | From 51a22c0e27f12fd6cb712606767dd47b19953afd Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 03:29:48 -0500 Subject: [PATCH 18/36] refactor(core): derive an event's d-tag value in one place --- crates/nostream-core/src/event.rs | 52 +++++++++++++++++++ .../strategies/parameterized_replaceable.rs | 7 +-- .../src/services/event_import_service.rs | 8 +-- 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/crates/nostream-core/src/event.rs b/crates/nostream-core/src/event.rs index 4a1dbb5bf..aa5afb343 100644 --- a/crates/nostream-core/src/event.rs +++ b/crates/nostream-core/src/event.rs @@ -383,6 +383,24 @@ impl Event { .iter() .any(|tag| tag.first().is_some_and(|name| name == "-")) } + + /// The event's addressable identifier: the value of its **first** `["d", ]` tag, `""` + /// when it carries none — or carries only a valueless `["d"]` — which is the key an address + /// without a `d` is stored and deleted under. + /// + /// One derivation, because this value is a storage key in three places: the addressable + /// strategy writes it, the import path writes it, and a NIP-09 `a` tag is resolved through it + /// (LMDB by its address pointer, Postgres by the `event_deduplication` column). A second copy + /// of the rule is how the two backends come to delete different rows for one `a` tag. + pub fn d_tag_value(&self) -> &str { + self.tags + .iter() + .find_map(|tag| match tag.as_slice() { + [name, value, ..] if name == "d" => Some(value.as_str()), + _ => None, + }) + .unwrap_or_default() + } } /// The six-element array the event id is hashed over (`serializeEvent`, @@ -442,6 +460,40 @@ mod tests { EventId::try_from(value) } + #[test] + fn d_tag_value_is_the_first_valued_d_tag_and_empty_when_there_is_none() { + // The rule every consumer shares: the first `["d", ]` tag, `""` when absent. A + // duplicate `d` takes the first, a valueless `["d"]` is skipped rather than yielding `""`, + // and a colon in the value is part of the value (the address parser splits at most twice). + let addresses = [ + (vec![vec!["d".to_owned(), "post-1".to_owned()]], "post-1"), + (vec![vec!["t".to_owned(), "x".to_owned()]], ""), + ( + vec![ + vec!["d".to_owned(), "a".to_owned()], + vec!["d".to_owned(), "b".to_owned()], + ], + "a", + ), + (vec![vec!["d".to_owned()]], ""), + ( + vec![ + vec!["d".to_owned()], + vec!["d".to_owned(), "post-2".to_owned()], + ], + "post-2", + ), + (vec![vec!["d".to_owned(), "a:b".to_owned()]], "a:b"), + ]; + + for (tags, expected) in addresses { + let mut event = event(); + event.tags = tags; + + assert_eq!(event.d_tag_value(), expected, "{:?}", event.tags); + } + } + #[test] fn id_round_trips_through_hex() { let id: EventId = ID.parse().unwrap(); diff --git a/crates/nostream-relay/src/strategies/parameterized_replaceable.rs b/crates/nostream-relay/src/strategies/parameterized_replaceable.rs index 512891d79..60d32af48 100644 --- a/crates/nostream-relay/src/strategies/parameterized_replaceable.rs +++ b/crates/nostream-relay/src/strategies/parameterized_replaceable.rs @@ -28,12 +28,7 @@ impl EventStrategy for ParameterizedReplaceableStrategy { context: &'a HandlerContext, ) -> Pin> + Send + 'a>> { Box::pin(async move { - let deduplication = event - .tags - .iter() - .find(|tag| tag.len() >= 2 && tag[0] == "d") - .map(|tag| tag[1].clone()) - .unwrap_or_default(); + let deduplication = event.d_tag_value().to_owned(); let record = event_record(event, context) .with_deduplication(vec![serde_json::Value::String(deduplication)]); diff --git a/crates/nostream-services/src/services/event_import_service.rs b/crates/nostream-services/src/services/event_import_service.rs index 0b060a931..5bc0a9d97 100644 --- a/crates/nostream-services/src/services/event_import_service.rs +++ b/crates/nostream-services/src/services/event_import_service.rs @@ -493,13 +493,7 @@ fn enrich(event: &Event) -> EventRecord { } if event.kind.is_parameterized_replaceable() { - let deduplication = event - .tags - .iter() - .find(|tag| tag.len() >= 2 && tag[0] == "d") - .map(|tag| tag[1].clone()) - .filter(|value| !value.is_empty()) - .unwrap_or_default(); + let deduplication = event.d_tag_value().to_owned(); record.event_deduplication = Some(vec![serde_json::Value::String(deduplication)]); } From c007d422400d1e95af570e524932ab295c57ecdb Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 03:29:48 -0500 Subject: [PATCH 19/36] perf(storage): resolve a NIP-09 address through the replaceable point index --- crates/nostream-relay/src/unavailable.rs | 5 +- crates/nostream-services/src/testing.rs | 5 +- crates/nostream-services/src/unavailable.rs | 5 +- crates/nostream-storage/src/lib.rs | 7 + crates/nostream-storage/src/lmdb/events.rs | 81 ++++---- crates/nostream-storage/src/lmdb/mod.rs | 17 +- .../nostream-storage/src/postgres/events.rs | 35 ++-- .../src/postgres/repository.rs | 7 +- crates/nostream-storage/src/repositories.rs | 51 ++++- .../nostream-storage/tests/address_delete.rs | 189 ++++++++++++++---- crates/nostream-storage/tests/page_bound.rs | 101 ++++++++++ crates/nostream-storage/tests/storage.rs | 7 +- 12 files changed, 377 insertions(+), 133 deletions(-) create mode 100644 crates/nostream-storage/tests/page_bound.rs diff --git a/crates/nostream-relay/src/unavailable.rs b/crates/nostream-relay/src/unavailable.rs index 11a25f5ed..8121828e7 100644 --- a/crates/nostream-relay/src/unavailable.rs +++ b/crates/nostream-relay/src/unavailable.rs @@ -113,11 +113,10 @@ impl EventRepository for UnavailableEventRepository { Box::pin(async { Err(unavailable()) }) } - fn delete_by_address<'a>( + fn delete_by_addresses<'a>( &'a self, _pubkey: &'a Pubkey, - _kind: Kind, - _identifier: &'a str, + _addresses: &'a [(Kind, String)], _not_after: i64, ) -> StorageFuture<'a, u64> { Box::pin(async { Err(unavailable()) }) diff --git a/crates/nostream-services/src/testing.rs b/crates/nostream-services/src/testing.rs index fc08e90ca..414122295 100644 --- a/crates/nostream-services/src/testing.rs +++ b/crates/nostream-services/src/testing.rs @@ -573,11 +573,10 @@ impl EventRepository for InMemoryEventRepository { }) } - fn delete_by_address<'a>( + fn delete_by_addresses<'a>( &'a self, _pubkey: &'a Pubkey, - _kind: Kind, - _identifier: &'a str, + _addresses: &'a [(Kind, String)], _not_after: i64, ) -> StorageFuture<'a, u64> { Box::pin(async { Ok(0) }) diff --git a/crates/nostream-services/src/unavailable.rs b/crates/nostream-services/src/unavailable.rs index 610696446..50c6beaf1 100644 --- a/crates/nostream-services/src/unavailable.rs +++ b/crates/nostream-services/src/unavailable.rs @@ -100,11 +100,10 @@ impl EventRepository for UnavailableEventRepository { Box::pin(async { Err(unavailable()) }) } - fn delete_by_address<'a>( + fn delete_by_addresses<'a>( &'a self, _pubkey: &'a nostream_core::Pubkey, - _kind: nostream_core::Kind, - _identifier: &'a str, + _addresses: &'a [(nostream_core::Kind, String)], _not_after: i64, ) -> StoreFuture<'a, u64> { Box::pin(async { Err(unavailable()) }) diff --git a/crates/nostream-storage/src/lib.rs b/crates/nostream-storage/src/lib.rs index b3df47e89..ddee1d904 100644 --- a/crates/nostream-storage/src/lib.rs +++ b/crates/nostream-storage/src/lib.rs @@ -51,6 +51,13 @@ pub use postgres::{PostgresBackend, PostgresTransaction}; /// against `min(limit, MAX_PAGE_ROWS)`: a stream that yields that many has a page whose end the /// relay cannot see, and must say nothing about what lies beyond it. /// +/// This is only the relay's **default** assumption. The obligation itself is stated on +/// [`EventRepository::find_by_filters`](repositories::EventRepository::find_by_filters), in both +/// directions — a backend must yield up to the bound and must not exceed it — and a backend that +/// clamps lower declares its own through +/// [`EventRepository::page_bound`](repositories::EventRepository::page_bound), which the relay +/// reads instead of this constant. +/// /// The two backends reach the bound differently. LMDB clamps every filter to it in code /// (`lmdb::MAX_PAGE_KEYS`, which is this constant, and the clamp's own test fails if the two /// drift apart). Postgres does **not** clamp by constant: it honours the filter's own `limit`, diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index e2aa58a1c..9a8ee4107 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -17,7 +17,7 @@ use chrono::{DateTime, Utc}; use futures_core::Stream; use heed::types::Bytes; use heed::{Database, Env, RoTxn, RwTxn}; -use nostream_core::{Event, EventId, Filter, Kind, KindSelector, Pubkey}; +use nostream_core::{EventId, Filter, Kind, KindSelector, Pubkey}; use tokio::task::JoinHandle; use super::encoding::{self, EventHeader}; @@ -351,62 +351,69 @@ pub(crate) fn delete_by_pubkey_and_ids( Ok(affected) } -/// `delete_by_address`: the author's live rows of one addressable kind whose first `d` tag is -/// `identifier` and whose `created_at` is no later than `not_after`, soft-deleted. +/// `delete_by_addresses`: the author's live row at each address, soft-deleted when its +/// `created_at` is no later than `not_after` — one write transaction for the whole batch. /// -/// The walk is the author-and-kind prefix of `events_by_author` — `pubkey ‖ kind ‖ created_at ‖ -/// id` — so it is one index range rather than a filter page, and no page bound applies: a page -/// is what would put the address outside the search. The NIP-56 gate is a read's, not a -/// delete's, so a reported row is reachable here. -pub(crate) fn delete_by_address( +/// The address is resolved through `databases.replaceable`, the point index every production path +/// writes for an addressable kind (`upsert`, `upsert_many`): its key is +/// `replaceable_key(pubkey, kind, [d])`, so one `get` names the row and no scan is involved. That +/// matters twice over. It is the difference between O(1) and O(K) in the author's rows of the +/// kind — measured at 0.0395 ms against 144 ms for a 5 000-article author — and it means a miss +/// costs one lookup rather than a walk. +/// +/// The one row the index names is decoded and checked for the same five predicates the pointer +/// cannot express: not already deleted, `created_at` not after the request's, not kind 62, the +/// address the caller asked for, and — as a second, independent guard beside the key's pubkey — the +/// row's own author and kind. +/// +/// A row written through `create` has no pointer (`create` stores `event_deduplication: None`, the +/// capture's `INSERT` omitting the column), so it is invisible here; so it is to Postgres' +/// `event_deduplication->>0` predicate. No production path creates an addressable kind — the relay +/// and the importer both upsert — so the two backends agree, which they would not if this walked +/// the tags instead. +/// +/// The NIP-56 gate is a read's, not a delete's, so a reported row is reachable here. +pub(crate) fn delete_by_addresses( txn: &mut RwTxn<'_>, databases: &Databases, pubkey: &Pubkey, - kind: Kind, - identifier: &str, + addresses: &[(Kind, String)], not_after: i64, now: DateTime, ) -> Result { - let kind_number = i32::try_from(kind.0).unwrap_or(i32::MAX); - let mut prefix = Vec::with_capacity(36); - prefix.extend_from_slice(pubkey.as_bytes()); - prefix.extend_from_slice(&encoding::i32_key(kind_number)); - - let mut keys = Vec::new(); - for entry in databases.events_by_author.prefix_iter(txn, &prefix)? { - let (key, _) = entry?; - keys.push(key.to_vec()); - } - let mut affected = 0; - for key in keys { - let header = author_key_header(&key)?; - let Some(value) = databases.events.get(txn, header.event_id.as_bytes())? else { + + for (kind, identifier) in addresses { + let kind_number = i32::try_from(kind.0).unwrap_or(i32::MAX); + let pointer = encoding::replaceable_key( + pubkey, + kind_number, + &[serde_json::Value::String(identifier.clone())], + )?; + let Some(stored) = databases.replaceable.get(txn, &pointer)? else { + continue; + }; + let id = encoding::event_id_from(stored)?; + let Some(value) = databases.events.get(txn, id.as_bytes())? else { continue; }; let record = rows::decode_event(value)?; + if record.deleted_at.is_some() || record.event.created_at > not_after - || kind_of(&record)? == REQUEST_TO_VANISH - || d_identifier(&record.event) != identifier + || record.event.kind == Kind::REQUEST_TO_VANISH + || record.event.kind != *kind + || record.event.pubkey != *pubkey + || record.event.d_tag_value() != identifier { continue; } + soft_delete(txn, databases, &record, now)?; affected += 1; } - Ok(affected) -} -/// The value of an event's first `d` tag, `""` when it carries none — the key the addressable -/// strategy deduplicates on (`parameterized_replaceable.rs:29-35`). -fn d_identifier(event: &Event) -> &str { - event - .tags - .iter() - .find(|tag| tag.len() >= 2 && tag[0] == "d") - .map(|tag| tag[1].as_str()) - .unwrap_or_default() + Ok(affected) } /// `delete_by_pubkey_except_kinds`: every other live row of one author, soft-deleted. diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index d20215e73..e0cbd2b6f 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -422,27 +422,18 @@ impl EventRepository for LmdbBackend { }) } - fn delete_by_address<'a>( + fn delete_by_addresses<'a>( &'a self, pubkey: &'a Pubkey, - kind: Kind, - identifier: &'a str, + addresses: &'a [(Kind, String)], not_after: i64, ) -> StorageFuture<'a, u64> { Box::pin(async move { let pubkey = *pubkey; - let identifier = identifier.to_owned(); + let addresses = addresses.to_vec(); let now = encoding::micros(Utc::now()); self.write(move |txn, databases| { - events::delete_by_address( - txn, - databases, - &pubkey, - kind, - &identifier, - not_after, - now, - ) + events::delete_by_addresses(txn, databases, &pubkey, &addresses, not_after, now) }) .await }) diff --git a/crates/nostream-storage/src/postgres/events.rs b/crates/nostream-storage/src/postgres/events.rs index 142f713c4..186450ba0 100644 --- a/crates/nostream-storage/src/postgres/events.rs +++ b/crates/nostream-storage/src/postgres/events.rs @@ -237,33 +237,44 @@ where .rows_affected()) } -/// `delete_by_address`: one statement that resolves and deletes, so a replacement landing +/// `delete_by_addresses`: one statement that resolves and deletes, so a replacement landing /// between a read and a delete cannot survive it. /// -/// The address is `(event_pubkey, event_kind, event_deduplication->>0)` — the key the -/// addressable strategy upserts on — and `event_created_at <= not_after` is `09.md`'s "all -/// versions up to the `created_at` timestamp of the deletion request". Kind 62 is excluded, as -/// in [`delete_by_pubkey_and_ids`]. -pub(crate) async fn delete_by_address<'e, E>( +/// Each address is `(event_pubkey, event_kind, event_deduplication->>0)` — the key the addressable +/// strategy upserts on — and the pairs are compared as row values, so a batch cannot match a kind +/// from one address against another address's identifier. `event_created_at <= not_after` is +/// `09.md`'s "all versions up to the `created_at` timestamp of the deletion request", and kind 62 +/// is excluded, as in [`delete_by_pubkey_and_ids`]. +pub(crate) async fn delete_by_addresses<'e, E>( executor: E, pubkey: &Pubkey, - kind: Kind, - identifier: &str, + addresses: &[(Kind, String)], not_after: i64, ) -> Result where E: Executor<'e, Database = Postgres>, { + if addresses.is_empty() { + return Ok(0); + } + let mut args = Args::default(); let pubkey_slot = args.push(Arg::Bytes(pubkey.as_bytes().to_vec())); - let kind_slot = args.push(Arg::Int(i32::try_from(kind.0).unwrap_or(i32::MAX))); - let identifier_slot = args.push(Arg::Text(identifier.to_owned())); let not_after_slot = args.push(Arg::BigInt(not_after)); + let mut pairs = Vec::with_capacity(addresses.len()); + + for (kind, identifier) in addresses { + let kind_slot = args.push(Arg::Int(i32::try_from(kind.0).unwrap_or(i32::MAX))); + let identifier_slot = args.push(Arg::Text(identifier.clone())); + pairs.push(format!("({kind_slot}, {identifier_slot})")); + } + let sql = format!( "UPDATE events SET deleted_at = now() WHERE event_pubkey = {pubkey_slot} \ - AND event_kind = {kind_slot} AND event_deduplication->>0 = {identifier_slot} \ + AND (event_kind, event_deduplication->>0) IN ({}) \ AND event_created_at <= {not_after_slot} AND event_kind <> {REQUEST_TO_VANISH} \ - AND deleted_at IS NULL" + AND deleted_at IS NULL", + pairs.join(", ") ); Ok(EventQuery { sql, args } diff --git a/crates/nostream-storage/src/postgres/repository.rs b/crates/nostream-storage/src/postgres/repository.rs index e2989bad8..1e74d9983 100644 --- a/crates/nostream-storage/src/postgres/repository.rs +++ b/crates/nostream-storage/src/postgres/repository.rs @@ -69,15 +69,14 @@ impl EventRepository for PostgresBackend { Box::pin(async move { events::delete_by_pubkey_and_ids(&self.master, pubkey, ids).await }) } - fn delete_by_address<'a>( + fn delete_by_addresses<'a>( &'a self, pubkey: &'a Pubkey, - kind: Kind, - identifier: &'a str, + addresses: &'a [(Kind, String)], not_after: i64, ) -> StorageFuture<'a, u64> { Box::pin(async move { - events::delete_by_address(&self.master, pubkey, kind, identifier, not_after).await + events::delete_by_addresses(&self.master, pubkey, addresses, not_after).await }) } diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index c3d13ae2d..f91fd3c59 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -407,6 +407,22 @@ pub trait EventRepository: Send + Sync { fn upsert_many<'a>(&'a self, records: &'a [EventRecord]) -> StorageFuture<'a, u64>; /// Streams the events matching the filters, in the query's order. + /// + /// **The page obligation.** A NIP-67 `finish` hint claims that every stored event matching a + /// subscription's filters has been sent, and the relay decides it by comparing the rows it + /// received against `min(limit, page_bound())` — so this method carries a contract that has a + /// direction in each half: + /// + /// - a backend **MUST** yield up to `min(limit, page_bound())` matching rows whenever that + /// many exist. Yielding *fewer* while more exist is indistinguishable, at the call site, + /// from a short result, and makes a false `finish` reach the wire; + /// - a backend **MUST NOT** yield more than `min(limit, page_bound())` rows for one filter. + /// Yielding more only suppresses a hint the relay could have proved — the harmless + /// direction, but it is still a contract, because a caller sizes its buffers by it. + /// + /// A backend whose own page is smaller than [`MAX_PAGE_ROWS`](crate::MAX_PAGE_ROWS) declares it through + /// [`EventRepository::page_bound`]; the relay then asks for no more than it will be given and + /// emits no hint where it cannot prove one. fn find_by_filters<'a>( &'a self, filters: &'a [nostream_core::Filter], @@ -418,6 +434,17 @@ pub trait EventRepository: Send + Sync { filters: &'a [nostream_core::Filter], ) -> StorageFuture<'a, u64>; + /// The largest page this backend will return for one filter, whatever `limit` it carries. + /// + /// Defaults to [`MAX_PAGE_ROWS`](crate::MAX_PAGE_ROWS), which both shipped backends satisfy. A backend that clamps + /// lower overrides it: [`EventRepository::find_by_filters`]' obligation is stated in terms of + /// this value, and a caller that must know whether a page was truncated compares the rows it + /// received against `min(limit, page_bound())`. Overriding it is what keeps a smaller page + /// from being read as a short result. + fn page_bound(&self) -> u64 { + crate::MAX_PAGE_ROWS + } + /// Streams the live `(created_at, event_id)` pairs a NIP-77 reconciliation compares. fn find_reconciliation_items<'a>( &'a self, @@ -431,26 +458,30 @@ pub trait EventRepository: Send + Sync { ids: &'a [EventId], ) -> StorageFuture<'a, u64>; - /// Soft-deletes the author's live addressable rows that one address names — the `d` - /// identifier and the kinds 30000..=39999, with `created_at` no later than `not_after` — - /// returning the count. Kind 62 is never touched, as in + /// Soft-deletes the author's live addressable rows that these addresses name — each one's + /// `d` identifier and kind, with `created_at` no later than `not_after` — returning the + /// count. Kind 62 is never touched, as in /// [`delete_by_pubkey_and_ids`](Self::delete_by_pubkey_and_ids). /// + /// **The caller checks the addresses are the author's to delete**: the pubkey and kind are + /// taken as given, exactly as `delete_by_pubkey_and_ids` takes the ids it is given, and one + /// address is one `(kind, d)` pair. The store's own guard is that every row it can reach is + /// keyed by this pubkey. + /// + /// The whole batch is **one transaction**, which is why this is a batch rather than one call + /// per address: a deletion request names its addresses in one event, and one event must not + /// become N write transactions at the store. + /// /// This exists as its own operation rather than as a read followed by /// `delete_by_pubkey_and_ids`, for two reasons a caller cannot work around: a read is /// bounded by the backend's page, so a narrow address must not be found by scanning a page; /// and a read and a delete are two operations, so a replacement landing between them /// survives the delete the client was told had happened. Both are properties of the store, /// so the store performs the resolution and the deletion as one. - /// - /// `shortcut: the addressable kind range is the caller's to check, not the store's; upgrade - /// when a caller other than the NIP-09 `a`-tag strategy passes a kind`, because the store - /// takes the kind it is given exactly as `delete_by_pubkey_and_ids` takes the ids it is given. - fn delete_by_address<'a>( + fn delete_by_addresses<'a>( &'a self, pubkey: &'a Pubkey, - kind: Kind, - identifier: &'a str, + addresses: &'a [(Kind, String)], not_after: i64, ) -> StorageFuture<'a, u64>; diff --git a/crates/nostream-storage/tests/address_delete.rs b/crates/nostream-storage/tests/address_delete.rs index e278863c6..3a0d6c7a8 100644 --- a/crates/nostream-storage/tests/address_delete.rs +++ b/crates/nostream-storage/tests/address_delete.rs @@ -1,8 +1,9 @@ -//! `delete_by_address`: the NIP-09 `a`-tag delete, over the real store. +//! `delete_by_addresses`: the NIP-09 `a`-tag delete, over the real store. //! -//! The seam resolves the address and soft-deletes it in one operation, walking the author's kind -//! index rather than a read page, so neither the page bound nor the gap between a read and a -//! delete can leave a named row alive. +//! The seam resolves each address through the `replaceable` point index and soft-deletes the +//! whole batch in one write transaction, so neither the page bound nor the gap between a lookup +//! and a delete can leave a named row alive — and an address costs one lookup rather than a walk +//! of the author's kind. #![cfg(feature = "lmdb")] #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] @@ -28,7 +29,8 @@ async fn open_store() -> (tempfile::TempDir, LmdbBackend) { (directory, backend) } -/// A kind-30023 row under the address `30023::`, by the key `secret`. +/// A kind-30023 row under the address `30023::`, by the key `secret`, carrying the +/// deduplication value the strategy writes. fn addressable(secret: u8, d: &str, created_at: i64) -> EventRecord { let mut event = Event { id: EventId::from_byte_array([0; 32]), @@ -40,19 +42,38 @@ fn addressable(secret: u8, d: &str, created_at: i64) -> EventRecord { sig: Signature::from_byte_array([0; 64]), }; nostream_core::crypto::sign_event(&[secret; 32], &mut event).expect("the event signs"); - EventRecord::from_event(&event) + + EventRecord::from_event(&event).with_deduplication(vec![serde_json::Value::String( + event.d_tag_value().to_owned(), + )]) +} + +/// One address, as the trait takes it. +fn address(d: &str) -> (Kind, String) { + (Kind(30_023), d.to_owned()) } -/// 501 addressable rows of one author, so the target sits past the backend's page bound: a -/// read-then-delete seam could not reach it. +/// How many live rows the store serves this author. +async fn live_rows(backend: &LmdbBackend, pubkey: &Pubkey) -> u64 { + backend + .count_by_filters(&[Filter { + authors: Some(vec![pubkey.to_string()]), + ..Default::default() + }]) + .await + .expect("a read") +} + +/// 501 addressable rows of one author, so the target is far past the backend's page bound: a +/// read-then-delete seam could not reach it, and a walk of the kind would cost 501 decodes. #[tokio::test] -async fn delete_by_address_reaches_a_row_past_the_page_bound() { +async fn delete_by_addresses_reaches_a_row_past_the_page_bound() { let (_directory, backend) = open_store().await; let mut target = None; for index in 0..501u64 { let record = addressable(7, &format!("post-{index}"), 1_700_000_000 + index as i64); - backend.create(&record).await.expect("the store creates"); + backend.upsert(&record).await.expect("the store upserts"); if index == 500 { target = Some(record); @@ -65,8 +86,6 @@ async fn delete_by_address_reaches_a_row_past_the_page_bound() { ..Default::default() }; - // A streamed read is what a read-then-delete seam would have used, and it stops at the - // backend's page, so the five hundred and first row is not in it. let served = collect( backend .find_by_filters(std::slice::from_ref(&filter)) @@ -82,28 +101,63 @@ async fn delete_by_address_reaches_a_row_past_the_page_bound() { ); let deleted = backend - .delete_by_address(&target.event.pubkey, Kind(30_023), "post-500", i64::MAX) + .delete_by_addresses(&target.event.pubkey, &[address("post-500")], i64::MAX) .await .expect("the store deletes"); assert_eq!(deleted, 1); assert_eq!( - backend - .count_by_filters(std::slice::from_ref(&filter)) - .await - .expect("a read after the delete"), + live_rows(&backend, &target.event.pubkey).await, 500, "the row named by the address is the one gone" ); } +/// One call is one transaction for the whole batch, and an address that names nothing costs +/// nothing: three addresses and one miss delete three rows, and an empty batch is zero. +#[tokio::test] +async fn delete_by_addresses_deletes_the_whole_batch() { + let (_directory, backend) = open_store().await; + + let mut author = None; + + for d in ["post-1", "post-2", "post-3"] { + let record = addressable(7, d, 1_700_000_000); + author = Some(record.event.pubkey); + backend.upsert(&record).await.expect("the store upserts"); + } + + let author = author.expect("one row"); + + assert_eq!( + backend + .delete_by_addresses( + &author, + &[address("post-1"), address("post-2"), address("missing")], + i64::MAX, + ) + .await + .expect("the store deletes"), + 2 + ); + assert_eq!( + backend + .delete_by_addresses(&author, &[], i64::MAX) + .await + .expect("the store deletes"), + 0, + "an empty batch is not an error" + ); + assert_eq!(live_rows(&backend, &author).await, 1); +} + /// A row NIP-56 withholds from reads is still a row the author's deletion request names: the -/// report gate lives on the read path, and the delete walks the author's index instead. +/// report gate lives on the read path, and the delete walks the address index instead. #[tokio::test] -async fn delete_by_address_reaches_a_reported_row() { +async fn delete_by_addresses_reaches_a_reported_row() { let (_directory, backend) = open_store().await; let target = addressable(7, "post-1", 1_700_000_000); - backend.create(&target).await.expect("the store creates"); + backend.upsert(&target).await.expect("the store upserts"); let repos = Repos::all(&backend); repos @@ -120,21 +174,14 @@ async fn delete_by_address_reaches_a_reported_row() { .await .expect("the report stores"); - let filter = Filter { - authors: Some(vec![target.event.pubkey.to_string()]), - ..Default::default() - }; assert_eq!( - backend - .count_by_filters(std::slice::from_ref(&filter)) - .await - .expect("a read"), + live_rows(&backend, &target.event.pubkey).await, 0, "the read path withholds the reported row" ); let deleted = backend - .delete_by_address(&target.event.pubkey, Kind(30_023), "post-1", i64::MAX) + .delete_by_addresses(&target.event.pubkey, &[address("post-1")], i64::MAX) .await .expect("the store deletes"); @@ -145,42 +192,96 @@ async fn delete_by_address_reaches_a_reported_row() { /// delete: a row newer than the request survives, and one no newer than it goes. Both rows sit /// under the same author and kind, one address apart, so only the bound decides. #[tokio::test] -async fn delete_by_address_keeps_a_row_newer_than_the_request() { +async fn delete_by_addresses_keeps_a_row_newer_than_the_request() { let (_directory, backend) = open_store().await; let author = addressable(7, "kept", 1_010); let at_the_bound = addressable(7, "gone", 1_000); - backend.create(&author).await.expect("the store creates"); + backend.upsert(&author).await.expect("the store upserts"); backend - .create(&at_the_bound) + .upsert(&at_the_bound) .await - .expect("the store creates"); + .expect("the store upserts"); + + let deleted = backend + .delete_by_addresses( + &author.event.pubkey, + &[address("kept"), address("gone")], + 1_000, + ) + .await + .expect("the store deletes"); + + assert_eq!( + deleted, 1, + "the newer row is not the request's to delete, the other is" + ); + assert_eq!( + live_rows(&backend, &author.event.pubkey).await, + 1, + "only the newer row is left" + ); +} + +/// The address's `d` is [`nostream_core::Event::d_tag_value`], the value every writer of the +/// deduplication column uses: the **first** `d` tag. A row carrying two `d` tags is therefore +/// addressable as its first and not as its second, on this backend and on Postgres, whose +/// predicate matches the column the same function filled. +#[tokio::test] +async fn delete_by_addresses_uses_the_first_d_tag() { + let (_directory, backend) = open_store().await; + let mut event = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at: 1_700_000_000, + kind: Kind(30_023), + tags: vec![ + vec!["d".to_owned(), "a".to_owned()], + vec!["d".to_owned(), "b".to_owned()], + ], + content: "two d tags".to_owned(), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[7; 32], &mut event).expect("the event signs"); + let record = + EventRecord::from_event(&event).with_deduplication(vec![serde_json::Value::String( + event.d_tag_value().to_owned(), + )]); + backend.upsert(&record).await.expect("the store upserts"); assert_eq!( backend - .delete_by_address(&author.event.pubkey, Kind(30_023), "kept", 1_000) + .delete_by_addresses(&event.pubkey, &[address("b")], i64::MAX) .await .expect("the store deletes"), 0, - "a replacement newer than the request is not its to delete" + "the second d tag is not the address" ); assert_eq!( backend - .delete_by_address(&at_the_bound.event.pubkey, Kind(30_023), "gone", 1_000) + .delete_by_addresses(&event.pubkey, &[address("a")], i64::MAX) .await .expect("the store deletes"), 1, - "a row at exactly the request's created_at is deleted" + "the first d tag is" ); +} + +/// A row written through `create` carries no deduplication value — the capture's `INSERT` omits +/// the column — so it has no address pointer and neither backend's address delete can name it. +/// No production path creates an addressable kind (the relay and the importer both upsert), so +/// this pins the shape rather than a live defect: it is why the tests here seed with `upsert`. +#[tokio::test] +async fn a_row_written_through_create_has_no_address_pointer() { + let (_directory, backend) = open_store().await; + let target = addressable(7, "post-1", 1_700_000_000); + backend.create(&target).await.expect("the store creates"); assert_eq!( backend - .count_by_filters(&[Filter { - authors: Some(vec![author.event.pubkey.to_string()]), - ..Default::default() - }]) + .delete_by_addresses(&target.event.pubkey, &[address("post-1")], i64::MAX) .await - .expect("a read"), - 1, - "only the newer row is left" + .expect("the store deletes"), + 0 ); + assert_eq!(live_rows(&backend, &target.event.pubkey).await, 1); } diff --git a/crates/nostream-storage/tests/page_bound.rs b/crates/nostream-storage/tests/page_bound.rs new file mode 100644 index 000000000..12e86568b --- /dev/null +++ b/crates/nostream-storage/tests/page_bound.rs @@ -0,0 +1,101 @@ +//! The page contract `EventRepository::find_by_filters` states, over the real backend. +//! +//! A caller that must know whether a page was truncated compares the rows it received against +//! `min(limit, page_bound())`. That is sound only while the backend fills its page and never +//! exceeds it, in both directions — a backend that yields fewer rows than that while more exist +//! makes a false NIP-67 `finish` reach the wire, and one that yields more suppresses a hint it +//! could have proved. + +#![cfg(feature = "lmdb")] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod common; + +use std::sync::Arc; + +use common::{collect, fixture, settings_from_fixture}; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; +use nostream_storage::repositories::{EventRecord, EventRepository}; +use nostream_storage::{LmdbBackend, MAX_PAGE_ROWS}; + +/// A store on a temporary directory with the fixture's settings gates. +async fn open_store() -> (tempfile::TempDir, LmdbBackend) { + let fixture = fixture(); + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens") + .with_settings(Arc::new(settings_from_fixture(&fixture))); + + (directory, backend) +} + +/// A signed kind-1 note by the key `secret`, at `created_at`. +fn note(secret: u8, created_at: i64) -> EventRecord { + let mut event = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(1), + tags: Vec::new(), + content: format!("{secret}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut event).expect("the event signs"); + EventRecord::from_event(&event) +} + +/// LMDB declares `MAX_PAGE_ROWS`, fills a page that asks for exactly that, and never exceeds it. +#[tokio::test] +async fn lmdb_declares_and_fills_the_page_bound() { + let (_directory, backend) = open_store().await; + assert_eq!(backend.page_bound(), MAX_PAGE_ROWS); + + let records: Vec = (1..=MAX_PAGE_ROWS as i64 + 1) + .map(|step| note(11, step)) + .collect(); + backend + .create_many(&records) + .await + .expect("the store creates"); + + let stored = filter(Kind(1)); + + let full = collect( + backend + .find_by_filters(&[Filter { + limit: Some(MAX_PAGE_ROWS + 1), + ..stored.clone() + }]) + .await + .expect("a read"), + ) + .await + .expect("a read drains"); + assert_eq!( + full.len() as u64, + MAX_PAGE_ROWS, + "a page asking for more than the bound gets the bound, not more" + ); + + let short = collect( + backend + .find_by_filters(&[Filter { + limit: Some(3), + ..stored + }]) + .await + .expect("a read"), + ) + .await + .expect("a read drains"); + assert_eq!(short.len(), 3, "a page asking for less is filled"); +} + +/// The filter the cases read through. +fn filter(kind: Kind) -> Filter { + Filter { + kinds: Some(vec![kind]), + ..Filter::default() + } +} diff --git a/crates/nostream-storage/tests/storage.rs b/crates/nostream-storage/tests/storage.rs index 9a0620905..a7c8715ae 100644 --- a/crates/nostream-storage/tests/storage.rs +++ b/crates/nostream-storage/tests/storage.rs @@ -105,14 +105,13 @@ async fn the_repository_traits_are_dyn_compatible() { unsupported("noop::events::delete_by_pubkey_and_ids") } - fn delete_by_address<'a>( + fn delete_by_addresses<'a>( &'a self, _pubkey: &'a Pubkey, - _kind: Kind, - _identifier: &'a str, + _addresses: &'a [(Kind, String)], _not_after: i64, ) -> StorageFuture<'a, u64> { - unsupported("noop::events::delete_by_address") + unsupported("noop::events::delete_by_addresses") } fn delete_by_pubkey_except_kinds<'a>( From 2f33f45a90e463750e3d5c7f45516aa43f7bdb53 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 03:29:48 -0500 Subject: [PATCH 20/36] fix(relay): dedupe and cap a deletion request's addresses, one call for the batch --- .../nostream-relay/src/strategies/delete.rs | 125 +++++++++++++----- crates/nostream-relay/src/testing.rs | 60 +++++---- crates/nostream-relay/tests/strategies.rs | 77 +++++++++++ 3 files changed, 204 insertions(+), 58 deletions(-) diff --git a/crates/nostream-relay/src/strategies/delete.rs b/crates/nostream-relay/src/strategies/delete.rs index 4b565ab9f..333152f5b 100644 --- a/crates/nostream-relay/src/strategies/delete.rs +++ b/crates/nostream-relay/src/strategies/delete.rs @@ -1,16 +1,17 @@ //! The `delete` event strategy. Mirrors `src/handlers/event-strategies/delete-event-strategy.ts`. //! -//! A NIP-09 request names its targets two ways, and each is one storage operation: +//! A NIP-09 request names its targets two ways, and each is one storage call for the whole +//! request: //! //! - an `e` tag whose value parses as an event id — exact 32 bytes of lowercase hex, which is //! what `EventId`'s own parser accepts — goes to `delete_by_pubkey_and_ids`; -//! - an `a` tag naming `::` goes to `delete_by_address`, which resolves and -//! deletes in the store (see [`EventRepository::delete_by_address`]). +//! - an `a` tag naming `::` goes to `delete_by_addresses`, which resolves and +//! deletes the whole batch in one transaction (see [`EventRepository::delete_by_addresses`]). //! //! The second is deliberately *not* "read the row, then delete its id": a read is bounded by //! the backend's page, so an address could sit outside it, and a read and a delete are two //! operations, so a replacement landing between them would survive the delete the client was -//! told had happened. One operation per address is what makes the `OK` true. +//! told had happened. One operation per request is what makes the `OK` true. //! //! The deletions run **before** the request itself is stored, so a client that re-reads after //! the `OK` sees them already applied. Then the common create-or-dedupe. @@ -25,6 +26,31 @@ use crate::error::RelayError; use crate::handlers::HandlerContext; use crate::strategies::{EventStrategy, event_record}; +/// The most distinct `(kind, d)` addresses one deletion request may name. +/// +/// This is a policy, and the number is derived rather than felt, so the next reader can re-check +/// it against the arithmetic instead of trusting it: +/// +/// - **what one admission gate allows.** `maxPayloadSize` is 524 288 bytes and an `a` tag costs +/// about 86 (`["a","30023:<64 hex>:"]`), so the transport alone admits ~6 096 tags. Nothing +/// else caps a tag list, so without this constant that is the count the store would see. +/// - **what one address costs.** One `replaceable` point lookup and one row decode, measured at +/// 0.0395 ms; 100 of them are ~4 ms of one write transaction. The batch is one transaction, so +/// the cost is one commit's latency, not 100. +/// - **what a key can ask for.** Kind 5 is limited to 30 events/min per free key, so 30 × 100 = +/// 3 000 address-deletes/min, ~120 ms of writer time per minute — 0.2 % of the single LMDB +/// writer. The same key at the transport's 6 096 would be ~180× that: 30 × 6 096 × 0.04 ms +/// ≈ 7.3 s of writer time per minute, i.e. seven writers' worth from one free key. That is the +/// number this cap exists to overrule. +/// +/// A request naming more is **refused**, `["OK",,false,"invalid: …"]`, before anything is +/// deleted or stored. Refusing is the kindest correct answer: the alternative — deleting the first +/// 100 and answering `true` — would tell the client its other articles were deleted when they were +/// not, and a partial delete answered `false` would already have changed the store while claiming +/// it had not. A refusal changes nothing, so the client can retry with the addresses split across +/// requests, which is what the per-request reading of `09.md` asks for anyway. +const MAX_ADDRESSES_PER_REQUEST: usize = 100; + /// The `delete` arm. #[derive(Debug, Clone, Copy, Default)] pub struct DeleteStrategy; @@ -36,6 +62,22 @@ impl EventStrategy for DeleteStrategy { context: &'a HandlerContext, ) -> Pin> + Send + 'a>> { Box::pin(async move { + let addresses = addresses(event); + + if addresses.len() > MAX_ADDRESSES_PER_REQUEST { + context.send_ok( + event.id, + false, + &format!( + "invalid: a deletion request may name at most \ + {MAX_ADDRESSES_PER_REQUEST} addresses" + ), + None, + ); + + return Ok(()); + } + let ids: Vec = event .tags .iter() @@ -51,11 +93,11 @@ impl EventStrategy for DeleteStrategy { .await?; } - for address in addresses(event) { + if !addresses.is_empty() { context .stores .events - .delete_by_address(&event.pubkey, address.kind, &address.d, event.created_at) + .delete_by_addresses(&event.pubkey, &addresses, event.created_at) .await?; } @@ -78,34 +120,47 @@ impl EventStrategy for DeleteStrategy { } } -/// One `a` tag's address, kept only when the request may name it. -struct Address { - kind: Kind, - /// The `d` identifier, `""` for a tag that omits it — the key the addressable strategy - /// stores a row without a `d` tag under. - d: String, -} - -/// The addresses a deletion request may act on: `::`, split into at most three -/// parts so a `d` may hold colons of its own. +/// The distinct `(kind, d)` addresses a deletion request may act on, in the order they appear. /// -/// `09.md`'s `a` branch is about addressable events, and only those carry a `d` in their -/// address, so a regular replaceable kind named by an `a` tag is ignored. An address naming -/// another author is not this request's to delete: the pubkey must be the request's own, and -/// the store's delete re-checks the author as a second, independent guard. -fn addresses(event: &Event) -> Vec
{ - event - .tags - .iter() - .filter(|tag| tag.len() >= 2 && tag[0] == "a") - .filter_map(|tag| { - let mut parts = tag[1].splitn(3, ':'); - let kind = Kind(parts.next()?.parse::().ok()?); - let pubkey = parts.next()?.parse::().ok()?; - let d = parts.next().unwrap_or_default().to_owned(); - - (kind.is_parameterized_replaceable() && pubkey == event.pubkey) - .then_some(Address { kind, d }) - }) - .collect() +/// `09.md`'s `a` branch is about addressable events, and only those carry a `d` in their address, +/// so a regular replaceable kind named by an `a` tag is ignored (`Kind::is_parameterized_replaceable` +/// is 30 000–39 999). An address naming another author is not this request's to delete: the pubkey +/// must be the request's own, and the store re-checks the row's author as a second, independent +/// guard. A tag whose kind does not parse, whose pubkey does not parse, or that carries no kind or +/// pubkey at all is ignored. +/// +/// The `d` is the third part and may hold colons of its own, which is why the split stops at three +/// parts; a tag with no third part names the empty `d`, the same address the addressable strategy +/// gives a row with no `d` tag. Duplicates collapse to one address, so `[["a",A],["a",A]]` costs the +/// store one lookup rather than two, and the count the cap bounds is the work's, not the tags'. +fn addresses(event: &Event) -> Vec<(Kind, String)> { + let mut addresses: Vec<(Kind, String)> = Vec::new(); + + for tag in &event.tags { + if tag.len() < 2 || tag[0] != "a" { + continue; + } + + let mut parts = tag[1].splitn(3, ':'); + let Some(kind) = parts.next().and_then(|part| part.parse::().ok()) else { + continue; + }; + let Some(pubkey) = parts.next().and_then(|part| part.parse::().ok()) else { + continue; + }; + let kind = Kind(kind); + + if !kind.is_parameterized_replaceable() || pubkey != event.pubkey { + continue; + } + + let d = parts.next().unwrap_or_default(); + let address = (kind, d.to_owned()); + + if !addresses.contains(&address) { + addresses.push(address); + } + } + + addresses } diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index c91cd38e5..4689af080 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -155,6 +155,7 @@ pub struct InMemoryEventRepository { fail_queries: AtomicBool, fail_delete: AtomicBool, fail_reads: AtomicBool, + page_bound: std::sync::atomic::AtomicU64, upsert_failure: Mutex>, upserts: Mutex>>, deletes: Mutex)>>, @@ -213,6 +214,13 @@ impl InMemoryEventRepository { self.fail_reads.store(fail, Ordering::SeqCst); } + /// Declares a smaller page than `MAX_PAGE_ROWS`, the way a backend that clamps lower would, + /// so a test can see the relay honour it instead of its own assumption. Zero restores the + /// default. + pub fn declare_page_bound(&self, bound: u64) { + self.page_bound.store(bound, Ordering::SeqCst); + } + /// Makes every read wait for a permit. Grant one with `Semaphore::add_permits` to let a /// read through. pub fn gate_reads(&self) -> Arc { @@ -325,17 +333,6 @@ impl InMemoryEventRepository { } } -/// The value of an event's first `d` tag, `""` when it carries none — the key the addressable -/// strategy deduplicates on. -fn d_identifier(event: &Event) -> &str { - event - .tags - .iter() - .find(|tag| tag.len() >= 2 && tag[0] == "d") - .map(|tag| tag[1].as_str()) - .unwrap_or_default() -} - impl EventRepository for InMemoryEventRepository { fn create<'a>(&'a self, record: &'a EventRecord) -> StorageFuture<'a, u64> { Box::pin(async move { @@ -433,6 +430,15 @@ impl EventRepository for InMemoryEventRepository { }) } + /// The page this double serves, which a test may lower with + /// [`InMemoryEventRepository::declare_page_bound`]; zero means "not declared". + fn page_bound(&self) -> u64 { + match self.page_bound.load(Ordering::SeqCst) { + 0 => MAX_PAGE_ROWS, + bound => bound, + } + } + fn find_by_filters<'a>( &'a self, filters: &'a [Filter], @@ -470,8 +476,10 @@ impl EventRepository for InMemoryEventRepository { let mut matched: Vec> = Vec::new(); let mut seen: Vec = Vec::new(); + let page_bound = self.page_bound(); + for filter in filters { - let bound = filter.limit.unwrap_or(u64::MAX).min(MAX_PAGE_ROWS); + let bound = filter.limit.unwrap_or(u64::MAX).min(page_bound); let mut taken = 0u64; for record in &records { @@ -599,20 +607,25 @@ impl EventRepository for InMemoryEventRepository { }) } - /// The addressable row one `(pubkey, kind, d)` names, soft-deleted when it is no newer than - /// `not_after` — one operation, so there is no window between a lookup and a delete. - fn delete_by_address<'a>( + /// The row at each address, soft-deleted when it is no newer than `not_after` — one + /// operation for the batch, and one recorded entry per address so a test can count the work. + fn delete_by_addresses<'a>( &'a self, pubkey: &'a Pubkey, - kind: Kind, - identifier: &'a str, + addresses: &'a [(Kind, String)], not_after: i64, ) -> StorageFuture<'a, u64> { Box::pin(async move { - self.address_deletes - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .push((*pubkey, kind, identifier.to_owned(), not_after)); + { + let mut recorded = self + .address_deletes + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + for (kind, identifier) in addresses { + recorded.push((*pubkey, *kind, identifier.clone(), not_after)); + } + } if self.fail_delete.load(Ordering::SeqCst) { return Err(StorageError::InvalidArgument { @@ -622,10 +635,11 @@ impl EventRepository for InMemoryEventRepository { Ok(self.soft_delete(|record| { record.event.pubkey == *pubkey - && record.event.kind == kind && record.event.created_at <= not_after && record.event.kind != Kind::REQUEST_TO_VANISH - && d_identifier(&record.event) == identifier + && addresses.iter().any(|(kind, identifier)| { + record.event.kind == *kind && record.event.d_tag_value() == identifier + }) })) }) } diff --git a/crates/nostream-relay/tests/strategies.rs b/crates/nostream-relay/tests/strategies.rs index 24ce4622b..7d4fbc128 100644 --- a/crates/nostream-relay/tests/strategies.rs +++ b/crates/nostream-relay/tests/strategies.rs @@ -606,6 +606,83 @@ async fn delete_matches_an_empty_d_identifier() { } } +/// A request naming the same address a thousand times makes **one** storage call: the addresses +/// are deduplicated before the seam, so the tag count is not the work count. Measured before the +/// change: 1 000 identical tags produced 1 000 `delete_by_address` calls, each its own write +/// transaction. +#[tokio::test] +async fn delete_names_each_distinct_address_once() { + let fixture = Fixture::new(settings()); + let target = addressable_row(Some("post-1"), NOW_SECS); + fixture.seed_event(target.clone()).await; + + let address = format!("30023:{}:post-1", author()); + let tags: Vec> = (0..1_000).map(|_| tag(&["a", &address])).collect(); + let event = event(5, tags, ""); + + let (context, mut sink) = socket(&fixture); + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!( + fixture.events.address_deletes(), + one_address_delete("post-1", NOW_SECS), + "a thousand identical tags are one address, so one call" + ); + assert_eq!(fixture.events.live_events(), vec![event.clone()]); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + +/// A request naming more distinct addresses than one event may is refused before anything is +/// deleted or stored: `OK false`, no storage call, no row written. Partial work with a `true` +/// would tell the client its other articles were gone. +#[tokio::test] +async fn delete_refuses_a_request_naming_too_many_addresses() { + let fixture = Fixture::new(settings()); + let target = addressable_row(Some("post-0"), NOW_SECS); + fixture.seed_event(target.clone()).await; + + let tags: Vec> = (0..101) + .map(|index| tag(&["a", &format!("30023:{}:post-{index}", author())])) + .collect(); + let event = event(5, tags, ""); + + let (context, mut sink) = socket(&fixture); + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!(fixture.events.address_deletes(), Vec::new()); + assert_eq!( + fixture.events.live_events(), + vec![target], + "nothing was deleted and the request itself was not stored" + ); + assert_eq!( + frames(&mut sink), + vec![ok_frame( + &event, + false, + "invalid: a deletion request may name at most 100 addresses" + )] + ); +} + +/// One hundred and one tags naming **one** address are admitted: the cap bounds distinct +/// addresses, which is the work, not the tag list the transport already bounds. +#[tokio::test] +async fn delete_admits_a_cap_sized_batch() { + let fixture = Fixture::new(settings()); + let tags: Vec> = (0..100) + .map(|index| tag(&["a", &format!("30023:{}:post-{index}", author())])) + .collect(); + let event = event(5, tags, ""); + + let (context, mut sink) = socket(&fixture); + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!(fixture.events.address_deletes().len(), 100); + assert_eq!(fixture.events.live_events(), vec![event.clone()]); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + // --------------------------------------------------------------------------- // dvm_job_request — `dvm-job-request-event-strategy.spec.ts` // --------------------------------------------------------------------------- From b10e7ba2c127e2db657413708d4a3e831828b903 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 03:29:48 -0500 Subject: [PATCH 21/36] fix(relay): ask the backend its page bound, and borrow the filters --- .../nostream-relay/src/handlers/subscribe.rs | 38 ++++++++++--------- crates/nostream-relay/tests/handlers.rs | 20 ++++++++++ 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index 7105c1b45..7de9b5544 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -28,7 +28,6 @@ use nostream_core::messages::{ }; use nostream_core::nip42; use nostream_core::{Filter, Kind}; -use nostream_storage::MAX_PAGE_ROWS; use crate::connection::InFlightRead; use crate::error::RelayError; @@ -274,23 +273,25 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // admits nothing beyond the maximum and so cannot hide a row. The backends already honour // `limit`; the in-memory double does not, and the loop below drops the extra row either way. let probe = limit.map(|limit| limit.saturating_add(1)); - let storage_filters: Vec = match probe { - Some(probe) => filters - .iter() - .map(|filter| Filter { - limit: Some(probe), - ..filter.clone() - }) - .collect(), - None => filters.to_vec(), + // Only a probing read needs new filters, and it only rewrites `limit`: every other read + // borrows the subscription's own list rather than copying it (a 1 000-author filter costs + // 63 µs to clone, and the copy buys nothing). + let probed; + let storage_filters: &[Filter] = match probe { + Some(probe) => { + probed = filters + .iter() + .map(|filter| Filter { + limit: Some(probe), + ..filter.clone() + }) + .collect::>(); + &probed + } + None => filters, }; - let stream = match context - .stores - .events - .find_by_filters(&storage_filters) - .await - { + let stream = match context.stores.events.find_by_filters(storage_filters).await { Ok(stream) => stream, Err(error) => { tracing::error!(error = %error, "unable to stream events"); @@ -421,7 +422,7 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // all three: // // 1. the stream ended **cleanly** — after a storage error, how far the read got is unknown; - // 2. the page was **not full** — a page that yielded `min(probe, MAX_PAGE_ROWS)` rows says + // 2. the page was **not full** — a page that yielded `min(probe, page_bound())` rows says // nothing about the rows beyond it, whether the bound came from the client's `limit` or // from the backend's own page cap, and a skipped row spends a page slot just as a match // does (which is why `rows`, not `matched`, decides this); @@ -435,7 +436,8 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // absent hint is honest and a wrong one is not. `more` needs one further **match** beyond // the delivered window, `auth` is unaffected, and `finish` is never sent beside `auth` // because a withheld event falsifies it. - let page_full = probe.is_some_and(|probe| rows >= probe.min(MAX_PAGE_ROWS)); + let page_bound = context.stores.events.page_bound(); + let page_full = probe.is_some_and(|probe| rows >= probe.min(page_bound)); let proven = !failed && !page_full; let mut hints = Vec::new(); diff --git a/crates/nostream-relay/tests/handlers.rs b/crates/nostream-relay/tests/handlers.rs index f37a50994..e23172ee7 100644 --- a/crates/nostream-relay/tests/handlers.rs +++ b/crates/nostream-relay/tests/handlers.rs @@ -1170,6 +1170,26 @@ async fn eose_hints_are_absent_when_the_backend_page_cap_was_reached() { assert_eq!(frames[5_000], r#"["EOSE","s1"]"#); } +/// A backend that declares a smaller page than `MAX_PAGE_ROWS` is believed: it served one row +/// and said it can serve no more, so the page came back full and `finish` would be a claim about +/// rows beyond the one it was given. Before the relay asked the backend, this read sent `finish` +/// over a store that still held a match. +#[tokio::test] +async fn eose_hints_are_absent_when_the_backend_declares_a_smaller_page() { + let fixture = Fixture::new(settings()); + seed_notes(&fixture, 2).await; + fixture.events.declare_page_bound(1); + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r#"["REQ","s1",{"kinds":[1],"limit":3}]"#).await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 2, "{frames:?}"); + assert!(events_then_eose(&frames), "{frames:?}"); + assert_eq!(frames[1], r#"["EOSE","s1"]"#); +} + /// A storage error leaves the read's extent unknown, so no hint is added after it. #[tokio::test] async fn eose_hints_are_absent_when_the_stream_failed() { From 19560328823b97664fc70cfab168298440eb8aec Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 03:29:48 -0500 Subject: [PATCH 22/36] docs(rewrite): record the address cap, the d-tag derivation and Postgres's untested arm --- docs/rewrite/nip-parity.md | 50 +++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/docs/rewrite/nip-parity.md b/docs/rewrite/nip-parity.md index 63fbb83a2..3418d469d 100644 --- a/docs/rewrite/nip-parity.md +++ b/docs/rewrite/nip-parity.md @@ -878,17 +878,45 @@ is a defect, and §10 is the same record in table form. whose tags are `a`-only deletes nothing. The rewrite either implements addressable-event deletion or records the omission deliberately — but §5 and the §7 row now say so, and Phase 2 must not inherit the silence. - - **Status:** done in this job — an `a` tag is handed to `EventRepository::delete_by_address`, - which resolves the address and soft-deletes it in **one** storage operation: LMDB walks the - author-and-kind prefix of `events_by_author` (one index range, no read page, so a narrow - address is never found by scanning a page that sorts it out of view) and Postgres runs one - `UPDATE … WHERE`; the request's `created_at` is compared inside that transaction, so a - replacement cannot land between a lookup and the delete, and the author-equality and - addressable-kind rules still gate the tag before it is read. Records with an actionable - NIP-56 report are reachable too, because the delete no longer goes through the read path - that withholds them. `strategies::delete_resolves_and_deletes_an_address_in_one_operation`, - `delete_by_address_reaches_a_row_past_the_page_bound`, - `delete_by_address_reaches_a_reported_row` and four more (§10). + - **Status:** done in this job — an `a` tag is handed to + `EventRepository::delete_by_addresses`, which resolves each address and soft-deletes the + batch in **one** storage operation: + - **LMDB** probes the `replaceable` point index every production path writes + (`upsert`/`upsert_many`), so an address costs one lookup and one row decode — measured + 0.0395 ms against 144 ms for the author-kind walk this replaced at 5 000 rows of the kind + — and applies the remaining predicates (not deleted, `created_at` not after the request's, + not kind 62, the row's own author and kind, the `d` value) to the one row the index names. + - **Postgres** runs one `UPDATE … WHERE (event_kind, event_deduplication->>0) IN (…)`, the + row-value form, so a batch cannot pair one address's kind with another's identifier. + - The request's `created_at` is compared inside that transaction, so a replacement cannot + land between a lookup and the delete; the author-equality and addressable-kind rules gate + the tag before it is read; and records with an actionable NIP-56 report are reachable, + because the delete does not go through the read path that withholds them. + - **The `d` is `Event::d_tag_value()`** — one derivation in `nostream-core`, used by the + addressable strategy, the import path, the LMDB delete and the relay's double, so the + column Postgres matches and the value LMDB compares cannot drift apart. + - **One request names at most 100 distinct addresses** (`delete.rs`'s + `MAX_ADDRESSES_PER_REQUEST`), and a request naming more is refused with + `["OK",,false,"invalid: …"]` before anything is deleted or stored. The number is + derived, not felt: `maxPayloadSize` 524 288 admits ≈6 096 `a` tags and nothing else caps a + tag list, one address is ~0.04 ms of one write transaction, and kind 5 is limited to 30 + events/min per free key — so 30 × 100 ≈ 3 000 address-deletes/min (~0.2 % of the single + LMDB writer), where 30 × 6 096 would be ≈7.3 s of writer time per minute from one free key. + A refusal changes nothing, so the client can retry with the addresses split across + requests; a partial delete answered `true` would claim deletions that did not happen. + - **Postgres's arm has never been executed.** No command in this repository runs it: the + capture has no such operation, so neither parity target can replay it, and + `tests/address_delete.rs` is `#![cfg(feature = "lmdb")]`. What would close it: a + hand-written case in `tests/parity.rs` (the target that CI runs against + `postgres:16-alpine`) seeding one row at the bound and one newer, or a Postgres-parameterised + `address_delete.rs`. + - Tests: `strategies::delete_names_each_distinct_address_once`, + `delete_refuses_a_request_naming_too_many_addresses`, `delete_admits_a_cap_sized_batch`, + `delete_resolves_and_deletes_an_address_in_one_operation`, + `delete_by_addresses_reaches_a_row_past_the_page_bound`, + `delete_by_addresses_reaches_a_reported_row`, + `delete_by_addresses_uses_the_first_d_tag`, + `delete_by_addresses_keeps_a_row_newer_than_the_request` (§10). ## 10. Phase 2: the advertisement and the harness map From 3d55bf825ce12288382129568ad4dd3f16323c2a Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 03:56:54 -0500 Subject: [PATCH 23/36] feat(storage): declare each backend's search rule beside its page bound --- .../src/postgres/repository.rs | 16 +++++- crates/nostream-storage/src/repositories.rs | 49 +++++++++++++++++-- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/crates/nostream-storage/src/postgres/repository.rs b/crates/nostream-storage/src/postgres/repository.rs index 1e74d9983..da02deee9 100644 --- a/crates/nostream-storage/src/postgres/repository.rs +++ b/crates/nostream-storage/src/postgres/repository.rs @@ -13,8 +13,8 @@ use crate::repositories::{ EventRetentionOptions, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, OutboxPayload, - QueryStream, ReconciliationItem, Report, ReportRepository, StorageFuture, User, UserPatch, - UserRepository, + QueryStream, ReconciliationItem, Report, ReportRepository, SearchRule, StorageFuture, User, + UserPatch, UserRepository, }; impl EventRepository for PostgresBackend { @@ -69,6 +69,18 @@ impl EventRepository for PostgresBackend { Box::pin(async move { events::delete_by_pubkey_and_ids(&self.master, pubkey, ids).await }) } + /// Postgres answers a search filter with `to_tsvector(@@) plainto_tsquery` when NIP-50 is + /// enabled (`sql.rs:317-323`), and with no text predicate at all when it is not — in which + /// case the relay's own matcher is what decides, which is `SubstringTerms`. + fn search_rule(&self) -> SearchRule { + match self.gates().nip50 { + Some(gate) => SearchRule::ToTsvector { + language: gate.language, + }, + None => SearchRule::SubstringTerms, + } + } + fn delete_by_addresses<'a>( &'a self, pubkey: &'a Pubkey, diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index f91fd3c59..e8940a72d 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -380,6 +380,36 @@ pub struct OutboxMessage { /// (`src/repositories/notification-outbox-repository.ts`). pub const NOTIFICATION_OUTBOX_PROCESSING_LEASE_MS: i64 = 5 * 60 * 1000; +/// The matching rule a backend applies to a filter that carries `search` (NIP-50). +/// +/// A completeness claim is a claim about the set a backend *enumerated*, so the caller has to +/// know what that backend's `search` predicate actually does. Saying only "my predicate may +/// differ" would let the next backend tick a box; each variant names the rule, so an implementor +/// has to read what it is asserting. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SearchRule { + /// Exactly [`nostream_core::Filter::matches`]: every whitespace-separated term of `search` + /// must appear as a lowercase **substring** of the content. The backend enumerates with the + /// relay's own matcher, so a completeness claim over such a read is one the relay can check. + SubstringTerms, + /// Postgres' prefilter: `to_tsvector(, content) @@ plainto_tsquery(, + /// query)`. Word **tokens**, stemmed by the `language` regconfig — not substrings, and not + /// the relay's rule. The two predicates contain neither the other: + /// + /// - the query `study` selects a row whose content is `studies` (both reduce to the one + /// English stem `studi`), and [`nostream_core::Filter::matches`] refuses it because + /// `"studies"` contains no `"study"`; + /// - `abcdef` is a substring of the content `xxabcdefyy`, which `Filter::matches` accepts and + /// the tsquery never yields, because that content is one token. + /// + /// The first is the direction a probe can see (a yielded row the relay refuses); the second + /// is why no probe is enough, and why a caller withholds rather than checks. + ToTsvector { + /// The regconfig the predicate is evaluated under (`nip50.language`, `simple` by default). + language: String, + }, +} + /// Event storage (`src/@types/repositories.ts` `IEventRepository`). pub trait EventRepository: Send + Sync { /// Inserts one event, returning the affected row count (0 on a duplicate `event_id`). @@ -416,9 +446,11 @@ pub trait EventRepository: Send + Sync { /// - a backend **MUST** yield up to `min(limit, page_bound())` matching rows whenever that /// many exist. Yielding *fewer* while more exist is indistinguishable, at the call site, /// from a short result, and makes a false `finish` reach the wire; - /// - a backend **MUST NOT** yield more than `min(limit, page_bound())` rows for one filter. - /// Yielding more only suppresses a hint the relay could have proved — the harmless - /// direction, but it is still a contract, because a caller sizes its buffers by it. + /// - a backend **MUST NOT** yield more than `min(limit, page_bound())` rows for one filter, + /// with one exception the relay itself asks for: its probe passes `limit + 1`, so a backend + /// honouring that filter yields up to `min(limit + 1, page_bound())`. Yielding more only + /// suppresses a hint the relay could have proved — the harmless direction, but it is still + /// a contract, because a caller sizes its buffers by it. /// /// A backend whose own page is smaller than [`MAX_PAGE_ROWS`](crate::MAX_PAGE_ROWS) declares it through /// [`EventRepository::page_bound`]; the relay then asks for no more than it will be given and @@ -445,6 +477,17 @@ pub trait EventRepository: Send + Sync { crate::MAX_PAGE_ROWS } + /// The matching rule this backend applies to a filter that carries `search` (NIP-50). + /// + /// Defaults to [`SearchRule::SubstringTerms`], which is what a backend that filters with + /// [`nostream_core::Filter::matches`] applies — LMDB's filter path *is* that matcher. A + /// backend whose own predicate answers the filter overrides this, and a caller that must + /// claim completeness over such a read reads it rather than assuming the relay's matcher + /// enumerated the set. + fn search_rule(&self) -> SearchRule { + SearchRule::SubstringTerms + } + /// Streams the live `(created_at, event_id)` pairs a NIP-77 reconciliation compares. fn find_reconciliation_items<'a>( &'a self, From 693febdc374cd95f873393fad1072d47c9ae4a02 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 03:56:54 -0500 Subject: [PATCH 24/36] fix(relay): withhold finish over a search set the backend enumerated differently --- .../nostream-relay/src/handlers/subscribe.rs | 37 ++++++++- crates/nostream-relay/src/testing.rs | 48 ++++++++++- crates/nostream-relay/tests/handlers.rs | 79 ++++++++++++++++++- 3 files changed, 155 insertions(+), 9 deletions(-) diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index 7de9b5544..a205ce9bb 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -28,6 +28,7 @@ use nostream_core::messages::{ }; use nostream_core::nip42; use nostream_core::{Filter, Kind}; +use nostream_storage::repositories::SearchRule; use crate::connection::InFlightRead; use crate::error::RelayError; @@ -309,12 +310,15 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // `rows`: every row the page yielded, skipped ones included — a full page is what makes the // backend's remaining extent unknown. `matched`: rows that were a match inside the client's // window, which is what `limit` bounds. `trimmed`: the guard withheld a match. `more`: a - // match beyond the window was found. `failed`: the stream errored. + // match beyond the window was found. `failed`: the stream errored. `refused`: the backend + // yielded a row this relay's matcher refuses, which is what a backend whose predicate is not + // `Filter::matches` looks like from here. let mut rows = 0u64; let mut matched = 0u64; let mut trimmed = false; let mut more = false; let mut failed = false; + let mut refused = false; loop { if *read.cancellation().borrow() { @@ -372,7 +376,18 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha } let event = record.event; - if expiration::is_expired_event(&event, now) || !matches_any(&event) { + if expiration::is_expired_event(&event, now) { + continue; + } + + if !matches_any(&event) { + // The backend selected a row this relay's own matcher refuses. That is not a + // skip: it is evidence that the set the backend enumerated is not the set the + // relay would have, so completeness over this read cannot be claimed. It is + // tracked apart from `trimmed` because a refused row is one **no** client + // could see, while `auth` means "a client could see this". + refused = true; + continue; } @@ -426,7 +441,13 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // nothing about the rows beyond it, whether the bound came from the client's `limit` or // from the backend's own page cap, and a skipped row spends a page slot just as a match // does (which is why `rows`, not `matched`, decides this); - // 3. **no further match** was found — the `more` branch below is where this one lives: a + // 3. the set the backend enumerated is the set this relay would have — no row it yielded was + // refused by `Filter::matches`, and no filter carrying `search` was answered by a + // predicate other than that matcher. Both are needed and neither implies the other: a + // refusal is the direction a probe can see (`study` yields `studies`, which the matcher + // refuses), while a row the matcher *would* accept and the backend never yields + // (`abcdef` inside `xxabcdefyy`) leaves no trace at all. See `EventRepository::search_rule`. + // 4. **no further match** was found — the `more` branch below is where this one lives: a // further match ends the read with `more` set, so the code that can reach the hint is the // code where none was found. It is only meaningful once 1 and 2 hold, because the loop // keeps asking past deleted, expired, withheld and non-matching rows until it sees a @@ -438,7 +459,15 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // because a withheld event falsifies it. let page_bound = context.stores.events.page_bound(); let page_full = probe.is_some_and(|probe| rows >= probe.min(page_bound)); - let proven = !failed && !page_full; + let search_rule = context.stores.events.search_rule(); + let searched = filters.iter().any(|filter| { + filter + .search + .as_deref() + .is_some_and(|search| !search.trim().is_empty()) + }); + let search_agrees = search_rule == SearchRule::SubstringTerms || !searched; + let proven = !failed && !page_full && !refused && search_agrees; let mut hints = Vec::new(); diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index 4689af080..0b4438a5b 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -28,7 +28,7 @@ use nostream_storage::repositories::{ ActionableTarget, CreateInviteCodeOptions, DvmJob, DvmJobStatus, EventPurgeCounts, EventRecord, EventRepository, EventRetentionOptions, FieldUpdate, InviteCode, InviteCodeRepository, NewReport, Nip05Verification, Nip05VerificationRepository, QueryStream, ReconciliationItem, - Report, ReportRepository, StorageFuture, User, UserPatch, UserRepository, + Report, ReportRepository, SearchRule, StorageFuture, User, UserPatch, UserRepository, }; use tokio::sync::Notify; @@ -156,6 +156,11 @@ pub struct InMemoryEventRepository { fail_delete: AtomicBool, fail_reads: AtomicBool, page_bound: std::sync::atomic::AtomicU64, + search_rule: Mutex>, + /// Rows this double yields for **any** filter: what a backend whose `search` predicate is not + /// `Filter::matches` looks like from the handler's side, which is the case no honest filter + /// can produce. + unmatched: Mutex>, upsert_failure: Mutex>, upserts: Mutex>>, deletes: Mutex)>>, @@ -214,6 +219,30 @@ impl InMemoryEventRepository { self.fail_reads.store(fail, Ordering::SeqCst); } + /// Seeds a row this double yields whatever the filter says — a backend that selected it by a + /// predicate the relay's matcher does not share (NIP-50's text search is the one that ships). + pub fn seed_unmatched(&self, event: Event) { + let record = EventRecord::from_event(&event); + + self.records + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(record); + self.unmatched + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(event.id); + } + + /// Declares the matching rule this double answers a `search` filter with, the way a backend + /// whose predicate is its own would. + pub fn declare_search_rule(&self, rule: SearchRule) { + *self + .search_rule + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(rule); + } + /// Declares a smaller page than `MAX_PAGE_ROWS`, the way a backend that clamps lower would, /// so a test can see the relay honour it instead of its own assumption. Zero restores the /// default. @@ -284,7 +313,7 @@ impl InMemoryEventRepository { .clone() } - /// The `(pubkey, kind, d, not_after)` each `delete_by_address` was called with. + /// The `(pubkey, kind, d, not_after)` each `delete_by_addresses` entry was called with. pub fn address_deletes(&self) -> Vec<(Pubkey, Kind, String, i64)> { self.address_deletes .lock() @@ -439,6 +468,14 @@ impl EventRepository for InMemoryEventRepository { } } + fn search_rule(&self) -> SearchRule { + self.search_rule + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() + .unwrap_or(SearchRule::SubstringTerms) + } + fn find_by_filters<'a>( &'a self, filters: &'a [Filter], @@ -477,6 +514,11 @@ impl EventRepository for InMemoryEventRepository { let mut seen: Vec = Vec::new(); let page_bound = self.page_bound(); + let unmatched = self + .unmatched + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); for filter in filters { let bound = filter.limit.unwrap_or(u64::MAX).min(page_bound); @@ -487,7 +529,7 @@ impl EventRepository for InMemoryEventRepository { break; } - if !filter.matches(&record.event) { + if !filter.matches(&record.event) && !unmatched.contains(&record.event.id) { continue; } diff --git a/crates/nostream-relay/tests/handlers.rs b/crates/nostream-relay/tests/handlers.rs index e23172ee7..f05845b09 100644 --- a/crates/nostream-relay/tests/handlers.rs +++ b/crates/nostream-relay/tests/handlers.rs @@ -31,8 +31,10 @@ use nostream_core::messages::{SubscribeMessage, UnsubscribeMessage}; use nostream_relay::connection::Connection; use nostream_relay::relay_key::RelayKeySource; use nostream_relay::sink::FrameSink; -use nostream_relay::testing::{Fixture, nip42_settings, nip43_settings, nip45_settings, settings}; -use nostream_storage::repositories::{EventRepository, User}; +use nostream_relay::testing::{ + Fixture, nip42_settings, nip43_settings, nip45_settings, nip50_settings, settings, +}; +use nostream_storage::repositories::{EventRepository, SearchRule, User}; /// The relay's public key for [`RELAY_SECRET`]. fn relay_pubkey() -> nostream_core::Pubkey { @@ -1190,6 +1192,79 @@ async fn eose_hints_are_absent_when_the_backend_declares_a_smaller_page() { assert_eq!(frames[1], r#"["EOSE","s1"]"#); } +/// The backend's `search` predicate is not this relay's matcher, and the two disagree in **both** +/// directions — so a row the backend yielded and the matcher refused ends any completeness claim. +/// +/// The case: filter `search: "study"`, stored content `studies`. Postgres prefilter yields that +/// row (`plainto_tsquery` and the content's tsvector both reduce to the one English stem `studi`, +/// which is the `english` regconfig's documented stemming and not something this repository has +/// run), and `Filter::matches` refuses it because `"studies"` contains no `"study"` — that half is +/// arithmetic on the two strings. So the read is short, the page is not full, and before the fix +/// the relay said `["finish"]`: every matching event sent. The other direction (`abcdef` inside +/// `xxabcdefyy`, which the matcher accepts and the tsquery never yields) leaves no trace, which is +/// why the declared rule below is needed as well. +/// +/// Note what this test is built to catch: if someone "fixes" the asymmetry by loosening `matches` +/// instead of suppressing the hint, the row here starts being delivered and the exact frame +/// assertion below fails — because the predicate is a statement about the backend, not a bug in +/// the matcher. +#[tokio::test] +async fn eose_hints_are_absent_when_the_backend_yields_a_row_the_matcher_refuses() { + let mut settings = settings(); + settings.nip50 = Some(nip50_settings()); + settings.nip50 = Some(nostream_config::settings::Nip50Settings { + enabled: Some(true), + ..nip50_settings() + }); + let fixture = Fixture::new(settings); + fixture + .events + .seed_unmatched(signed_event(&CLIENT_SECRET, 1, vec![], "studies", NOW_SECS)); + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client + .send(r#"["REQ","s1",{"kinds":[1],"search":"study","limit":5}]"#) + .await; + let frames = client.read_until_eose().await; + + assert_eq!( + frames, + vec![r#"["EOSE","s1"]"#.to_owned()], + "one row was yielded and refused, so the page is not a proof of anything" + ); +} + +/// A backend whose `search` rule is its own withholds `finish` for that filter even where every +/// row agreed: the set it enumerated is not the set the relay would have enumerated, so no check +/// the relay can run proves the claim. This is the half that closes the direction no probe sees. +#[tokio::test] +async fn eose_hints_are_absent_when_the_search_rule_is_the_backends_own() { + let mut settings = settings(); + settings.nip50 = Some(nostream_config::settings::Nip50Settings { + enabled: Some(true), + ..nip50_settings() + }); + let fixture = Fixture::new(settings); + fixture.events.declare_search_rule(SearchRule::ToTsvector { + language: "english".to_owned(), + }); + fixture + .seed_event(signed_event(&CLIENT_SECRET, 1, vec![], "a study", NOW_SECS)) + .await; + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client + .send(r#"["REQ","s1",{"kinds":[1],"search":"study","limit":5}]"#) + .await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 2, "{frames:?}"); + assert!(events_then_eose(&frames), "{frames:?}"); + assert_eq!(frames[1], r#"["EOSE","s1"]"#); +} + /// A storage error leaves the read's extent unknown, so no hint is added after it. #[tokio::test] async fn eose_hints_are_absent_when_the_stream_failed() { From d4bbe4aa9845d81aa72f4bdec21ec919c2f01f62 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 04:35:57 -0500 Subject: [PATCH 25/36] perf(relay): stop collecting a deletion request's addresses once it exceeds the cap --- .../nostream-relay/src/strategies/delete.rs | 17 +++++++++-- crates/nostream-relay/tests/strategies.rs | 30 +++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/crates/nostream-relay/src/strategies/delete.rs b/crates/nostream-relay/src/strategies/delete.rs index 333152f5b..6a259b875 100644 --- a/crates/nostream-relay/src/strategies/delete.rs +++ b/crates/nostream-relay/src/strategies/delete.rs @@ -157,8 +157,21 @@ fn addresses(event: &Event) -> Vec<(Kind, String)> { let d = parts.next().unwrap_or_default(); let address = (kind, d.to_owned()); - if !addresses.contains(&address) { - addresses.push(address); + if addresses.contains(&address) { + continue; + } + + addresses.push(address); + + // Stop as soon as the request is over the cap: it will be refused, and the tags left are + // not worth parsing. The check is on the **distinct** count, so a run of repeated tags + // (which the `contains` above drops without pushing) never trips it — a request that names + // one address 6 096 times is legal, and this breaks only on the (cap + 1)th new address. + // Without this the collection is O(T²) in the tag count, spent before the cap is even + // consulted: 6 096 addresses cost 42.5 ms (release) against 2.4 ms to decode the same + // frame, and the work happens on a tokio worker. + if addresses.len() > MAX_ADDRESSES_PER_REQUEST { + break; } } diff --git a/crates/nostream-relay/tests/strategies.rs b/crates/nostream-relay/tests/strategies.rs index 7d4fbc128..885844515 100644 --- a/crates/nostream-relay/tests/strategies.rs +++ b/crates/nostream-relay/tests/strategies.rs @@ -665,6 +665,36 @@ async fn delete_refuses_a_request_naming_too_many_addresses() { ); } +/// A legal request stays legal when its addresses repeat: the cap counts distinct addresses, and the +/// early break that keeps an over-cap list cheap must not turn one of these into a refusal. Twenty +/// tags per address, a total far past the cap, plus a repeat of the first address at the end. +#[tokio::test] +async fn delete_admits_a_cap_sized_batch_with_repeats() { + let fixture = Fixture::new(settings()); + let mut tags: Vec> = Vec::new(); + + for _ in 0..20 { + for index in 0..100 { + tags.push(tag(&["a", &format!("30023:{}:post-{index}", author())])); + } + } + + tags.push(tag(&["a", &format!("30023:{}:post-0", author())])); + assert!(tags.len() > 101, "the tag count is past the cap on purpose"); + let event = event(5, tags, ""); + + let (context, mut sink) = socket(&fixture); + DeleteStrategy.execute(&event, &context).await.unwrap(); + + assert_eq!( + fixture.events.address_deletes().len(), + 100, + "one entry per distinct address, and no refusal" + ); + assert_eq!(fixture.events.live_events(), vec![event.clone()]); + assert_eq!(frames(&mut sink), vec![ok_frame(&event, true, "")]); +} + /// One hundred and one tags naming **one** address are admitted: the cap bounds distinct /// addresses, which is the work, not the tag list the transport already bounds. #[tokio::test] From 14ada511a2b9ab5f0e4ad8ed91b2141384658fe3 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 08:53:56 -0500 Subject: [PATCH 26/36] feat(storage): report a vanished page slot so finish cannot claim it find_by_filters now yields StreamedEvent: a Record, or Vanished for a slot the backend selected and could not read. LMDB's batch loop emitted nothing there, so a page that lost a key to a concurrent replacement read as a short result and the relay sent a false NIP-67 finish (limit 1 selects two keys, one is replaced, one row is yielded and a third match is never sent). collect_records is the one helper for the call sites that only want rows, so the marker cannot reach a consumer as a record. subscribe.rs counts a Vanished as a spent slot and withholds finish on it. --- .../nostream-relay/src/handlers/subscribe.rs | 38 +++-- crates/nostream-relay/src/testing.rs | 65 +++++--- crates/nostream-relay/src/unavailable.rs | 4 +- crates/nostream-relay/tests/handlers.rs | 34 +++++ .../src/services/wot_graph_service.rs | 9 +- crates/nostream-services/src/testing.rs | 15 +- crates/nostream-services/src/unavailable.rs | 2 +- .../src/workers/dvm_orchestrator.rs | 8 +- crates/nostream-storage/src/lmdb/events.rs | 17 ++- crates/nostream-storage/src/lmdb/mod.rs | 8 +- .../nostream-storage/src/postgres/events.rs | 12 +- .../src/postgres/repository.rs | 6 +- crates/nostream-storage/src/repositories.rs | 120 ++++++++++++++- .../nostream-storage/tests/address_delete.rs | 8 +- crates/nostream-storage/tests/common/mod.rs | 4 +- crates/nostream-storage/tests/page_bound.rs | 8 +- crates/nostream-storage/tests/parity_lmdb.rs | 22 +-- crates/nostream-storage/tests/storage.rs | 6 +- crates/nostream-storage/tests/vanished.rs | 141 ++++++++++++++++++ 19 files changed, 442 insertions(+), 85 deletions(-) create mode 100644 crates/nostream-storage/tests/vanished.rs diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index a205ce9bb..bea6e9ee5 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -28,7 +28,7 @@ use nostream_core::messages::{ }; use nostream_core::nip42; use nostream_core::{Filter, Kind}; -use nostream_storage::repositories::SearchRule; +use nostream_storage::repositories::{SearchRule, StreamedEvent}; use crate::connection::InFlightRead; use crate::error::RelayError; @@ -312,13 +312,15 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // window, which is what `limit` bounds. `trimmed`: the guard withheld a match. `more`: a // match beyond the window was found. `failed`: the stream errored. `refused`: the backend // yielded a row this relay's matcher refuses, which is what a backend whose predicate is not - // `Filter::matches` looks like from here. + // `Filter::matches` looks like from here. `vanished`: the backend spent a page slot on a key + // that was gone when it came to read it. let mut rows = 0u64; let mut matched = 0u64; let mut trimmed = false; let mut more = false; let mut failed = false; let mut refused = false; + let mut vanished = false; loop { if *read.cancellation().borrow() { @@ -366,7 +368,7 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha }; match record { - Ok(record) => { + Ok(StreamedEvent::Record(record)) => { // Every row the page yielded counts towards whether the page was full, whether // or not it turns out to be a match. rows += 1; @@ -420,6 +422,17 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha event: std::sync::Arc::new(event), })); } + Ok(StreamedEvent::Vanished) => { + // The backend selected this slot and the key was gone when it read the value, so + // the slot is spent without a record. It counts as a row, exactly as a skipped + // row does, and it sets `vanished` on its own: `page_full` compares the rows the + // read *yielded* against the probe, so a page that lost a key arrives short of it + // and reads as innocent, and where no probe was built `page_full` is `false` + // outright. The marker is the one signal in the read that says a selected slot + // produced nothing, so `finish` is withheld on it rather than argued about. + rows += 1; + vanished = true; + } Err(error) => { tracing::error!(error = %error, "error streaming events"); failed = true; @@ -433,27 +446,34 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // The rule, in one place, so the next reader does not have to re-derive it. `finish` claims // that every stored event matching the subscription's filters has been sent (`67.md` - // §"Specification"), and this read can fail to establish that three ways, so `finish` needs - // all three: + // §"Specification"), and this read can fail to establish that four ways, so `finish` needs + // all four: // // 1. the stream ended **cleanly** — after a storage error, how far the read got is unknown; // 2. the page was **not full** — a page that yielded `min(probe, page_bound())` rows says // nothing about the rows beyond it, whether the bound came from the client's `limit` or // from the backend's own page cap, and a skipped row spends a page slot just as a match // does (which is why `rows`, not `matched`, decides this); - // 3. the set the backend enumerated is the set this relay would have — no row it yielded was + // 3. **no page slot was spent without its row** — a backend reports a slot it selected and + // then could not read as `StreamedEvent::Vanished`, and that slot is a row the read was + // promised. The count of yielded rows cannot carry that on its own: `page_full` compares + // that count against the probe, so a page that lost a key to a concurrent replacement + // arrives short of it and reads as innocent, and where no probe was built `page_full` is + // `false` by construction. The marker withholds `finish` directly, rather than relying on + // the arithmetic a vanished row happens to disturb; + // 4. the set the backend enumerated is the set this relay would have — no row it yielded was // refused by `Filter::matches`, and no filter carrying `search` was answered by a // predicate other than that matcher. Both are needed and neither implies the other: a // refusal is the direction a probe can see (`study` yields `studies`, which the matcher // refuses), while a row the matcher *would* accept and the backend never yields // (`abcdef` inside `xxabcdefyy`) leaves no trace at all. See `EventRepository::search_rule`. - // 4. **no further match** was found — the `more` branch below is where this one lives: a + // 5. **no further match** was found — the `more` branch below is where this one lives: a // further match ends the read with `more` set, so the code that can reach the hint is the // code where none was found. It is only meaningful once 1 and 2 hold, because the loop // keeps asking past deleted, expired, withheld and non-matching rows until it sees a // match, the genuine end, or the page cap. // - // Where any of the three cannot be established the relay sends no completeness hint: an + // Where any of the four cannot be established the relay sends no completeness hint: an // absent hint is honest and a wrong one is not. `more` needs one further **match** beyond // the delivered window, `auth` is unaffected, and `finish` is never sent beside `auth` // because a withheld event falsifies it. @@ -467,7 +487,7 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha .is_some_and(|search| !search.trim().is_empty()) }); let search_agrees = search_rule == SearchRule::SubstringTerms || !searched; - let proven = !failed && !page_full && !refused && search_agrees; + let proven = !failed && !page_full && !vanished && !refused && search_agrees; let mut hints = Vec::new(); diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index 0b4438a5b..05998ae53 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -28,7 +28,8 @@ use nostream_storage::repositories::{ ActionableTarget, CreateInviteCodeOptions, DvmJob, DvmJobStatus, EventPurgeCounts, EventRecord, EventRepository, EventRetentionOptions, FieldUpdate, InviteCode, InviteCodeRepository, NewReport, Nip05Verification, Nip05VerificationRepository, QueryStream, ReconciliationItem, - Report, ReportRepository, SearchRule, StorageFuture, User, UserPatch, UserRepository, + Report, ReportRepository, SearchRule, StorageFuture, StreamedEvent, User, UserPatch, + UserRepository, }; use tokio::sync::Notify; @@ -161,6 +162,10 @@ pub struct InMemoryEventRepository { /// `Filter::matches` looks like from the handler's side, which is the case no honest filter /// can produce. unmatched: Mutex>, + /// Rows this double yields as [`StreamedEvent::Vanished`] instead of as a record: a page slot + /// the backend selected and then could not read, which is what a concurrent replacement + /// leaves behind in LMDB. + vanishing: Mutex>, upsert_failure: Mutex>, upserts: Mutex>>, deletes: Mutex)>>, @@ -234,6 +239,16 @@ impl InMemoryEventRepository { .push(event.id); } + /// Makes the read yield [`StreamedEvent::Vanished`] in place of this row's slot, after the + /// row has counted towards the page: the case a replacement removes a selected key before + /// its batch is read. + pub fn vanish(&self, id: EventId) { + self.vanishing + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(id); + } + /// Declares the matching rule this double answers a `search` filter with, the way a backend /// whose predicate is its own would. pub fn declare_search_rule(&self, rule: SearchRule) { @@ -479,7 +494,7 @@ impl EventRepository for InMemoryEventRepository { fn find_by_filters<'a>( &'a self, filters: &'a [Filter], - ) -> StorageFuture<'a, QueryStream<'a, EventRecord>> { + ) -> StorageFuture<'a, QueryStream<'a, StreamedEvent>> { let gate = self .read_gate .lock() @@ -490,12 +505,12 @@ impl EventRepository for InMemoryEventRepository { Box::pin(async move { if failing { - let failed: Vec> = + let failed: Vec> = vec![Err(StorageError::InvalidArgument { message: "in-memory double: the read failed", })]; - return Ok(Box::pin(stream::iter(failed)) as QueryStream<'a, EventRecord>); + return Ok(Box::pin(stream::iter(failed)) as QueryStream<'a, StreamedEvent>); } let records = self @@ -510,7 +525,7 @@ impl EventRepository for InMemoryEventRepository { // filter's limit, exactly as it does in both backends. The double used to answer // every match, and then to not charge an overlap against the limit, which is how a // handler test stops meaning anything. - let mut matched: Vec> = Vec::new(); + let mut matched: Vec = Vec::new(); let mut seen: Vec = Vec::new(); let page_bound = self.page_bound(); @@ -540,19 +555,38 @@ impl EventRepository for InMemoryEventRepository { } seen.push(record.event.id); - matched.push(Ok(record.clone())); + matched.push(record.clone()); } } + // A row the test marked as vanished is a slot this double selected and cannot read: + // it keeps its place in the page and yields the marker instead of the record, which + // is what LMDB does when a replacement removes a selected key before its batch. + let vanishing = self + .vanishing + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone(); + let items: Vec> = matched + .into_iter() + .map(|record| { + Ok(if vanishing.contains(&record.event.id) { + StreamedEvent::Vanished + } else { + StreamedEvent::Record(record) + }) + }) + .collect(); + let Some(gate) = gate else { - return Ok(Box::pin(stream::iter(matched)) as QueryStream<'a, EventRecord>); + return Ok(Box::pin(stream::iter(items)) as QueryStream<'a, StreamedEvent>); }; if per_item { // One permit per record, and the permits are held rather than returned, so // one release lets exactly one record through. let held = Arc::new(Mutex::new(Vec::new())); - let gated = stream::iter(matched).then(move |item| { + let gated = stream::iter(items).then(move |item| { let gate = Arc::clone(&gate); let held = Arc::clone(&held); @@ -572,7 +606,7 @@ impl EventRepository for InMemoryEventRepository { } }); - return Ok(Box::pin(gated) as QueryStream<'a, EventRecord>); + return Ok(Box::pin(gated) as QueryStream<'a, StreamedEvent>); } let permit = gate @@ -584,13 +618,13 @@ impl EventRepository for InMemoryEventRepository { // The permit is held by the stream, so the read stays gated until it completes or // the handler drops it. - let gated = stream::iter(matched).map(move |item| { + let gated = stream::iter(items).map(move |item| { let _held = &permit; item }); - Ok(Box::pin(gated) as QueryStream<'a, EventRecord>) + Ok(Box::pin(gated) as QueryStream<'a, StreamedEvent>) }) } @@ -1805,12 +1839,9 @@ mod tests { ]; let stream = events.find_by_filters(&filters).await.unwrap(); - futures_util::pin_mut!(stream); - - let mut rows = Vec::new(); - while let Some(record) = futures_util::StreamExt::next(&mut stream).await { - rows.push(record.unwrap()); - } + let rows = nostream_storage::repositories::collect_records(stream) + .await + .unwrap(); assert_eq!( rows.len(), diff --git a/crates/nostream-relay/src/unavailable.rs b/crates/nostream-relay/src/unavailable.rs index 8121828e7..5b7ef024b 100644 --- a/crates/nostream-relay/src/unavailable.rs +++ b/crates/nostream-relay/src/unavailable.rs @@ -14,7 +14,7 @@ use nostream_storage::repositories::{ ActionableTarget, CreateInviteCodeOptions, DvmJob, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, EventRetentionOptions, InviteCode, InviteCodeRepository, Nip05Verification, Nip05VerificationRepository, QueryStream, ReconciliationItem, Report, ReportRepository, - StorageFuture, User, UserPatch, UserRepository, + StorageFuture, StreamedEvent, User, UserPatch, UserRepository, }; use crate::handlers::Stores; @@ -90,7 +90,7 @@ impl EventRepository for UnavailableEventRepository { fn find_by_filters<'a>( &'a self, _filters: &'a [Filter], - ) -> StorageFuture<'a, QueryStream<'a, EventRecord>> { + ) -> StorageFuture<'a, QueryStream<'a, StreamedEvent>> { Box::pin(async { Err(unavailable()) }) } diff --git a/crates/nostream-relay/tests/handlers.rs b/crates/nostream-relay/tests/handlers.rs index f05845b09..dc1a20a21 100644 --- a/crates/nostream-relay/tests/handlers.rs +++ b/crates/nostream-relay/tests/handlers.rs @@ -1280,6 +1280,40 @@ async fn eose_hints_are_absent_when_the_stream_failed() { assert_eq!(frames, vec![r#"["EOSE","s1"]"#.to_owned()], "{frames:?}"); } +/// A row the backend selected and could not read spends a page slot without a record. The page +/// comes back one row short of the probe, so `page_full` is `false` and `finish` would otherwise +/// claim that every matching event had been sent over a slot that never yielded one. The case is +/// real on LMDB, where a replacement hard-deletes the row it supersedes between the page and the +/// batch that would have read it. +/// +/// The other direction is `eose_hints_finish_when_the_limit_was_not_reached`: an honest short +/// page still proves `finish`, so the suppression is the marker's and not the read's. +#[tokio::test] +async fn eose_hints_are_absent_when_a_selected_row_vanished() { + let fixture = Fixture::new(settings()); + let survivor = signed_event(&CLIENT_SECRET, 1, vec![], "note 0", NOW_SECS); + let vanished = signed_event(&CLIENT_SECRET, 1, vec![], "note 1", NOW_SECS); + fixture.seed_event(survivor.clone()).await; + fixture.seed_event(vanished.clone()).await; + fixture.events.vanish(vanished.id); + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r#"["REQ","s1",{"kinds":[1],"limit":3}]"#).await; + let frames = client.read_until_eose().await; + + assert_eq!( + frames.len(), + 2, + "one row was spent without a record: {frames:?}" + ); + assert!( + frames[0].starts_with(r#"["EVENT","s1","#), + "the survivor is still delivered: {frames:?}" + ); + assert_eq!(frames[1], r#"["EOSE","s1"]"#, "{frames:?}"); +} + /// `auth` and the completeness hint compose: a withheld match falsifies `finish` even where the /// page was neither full nor capped, and `auth` is what is sent. #[tokio::test] diff --git a/crates/nostream-services/src/services/wot_graph_service.rs b/crates/nostream-services/src/services/wot_graph_service.rs index 3749b56c2..1d30f3ee9 100644 --- a/crates/nostream-services/src/services/wot_graph_service.rs +++ b/crates/nostream-services/src/services/wot_graph_service.rs @@ -10,7 +10,6 @@ use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; -use futures_util::StreamExt; use nostream_config::Settings; use nostream_core::{Filter, Kind, Pubkey}; use nostream_relay::wot::{WotFuture, WotGraph}; @@ -272,11 +271,13 @@ impl WotGraphService { }; let filters = [filter]; - let mut stream = self.inner.events.find_by_filters(&filters).await?; - let Some(record) = stream.next().await else { + let rows = nostream_storage::repositories::collect_records( + self.inner.events.find_by_filters(&filters).await?, + ) + .await?; + let Some(record) = rows.into_iter().next() else { return Ok(Vec::new()); }; - let record = record?; let follows = deduplicated( record diff --git a/crates/nostream-services/src/testing.rs b/crates/nostream-services/src/testing.rs index 414122295..d50ea407e 100644 --- a/crates/nostream-services/src/testing.rs +++ b/crates/nostream-services/src/testing.rs @@ -26,8 +26,8 @@ use nostream_storage::repositories::{ EventPurgeCounts, EventRecord, EventRepository, EventRetentionOptions, FieldUpdate, Invoice, InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, Nip05Verification, Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, - OutboxMessage, OutboxPayload, QueryStream, ReconciliationItem, StorageFuture, User, UserPatch, - UserRepository, + OutboxMessage, OutboxPayload, QueryStream, ReconciliationItem, StorageFuture, StreamedEvent, + User, UserPatch, UserRepository, }; use crate::error::ServicesError; @@ -497,7 +497,7 @@ impl EventRepository for InMemoryEventRepository { fn find_by_filters<'a>( &'a self, filters: &'a [Filter], - ) -> StorageFuture<'a, QueryStream<'a, EventRecord>> { + ) -> StorageFuture<'a, QueryStream<'a, StreamedEvent>> { Box::pin(async move { let gate = self .read_gate @@ -535,8 +535,13 @@ impl EventRepository for InMemoryEventRepository { selected.truncate(usize::try_from(limit).unwrap_or(usize::MAX)); } - let stream: QueryStream<'a, EventRecord> = - Box::pin(stream::iter(selected.into_iter().map(Ok))); + // This double holds its page, so it has no slot it selected and could not read: + // every item is a record. + let stream: QueryStream<'a, StreamedEvent> = Box::pin(stream::iter( + selected + .into_iter() + .map(|record| Ok(StreamedEvent::Record(record))), + )); Ok(stream) }) diff --git a/crates/nostream-services/src/unavailable.rs b/crates/nostream-services/src/unavailable.rs index 50c6beaf1..d57efd2bb 100644 --- a/crates/nostream-services/src/unavailable.rs +++ b/crates/nostream-services/src/unavailable.rs @@ -74,7 +74,7 @@ impl EventRepository for UnavailableEventRepository { fn find_by_filters<'a>( &'a self, _filters: &'a [nostream_core::Filter], - ) -> StoreFuture<'a, QueryStream<'a, EventRecord>> { + ) -> StoreFuture<'a, QueryStream<'a, nostream_storage::repositories::StreamedEvent>> { Box::pin(async { Err(unavailable()) }) } diff --git a/crates/nostream-services/src/workers/dvm_orchestrator.rs b/crates/nostream-services/src/workers/dvm_orchestrator.rs index 5c2ae3d93..7e2ddc13a 100644 --- a/crates/nostream-services/src/workers/dvm_orchestrator.rs +++ b/crates/nostream-services/src/workers/dvm_orchestrator.rs @@ -302,10 +302,10 @@ impl DvmOrchestrator { ..Filter::default() }; let record = match self.stores.events.find_by_filters(&[filter]).await { - Ok(mut stream) => { - use futures_util::StreamExt; - stream.next().await.and_then(Result::ok) - } + Ok(stream) => nostream_storage::repositories::collect_records(stream) + .await + .ok() + .and_then(|rows| rows.into_iter().next()), Err(error) => { tracing::error!(error = %error, "unable to read a dvm job's source event"); None diff --git a/crates/nostream-storage/src/lmdb/events.rs b/crates/nostream-storage/src/lmdb/events.rs index 9a8ee4107..c1281b4f4 100644 --- a/crates/nostream-storage/src/lmdb/events.rs +++ b/crates/nostream-storage/src/lmdb/events.rs @@ -29,7 +29,7 @@ use super::{ use crate::error::StorageError; use crate::records::{distinct_replacements, replaces_fields}; use crate::repositories::{ - EventPurgeCounts, EventRecord, EventRetentionOptions, ReconciliationItem, + EventPurgeCounts, EventRecord, EventRetentionOptions, ReconciliationItem, StreamedEvent, }; /// The kind of the capture's `REQUEST_TO_VANISH`, which every retention whitelist keeps and @@ -1682,6 +1682,12 @@ impl Position { /// copies each batch's values out of it; it is the reconciliation stream, not this one, that /// holds only a cursor key. /// +/// A key the page selected may be gone when its batch is read: a replacement hard-deletes the +/// row it supersedes (`hard_delete`), a retention pass removes a soft-deleted one, and this +/// stream opens a fresh read transaction per batch, so the removal can land between the two. +/// The slot is still spent, and it is reported as [`StreamedEvent::Vanished`] rather than +/// skipped — a silent skip reads at the call site as a short page. +/// /// shortcut: a stream is not a snapshot — a row written or deleted between two batches may /// appear or vanish; upgrade when a caller needs a stable view, per ADR 0003 trigger 1 and the /// benchmark job, which would buy the snapshot back with a held transaction. @@ -1689,8 +1695,8 @@ pub(crate) struct ValuesStream { env: Env, databases: Databases, keys: std::vec::IntoIter, - pending: Option, StorageError>>>, - buffer: std::vec::IntoIter, + pending: Option, StorageError>>>, + buffer: std::vec::IntoIter, finished: bool, } @@ -1712,7 +1718,7 @@ impl ValuesStream { } impl Stream for ValuesStream { - type Item = Result; + type Item = Result; fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { let this = self.get_mut(); @@ -1755,9 +1761,10 @@ impl Stream for ValuesStream { let mut records = Vec::with_capacity(batch.len()); for id in batch { let Some(value) = database.get(&transaction, id.as_bytes())? else { + records.push(StreamedEvent::Vanished); continue; }; - records.push(rows::decode_event(value)?); + records.push(StreamedEvent::Record(rows::decode_event(value)?)); } Ok(records) })); diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index e0cbd2b6f..999438295 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -32,8 +32,8 @@ use crate::repositories::{ EventRetentionOptions, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, OutboxPayload, - QueryStream, ReconciliationItem, Report, ReportRepository, StorageFuture, User, UserPatch, - UserRepository, + QueryStream, ReconciliationItem, Report, ReportRepository, StorageFuture, StreamedEvent, User, + UserPatch, UserRepository, }; /// The fixed map size every environment is opened with: `Env::resize` is `unsafe` and this @@ -343,7 +343,7 @@ impl EventRepository for LmdbBackend { fn find_by_filters<'a>( &'a self, filters: &'a [Filter], - ) -> StorageFuture<'a, QueryStream<'a, EventRecord>> { + ) -> StorageFuture<'a, QueryStream<'a, StreamedEvent>> { Box::pin(async move { let filters = filters.to_vec(); let gates = self.gates(); @@ -351,7 +351,7 @@ impl EventRepository for LmdbBackend { let page = self .read(move |txn, databases| events::page(txn, databases, &filters, &gates, now)) .await?; - let stream: QueryStream<'_, EventRecord> = Box::pin(events::ValuesStream::new( + let stream: QueryStream<'_, StreamedEvent> = Box::pin(events::ValuesStream::new( self.env.clone(), self.databases, &page, diff --git a/crates/nostream-storage/src/postgres/events.rs b/crates/nostream-storage/src/postgres/events.rs index 186450ba0..8d10370cd 100644 --- a/crates/nostream-storage/src/postgres/events.rs +++ b/crates/nostream-storage/src/postgres/events.rs @@ -16,6 +16,7 @@ use crate::error::StorageError; use crate::records::distinct_replacements; use crate::repositories::{ EventPurgeCounts, EventRecord, EventRetentionOptions, QueryStream, ReconciliationItem, + StreamedEvent, }; /// The conflict target of every replaceable write, exactly as the capture spells it. @@ -174,7 +175,7 @@ pub(crate) fn find_by_filters<'a>( reader: &'a sqlx::PgPool, filters: &[Filter], gates: &Gates, -) -> Result, StorageError> { +) -> Result, StorageError> { let query = sql::find_by_filters(filters, gates)?; Ok(Box::pin(EventStream { inner: query.into_query().fetch(reader), @@ -490,17 +491,22 @@ fn tags_json(record: &EventRecord) -> Result { } /// A stream of stored events, mapping each driver row. +/// +/// Postgres streams one query over rows it has already read, so it has no slot it selected and +/// could not read: every item is [`StreamedEvent::Record`] and the marker never appears here. struct EventStream<'a> { inner: Pin> + Send + 'a>>, } impl Stream for EventStream<'_> { - type Item = Result; + type Item = Result; fn poll_next(self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { let this = self.get_mut(); match this.inner.as_mut().poll_next(context) { - Poll::Ready(Some(Ok(row))) => Poll::Ready(Some(super::rows::event_record(&row))), + Poll::Ready(Some(Ok(row))) => Poll::Ready(Some( + super::rows::event_record(&row).map(StreamedEvent::Record), + )), Poll::Ready(Some(Err(error))) => Poll::Ready(Some(Err(StorageError::Database(error)))), Poll::Ready(None) => Poll::Ready(None), Poll::Pending => Poll::Pending, diff --git a/crates/nostream-storage/src/postgres/repository.rs b/crates/nostream-storage/src/postgres/repository.rs index da02deee9..21a67565c 100644 --- a/crates/nostream-storage/src/postgres/repository.rs +++ b/crates/nostream-storage/src/postgres/repository.rs @@ -13,8 +13,8 @@ use crate::repositories::{ EventRetentionOptions, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, OutboxPayload, - QueryStream, ReconciliationItem, Report, ReportRepository, SearchRule, StorageFuture, User, - UserPatch, UserRepository, + QueryStream, ReconciliationItem, Report, ReportRepository, SearchRule, StorageFuture, + StreamedEvent, User, UserPatch, UserRepository, }; impl EventRepository for PostgresBackend { @@ -37,7 +37,7 @@ impl EventRepository for PostgresBackend { fn find_by_filters<'a>( &'a self, filters: &'a [Filter], - ) -> StorageFuture<'a, QueryStream<'a, EventRecord>> { + ) -> StorageFuture<'a, QueryStream<'a, StreamedEvent>> { Box::pin(async move { let gates = self.gates(); events::find_by_filters(&self.reader, filters, &gates) diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index e8940a72d..a777beb18 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -20,10 +20,11 @@ use crate::error::StorageError; /// `src/@types/repositories.ts`, whose methods take the ambient `DatabaseClient`. pub type StorageFuture<'a, T> = Pin> + Send + 'a>>; -/// One row per item, in exactly the query's `ORDER BY`. The borrow of the store is held for -/// the stream's life, which is what both backends need: Postgres holds a portal and a pooled -/// connection, and LMDB holds whatever its own query kept, opening a short read transaction -/// per batch (256 values) so a dropped stream pins no pages. +/// One item per page slot, in exactly the query's `ORDER BY`. The borrow of the store is held +/// for the stream's life, which is what both backends need: Postgres holds a portal and a +/// pooled connection, and LMDB holds whatever its own query kept, opening a short read +/// transaction per batch (256 values) so a dropped stream pins no pages. A filter read's item +/// is a [`StreamedEvent`], because a slot the backend selected may have no row to yield. /// /// An LMDB backend's two streams keep different things, and a caller plans memory from the /// difference: `find_by_filters` collects its page first — at most each filter's limit, clamped @@ -82,6 +83,48 @@ pub struct EventRecord { pub remote_address: Option, } +/// One item of a filter read: the row for a page slot, or the marker for a slot the backend +/// selected and could not read. +/// +/// The distinction exists because a `limit` read is a claim about a **count of slots**, and the +/// page obligation on [`EventRepository::find_by_filters`] is stated in rows. A backend that +/// enumerated a key and found it gone when it came to read the value has spent the slot without +/// a row; reporting that as "fewer rows" is indistinguishable at the call site from a short +/// result, which is how a false NIP-67 `finish` reaches the wire. [`StreamedEvent::Vanished`] is +/// the backend saying so, and the relay counts it as a spent slot. +// The row is the item, and it moves by value exactly as it did inside `Result` +// before the marker existed; boxing it would add one allocation per streamed row to the read +// path to make a two-variant enum smaller than the rows it carries. +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Clone, PartialEq)] +pub enum StreamedEvent { + /// One stored row, as the query ordered it. + Record(EventRecord), + /// The backend selected this slot and then could not read it: the key was in the page when + /// the backend enumerated it and gone when the backend read its value. + Vanished, +} + +/// Drains a filter read into the rows it yielded, dropping [`StreamedEvent::Vanished`]. +/// +/// This is the helper for every call site that does not account for vanished slots: the marker +/// is a fact about the page, not a row, so a consumer that only wants rows cannot inherit it. +pub fn collect_records<'a>( + mut stream: QueryStream<'a, StreamedEvent>, +) -> StorageFuture<'a, Vec> { + Box::pin(async move { + let mut records = Vec::new(); + while let Some(item) = + std::future::poll_fn(|context| stream.as_mut().poll_next(context)).await + { + if let StreamedEvent::Record(record) = item? { + records.push(record); + } + } + Ok(records) + }) +} + /// One NIP-77 reconciliation item (`src/@types/repositories.ts` `ReconciliationItem`). #[derive(Debug, Clone, PartialEq, Eq)] pub struct ReconciliationItem { @@ -455,10 +498,17 @@ pub trait EventRepository: Send + Sync { /// A backend whose own page is smaller than [`MAX_PAGE_ROWS`](crate::MAX_PAGE_ROWS) declares it through /// [`EventRepository::page_bound`]; the relay then asks for no more than it will be given and /// emits no hint where it cannot prove one. + /// + /// **The marker.** A backend that cannot honour the page at read time — it selected a key and + /// the key was gone when it came to read the value — **MUST** yield + /// [`StreamedEvent::Vanished`] for that slot rather than skipping it. The obligation above + /// forbids the silent short page; this names the signal that keeps it honest when a + /// concurrent write removes a selected row. The slot is spent either way, so a caller that + /// only wants rows reads them through [`collect_records`], which drops the marker. fn find_by_filters<'a>( &'a self, filters: &'a [nostream_core::Filter], - ) -> StorageFuture<'a, QueryStream<'a, EventRecord>>; + ) -> StorageFuture<'a, QueryStream<'a, StreamedEvent>>; /// Counts the events matching the filters. fn count_by_filters<'a>( @@ -747,3 +797,63 @@ pub trait NotificationOutboxRepository: Send + Sync { /// Deletes delivered and dead messages older than the cutoff. fn delete_terminal_older_than<'a>(&'a self, cutoff: DateTime) -> StorageFuture<'a, u64>; } + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + + use std::pin::Pin; + use std::task::{Context, Poll}; + + use futures_core::Stream; + use nostream_core::{Event, EventId, Kind, Pubkey, Signature}; + + use super::*; + + /// A stream of the items it was built from. + struct Items(std::vec::IntoIter>); + + impl Stream for Items { + type Item = Result; + + fn poll_next( + mut self: Pin<&mut Self>, + _context: &mut Context<'_>, + ) -> Poll> { + Poll::Ready(self.0.next()) + } + } + + /// A kind-1 row whose content is `content`. + fn record(content: &str) -> EventRecord { + EventRecord::from_event(&Event { + id: EventId::from_byte_array([content.len() as u8; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at: 1_700_000_000, + kind: Kind(1), + tags: Vec::new(), + content: content.to_owned(), + sig: Signature::from_byte_array([0; 64]), + }) + } + + /// The helper hands a consumer only rows: the vanished slot is dropped, and the rows keep + /// the page's order, so the marker cannot be read as if it were a record. + #[tokio::test] + async fn collect_records_drops_the_marker_and_keeps_every_row() { + let items = vec![ + Ok(StreamedEvent::Record(record("one"))), + Ok(StreamedEvent::Vanished), + Ok(StreamedEvent::Record(record("two"))), + ]; + let stream: QueryStream<'_, StreamedEvent> = Box::pin(Items(items.into_iter())); + + let rows = collect_records(stream).await.expect("the read drains"); + let contents: Vec<&str> = rows + .iter() + .map(|row| row.event.content.as_str()) + .collect::>(); + + assert_eq!(contents, vec!["one", "two"]); + } +} diff --git a/crates/nostream-storage/tests/address_delete.rs b/crates/nostream-storage/tests/address_delete.rs index 3a0d6c7a8..42ba6fc52 100644 --- a/crates/nostream-storage/tests/address_delete.rs +++ b/crates/nostream-storage/tests/address_delete.rs @@ -12,10 +12,12 @@ mod common; use std::sync::Arc; -use common::{Repos, collect, fixture, settings_from_fixture}; +use common::{Repos, fixture, settings_from_fixture}; use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; use nostream_storage::LmdbBackend; -use nostream_storage::repositories::{EventRecord, EventRepository, NewReport, ReportType}; +use nostream_storage::repositories::{ + EventRecord, EventRepository, NewReport, ReportType, collect_records, +}; /// A store on a temporary directory with the fixture's settings gates. async fn open_store() -> (tempfile::TempDir, LmdbBackend) { @@ -86,7 +88,7 @@ async fn delete_by_addresses_reaches_a_row_past_the_page_bound() { ..Default::default() }; - let served = collect( + let served = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await diff --git a/crates/nostream-storage/tests/common/mod.rs b/crates/nostream-storage/tests/common/mod.rs index ffba90522..2619fb3a3 100644 --- a/crates/nostream-storage/tests/common/mod.rs +++ b/crates/nostream-storage/tests/common/mod.rs @@ -23,7 +23,7 @@ use nostream_storage::repositories::{ InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, QueryStream, ReconciliationItem, Report, ReportRepository, User, UserPatch, - UserRepository, + UserRepository, collect_records, }; /// The nine repositories one replay calls, as the traits rather than a concrete backend. @@ -412,7 +412,7 @@ pub async fn run_op( let failed = repos.events.find_by_filters(&[]).await.is_err(); return Ok(json!({ "error": failed })); } - let rows = collect(repos.events.find_by_filters(&filters).await?).await?; + let rows = collect_records(repos.events.find_by_filters(&filters).await?).await?; Ok(json!({ "rows": rows.iter().map(event_row).collect::>() })) } "events.count_by_filters" => { diff --git a/crates/nostream-storage/tests/page_bound.rs b/crates/nostream-storage/tests/page_bound.rs index 12e86568b..708ccfbc0 100644 --- a/crates/nostream-storage/tests/page_bound.rs +++ b/crates/nostream-storage/tests/page_bound.rs @@ -13,9 +13,9 @@ mod common; use std::sync::Arc; -use common::{collect, fixture, settings_from_fixture}; +use common::{fixture, settings_from_fixture}; use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; -use nostream_storage::repositories::{EventRecord, EventRepository}; +use nostream_storage::repositories::{EventRecord, EventRepository, collect_records}; use nostream_storage::{LmdbBackend, MAX_PAGE_ROWS}; /// A store on a temporary directory with the fixture's settings gates. @@ -61,7 +61,7 @@ async fn lmdb_declares_and_fills_the_page_bound() { let stored = filter(Kind(1)); - let full = collect( + let full = collect_records( backend .find_by_filters(&[Filter { limit: Some(MAX_PAGE_ROWS + 1), @@ -78,7 +78,7 @@ async fn lmdb_declares_and_fills_the_page_bound() { "a page asking for more than the bound gets the bound, not more" ); - let short = collect( + let short = collect_records( backend .find_by_filters(&[Filter { limit: Some(3), diff --git a/crates/nostream-storage/tests/parity_lmdb.rs b/crates/nostream-storage/tests/parity_lmdb.rs index 5cc20f545..804e0859b 100644 --- a/crates/nostream-storage/tests/parity_lmdb.rs +++ b/crates/nostream-storage/tests/parity_lmdb.rs @@ -27,7 +27,7 @@ use nostream_storage::repositories::{ EventRetentionOptions, FieldUpdate, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, InvoiceUnit, NewDeliveryLogEntry, NewReport, Nip05Verification, NotificationDeliveryLogRepository, NotificationDeliveryStatus, NotificationOutboxRepository, - NotificationTargetType, ReconciliationItem, ReportType, UserRepository, + NotificationTargetType, ReconciliationItem, ReportType, UserRepository, collect_records, }; mod common; @@ -216,7 +216,7 @@ async fn a_soft_deleted_event_is_readable_by_filter_and_excluded_from_count() { ids: Some(vec![record.event.id.to_string()]), ..Filter::default() }; - let live = common::collect( + let live = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await @@ -235,7 +235,7 @@ async fn a_soft_deleted_event_is_readable_by_filter_and_excluded_from_count() { 1 ); - let after = common::collect( + let after = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await @@ -282,7 +282,7 @@ async fn a_filter_result_larger_than_one_batch_arrives_whole_and_in_order() { kinds: Some(vec![Kind(1)]), ..Filter::default() }; - let seen = common::collect( + let seen = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await @@ -694,7 +694,7 @@ async fn a_limited_ids_filter_returns_the_newest_events() { ..Filter::default() }; - let page = common::collect( + let page = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await @@ -727,7 +727,7 @@ async fn a_limited_author_filter_returns_the_newest_events() { ..Filter::default() }; - let page = common::collect( + let page = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await @@ -744,7 +744,7 @@ async fn a_limited_author_filter_returns_the_newest_events() { /// The page and count of one filter: `(rows, count)`. async fn page_and_count(backend: &LmdbBackend, filter: &Filter) -> (usize, u64) { - let page = common::collect( + let page = collect_records( backend .find_by_filters(std::slice::from_ref(filter)) .await @@ -1014,7 +1014,7 @@ async fn an_empty_tag_value_criterion_matches_nothing() { limit: Some(10), ..Filter::default() }; - let reachable = common::collect( + let reachable = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await @@ -1025,7 +1025,7 @@ async fn an_empty_tag_value_criterion_matches_nothing() { assert_eq!(reachable.len(), 1, "the row is reachable by its id"); filter.tags.insert("#e".to_owned(), vec![String::new()]); - let page = common::collect( + let page = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await @@ -1061,7 +1061,7 @@ async fn an_empty_criterion_family_matches_nothing() { ..Filter::default() }, ] { - let page = common::collect( + let page = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await @@ -1183,7 +1183,7 @@ async fn a_retention_pass_classifies_a_row_once_and_spares_the_whitelist() { ids: Some(vec![vanish.event.id.to_string()]), ..Filter::default() }; - let survivors = common::collect( + let survivors = collect_records( backend .find_by_filters(std::slice::from_ref(&filter)) .await diff --git a/crates/nostream-storage/tests/storage.rs b/crates/nostream-storage/tests/storage.rs index a7c8715ae..52fba6058 100644 --- a/crates/nostream-storage/tests/storage.rs +++ b/crates/nostream-storage/tests/storage.rs @@ -52,8 +52,8 @@ async fn the_repository_traits_are_dyn_compatible() { InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, OutboxPayload, QueryStream, - ReconciliationItem, Report, ReportRepository, StorageFuture, User, UserPatch, - UserRepository, + ReconciliationItem, Report, ReportRepository, StorageFuture, StreamedEvent, User, + UserPatch, UserRepository, }; struct Noop; @@ -82,7 +82,7 @@ async fn the_repository_traits_are_dyn_compatible() { fn find_by_filters<'a>( &'a self, _filters: &'a [Filter], - ) -> StorageFuture<'a, QueryStream<'a, EventRecord>> { + ) -> StorageFuture<'a, QueryStream<'a, StreamedEvent>> { unsupported("noop::events::find_by_filters") } diff --git a/crates/nostream-storage/tests/vanished.rs b/crates/nostream-storage/tests/vanished.rs new file mode 100644 index 000000000..ba68c56fd --- /dev/null +++ b/crates/nostream-storage/tests/vanished.rs @@ -0,0 +1,141 @@ +//! `StreamedEvent::Vanished`: a page slot the store selected and could not read. +//! +//! `find_by_filters` fixes its page first — `ValuesStream::new` takes the page the read +//! collected — and then reads the values in batches of `STREAM_BATCH_ITEMS`, one read +//! transaction per batch. A key removed between the page and its batch therefore has a slot +//! with no row, and the stream reports that slot instead of skipping it: a silent skip is a +//! short page, which is what the relay reads as a complete result. + +#![cfg(feature = "lmdb")] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod common; + +use std::sync::Arc; + +use chrono::Utc; +use common::{fixture, settings_from_fixture}; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; +use nostream_storage::LmdbBackend; +use nostream_storage::repositories::{ + EventRecord, EventRepository, EventRetentionOptions, StreamedEvent, +}; + +/// A store on a temporary directory with the fixture's settings gates. +async fn open_store() -> (tempfile::TempDir, LmdbBackend) { + let fixture = fixture(); + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens") + .with_settings(Arc::new(settings_from_fixture(&fixture))); + + (directory, backend) +} + +/// A signed kind-1 note by the key `secret`, at `created_at`. +fn note(secret: u8, created_at: i64) -> EventRecord { + let mut event = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(1), + tags: Vec::new(), + content: format!("{secret}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut event).expect("the event signs"); + EventRecord::from_event(&event) +} + +/// A key the page selected and a later read cannot find is reported where it sat in the page, +/// and every surviving row still arrives in the page's order. +/// +/// The ordering is guaranteed rather than hoped for: `find_by_filters` collects the page inside +/// its own read (`lmdb/mod.rs` calls `events::page`), and `ValuesStream` does not read a value +/// until it is polled, in batches of 256 with one read transaction each. So a delete that +/// happens after `find_by_filters` returned lands between the page and every batch, and the +/// target sits at index 270 — past the first batch — so the case proved is the one the review +/// named rather than a deletion inside batch one. +#[tokio::test] +async fn a_key_removed_before_its_batch_is_read_yields_vanished_in_its_place() { + let (_directory, backend) = open_store().await; + + // Recent `created_at`s, so the retention pass below takes the target because it is deleted + // and not because every row is past the age limit. Distinct values, so the page's order is + // the fixture's order. + let base = Utc::now().timestamp() - 301; + let records: Vec = (0..300i64).map(|index| note(7, base + index)).collect(); + backend + .create_many(&records) + .await + .expect("the events store"); + + let filter = Filter { + kinds: Some(vec![Kind(1)]), + ..Filter::default() + }; + let stream = backend + .find_by_filters(std::slice::from_ref(&filter)) + .await + .expect("a read"); + + // A soft delete alone keeps the key; the retention pass is what removes a soft-deleted row, + // which is one of the two ways LMDB drops a key (the other is a replacement's hard delete). + let target = records[270].clone(); + assert_eq!( + backend + .delete_by_pubkey_and_ids(&target.event.pubkey, &[target.event.id]) + .await + .expect("the row soft-deletes"), + 1 + ); + let purged = backend + .delete_expired_and_retained(&EventRetentionOptions { + max_days: Some(1), + ..EventRetentionOptions::default() + }) + .await + .expect("the retention pass runs"); + assert_eq!(purged.deleted, 1, "the deleted row is hard-deleted"); + assert_eq!( + purged.retained, 0, + "no row is over the age limit, so the target is the only candidate" + ); + + let items = common::collect(stream).await.expect("the read drains"); + + assert_eq!( + items.len(), + records.len(), + "every selected slot is yielded, the vanished one included" + ); + assert_eq!( + items[270], + StreamedEvent::Vanished, + "the removed key is the slot that vanished" + ); + assert_eq!( + items + .iter() + .filter(|item| **item == StreamedEvent::Vanished) + .count(), + 1, + "no other slot lost its row" + ); + + let yielded: Vec = items + .iter() + .filter_map(|item| match item { + StreamedEvent::Record(record) => Some(record.event.id), + StreamedEvent::Vanished => None, + }) + .collect(); + let expected: Vec = records + .iter() + .map(|record| record.event.id) + .filter(|id| *id != target.event.id) + .collect(); + + assert_eq!(yielded, expected, "every survivor, in the page's order"); +} From 40c7545a0823f4b519b20920ec652b3c6f946d2c Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 09:24:33 -0500 Subject: [PATCH 27/36] feat(storage): declare whether a tag index can answer an empty criterion --- crates/nostream-storage/src/lmdb/mod.rs | 9 +- .../src/postgres/repository.rs | 10 +- crates/nostream-storage/src/repositories.rs | 34 +++++ crates/nostream-storage/tests/empty_tag.rs | 123 ++++++++++++++++++ 4 files changed, 174 insertions(+), 2 deletions(-) create mode 100644 crates/nostream-storage/tests/empty_tag.rs diff --git a/crates/nostream-storage/src/lmdb/mod.rs b/crates/nostream-storage/src/lmdb/mod.rs index 999438295..61b89184b 100644 --- a/crates/nostream-storage/src/lmdb/mod.rs +++ b/crates/nostream-storage/src/lmdb/mod.rs @@ -28,7 +28,7 @@ use crate::backend::LmdbEnv; use crate::error::StorageError; use crate::repositories::{ ActionableTarget, CreateInviteCodeOptions, DeliveryLogEntry, DeliveryLogFilters, DvmJob, - DvmJobRepository, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, + DvmJobRepository, DvmJobUpdate, EmptyTagRule, EventPurgeCounts, EventRecord, EventRepository, EventRetentionOptions, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, OutboxPayload, @@ -370,6 +370,13 @@ impl EventRepository for LmdbBackend { }) } + /// LMDB's tag index holds no empty value: `encoding::tag_entries` writes a key only when the + /// value is non-empty (`encoding.rs:387`), so the criterion `""` — which `Filter::matches` + /// accepts — selects nothing. + fn empty_tag_rule(&self) -> EmptyTagRule { + EmptyTagRule::IndexesNoEmptyValue + } + fn find_reconciliation_items<'a>( &'a self, filter: &'a Filter, diff --git a/crates/nostream-storage/src/postgres/repository.rs b/crates/nostream-storage/src/postgres/repository.rs index 21a67565c..05c01875a 100644 --- a/crates/nostream-storage/src/postgres/repository.rs +++ b/crates/nostream-storage/src/postgres/repository.rs @@ -9,7 +9,7 @@ use super::{ use crate::error::StorageError; use crate::repositories::{ ActionableTarget, CreateInviteCodeOptions, DeliveryLogEntry, DeliveryLogFilters, DvmJob, - DvmJobRepository, DvmJobUpdate, EventPurgeCounts, EventRecord, EventRepository, + DvmJobRepository, DvmJobUpdate, EmptyTagRule, EventPurgeCounts, EventRecord, EventRepository, EventRetentionOptions, InviteCode, InviteCodeRepository, Invoice, InvoiceRepository, InvoiceStatus, NewDeliveryLogEntry, NewReport, Nip05Verification, Nip05VerificationRepository, NotificationDeliveryLogRepository, NotificationOutboxRepository, OutboxMessage, OutboxPayload, @@ -81,6 +81,14 @@ impl EventRepository for PostgresBackend { } } + /// Postgres answers a tag criterion from `event_tags` (`sql.rs:347-354`), and that table + /// holds no empty value: its trigger inserts a row only where `tag_value <> ''` + /// (`migrations/20240108130100_add_event_tags_table.sql:24`), so `tag_value = ANY('{""}')` + /// matches nothing while `Filter::matches` accepts `["d", ""]`. + fn empty_tag_rule(&self) -> EmptyTagRule { + EmptyTagRule::IndexesNoEmptyValue + } + fn delete_by_addresses<'a>( &'a self, pubkey: &'a Pubkey, diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index a777beb18..869eb1024 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -453,6 +453,28 @@ pub enum SearchRule { }, } +/// Whether a backend's tag index can answer a criterion that is the empty string. +/// +/// [`nostream_core::Filter::matches`] compares a tag value to the criterion with `==`, so the +/// criterion `""` accepts a tag stored as `["d", ""]`. A backend that answers tag criteria from +/// an index of non-empty values holds no entry for the empty one, and then the read selects +/// nothing while the matcher would have accepted every such row — a short page that proves +/// nothing about the rows it never enumerated. Each variant names the rule, as [`SearchRule`]'s +/// do, so an implementor has to read what it is asserting. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EmptyTagRule { + /// A criterion that is the empty string selects what [`nostream_core::Filter::matches`] + /// selects. The in-memory backends' tag predicate *is* that matcher, so a completeness claim + /// over such a read is one the relay can check. + AnswersEmptyCriterion, + /// The tag index holds no entry for an empty value, so the criterion `""` selects nothing + /// however many rows carry `["d", ""]`. LMDB: `tag_entries` writes a key only when the value + /// is non-empty (`crates/nostream-storage/src/lmdb/encoding.rs:387`). Postgres: the + /// `event_tags` trigger inserts a row only where `tag_value <> ''` + /// (`migrations/20240108130100_add_event_tags_table.sql:24`). + IndexesNoEmptyValue, +} + /// Event storage (`src/@types/repositories.ts` `IEventRepository`). pub trait EventRepository: Send + Sync { /// Inserts one event, returning the affected row count (0 on a duplicate `event_id`). @@ -538,6 +560,18 @@ pub trait EventRepository: Send + Sync { SearchRule::SubstringTerms } + /// Whether a tag criterion that is the empty string can be answered from this backend's tag + /// index (see [`EmptyTagRule`]). + /// + /// Defaults to [`EmptyTagRule::AnswersEmptyCriterion`], which is what a backend that filters + /// with [`nostream_core::Filter::matches`] answers — its tag predicate *is* that matcher. A + /// backend that answers tag criteria from an index of non-empty values overrides this, and a + /// caller that must claim completeness over such a read reads it rather than assuming the + /// criterion enumerated the set. + fn empty_tag_rule(&self) -> EmptyTagRule { + EmptyTagRule::AnswersEmptyCriterion + } + /// Streams the live `(created_at, event_id)` pairs a NIP-77 reconciliation compares. fn find_reconciliation_items<'a>( &'a self, diff --git a/crates/nostream-storage/tests/empty_tag.rs b/crates/nostream-storage/tests/empty_tag.rs new file mode 100644 index 000000000..1f9202e83 --- /dev/null +++ b/crates/nostream-storage/tests/empty_tag.rs @@ -0,0 +1,123 @@ +//! A tag criterion that is the empty string, over the real store. +//! +//! `Filter::matches` accepts a tag stored as `["d", ""]` — the criterion and the value compare +//! equal — while LMDB's `tags` index holds no entry for an empty value +//! (`encoding.rs:387`), so the read selects nothing. That disagreement is what +//! `EventRepository::empty_tag_rule` declares as `IndexesNoEmptyValue`, and it is why the relay +//! withholds `finish` for such a filter: the stored article matches the subscription and would +//! never be sent. + +#![cfg(feature = "lmdb")] +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] + +mod common; + +use std::collections::BTreeMap; +use std::sync::Arc; + +use common::{fixture, settings_from_fixture}; +use nostream_core::{Event, EventId, Filter, Kind, Pubkey, Signature}; +use nostream_storage::LmdbBackend; +use nostream_storage::repositories::{EmptyTagRule, EventRecord, EventRepository, collect_records}; + +/// A store on a temporary directory with the fixture's settings gates. +async fn open_store() -> (tempfile::TempDir, LmdbBackend) { + let fixture = fixture(); + let directory = tempfile::tempdir().expect("tempdir"); + let backend = LmdbBackend::open(directory.path()) + .await + .expect("the store opens") + .with_settings(Arc::new(settings_from_fixture(&fixture))); + + (directory, backend) +} + +/// A signed kind-30023 row carrying one `d` tag, with the deduplication value the addressable +/// write paths store so two values are two rows rather than one replacement. +fn article(secret: u8, d: &str, created_at: i64) -> Event { + let mut event = Event { + id: EventId::from_byte_array([0; 32]), + pubkey: Pubkey::from_byte_array([0; 32]), + created_at, + kind: Kind(30_023), + tags: vec![vec!["d".to_owned(), d.to_owned()]], + content: format!("{secret}-{d}-{created_at}"), + sig: Signature::from_byte_array([0; 64]), + }; + nostream_core::crypto::sign_event(&[secret; 32], &mut event).expect("the event signs"); + + event +} + +/// The row to write for `event`. +fn record(event: &Event) -> EventRecord { + EventRecord::from_event(event).with_deduplication(vec![serde_json::Value::String( + event.d_tag_value().to_owned(), + )]) +} + +/// One tag family, as `Filter` takes it. +fn tag_query(criterion: &str) -> Filter { + Filter { + tags: BTreeMap::from([("#d".to_owned(), vec![criterion.to_owned()])]), + ..Default::default() + } +} + +/// The fact `EventRepository::empty_tag_rule` claims: the index cannot answer a criterion that is +/// the empty string, even over a row that carries one, while the relay's matcher accepts that +/// row. The second read is the control — an ordinary `#d` criterion still selects its row — so +/// the empty page is the criterion's and not a broken tag path. +#[tokio::test] +async fn an_empty_tag_value_is_not_selectable_through_the_index() { + let (_directory, backend) = open_store().await; + + let empty = article(7, "", 1_700_000_000); + assert!( + tag_query("").matches(&empty), + "the matcher accepts a [\"d\", \"\"] tag" + ); + + backend + .upsert(&record(&empty)) + .await + .expect("the store upserts"); + let named_row = article(7, "post-1", 1_700_000_001); + backend + .upsert(&record(&named_row)) + .await + .expect("the store upserts"); + + assert_eq!( + backend.empty_tag_rule(), + EmptyTagRule::IndexesNoEmptyValue, + "the backend declares the rule this test measures" + ); + + let selected = collect_records( + backend + .find_by_filters(&[tag_query("")]) + .await + .expect("a read"), + ) + .await + .expect("the stream"); + + assert!( + selected.is_empty(), + "the tag index holds no empty value, so the criterion selects nothing: {} row(s)", + selected.len() + ); + + let named = collect_records( + backend + .find_by_filters(&[tag_query("post-1")]) + .await + .expect("a read"), + ) + .await + .expect("the stream"); + + assert_eq!(named.len(), 1, "an ordinary `#d` criterion still selects"); + assert_eq!(named[0].event.id, named_row.id); +} From 8d4147bfe47f5b670b518a1103d495c9eea92931 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 09:24:33 -0500 Subject: [PATCH 28/36] fix(relay): withhold finish when the tag index cannot answer an empty criterion --- .../nostream-relay/src/handlers/subscribe.rs | 89 +++++++++++------ crates/nostream-relay/src/testing.rs | 50 +++++++++- crates/nostream-relay/tests/handlers.rs | 97 ++++++++++++++++++- 3 files changed, 196 insertions(+), 40 deletions(-) diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index bea6e9ee5..0193eaa78 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -21,14 +21,14 @@ use std::sync::Arc; use futures_util::StreamExt; use nostream_core::expiration; -use nostream_core::filter::{count_filter_values, normalise_subscription}; +use nostream_core::filter::{count_filter_values, is_generic_tag_query, normalise_subscription}; use nostream_core::messages::{ EOSE_HINT_AUTH, EOSE_HINT_FINISH, EOSE_HINT_MORE, EoseMessage, OutgoingEventMessage, RelayMessage, SubscribeMessage, }; use nostream_core::nip42; use nostream_core::{Filter, Kind}; -use nostream_storage::repositories::{SearchRule, StreamedEvent}; +use nostream_storage::repositories::{EmptyTagRule, SearchRule, StreamedEvent}; use crate::connection::InFlightRead; use crate::error::RelayError; @@ -424,12 +424,15 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha } Ok(StreamedEvent::Vanished) => { // The backend selected this slot and the key was gone when it read the value, so - // the slot is spent without a record. It counts as a row, exactly as a skipped - // row does, and it sets `vanished` on its own: `page_full` compares the rows the - // read *yielded* against the probe, so a page that lost a key arrives short of it - // and reads as innocent, and where no probe was built `page_full` is `false` - // outright. The marker is the one signal in the read that says a selected slot - // produced nothing, so `finish` is withheld on it rather than argued about. + // the slot is spent without a record. It counts as a row, exactly as a skipped row + // does, so a page that was full at enumeration is full by count and `page_full` + // withholds on its own. What the count cannot carry is the page that **never + // filled**: fewer rows than the probe, so `page_full` is false, and a slot that + // was a match when the page was enumerated produced nothing when it was read. That + // slot may have been deleted — `finish` would still be true over the current set — + // or replaced, in which case the new version was never enumerated and `finish` is + // false, and the read cannot tell the two apart, so it declines rather than + // argues. rows += 1; vanished = true; } @@ -446,8 +449,8 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // The rule, in one place, so the next reader does not have to re-derive it. `finish` claims // that every stored event matching the subscription's filters has been sent (`67.md` - // §"Specification"), and this read can fail to establish that four ways, so `finish` needs - // all four: + // §"Specification"), and this read can fail to establish that six ways, so `finish` needs + // all six: // // 1. the stream ended **cleanly** — after a storage error, how far the read got is unknown; // 2. the page was **not full** — a page that yielded `min(probe, page_bound())` rows says @@ -455,28 +458,40 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // from the backend's own page cap, and a skipped row spends a page slot just as a match // does (which is why `rows`, not `matched`, decides this); // 3. **no page slot was spent without its row** — a backend reports a slot it selected and - // then could not read as `StreamedEvent::Vanished`, and that slot is a row the read was - // promised. The count of yielded rows cannot carry that on its own: `page_full` compares - // that count against the probe, so a page that lost a key to a concurrent replacement - // arrives short of it and reads as innocent, and where no probe was built `page_full` is - // `false` by construction. The marker withholds `finish` directly, rather than relying on - // the arithmetic a vanished row happens to disturb; - // 4. the set the backend enumerated is the set this relay would have — no row it yielded was - // refused by `Filter::matches`, and no filter carrying `search` was answered by a - // predicate other than that matcher. Both are needed and neither implies the other: a - // refusal is the direction a probe can see (`study` yields `studies`, which the matcher - // refuses), while a row the matcher *would* accept and the backend never yields - // (`abcdef` inside `xxabcdefyy`) leaves no trace at all. See `EventRepository::search_rule`. - // 5. **no further match** was found — the `more` branch below is where this one lives: a - // further match ends the read with `more` set, so the code that can reach the hint is the - // code where none was found. It is only meaningful once 1 and 2 hold, because the loop - // keeps asking past deleted, expired, withheld and non-matching rows until it sees a - // match, the genuine end, or the page cap. + // then could not read as `StreamedEvent::Vanished`, and that slot is counted as a row, so + // a page that was full at enumeration is full by count and item 2 withholds on its own. + // The marker's own work is the page that **never filled**: fewer rows than the probe, so + // `page_full` is false, and a slot that was a match when the page was enumerated produced + // nothing when it was read. That slot may have been deleted — `finish` would still be + // true over the current set — or replaced, in which case the new version was never + // enumerated and `finish` is false, and the read cannot tell the two apart, so it + // declines rather than argues; + // 4. **no yielded row was refused** — the backend selected a row `Filter::matches` refuses, + // which proves the two predicates disagree, so a short page proves nothing about the rows + // the backend never yielded; + // 5. **no `search` filter was answered by another predicate** — Postgres prefilters with + // `to_tsvector`, and a row the matcher *would* accept and the backend never yields + // (`abcdef` inside `xxabcdefyy`) leaves no trace at all, so the backend declares its rule + // rather than the read probing for it. See `EventRepository::search_rule`; + // 6. **no tag criterion that is the empty string was read from an index without one** — the + // matcher accepts a `["d", ""]` tag, an index of non-empty values selects nothing for + // the criterion `""`, and nothing is yielded to refuse, so like item 5 the enumeration + // could not answer a criterion the matcher accepts and no probe shows it. See + // `EventRepository::empty_tag_rule`. // - // Where any of the four cannot be established the relay sends no completeness hint: an - // absent hint is honest and a wrong one is not. `more` needs one further **match** beyond - // the delivered window, `auth` is unaffected, and `finish` is never sent beside `auth` - // because a withheld event falsifies it. + // Items 4 and 5 are both needed and neither implies the other: a refusal is the direction a + // probe can see (`study` yields `studies`, which the matcher refuses), while a row the + // matcher would accept and the backend never yields leaves no trace at all. + // + // Beyond the six, `more` needs one further **match** beyond the delivered window: that branch + // ends the read with `more` set, so the code that can reach the hint is the code where none + // was found. It is only meaningful once items 1 and 2 hold, because the loop keeps asking + // past deleted, expired, withheld and non-matching rows until it sees a match, the genuine + // end, or the page cap. + // + // Where any of the six cannot be established the relay sends no completeness hint: an absent + // hint is honest and a wrong one is not. `auth` is unaffected, and `finish` is never sent + // beside `auth` because a withheld event falsifies it. let page_bound = context.stores.events.page_bound(); let page_full = probe.is_some_and(|probe| rows >= probe.min(page_bound)); let search_rule = context.stores.events.search_rule(); @@ -487,7 +502,17 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha .is_some_and(|search| !search.trim().is_empty()) }); let search_agrees = search_rule == SearchRule::SubstringTerms || !searched; - let proven = !failed && !page_full && !vanished && !refused && search_agrees; + // A tag criterion that is the empty string is one `Filter::matches` accepts and a tag index + // built from non-empty values cannot answer: nothing is yielded, so neither a refusal nor a + // short page shows it, and the enumeration could not answer a criterion the matcher accepts. + let empty_tag_query = filters.iter().any(|filter| { + filter.tags.iter().any(|(key, criteria)| { + is_generic_tag_query(key) && criteria.iter().any(String::is_empty) + }) + }); + let tag_agrees = context.stores.events.empty_tag_rule() == EmptyTagRule::AnswersEmptyCriterion + || !empty_tag_query; + let proven = !failed && !page_full && !vanished && !refused && search_agrees && tag_agrees; let mut hints = Vec::new(); diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index 05998ae53..61c068910 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -21,15 +21,16 @@ use nostream_config::settings::{ RateLimiterSettings, Settings, }; use nostream_core::clock::Clock; +use nostream_core::filter::is_generic_tag_query; use nostream_core::{Event, EventId, Filter, Kind, Pubkey}; use nostream_storage::MAX_PAGE_ROWS; use nostream_storage::StorageError; use nostream_storage::repositories::{ - ActionableTarget, CreateInviteCodeOptions, DvmJob, DvmJobStatus, EventPurgeCounts, EventRecord, - EventRepository, EventRetentionOptions, FieldUpdate, InviteCode, InviteCodeRepository, - NewReport, Nip05Verification, Nip05VerificationRepository, QueryStream, ReconciliationItem, - Report, ReportRepository, SearchRule, StorageFuture, StreamedEvent, User, UserPatch, - UserRepository, + ActionableTarget, CreateInviteCodeOptions, DvmJob, DvmJobStatus, EmptyTagRule, + EventPurgeCounts, EventRecord, EventRepository, EventRetentionOptions, FieldUpdate, InviteCode, + InviteCodeRepository, NewReport, Nip05Verification, Nip05VerificationRepository, QueryStream, + ReconciliationItem, Report, ReportRepository, SearchRule, StorageFuture, StreamedEvent, User, + UserPatch, UserRepository, }; use tokio::sync::Notify; @@ -158,6 +159,10 @@ pub struct InMemoryEventRepository { fail_reads: AtomicBool, page_bound: std::sync::atomic::AtomicU64, search_rule: Mutex>, + /// When set, the double models a backend whose tag index holds no entry for an empty value: + /// a filter carrying a generic tag criterion of `""` selects nothing, and `empty_tag_rule` + /// declares it. LMDB and Postgres are both that backend. + empty_tag_index: AtomicBool, /// Rows this double yields for **any** filter: what a backend whose `search` predicate is not /// `Filter::matches` looks like from the handler's side, which is the case no honest filter /// can produce. @@ -265,6 +270,13 @@ impl InMemoryEventRepository { self.page_bound.store(bound, Ordering::SeqCst); } + /// Declares the tag index this double answers from, the way LMDB and Postgres do: no entry + /// for an empty value, so a filter carrying a generic tag criterion of `""` selects nothing + /// even from a row that holds one. + pub fn declare_no_empty_tag_values(&self) { + self.empty_tag_index.store(true, Ordering::SeqCst); + } + /// Makes every read wait for a permit. Grant one with `Semaphore::add_permits` to let a /// read through. pub fn gate_reads(&self) -> Arc { @@ -377,6 +389,15 @@ impl InMemoryEventRepository { } } +/// Whether a filter carries a generic tag criterion that is the empty string — the one criterion +/// a tag index built from non-empty values cannot answer. +fn queries_an_empty_tag_value(filter: &Filter) -> bool { + filter + .tags + .iter() + .any(|(key, criteria)| is_generic_tag_query(key) && criteria.iter().any(String::is_empty)) +} + impl EventRepository for InMemoryEventRepository { fn create<'a>(&'a self, record: &'a EventRecord) -> StorageFuture<'a, u64> { Box::pin(async move { @@ -491,6 +512,16 @@ impl EventRepository for InMemoryEventRepository { .unwrap_or(SearchRule::SubstringTerms) } + /// The double's filter path *is* `Filter::matches`, so it answers the empty criterion; a test + /// that models an indexed backend declares the other rule with + /// [`InMemoryEventRepository::declare_no_empty_tag_values`]. + fn empty_tag_rule(&self) -> EmptyTagRule { + match self.empty_tag_index.load(Ordering::SeqCst) { + true => EmptyTagRule::IndexesNoEmptyValue, + false => EmptyTagRule::AnswersEmptyCriterion, + } + } + fn find_by_filters<'a>( &'a self, filters: &'a [Filter], @@ -502,6 +533,7 @@ impl EventRepository for InMemoryEventRepository { .clone(); let per_item = self.gate_per_item.load(std::sync::atomic::Ordering::SeqCst); let failing = self.fail_reads.load(Ordering::SeqCst); + let empty_tag_index = self.empty_tag_index.load(Ordering::SeqCst); Box::pin(async move { if failing { @@ -536,6 +568,14 @@ impl EventRepository for InMemoryEventRepository { .clone(); for filter in filters { + // A tag index that holds no empty value cannot answer a criterion that is one, so + // the filter selects nothing — the whole filter, not just that family, because + // both shipped backends seek that index for it and their matchers refuse the + // non-empty values it does hold. + if empty_tag_index && queries_an_empty_tag_value(filter) { + continue; + } + let bound = filter.limit.unwrap_or(u64::MAX).min(page_bound); let mut taken = 0u64; diff --git a/crates/nostream-relay/tests/handlers.rs b/crates/nostream-relay/tests/handlers.rs index dc1a20a21..9b31c55a5 100644 --- a/crates/nostream-relay/tests/handlers.rs +++ b/crates/nostream-relay/tests/handlers.rs @@ -1280,12 +1280,18 @@ async fn eose_hints_are_absent_when_the_stream_failed() { assert_eq!(frames, vec![r#"["EOSE","s1"]"#.to_owned()], "{frames:?}"); } -/// A row the backend selected and could not read spends a page slot without a record. The page -/// comes back one row short of the probe, so `page_full` is `false` and `finish` would otherwise -/// claim that every matching event had been sent over a slot that never yielded one. The case is +/// A row the backend selected and could not read spends a page slot without a record. The case is /// real on LMDB, where a replacement hard-deletes the row it supersedes between the page and the /// batch that would have read it. /// +/// This is the page that **never filled**, which is the case the marker exists for: two rows +/// against a probe of four (`limit: 3`), so `page_full` is `false` and the only thing withholding +/// `finish` is `vanished`. The read cannot tell whether the lost key was deleted — where `finish` +/// would still be honest over the current set — or replaced, where the new version was never +/// enumerated and `finish` would be false, which is why it declines rather than argues. Its pair +/// is `eose_hints_are_absent_when_the_page_was_full_at_enumeration`, where `page_full` carries the +/// suppression and `!vanished` could be removed without failing anything. +/// /// The other direction is `eose_hints_finish_when_the_limit_was_not_reached`: an honest short /// page still proves `finish`, so the suppression is the marker's and not the read's. #[tokio::test] @@ -1314,6 +1320,91 @@ async fn eose_hints_are_absent_when_a_selected_row_vanished() { assert_eq!(frames[1], r#"["EOSE","s1"]"#, "{frames:?}"); } +/// The same vanished slot in a page that **was** full at enumeration: `limit: 1` probes two, two +/// rows are stored, and one of them vanishes. The marker is counted as a row, so `rows` reaches +/// two and `page_full` is what withholds `finish` — removing `!vanished` would not fail this +/// test, and its purpose is the pair with +/// `eose_hints_are_absent_when_a_selected_row_vanished`: each names the mechanism carrying it, so +/// neither `rows += 1` nor `!vanished` can be dropped as redundant on the other's evidence. +#[tokio::test] +async fn eose_hints_are_absent_when_the_page_was_full_at_enumeration() { + let fixture = Fixture::new(settings()); + let survivor = signed_event(&CLIENT_SECRET, 1, vec![], "note 0", NOW_SECS); + let vanished = signed_event(&CLIENT_SECRET, 1, vec![], "note 1", NOW_SECS); + fixture.seed_event(survivor).await; + fixture.seed_event(vanished.clone()).await; + fixture.events.vanish(vanished.id); + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r#"["REQ","s1",{"kinds":[1],"limit":1}]"#).await; + let frames = client.read_until_eose().await; + + assert_eq!( + frames.len(), + 2, + "one row was spent without a record: {frames:?}" + ); + assert!( + frames[0].starts_with(r#"["EVENT","s1","#), + "the survivor is still delivered: {frames:?}" + ); + assert_eq!(frames[1], r#"["EOSE","s1"]"#, "{frames:?}"); +} + +/// A tag criterion that is the empty string is one `Filter::matches` accepts and the tag index +/// cannot answer: `#d: [""]` matches a stored `["d", ""]`, while both shipped backends index no +/// empty value, so the read yields nothing. With no row yielded there is no refusal and no short +/// page to see, so before this round the relay finished over a stored article it never sent. The +/// second request is the same fixture's ordinary `#d` filter, which still finishes: the +/// suppression is the empty criterion's and not a blanket refusal of tag filters. +#[tokio::test] +async fn eose_hints_are_absent_for_an_empty_tag_criterion() { + let fixture = Fixture::new(settings()); + fixture + .seed_event(signed_event( + &CLIENT_SECRET, + 30_023, + vec![vec!["d", ""]], + "empty d", + NOW_SECS, + )) + .await; + fixture + .seed_event(signed_event( + &CLIENT_SECRET, + 30_023, + vec![vec!["d", "post-1"]], + "named d", + NOW_SECS, + )) + .await; + fixture.events.declare_no_empty_tag_values(); + let server = start(fixture).await; + let (mut client, _challenge) = server.connect().await; + + client.send(r##"["REQ","s1",{"#d":[""],"limit":5}]"##).await; + let frames = client.read_until_eose().await; + + assert_eq!( + frames, + vec![r#"["EOSE","s1"]"#.to_owned()], + "the index cannot answer `\"\"`, so nothing proves completion: {frames:?}" + ); + + client + .send(r##"["REQ","s2",{"#d":["post-1"],"limit":5}]"##) + .await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 2, "{frames:?}"); + assert!( + frames[0].starts_with(r#"["EVENT","s2","#), + "the named row is delivered: {frames:?}" + ); + assert_eq!(frames[1], r#"["EOSE","s2",["finish"]]"#, "{frames:?}"); +} + /// `auth` and the completeness hint compose: a withheld match falsifies `finish` even where the /// page was neither full nor capped, and `auth` is what is sent. #[tokio::test] From 74304510a99a3831550747d25f2d9698ef18d667 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 09:28:47 -0500 Subject: [PATCH 29/36] docs(relay): say which mechanism withholds finish in the vanished-slot pair --- crates/nostream-relay/tests/handlers.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/crates/nostream-relay/tests/handlers.rs b/crates/nostream-relay/tests/handlers.rs index 9b31c55a5..604263ffd 100644 --- a/crates/nostream-relay/tests/handlers.rs +++ b/crates/nostream-relay/tests/handlers.rs @@ -1322,10 +1322,14 @@ async fn eose_hints_are_absent_when_a_selected_row_vanished() { /// The same vanished slot in a page that **was** full at enumeration: `limit: 1` probes two, two /// rows are stored, and one of them vanishes. The marker is counted as a row, so `rows` reaches -/// two and `page_full` is what withholds `finish` — removing `!vanished` would not fail this -/// test, and its purpose is the pair with -/// `eose_hints_are_absent_when_a_selected_row_vanished`: each names the mechanism carrying it, so -/// neither `rows += 1` nor `!vanished` can be dropped as redundant on the other's evidence. +/// two and the page is full by count. +/// +/// It is the pair of `eose_hints_are_absent_when_a_selected_row_vanished`, which is the page that +/// never filled: removing `!vanished` leaves this test green and turns that one red, so the marker +/// is what that case needs and not this one. Removing `rows += 1` leaves this test green too — +/// a marker always sets `vanished`, so no frame separates the two here — which is why the count is +/// justified by the rule's arithmetic rather than by an assertion. The pair exists so a reader +/// sees both shapes and neither rule item is dropped as covered by the other. #[tokio::test] async fn eose_hints_are_absent_when_the_page_was_full_at_enumeration() { let fixture = Fixture::new(settings()); From 2662008d718d69a7c31124e142547e473e48f76d Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 10:03:18 -0500 Subject: [PATCH 30/36] fix(relay): strip an empty tag criterion instead of the whole filter --- crates/nostream-relay/src/testing.rs | 112 ++++++++++++++++++++---- crates/nostream-relay/tests/handlers.rs | 21 ++++- 2 files changed, 117 insertions(+), 16 deletions(-) diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index 61c068910..2ebfe2317 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -21,7 +21,7 @@ use nostream_config::settings::{ RateLimiterSettings, Settings, }; use nostream_core::clock::Clock; -use nostream_core::filter::is_generic_tag_query; +use nostream_core::filter::{has_empty_tag_criterion, is_generic_tag_query}; use nostream_core::{Event, EventId, Filter, Kind, Pubkey}; use nostream_storage::MAX_PAGE_ROWS; use nostream_storage::StorageError; @@ -389,13 +389,25 @@ impl InMemoryEventRepository { } } -/// Whether a filter carries a generic tag criterion that is the empty string — the one criterion -/// a tag index built from non-empty values cannot answer. -fn queries_an_empty_tag_value(filter: &Filter) -> bool { - filter - .tags - .iter() - .any(|(key, criteria)| is_generic_tag_query(key) && criteria.iter().any(String::is_empty)) +/// The filter a tag index built from non-empty values answers: the empty criteria removed from +/// every generic tag family, or `None` when a family is left with no criterion — the one shape +/// that selects nothing. +fn without_empty_tag_criteria(filter: &Filter) -> Option { + let mut stripped = filter.clone(); + + for (key, criteria) in &mut stripped.tags { + if !is_generic_tag_query(key) { + continue; + } + + criteria.retain(|criterion| !criterion.is_empty()); + + if criteria.is_empty() { + return None; + } + } + + Some(stripped) } impl EventRepository for InMemoryEventRepository { @@ -568,13 +580,25 @@ impl EventRepository for InMemoryEventRepository { .clone(); for filter in filters { - // A tag index that holds no empty value cannot answer a criterion that is one, so - // the filter selects nothing — the whole filter, not just that family, because - // both shipped backends seek that index for it and their matchers refuse the - // non-empty values it does hold. - if empty_tag_index && queries_an_empty_tag_value(filter) { - continue; - } + // An index built from non-empty values cannot answer the criterion `""`, but both + // shipped backends still answer the rest of that filter: LMDB skips an empty tag + // value and compares the remaining criteria + // (`crates/nostream-storage/src/lmdb/events.rs:1621-1626`), and Postgres binds + // every criterion left into one `tag_value = ANY(…)` + // (`crates/nostream-storage/src/postgres/sql.rs:338-354`). So the double matches + // the stripped filter — and a family the strip leaves with no criterion selects + // nothing, which is also what both answer for `{"#d": [""]}`. Dropping the whole + // filter, as this did, answered none of the rows both answer for + // `{"#d": ["", "post-1"]}`. + let stripped = if empty_tag_index && has_empty_tag_criterion(filter) { + match without_empty_tag_criteria(filter) { + Some(stripped) => Some(stripped), + None => continue, + } + } else { + None + }; + let filter = stripped.as_ref().unwrap_or(filter); let bound = filter.limit.unwrap_or(u64::MAX).min(page_bound); let mut taken = 0u64; @@ -1890,6 +1914,64 @@ mod tests { ); } + #[tokio::test] + async fn an_index_without_an_empty_value_answers_the_rest_of_the_filter() { + // Both backends keep the non-empty criteria of the family and compare those: LMDB skips + // the empty tag value and matches the others (`lmdb/events.rs:1621-1626`), and Postgres + // binds everything left into one `tag_value = ANY(…)` (`postgres/sql.rs:338-354`). So + // `{"#d": ["", "post-1"]}` answers the `post-1` row, and `{"#d": [""]}` answers nothing. + // The double used to drop the whole filter, which for the mixed one answered none of the + // rows both backends answer. + let events = InMemoryEventRepository::new(); + events.declare_no_empty_tag_values(); + + for (index, d) in ["", "post-1"].into_iter().enumerate() { + let event = Event { + id: EventId::from_byte_array([index as u8 + 1; 32]), + pubkey: Pubkey::from_byte_array([7; 32]), + created_at: index as i64 + 1, + kind: Kind(30_023), + tags: vec![vec!["d".to_owned(), d.to_owned()]], + content: d.to_owned(), + sig: Signature::from_byte_array([0; 64]), + }; + events + .create(&EventRecord::from_event(&event)) + .await + .unwrap(); + } + + let family = |criteria: &[&str]| Filter { + tags: BTreeMap::from([( + "#d".to_owned(), + criteria + .iter() + .map(|criterion| (*criterion).to_owned()) + .collect(), + )]), + ..Filter::default() + }; + + let mixed = [family(&["", "post-1"])]; + let rows = nostream_storage::repositories::collect_records( + events.find_by_filters(&mixed).await.unwrap(), + ) + .await + .unwrap(); + + assert_eq!(rows.len(), 1, "the `post-1` criterion is still answered"); + assert_eq!(rows[0].event.content, "post-1"); + + let empty = [family(&[""])]; + let rows = nostream_storage::repositories::collect_records( + events.find_by_filters(&empty).await.unwrap(), + ) + .await + .unwrap(); + + assert!(rows.is_empty(), "no index entry holds `\"\"`: {rows:?}"); + } + #[tokio::test] async fn a_shared_invite_code_admits_a_second_pubkey_until_its_uses_run_out() { // The capture's `claimCode` conditions on `code`, the remaining uses and the expiry diff --git a/crates/nostream-relay/tests/handlers.rs b/crates/nostream-relay/tests/handlers.rs index 604263ffd..341184649 100644 --- a/crates/nostream-relay/tests/handlers.rs +++ b/crates/nostream-relay/tests/handlers.rs @@ -1361,7 +1361,10 @@ async fn eose_hints_are_absent_when_the_page_was_full_at_enumeration() { /// empty value, so the read yields nothing. With no row yielded there is no refusal and no short /// page to see, so before this round the relay finished over a stored article it never sent. The /// second request is the same fixture's ordinary `#d` filter, which still finishes: the -/// suppression is the empty criterion's and not a blanket refusal of tag filters. +/// suppression is the empty criterion's and not a blanket refusal of tag filters. The third is +/// the mixed one — the index answers `post-1` and cannot answer `""`, so that row is delivered +/// and `finish` is still withheld, because the `["d", ""]` row the matcher accepts was never +/// enumerated. #[tokio::test] async fn eose_hints_are_absent_for_an_empty_tag_criterion() { let fixture = Fixture::new(settings()); @@ -1407,6 +1410,22 @@ async fn eose_hints_are_absent_for_an_empty_tag_criterion() { "the named row is delivered: {frames:?}" ); assert_eq!(frames[1], r#"["EOSE","s2",["finish"]]"#, "{frames:?}"); + + client + .send(r##"["REQ","s3",{"#d":["","post-1"],"limit":5}]"##) + .await; + let frames = client.read_until_eose().await; + + assert_eq!(frames.len(), 2, "{frames:?}"); + assert!( + frames[0].starts_with(r#"["EVENT","s3","#) && frames[0].contains("named d"), + "the `post-1` row answers the criterion: {frames:?}" + ); + assert!( + !frames[0].contains("empty d"), + "the empty value is not a row the index can answer: {frames:?}" + ); + assert_eq!(frames[1], r#"["EOSE","s3"]"#, "{frames:?}"); } /// `auth` and the completeness hint compose: a withheld match falsifies `finish` even where the From 01d1bfdd65ee3578c9ec559f89e748e6b6fd3304 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 10:03:29 -0500 Subject: [PATCH 31/36] refactor(core): one home for the empty tag criterion --- crates/nostream-core/src/filter.rs | 20 +++++++++++++++++++ .../nostream-relay/src/handlers/subscribe.rs | 8 ++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/crates/nostream-core/src/filter.rs b/crates/nostream-core/src/filter.rs index 507ad692a..c8dccc80f 100644 --- a/crates/nostream-core/src/filter.rs +++ b/crates/nostream-core/src/filter.rs @@ -186,6 +186,15 @@ pub fn is_generic_tag_query(key: &str) -> bool { && characters.next().is_none() } +/// Whether a filter carries a generic tag criterion that is the empty string — the one criterion a +/// tag index built from non-empty values cannot answer, while [`Filter::matches`] accepts it. +pub fn has_empty_tag_criterion(filter: &Filter) -> bool { + filter + .tags + .iter() + .any(|(key, criteria)| is_generic_tag_query(key) && criteria.iter().any(String::is_empty)) +} + /// Whether the lowercase hex rendering of `bytes` starts with `prefix`. /// /// The comparison is on nibbles of the byte array, so no string is built. It is @@ -729,6 +738,17 @@ mod tests { assert!(!is_generic_tag_query("e")); } + #[test] + fn only_a_generic_tag_family_can_carry_an_empty_criterion() { + let empty = |json: &str| has_empty_tag_criterion(&serde_json::from_str(json).unwrap()); + + assert!(empty(r##"{"#d":[""]}"##)); + assert!(empty(r##"{"#d":["","post-1"]}"##)); + assert!(!empty(r##"{"#d":["post-1"]}"##)); + assert!(!empty(r##"{"#expiration":[""]}"##)); + assert!(!empty(r#"{"kinds":[1]}"#)); + } + #[test] fn hex_prefixes_are_case_sensitive_and_length_bound() { let bytes = [0xaa, 0xbb]; diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index 0193eaa78..204004045 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -21,7 +21,7 @@ use std::sync::Arc; use futures_util::StreamExt; use nostream_core::expiration; -use nostream_core::filter::{count_filter_values, is_generic_tag_query, normalise_subscription}; +use nostream_core::filter::{count_filter_values, has_empty_tag_criterion, normalise_subscription}; use nostream_core::messages::{ EOSE_HINT_AUTH, EOSE_HINT_FINISH, EOSE_HINT_MORE, EoseMessage, OutgoingEventMessage, RelayMessage, SubscribeMessage, @@ -505,11 +505,7 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // A tag criterion that is the empty string is one `Filter::matches` accepts and a tag index // built from non-empty values cannot answer: nothing is yielded, so neither a refusal nor a // short page shows it, and the enumeration could not answer a criterion the matcher accepts. - let empty_tag_query = filters.iter().any(|filter| { - filter.tags.iter().any(|(key, criteria)| { - is_generic_tag_query(key) && criteria.iter().any(String::is_empty) - }) - }); + let empty_tag_query = filters.iter().any(has_empty_tag_criterion); let tag_agrees = context.stores.events.empty_tag_rule() == EmptyTagRule::AnswersEmptyCriterion || !empty_tag_query; let proven = !failed && !page_full && !vanished && !refused && search_agrees && tag_agrees; From 6f8ecba2723fa35d3a5d8ce5a74e503d2804cf18 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 10:03:36 -0500 Subject: [PATCH 32/36] docs(relay): count the page's slots, not its yielded rows --- crates/nostream-relay/src/handlers/subscribe.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/nostream-relay/src/handlers/subscribe.rs b/crates/nostream-relay/src/handlers/subscribe.rs index 204004045..c09f8a29c 100644 --- a/crates/nostream-relay/src/handlers/subscribe.rs +++ b/crates/nostream-relay/src/handlers/subscribe.rs @@ -307,13 +307,13 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha let matches_any = |event: &nostream_core::Event| filters.iter().any(|filter| filter.matches(event)); - // `rows`: every row the page yielded, skipped ones included — a full page is what makes the - // backend's remaining extent unknown. `matched`: rows that were a match inside the client's - // window, which is what `limit` bounds. `trimmed`: the guard withheld a match. `more`: a - // match beyond the window was found. `failed`: the stream errored. `refused`: the backend - // yielded a row this relay's matcher refuses, which is what a backend whose predicate is not - // `Filter::matches` looks like from here. `vanished`: the backend spent a page slot on a key - // that was gone when it came to read it. + // `rows`: every page slot the backend spent, skipped and vanished ones included — a full + // page is what makes the backend's remaining extent unknown. `matched`: rows that were a match + // inside the client's window, which is what `limit` bounds. `trimmed`: the guard withheld a + // match. `more`: a match beyond the window was found. `failed`: the stream errored. `refused`: + // the backend yielded a row this relay's matcher refuses, which is what a backend whose + // predicate is not `Filter::matches` looks like from here. `vanished`: the backend spent a + // page slot on a key that was gone when it came to read it. let mut rows = 0u64; let mut matched = 0u64; let mut trimmed = false; @@ -453,7 +453,7 @@ async fn fetch_and_send(mut read: InFlightRead, filters: &[Filter], context: &Ha // all six: // // 1. the stream ended **cleanly** — after a storage error, how far the read got is unknown; - // 2. the page was **not full** — a page that yielded `min(probe, page_bound())` rows says + // 2. the page was **not full** — a page that spent `min(probe, page_bound())` slots says // nothing about the rows beyond it, whether the bound came from the client's `limit` or // from the backend's own page cap, and a skipped row spends a page slot just as a match // does (which is why `rows`, not `matched`, decides this); From 9d2112010b8c7b4a18d7bdad4dfcf1ba2045c709 Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 10:03:36 -0500 Subject: [PATCH 33/36] docs(storage): say what an empty-tag claim asserts --- crates/nostream-storage/src/repositories.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index 869eb1024..0c9b6fd1d 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -463,9 +463,11 @@ pub enum SearchRule { /// do, so an implementor has to read what it is asserting. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EmptyTagRule { - /// A criterion that is the empty string selects what [`nostream_core::Filter::matches`] - /// selects. The in-memory backends' tag predicate *is* that matcher, so a completeness claim - /// over such a read is one the relay can check. + /// A criterion that is the empty string selects at least what + /// [`nostream_core::Filter::matches`] selects: the implementor asserts its tag predicate + /// accepts everything that matcher accepts on the tag criteria — a wider predicate accepts + /// more, and cannot hide a row, because a yielded row the matcher refuses is what the caller + /// reads as a failed claim. AnswersEmptyCriterion, /// The tag index holds no entry for an empty value, so the criterion `""` selects nothing /// however many rows carry `["d", ""]`. LMDB: `tag_entries` writes a key only when the value @@ -563,11 +565,11 @@ pub trait EventRepository: Send + Sync { /// Whether a tag criterion that is the empty string can be answered from this backend's tag /// index (see [`EmptyTagRule`]). /// - /// Defaults to [`EmptyTagRule::AnswersEmptyCriterion`], which is what a backend that filters - /// with [`nostream_core::Filter::matches`] answers — its tag predicate *is* that matcher. A - /// backend that answers tag criteria from an index of non-empty values overrides this, and a - /// caller that must claim completeness over such a read reads it rather than assuming the - /// criterion enumerated the set. + /// Defaults to [`EmptyTagRule::AnswersEmptyCriterion`], which is what a backend whose tag + /// predicate is [`nostream_core::Filter::matches`] on the tag criteria, or a wider one, + /// answers. A backend that answers tag criteria from an index of non-empty values overrides + /// this, and a caller that must claim completeness over such a read reads it rather than + /// assuming the criterion enumerated the set. fn empty_tag_rule(&self) -> EmptyTagRule { EmptyTagRule::AnswersEmptyCriterion } From 9194b2868e973d90476d6f6f5a3058bacfcc373a Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 10:34:56 -0500 Subject: [PATCH 34/36] docs(relay): say what the empty tag criterion selects in the double --- crates/nostream-relay/src/testing.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index 2ebfe2317..f62979ea4 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -160,8 +160,10 @@ pub struct InMemoryEventRepository { page_bound: std::sync::atomic::AtomicU64, search_rule: Mutex>, /// When set, the double models a backend whose tag index holds no entry for an empty value: - /// a filter carrying a generic tag criterion of `""` selects nothing, and `empty_tag_rule` - /// declares it. LMDB and Postgres are both that backend. + /// the criterion `""` selects nothing, a family left with no other criterion selects nothing, + /// and the family's remaining criteria still answer — the filter is matched with its empty + /// criteria dropped, as both backends answer it. `empty_tag_rule` declares it; LMDB and + /// Postgres are both that backend. empty_tag_index: AtomicBool, /// Rows this double yields for **any** filter: what a backend whose `search` predicate is not /// `Filter::matches` looks like from the handler's side, which is the case no honest filter @@ -271,8 +273,9 @@ impl InMemoryEventRepository { } /// Declares the tag index this double answers from, the way LMDB and Postgres do: no entry - /// for an empty value, so a filter carrying a generic tag criterion of `""` selects nothing - /// even from a row that holds one. + /// for an empty value, so the criterion `""` selects nothing even from a row that holds one, + /// and a filter is matched with its empty criteria dropped — a family left with no other + /// criterion selects nothing, and the rest of that family still answers. pub fn declare_no_empty_tag_values(&self) { self.empty_tag_index.store(true, Ordering::SeqCst); } From 9f9fdc300ef83203560969dad2a5e3ec293b5b4b Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 10:34:59 -0500 Subject: [PATCH 35/36] test(relay): pin the non-generic tag family the empty-tag strip skips --- crates/nostream-relay/src/testing.rs | 49 ++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/nostream-relay/src/testing.rs b/crates/nostream-relay/src/testing.rs index f62979ea4..d4dac193c 100644 --- a/crates/nostream-relay/src/testing.rs +++ b/crates/nostream-relay/src/testing.rs @@ -1973,6 +1973,55 @@ mod tests { .unwrap(); assert!(rows.is_empty(), "no index entry holds `\"\"`: {rows:?}"); + + // A non-generic family is not a criterion a tag index answers at all: the matcher skips + // the key (`filter.rs:110-113`) and both backends skip it (`lmdb/events.rs:1646-1648`, + // `postgres/sql.rs:327-329`), so a strip that emptied it instead would answer nothing for + // a filter all three answer. `validate_filter` refuses the key (`filter.rs:281-285`), so + // only a hand-built filter reaches this shape. + let non_generic = [Filter { + tags: BTreeMap::from([ + ("#expiration".to_owned(), vec![String::new()]), + ("#d".to_owned(), vec!["post-1".to_owned()]), + ]), + ..Filter::default() + }]; + let rows = nostream_storage::repositories::collect_records( + events.find_by_filters(&non_generic).await.unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + rows.len(), + 1, + "a non-generic family is skipped, not emptied: {rows:?}" + ); + assert_eq!(rows[0].event.content, "post-1"); + + // The strip runs only when the filter carries an empty criterion in a *generic* family, + // so that family is what makes the guard observable: without it the non-generic one + // strips to an empty list, `without_empty_tag_criteria` answers `None`, and the whole + // filter selects nothing where every real predicate answers the `post-1` row. + let straddling = [Filter { + tags: BTreeMap::from([ + ("#expiration".to_owned(), vec![String::new()]), + ("#d".to_owned(), vec![String::new(), "post-1".to_owned()]), + ]), + ..Filter::default() + }]; + let rows = nostream_storage::repositories::collect_records( + events.find_by_filters(&straddling).await.unwrap(), + ) + .await + .unwrap(); + + assert_eq!( + rows.len(), + 1, + "the generic family's `post-1` criterion still answers: {rows:?}" + ); + assert_eq!(rows[0].event.content, "post-1"); } #[tokio::test] From c2e3db8e393f3166173e77fac7ab005fedaf7dcb Mon Sep 17 00:00:00 2001 From: chappie-daemon Date: Mon, 9 Nov 2026 10:35:01 -0500 Subject: [PATCH 36/36] docs(storage): scope the empty-criterion sentence to the criterion --- crates/nostream-storage/src/repositories.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/nostream-storage/src/repositories.rs b/crates/nostream-storage/src/repositories.rs index 0c9b6fd1d..9e6e7240b 100644 --- a/crates/nostream-storage/src/repositories.rs +++ b/crates/nostream-storage/src/repositories.rs @@ -457,10 +457,10 @@ pub enum SearchRule { /// /// [`nostream_core::Filter::matches`] compares a tag value to the criterion with `==`, so the /// criterion `""` accepts a tag stored as `["d", ""]`. A backend that answers tag criteria from -/// an index of non-empty values holds no entry for the empty one, and then the read selects -/// nothing while the matcher would have accepted every such row — a short page that proves -/// nothing about the rows it never enumerated. Each variant names the rule, as [`SearchRule`]'s -/// do, so an implementor has to read what it is asserting. +/// an index of non-empty values holds no entry for the empty one, and then that criterion +/// selects nothing while the matcher would have accepted every such row — a short page that +/// proves nothing about the rows it never enumerated. Each variant names the rule, as +/// [`SearchRule`]'s do, so an implementor has to read what it is asserting. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum EmptyTagRule { /// A criterion that is the empty string selects at least what