From 5c4b4f0c246f4b83368afa63c67e65630936c7b5 Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Thu, 12 Nov 2026 09:34:38 +0530 Subject: [PATCH 1/5] fix(admin): hold the settings write lock for every writer A settings PATCH read settings.yaml before saving it, while the moderation writer read and rewrote the file under the settings write lock. The two could interleave, so an ordinary admin save silently reverted a ban written by the other request. - read, validate and save inside withSettingsWriteLock in the settings, notifications, restore and CLI writers - keep audit and outbox work outside the lock so it is not held across I/O - add regression tests showing the read happens inside the lock and a ban written before acquisition survives the save Fixes #1039 --- .changeset/settings-write-lock-1039.md | 5 + src/cli/commands/config.ts | 42 +++-- .../admin/patch-notifications-controller.ts | 50 ++++-- .../admin/patch-settings-controller.ts | 47 ++++-- .../admin/post-settings-restore-controller.ts | 2 +- src/utils/settings-config.ts | 26 +-- .../patch-notifications-controller.spec.ts | 25 +++ .../admin/patch-settings-controller.spec.ts | 150 ++++++++++++++++++ test/unit/utils/settings-config.spec.ts | 22 +++ 9 files changed, 311 insertions(+), 58 deletions(-) create mode 100644 .changeset/settings-write-lock-1039.md create mode 100644 test/unit/controllers/admin/patch-settings-controller.spec.ts diff --git a/.changeset/settings-write-lock-1039.md b/.changeset/settings-write-lock-1039.md new file mode 100644 index 000000000..6933b07a8 --- /dev/null +++ b/.changeset/settings-write-lock-1039.md @@ -0,0 +1,5 @@ +--- +"nostream": patch +--- + +fix(admin): serialize every settings-file writer with the settings write lock diff --git a/src/cli/commands/config.ts b/src/cli/commands/config.ts index c851ff7ce..6391de710 100644 --- a/src/cli/commands/config.ts +++ b/src/cli/commands/config.ts @@ -11,6 +11,7 @@ import { toCategoryLabel, validatePathAgainstDefaults, validateSettings, + withSettingsWriteLock, } from '../../utils/settings-config' import { isSecretEnvKey, @@ -128,25 +129,38 @@ export const runConfigSet = async ( return 1 } - const settings = loadUserSettings() as unknown as Record - const next = setByPath(settings, path, parseTypedValue(rawValue, valueType)) + // Read and write under the settings-file lock so a CLI update cannot clobber + // a moderation ban written by a running relay in between. + const outcome = await withSettingsWriteLock< + | { ok: true } + | { ok: false; validationIssues: Array<{ path: string; message: string }> } + >(() => { + const settings = loadUserSettings() as unknown as Record + const next = setByPath(settings, path, parseTypedValue(rawValue, valueType)) + + if (options.validate !== false) { + const merged = loadMergedSettings() as unknown as Record + const mergedNext = setByPath(merged, path, getByPath(next, path)) + const validationIssues = validateSettings(mergedNext as any) + + if (validationIssues.length > 0) { + return { ok: false as const, validationIssues } + } + } - if (options.validate !== false) { - const merged = loadMergedSettings() as unknown as Record - const mergedNext = setByPath(merged, path, getByPath(next, path)) - const validationIssues = validateSettings(mergedNext as any) + saveSettings(next as any) - if (validationIssues.length > 0) { - logError('Config update rejected by validation:') - for (const issue of validationIssues) { - logError(`- ${issue.path}: ${issue.message}`) - } + return { ok: true as const } + }) - return 1 + if ('validationIssues' in outcome) { + logError('Config update rejected by validation:') + for (const issue of outcome.validationIssues) { + logError(`- ${issue.path}: ${issue.message}`) } - } - saveSettings(next as any) + return 1 + } logInfo(`Updated ${path}`) diff --git a/src/controllers/admin/patch-notifications-controller.ts b/src/controllers/admin/patch-notifications-controller.ts index 2e3545d49..d64e1dabb 100644 --- a/src/controllers/admin/patch-notifications-controller.ts +++ b/src/controllers/admin/patch-notifications-controller.ts @@ -19,6 +19,7 @@ import { loadUserSettings, saveSettings, validateSettings, + withSettingsWriteLock, } from '../../utils/settings-config' import { validateSchema } from '../../utils/validation' import { recordAdminAuditFromRequest } from '../../utils/record-admin-audit' @@ -35,27 +36,44 @@ export class PatchAdminNotificationsController implements IController { return } - const merged = loadMergedSettings() - const current = getMergedAdminNotifications(merged) - const nextNotifications = mergeAdminNotificationsPatch(current, validation.value) + // Read and write under the same settings-file lock the moderation writer + // uses, so a notification save cannot clobber a ban written meanwhile. + const outcome = await withSettingsWriteLock< + | { ok: true; redacted: ReturnType } + | { ok: false; issues: Array<{ path: string; message: string }> } + >(() => { + const merged = loadMergedSettings() + const current = getMergedAdminNotifications(merged) + const nextNotifications = mergeAdminNotificationsPatch(current, validation.value) - const userSettings = loadUserSettings() as Settings - const nextUserSettings = mergeDeepRight(userSettings, { - admin: { - ...userSettings.admin, - notifications: nextNotifications, - }, - }) as Settings + const userSettings = loadUserSettings() as Settings + const nextUserSettings = mergeDeepRight(userSettings, { + admin: { + ...userSettings.admin, + notifications: nextNotifications, + }, + }) as Settings - const mergedNext = mergeDeepRight(loadDefaults(), nextUserSettings) as Settings + const mergedNext = mergeDeepRight(loadDefaults(), nextUserSettings) as Settings - const issues = validateSettings(mergedNext) - if (issues.length > 0) { - response.status(400).setHeader('content-type', 'application/json').send({ error: 'Validation failed', issues }) + const issues = validateSettings(mergedNext) + if (issues.length > 0) { + return { ok: false as const, issues } + } + + saveSettings(nextUserSettings) + + return { ok: true as const, redacted: getRedactedAdminNotifications() } + }) + + if ('issues' in outcome) { + response + .status(400) + .setHeader('content-type', 'application/json') + .send({ error: 'Validation failed', issues: outcome.issues }) return } - saveSettings(nextUserSettings) appendSettingsAuditLog({ action: 'settings.updated', changes: [{ path: 'admin.notifications', reload: 'hot-reload' }], @@ -74,7 +92,7 @@ export class PatchAdminNotificationsController implements IController { response.status(200).setHeader('content-type', 'application/json').send({ ok: true, - notifications: getRedactedAdminNotifications(), + notifications: outcome.redacted, }) } } diff --git a/src/controllers/admin/patch-settings-controller.ts b/src/controllers/admin/patch-settings-controller.ts index f67eb099e..ac624c6e7 100644 --- a/src/controllers/admin/patch-settings-controller.ts +++ b/src/controllers/admin/patch-settings-controller.ts @@ -12,6 +12,7 @@ import { setByPath, validatePathAgainstDefaults, validateSettings, + withSettingsWriteLock, } from '../../utils/settings-config' import { isWriteProtectedSettingsPath, @@ -50,27 +51,43 @@ export class PatchAdminSettingsController implements IController { return } - const userSettings = loadUserSettings() as unknown as Record - const nextUserSettings = changes.reduce( - (settings, change) => setByPath(settings, change.path, change.value), - userSettings, - ) - const merged = loadMergedSettings() as unknown as Record - const mergedNext = changes.reduce( - (settings, change) => setByPath(settings, change.path, getByPath(nextUserSettings, change.path)), - merged, - ) - const validationIssues = validateSettings(mergedNext as unknown as Settings) - - if (validationIssues.length > 0) { + // Every writer of settings.yaml takes this lock. Reading inside it is what + // stops an ordinary settings save from writing a document read before a + // moderation ban was applied, which would silently undo the ban. + const outcome = await withSettingsWriteLock< + | { ok: true; nextUserSettings: Record } + | { ok: false; issues: Array<{ path: string; message: string }> } + >(() => { + const userSettings = loadUserSettings() as unknown as Record + const nextUserSettings = changes.reduce( + (settings, change) => setByPath(settings, change.path, change.value), + userSettings, + ) + const merged = loadMergedSettings() as unknown as Record + const mergedNext = changes.reduce( + (settings, change) => setByPath(settings, change.path, getByPath(nextUserSettings, change.path)), + merged, + ) + const validationIssues = validateSettings(mergedNext as unknown as Settings) + + if (validationIssues.length > 0) { + return { ok: false as const, issues: validationIssues } + } + + saveSettings(nextUserSettings as unknown as Settings) + + return { ok: true as const, nextUserSettings } + }) + + if ('issues' in outcome) { response .status(400) .setHeader('content-type', 'application/json') - .send({ error: 'Validation failed', issues: validationIssues }) + .send({ error: 'Validation failed', issues: outcome.issues }) return } - saveSettings(nextUserSettings as unknown as Settings) + const nextUserSettings = outcome.nextUserSettings const updatedChanges = changes.map(({ path }) => ({ path, value: redactSettingsValue(path, getByPath(nextUserSettings, path)), diff --git a/src/controllers/admin/post-settings-restore-controller.ts b/src/controllers/admin/post-settings-restore-controller.ts index aded18e40..1bb3c65e9 100644 --- a/src/controllers/admin/post-settings-restore-controller.ts +++ b/src/controllers/admin/post-settings-restore-controller.ts @@ -15,7 +15,7 @@ export class PostAdminSettingsRestoreController implements IController { } try { - restoreSettingsBackup(validation.value.filename) + await restoreSettingsBackup(validation.value.filename) appendSettingsAuditLog({ action: 'settings.restored', filename: validation.value.filename, diff --git a/src/utils/settings-config.ts b/src/utils/settings-config.ts index 7f326f4e4..65679c223 100644 --- a/src/utils/settings-config.ts +++ b/src/utils/settings-config.ts @@ -585,24 +585,26 @@ export const listSettingsBackups = (): SettingsBackupInfo[] => { .sort((left, right) => right.createdAt.localeCompare(left.createdAt)) } -export const restoreSettingsBackup = (filename: string): Settings => { +export const restoreSettingsBackup = async (filename: string): Promise => { if (!isSafeBackupFilename(filename)) { throw new Error('Invalid backup filename') } - const backupPath = join(getSettingsBackupDir(), filename) - if (!fs.existsSync(backupPath)) { - throw new Error('Backup not found') - } + return withSettingsWriteLock(() => { + const backupPath = join(getSettingsBackupDir(), filename) + if (!fs.existsSync(backupPath)) { + throw new Error('Backup not found') + } - const restored = yaml.load(fs.readFileSync(backupPath, 'utf-8')) as Settings - const issues = validateSettings(mergeDeepRight(loadDefaults(), restored) as Settings) - if (issues.length > 0) { - throw new Error(`Backup failed validation: ${issues.map((issue) => issue.path).join(', ')}`) - } + const restored = yaml.load(fs.readFileSync(backupPath, 'utf-8')) as Settings + const issues = validateSettings(mergeDeepRight(loadDefaults(), restored) as Settings) + if (issues.length > 0) { + throw new Error(`Backup failed validation: ${issues.map((issue) => issue.path).join(', ')}`) + } - saveSettings(restored) - return restored + saveSettings(restored) + return restored + }) } export const getTopLevelSettingCategories = (): string[] => { diff --git a/test/unit/controllers/admin/patch-notifications-controller.spec.ts b/test/unit/controllers/admin/patch-notifications-controller.spec.ts index 9698bcfae..a7b649fe7 100644 --- a/test/unit/controllers/admin/patch-notifications-controller.spec.ts +++ b/test/unit/controllers/admin/patch-notifications-controller.spec.ts @@ -47,6 +47,7 @@ describe('PatchAdminNotificationsController', () => { send: sandbox.stub().returnsThis(), } + sandbox.stub(settingsConfig, 'withSettingsWriteLock').callsFake(async (operation: () => unknown) => operation()) sandbox.stub(settingsConfig, 'loadMergedSettings').returns(baseMergedSettings as any) sandbox.stub(settingsConfig, 'loadUserSettings').returns({ admin: { notifications: baseNotifications } } as any) sandbox.stub(settingsConfig, 'saveSettings') @@ -130,6 +131,30 @@ describe('PatchAdminNotificationsController', () => { expect(response.send.firstCall.args[0].ok).to.equal(true) }) + it('reads and saves inside the settings write lock', async () => { + const order: string[] = [] + ;(settingsConfig.withSettingsWriteLock as Sinon.SinonStub).callsFake(async (operation: () => unknown) => { + order.push('lock:enter') + try { + return operation() + } finally { + order.push('lock:exit') + } + }) + ;(settingsConfig.loadUserSettings as Sinon.SinonStub).callsFake(() => { + order.push('read') + return { admin: { notifications: baseNotifications } } + }) + ;(settingsConfig.saveSettings as Sinon.SinonStub).callsFake(() => { + order.push('save') + }) + + const controller = new PatchAdminNotificationsController(outboxRepository as any) + await controller.handleRequest({ body: { enabled: true }, ip: '127.0.0.1' } as any, response as any) + + expect(order).to.deep.equal(['lock:enter', 'read', 'save', 'lock:exit']) + }) + it('still returns 200 when outbox enqueue fails', async () => { outboxRepository.enqueue.rejects(new Error('db unavailable')) const controller = new PatchAdminNotificationsController(outboxRepository as any) diff --git a/test/unit/controllers/admin/patch-settings-controller.spec.ts b/test/unit/controllers/admin/patch-settings-controller.spec.ts new file mode 100644 index 000000000..70dad7f3f --- /dev/null +++ b/test/unit/controllers/admin/patch-settings-controller.spec.ts @@ -0,0 +1,150 @@ +import chai from 'chai' +import Sinon from 'sinon' +import sinonChai from 'sinon-chai' + +import * as recordAdminAudit from '../../../../src/utils/record-admin-audit' +import * as settingsConfig from '../../../../src/utils/settings-config' + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { PatchAdminSettingsController } = require('../../../../src/controllers/admin/patch-settings-controller') + +chai.use(sinonChai) + +const { expect } = chai + +const baseSettings = { + payments: { enabled: false }, + info: { relay_url: 'wss://relay.example', name: 'test' }, +} + +describe('PatchAdminSettingsController settings write lock', () => { + let sandbox: Sinon.SinonSandbox + let outboxRepository: { enqueue: Sinon.SinonStub } + let response: { status: Sinon.SinonStub; setHeader: Sinon.SinonStub; send: Sinon.SinonStub } + let request: Record + + beforeEach(() => { + sandbox = Sinon.createSandbox() + outboxRepository = { enqueue: sandbox.stub().resolves() } + response = { + status: sandbox.stub().returnsThis(), + setHeader: sandbox.stub().returnsThis(), + send: sandbox.stub().returnsThis(), + } + request = { body: { path: 'payments.enabled', value: true }, ip: '127.0.0.1' } + + sandbox.stub(settingsConfig, 'withSettingsWriteLock').callsFake(async (operation: () => unknown) => operation()) + sandbox.stub(settingsConfig, 'loadUserSettings').returns(baseSettings as any) + sandbox.stub(settingsConfig, 'loadMergedSettings').returns(baseSettings as any) + sandbox.stub(settingsConfig, 'saveSettings') + sandbox.stub(settingsConfig, 'appendSettingsAuditLog') + sandbox.stub(settingsConfig, 'validatePathAgainstDefaults').returns([]) + sandbox.stub(settingsConfig, 'validateSettings').returns([]) + sandbox.stub(recordAdminAudit, 'recordAdminAuditFromRequest').resolves() + }) + + afterEach(() => { + sandbox.restore() + }) + + it('re-reads and saves inside the settings write lock', async () => { + const order: string[] = [] + ;(settingsConfig.withSettingsWriteLock as Sinon.SinonStub).callsFake(async (operation: () => unknown) => { + order.push('lock:enter') + try { + return operation() + } finally { + order.push('lock:exit') + } + }) + ;(settingsConfig.loadUserSettings as Sinon.SinonStub).callsFake(() => { + order.push('read') + return baseSettings + }) + ;(settingsConfig.loadMergedSettings as Sinon.SinonStub).callsFake(() => { + order.push('merged') + return baseSettings + }) + ;(settingsConfig.saveSettings as Sinon.SinonStub).callsFake(() => { + order.push('save') + }) + + const controller = new PatchAdminSettingsController(outboxRepository as any) + await controller.handleRequest(request as any, response as any) + + // The read has to happen after the lock is acquired. Otherwise a ban written + // by another worker between the read and the save would be overwritten. + expect(order).to.deep.equal(['lock:enter', 'read', 'merged', 'save', 'lock:exit']) + expect(response.status).to.have.been.calledOnceWithExactly(200) + }) + + it('keeps a moderation ban written before the lock is acquired', async () => { + let stored = { + payments: { enabled: false }, + limits: { event: { pubkey: { blacklist: [] as string[] } } }, + info: { relay_url: 'wss://relay.example', name: 'test' }, + } + ;(settingsConfig.withSettingsWriteLock as Sinon.SinonStub).callsFake(async (operation: () => unknown) => { + // Another worker bans a pubkey just before this writer gets the lock. + stored = { + ...stored, + limits: { event: { pubkey: { blacklist: ['banned-pubkey'] } } }, + } + return operation() + }) + ;(settingsConfig.loadUserSettings as Sinon.SinonStub).callsFake(() => stored) + ;(settingsConfig.loadMergedSettings as Sinon.SinonStub).callsFake(() => stored) + ;(settingsConfig.saveSettings as Sinon.SinonStub).callsFake((next: typeof stored) => { + stored = next + }) + + const controller = new PatchAdminSettingsController(outboxRepository as any) + await controller.handleRequest(request as any, response as any) + + // The ordinary save must not revert the ban; it also applies its own change. + expect(stored.limits.event.pubkey.blacklist).to.deep.equal(['banned-pubkey']) + expect(stored.payments.enabled).to.equal(true) + }) + + it('never reads settings outside the lock', async () => { + let locked = false + const readsWhileLocked: boolean[] = [] + ;(settingsConfig.withSettingsWriteLock as Sinon.SinonStub).callsFake(async (operation: () => unknown) => { + locked = true + try { + return operation() + } finally { + locked = false + } + }) + ;(settingsConfig.loadUserSettings as Sinon.SinonStub).callsFake(() => { + readsWhileLocked.push(locked) + return baseSettings + }) + ;(settingsConfig.loadMergedSettings as Sinon.SinonStub).callsFake(() => { + readsWhileLocked.push(locked) + return baseSettings + }) + + const controller = new PatchAdminSettingsController(outboxRepository as any) + await controller.handleRequest(request as any, response as any) + + expect(readsWhileLocked).to.deep.equal([true, true]) + }) + + it('does not save when merged validation fails', async () => { + ;(settingsConfig.validateSettings as Sinon.SinonStub).returns([{ path: 'payments', message: 'bad' }]) + + const controller = new PatchAdminSettingsController(outboxRepository as any) + await controller.handleRequest(request as any, response as any) + + expect(response.status).to.have.been.calledOnceWithExactly(400) + expect(response.send.firstCall.args[0]).to.deep.equal({ + error: 'Validation failed', + issues: [{ path: 'payments', message: 'bad' }], + }) + expect(settingsConfig.saveSettings).not.to.have.been.called + expect(settingsConfig.appendSettingsAuditLog).not.to.have.been.called + expect(outboxRepository.enqueue).not.to.have.been.called + }) +}) diff --git a/test/unit/utils/settings-config.spec.ts b/test/unit/utils/settings-config.spec.ts index 8bd0cf5e3..8d80479cf 100644 --- a/test/unit/utils/settings-config.spec.ts +++ b/test/unit/utils/settings-config.spec.ts @@ -3,10 +3,14 @@ import fs from 'fs' import os from 'os' import path from 'path' import Sinon from 'sinon' +import yaml from 'js-yaml' import { ensureSettingsExists, + getSettingsBackupDir, loadUserSettings, + restoreSettingsBackup, + saveSettings, toCategoryLabel, getByPath, getTopLevelSettingCategories, @@ -1048,4 +1052,22 @@ describe('settings write lock', () => { expect(error?.message).to.equal('Timed out waiting for settings write lock') expect(fs.existsSync(lockPath)).to.equal(true) }) + + it('restores a backup while holding the settings write lock', async () => { + const backupDir = getSettingsBackupDir() + fs.mkdirSync(backupDir, { recursive: true }) + const filename = 'settings.2026-11-02T12-00-00-000Z.yaml' + const defaults = loadDefaults() + + fs.writeFileSync( + path.join(backupDir, filename), + yaml.dump({ ...defaults, info: { ...defaults.info, name: 'restored-from-backup' } }), + ) + + const restored = await restoreSettingsBackup(filename) + + expect(restored.info?.name).to.equal('restored-from-backup') + expect(loadUserSettings().info?.name).to.equal('restored-from-backup') + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) + }) }) From 4e0c06e7be1c2f27f4e81d89828201dd14c0f6e0 Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Thu, 12 Nov 2026 10:47:34 +0530 Subject: [PATCH 2/5] test(admin): prove restore holds the settings write lock Greptile pointed out that the restore test only checked the result and the absence of the lock file, so it would still pass if restoreSettingsBackup stopped taking the lock. Assert that the lock file exists during the rename that writes the restored settings. Removing withSettingsWriteLock from restoreSettingsBackup now fails the test. --- test/unit/utils/settings-config.spec.ts | 32 +++++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/test/unit/utils/settings-config.spec.ts b/test/unit/utils/settings-config.spec.ts index 8d80479cf..4bb110525 100644 --- a/test/unit/utils/settings-config.spec.ts +++ b/test/unit/utils/settings-config.spec.ts @@ -22,7 +22,7 @@ import { validateSettings, withSettingsWriteLock, } from '../../../src/utils/settings-config' -import { getSettingsWriteLockPath } from '../../../src/utils/settings-paths' +import { getSettingsFilePath, getSettingsWriteLockPath } from '../../../src/utils/settings-paths' import { toBech32 } from '../../../src/utils/transform' import { getGuidedSettingCategory, @@ -1064,10 +1064,32 @@ describe('settings write lock', () => { yaml.dump({ ...defaults, info: { ...defaults.info, name: 'restored-from-backup' } }), ) - const restored = await restoreSettingsBackup(filename) + const lockPath = getSettingsWriteLockPath() + const settingsPath = getSettingsFilePath() + const renameSync = fs.renameSync.bind(fs) + let lockHeldDuringWrite = false + + // saveSettings() swaps the new document into place with renameSync(). Seeing + // the lock during that call proves restoreSettingsBackup() still holds it; + // the test would fail if the lock were removed from the restore path. + const renameStub = Sinon.stub(fs, 'renameSync').callsFake((from, to) => { + if (to === settingsPath) { + lockHeldDuringWrite = fs.existsSync(lockPath) + } + + return renameSync(from, to) + }) + + try { + const restored = await restoreSettingsBackup(filename) - expect(restored.info?.name).to.equal('restored-from-backup') - expect(loadUserSettings().info?.name).to.equal('restored-from-backup') - expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) + expect(restored.info?.name).to.equal('restored-from-backup') + expect(loadUserSettings().info?.name).to.equal('restored-from-backup') + } finally { + renameStub.restore() + } + + expect(lockHeldDuringWrite).to.equal(true) + expect(fs.existsSync(lockPath)).to.equal(false) }) }) From 8907b099d3b49c00c7bfab74f3c8ac5c736e85f8 Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Mon, 16 Nov 2026 22:19:53 +0530 Subject: [PATCH 3/5] fix(admin): keep the settings lock safe across hosts and async writers - record the owner hostname and heartbeat so a relay container does not trust a\n host PID check across PID namespaces and steal a live CLI lock\n- only trust ESRCH on the same host; fall back to heartbeat age otherwise\n- hold the lock until an async operation settles, and only remove the lock when\n the metadata still belongs to this writer\n- cover the cross-host, heartbeat, ownership and async cases with unit and\n integration tests --- .changeset/settings-write-lock-1039.md | 3 +- src/utils/settings-config.ts | 4 +- .../settings/settings-write-lock.feature | 23 +++ .../settings/settings-write-lock.feature.ts | 190 ++++++++++++++++++ test/unit/utils/settings-config.spec.ts | 28 +++ 5 files changed, 246 insertions(+), 2 deletions(-) create mode 100644 test/integration/features/settings/settings-write-lock.feature create mode 100644 test/integration/features/settings/settings-write-lock.feature.ts diff --git a/.changeset/settings-write-lock-1039.md b/.changeset/settings-write-lock-1039.md index 2222ea0c8..2b300c6c8 100644 --- a/.changeset/settings-write-lock-1039.md +++ b/.changeset/settings-write-lock-1039.md @@ -6,4 +6,5 @@ fix(admin): serialize every settings-file writer with the settings write lock The lock records its owner hostname and refreshes a heartbeat, so a relay in a container no longer steals a live CLI lock just because the host PID does not -exist in its own PID namespace. +exist in its own PID namespace. The lock is also held until async writers +settle. diff --git a/src/utils/settings-config.ts b/src/utils/settings-config.ts index 453322d09..293df7c30 100644 --- a/src/utils/settings-config.ts +++ b/src/utils/settings-config.ts @@ -229,7 +229,9 @@ export const withSettingsWriteLock = async (operation: () => T): Promise = heartbeat.unref() try { - return operation() + // Await the result so an async writer keeps the lock until it settles; a plain + // `return operation()` would run the finally block as soon as the promise is created. + return await operation() } finally { if (heartbeat) { clearInterval(heartbeat) diff --git a/test/integration/features/settings/settings-write-lock.feature b/test/integration/features/settings/settings-write-lock.feature new file mode 100644 index 000000000..3d77a4016 --- /dev/null +++ b/test/integration/features/settings/settings-write-lock.feature @@ -0,0 +1,23 @@ +@settings-write-lock +Feature: Settings write lock + + Every settings-file writer takes the same lock, and a lock held by a process + in another container or on the host is never stolen while its heartbeat is + fresh. + + Scenario: Two writers serialize and expose their lock owner + When two settings writers queue for the lock + Then the writers run one after the other + And the lock records the owner host and token + + Scenario: The notifications writer saves inside the lock + When the admin notifications controller saves settings + Then the notifications are saved and the lock is released + + Scenario: The settings writer saves inside the lock + When the admin settings controller saves a client subscription limit + Then the saved settings contain that limit and the lock is released + + Scenario: The restore writer saves inside the lock + When an admin settings backup is restored + Then the restored settings are saved and the lock is released diff --git a/test/integration/features/settings/settings-write-lock.feature.ts b/test/integration/features/settings/settings-write-lock.feature.ts new file mode 100644 index 000000000..fa0243936 --- /dev/null +++ b/test/integration/features/settings/settings-write-lock.feature.ts @@ -0,0 +1,190 @@ +import { After, Before, Then, When, World } from '@cucumber/cucumber' +import { expect } from 'chai' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import Sinon from 'sinon' + +import { PatchAdminNotificationsController } from '../../../../src/controllers/admin/patch-notifications-controller' +import { PatchAdminSettingsController } from '../../../../src/controllers/admin/patch-settings-controller' +import { PostAdminSettingsRestoreController } from '../../../../src/controllers/admin/post-settings-restore-controller' +import { + ensureSettingsExists, + getSettingsBackupDir, + getSettingsFilePath, + getSettingsWriteLockPath, + loadUserSettings, + saveSettings, + withSettingsWriteLock, +} from '../../../../src/utils/settings-config' +import * as audit from '../../../../src/utils/record-admin-audit' + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +const makeResponse = () => { + const response: any = { + status: Sinon.stub(), + setHeader: Sinon.stub(), + send: Sinon.stub(), + } + response.status.returns(response) + response.setHeader.returns(response) + response.send.returns(response) + + return response +} + +Before({ tags: '@settings-write-lock' }, function (this: World>) { + this.parameters.settingsConfigDir = process.env.NOSTR_CONFIG_DIR + this.parameters.settingsTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'nostream-settings-integration-')) + process.env.NOSTR_CONFIG_DIR = this.parameters.settingsTmpDir + this.parameters.settingsSandbox = Sinon.createSandbox() + this.parameters.settingsSandbox.stub(audit, 'recordAdminAuditFromRequest').resolves() + this.parameters.settingsOutbox = { enqueue: Sinon.stub().resolves() } + this.parameters.settingsResponses = {} + + ensureSettingsExists() +}) + +After({ tags: '@settings-write-lock' }, function (this: World>) { + this.parameters.settingsSandbox?.restore() + fs.rmSync(this.parameters.settingsTmpDir, { recursive: true, force: true }) + if (this.parameters.settingsConfigDir === undefined) { + delete process.env.NOSTR_CONFIG_DIR + } else { + process.env.NOSTR_CONFIG_DIR = this.parameters.settingsConfigDir + } + + this.parameters.settingsConfigDir = undefined + this.parameters.settingsTmpDir = undefined + this.parameters.settingsSandbox = undefined + this.parameters.settingsOutbox = undefined + this.parameters.settingsResponses = {} + this.parameters.settingsLockOrder = undefined + this.parameters.settingsLockMetadata = undefined +}) + +When('two settings writers queue for the lock', async function (this: World>) { + const lockPath = getSettingsWriteLockPath() + const order: string[] = [] + let acquired!: () => void + let release!: () => void + const firstAcquired = new Promise((resolve) => { + acquired = resolve + }) + const holdFirst = new Promise((resolve) => { + release = resolve + }) + + const first = withSettingsWriteLock(async () => { + order.push('first') + this.parameters.settingsLockMetadata = fs.readFileSync(lockPath, 'utf-8').trim().split('\n') + acquired() + await holdFirst + }) + + await firstAcquired + const second = withSettingsWriteLock(() => { + order.push('second') + }) + + await delay(50) + expect(order).to.deep.equal(['first']) + + release() + await Promise.all([first, second]) + this.parameters.settingsLockOrder = order +}) + +Then('the writers run one after the other', function (this: World>) { + expect(this.parameters.settingsLockOrder).to.deep.equal(['first', 'second']) +}) + +Then('the lock records the owner host and token', function (this: World>) { + const [pid, acquiredAt, hostname, token] = this.parameters.settingsLockMetadata as string[] + + expect(Number(pid)).to.equal(process.pid) + expect(Number(acquiredAt)).to.be.greaterThan(0) + expect(hostname).to.equal(os.hostname()) + expect(token).to.match(/^[0-9a-f-]{36}$/) + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) +}) + +When( + 'the admin notifications controller saves settings', + async function (this: World>) { + const controller = new PatchAdminNotificationsController(this.parameters.settingsOutbox) + const response = makeResponse() + this.parameters.settingsResponses.notifications = response + + await controller.handleRequest({ body: { enabled: true }, ip: '127.0.0.1' } as any, response) + }, +) + +Then( + 'the notifications are saved and the lock is released', + function (this: World>) { + const response = this.parameters.settingsResponses.notifications + const saved = loadUserSettings() as any + + Sinon.assert.calledWith(response.status, 200) + expect(saved.admin.notifications.enabled).to.equal(true) + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) + }, +) + +When( + 'the admin settings controller saves a client subscription limit', + async function (this: World>) { + const controller = new PatchAdminSettingsController(this.parameters.settingsOutbox) + const response = makeResponse() + this.parameters.settingsResponses.settings = response + + await controller.handleRequest( + { + body: { path: 'limits.client.subscription.maxSubscriptions', value: 12 }, + ip: '127.0.0.1', + } as any, + response, + ) + }, +) + +Then( + 'the saved settings contain that limit and the lock is released', + function (this: World>) { + const response = this.parameters.settingsResponses.settings + const saved = loadUserSettings() as any + + Sinon.assert.calledWith(response.status, 200) + expect(saved.limits.client.subscription.maxSubscriptions).to.equal(12) + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) + }, +) + +When('an admin settings backup is restored', async function (this: World>) { + const backupDir = getSettingsBackupDir() + fs.mkdirSync(backupDir, { recursive: true }) + const filename = 'settings.2026-11-02T12-00-00-000Z.yaml' + const current = loadUserSettings() + saveSettings({ ...current, info: { ...current.info, name: 'restored-from-integration' } } as any) + fs.copyFileSync(getSettingsFilePath(), path.join(backupDir, filename)) + + const controller = new PostAdminSettingsRestoreController() + const response = makeResponse() + this.parameters.settingsResponses.restore = response + + await controller.handleRequest({ body: { filename }, ip: '127.0.0.1' } as any, response) +}) + +Then( + 'the restored settings are saved and the lock is released', + function (this: World>) { + const response = this.parameters.settingsResponses.restore + const saved = loadUserSettings() as any + + Sinon.assert.calledWith(response.status, 200) + expect(saved.info.name).to.equal('restored-from-integration') + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) + }, +) diff --git a/test/unit/utils/settings-config.spec.ts b/test/unit/utils/settings-config.spec.ts index c33eff039..56778f5f3 100644 --- a/test/unit/utils/settings-config.spec.ts +++ b/test/unit/utils/settings-config.spec.ts @@ -1107,6 +1107,34 @@ describe('settings write lock', () => { expect(fs.readFileSync(lockPath, 'utf-8')).to.contain('replacement-token') }) + it('holds the lock until an async operation settles', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + let release!: () => void + const held = new Promise((resolve) => { + release = resolve + }) + + const first = withSettingsWriteLock(async () => { + await held + return 'first' + }) + + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(fs.existsSync(lockPath)).to.equal(true) + + let secondRan = false + const second = withSettingsWriteLock(() => { + secondRan = true + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(secondRan).to.equal(false) + + release() + expect(await Promise.all([first, second])).to.deep.equal(['first', undefined]) + expect(secondRan).to.equal(true) + }) + it('does not steal a live lock when the liveness check fails with EPERM', async () => { ensureSettingsExists() const lockPath = getSettingsWriteLockPath() From 7085da7c536787f99a2b45f1507b4f2882b6ab29 Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Mon, 16 Nov 2026 22:29:59 +0530 Subject: [PATCH 4/5] fix(admin): reclaim a lock whose heartbeat expired even if its pid was reused A restart can give an abandoned lock's PID to a new process. Check the heartbeat before trusting process.kill, so a same-host success no longer keeps a dead lock alive forever. --- .changeset/settings-write-lock-1039.md | 4 ++-- src/utils/settings-config.ts | 12 +++++++++--- test/unit/utils/settings-config.spec.ts | 13 +++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/.changeset/settings-write-lock-1039.md b/.changeset/settings-write-lock-1039.md index 2b300c6c8..e3cb10689 100644 --- a/.changeset/settings-write-lock-1039.md +++ b/.changeset/settings-write-lock-1039.md @@ -6,5 +6,5 @@ fix(admin): serialize every settings-file writer with the settings write lock The lock records its owner hostname and refreshes a heartbeat, so a relay in a container no longer steals a live CLI lock just because the host PID does not -exist in its own PID namespace. The lock is also held until async writers -settle. +exist in its own PID namespace. An expired heartbeat always wins over a reused +PID, and the lock is held until async writers settle. diff --git a/src/utils/settings-config.ts b/src/utils/settings-config.ts index 293df7c30..d11b411a5 100644 --- a/src/utils/settings-config.ts +++ b/src/utils/settings-config.ts @@ -78,10 +78,16 @@ const getSettingsWriteLockAgeMs = (lockPath: string): number => { const isSettingsWriteLockStale = (lockPath: string, metadata: SettingsWriteLockMetadata): boolean => { const age = getSettingsWriteLockAgeMs(lockPath) + // A live holder refreshes the mtime every heartbeat. An expired heartbeat means + // the owner is gone even if its PID was reused by a new process after a restart. + if (age > SETTINGS_WRITE_LOCK_STALE_MS) { + return true + } + // A PID means nothing across container/host namespaces: the relay's PID 5 is // not the host CLI's PID 5. Only the heartbeat age can prove abandonment there. if (metadata.hostname !== os.hostname()) { - return age > SETTINGS_WRITE_LOCK_STALE_MS + return false } try { @@ -89,8 +95,8 @@ const isSettingsWriteLockStale = (lockPath: string, metadata: SettingsWriteLockM return false } catch (error) { // Only ESRCH proves the owner is gone. EPERM means the process exists under - // another user; the heartbeat age remains the fallback for other errors. - return (error as NodeJS.ErrnoException).code === 'ESRCH' || age > SETTINGS_WRITE_LOCK_STALE_MS + // another user; treat other errors as alive until the heartbeat expires. + return (error as NodeJS.ErrnoException).code === 'ESRCH' } } diff --git a/test/unit/utils/settings-config.spec.ts b/test/unit/utils/settings-config.spec.ts index 56778f5f3..716f652d4 100644 --- a/test/unit/utils/settings-config.spec.ts +++ b/test/unit/utils/settings-config.spec.ts @@ -1094,6 +1094,19 @@ describe('settings write lock', () => { expect(fs.existsSync(lockPath)).to.equal(false) }) + it('reclaims a lock whose heartbeat stopped even when the pid was reused', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + // Same host and a live PID, but no heartbeat for over a minute: the owner died + // and a restart gave its PID to a new process. + fs.writeFileSync(lockPath, `${process.pid}\n${Date.now()}\n${os.hostname()}\nowner-token\n`) + const old = new Date(Date.now() - 61_000) + fs.utimesSync(lockPath, old, old) + + expect(await withSettingsWriteLock(() => 'ran')).to.equal('ran') + expect(fs.existsSync(lockPath)).to.equal(false) + }) + it('does not remove a lock that was replaced while it was held', async () => { ensureSettingsExists() const lockPath = getSettingsWriteLockPath() From 974724334a9dcb996be4b20351011eeef02d113e Mon Sep 17 00:00:00 2001 From: ABHAY PANDEY Date: Mon, 16 Nov 2026 22:59:13 +0530 Subject: [PATCH 5/5] fix(admin): identify lock owners by process start time, not heartbeat age Writers are synchronous, so the heartbeat cannot tick during a slow save. Heartbeat age alone would let another same-host writer evict a live lock. Store /proc start time in the lock and compare it to distinguish a reused PID from the original writer. Cross-host locks still use heartbeat age because PIDs are not comparable. --- .changeset/settings-write-lock-1039.md | 5 ++- src/utils/settings-config.ts | 54 +++++++++++++++++++------ test/unit/utils/settings-config.spec.ts | 34 +++++++++++++--- 3 files changed, 73 insertions(+), 20 deletions(-) diff --git a/.changeset/settings-write-lock-1039.md b/.changeset/settings-write-lock-1039.md index e3cb10689..d16d92172 100644 --- a/.changeset/settings-write-lock-1039.md +++ b/.changeset/settings-write-lock-1039.md @@ -6,5 +6,6 @@ fix(admin): serialize every settings-file writer with the settings write lock The lock records its owner hostname and refreshes a heartbeat, so a relay in a container no longer steals a live CLI lock just because the host PID does not -exist in its own PID namespace. An expired heartbeat always wins over a reused -PID, and the lock is held until async writers settle. +exist in its own PID namespace. On the same host the lock checks the owner's +process start time, so a reused PID is reclaimed without ever evicting a slow +writer, and async writers keep the lock until they settle. diff --git a/src/utils/settings-config.ts b/src/utils/settings-config.ts index d11b411a5..338af45cf 100644 --- a/src/utils/settings-config.ts +++ b/src/utils/settings-config.ts @@ -45,12 +45,32 @@ type SettingsWriteLockMetadata = { hostname?: string /** Missing on locks written by older versions. */ token?: string + /** Missing on locks written by older versions or platforms without /proc. */ + pidStartedAt?: string +} + +const readProcessStartTime = (pid: number): string | undefined => { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf-8') + // The comm field can contain spaces and parentheses, so split after its last ')'. + const end = stat.lastIndexOf(')') + if (end < 0) { + return undefined + } + // Field 22 is the start time; the remainder starts at field 3. + return stat + .slice(end + 1) + .trim() + .split(/\s+/)[19] + } catch { + return undefined + } } const readSettingsWriteLockMetadata = (lockPath: string): SettingsWriteLockMetadata | undefined => { try { const raw = fs.readFileSync(lockPath, 'utf-8') - const [pidLine, acquiredLine, hostnameLine, tokenLine] = raw.split('\n') + const [pidLine, acquiredLine, hostnameLine, tokenLine, startedLine] = raw.split('\n') const pid = Number(pidLine) const acquiredAt = Number(acquiredLine) if (!Number.isFinite(pid) || !Number.isFinite(acquiredAt)) { @@ -61,6 +81,7 @@ const readSettingsWriteLockMetadata = (lockPath: string): SettingsWriteLockMetad acquiredAt, hostname: hostnameLine || undefined, token: tokenLine || undefined, + pidStartedAt: startedLine || undefined, } } catch { return undefined @@ -78,26 +99,32 @@ const getSettingsWriteLockAgeMs = (lockPath: string): number => { const isSettingsWriteLockStale = (lockPath: string, metadata: SettingsWriteLockMetadata): boolean => { const age = getSettingsWriteLockAgeMs(lockPath) - // A live holder refreshes the mtime every heartbeat. An expired heartbeat means - // the owner is gone even if its PID was reused by a new process after a restart. - if (age > SETTINGS_WRITE_LOCK_STALE_MS) { - return true - } - // A PID means nothing across container/host namespaces: the relay's PID 5 is // not the host CLI's PID 5. Only the heartbeat age can prove abandonment there. if (metadata.hostname !== os.hostname()) { - return false + return age > SETTINGS_WRITE_LOCK_STALE_MS } try { process.kill(metadata.pid, 0) - return false } catch (error) { - // Only ESRCH proves the owner is gone. EPERM means the process exists under - // another user; treat other errors as alive until the heartbeat expires. - return (error as NodeJS.ErrnoException).code === 'ESRCH' + if ((error as NodeJS.ErrnoException).code === 'ESRCH') { + return true + } + // EPERM means the process exists under another user; fall through to identity. } + + if (metadata.pidStartedAt) { + const currentStartedAt = readProcessStartTime(metadata.pid) + if (currentStartedAt) { + // A restarted process can reuse the old PID but never its start time. + return currentStartedAt !== metadata.pidStartedAt + } + } + + // Writers are synchronous, so the heartbeat cannot tick during a slow save. + // A live same-host PID is never evicted on age alone. + return false } const removeSettingsWriteLockIfOwner = (lockPath: string, owner: SettingsWriteLockMetadata): void => { @@ -174,6 +201,7 @@ export const withSettingsWriteLock = async (operation: () => T): Promise = acquiredAt: Date.now(), hostname: os.hostname(), token: randomUUID(), + pidStartedAt: readProcessStartTime(process.pid), } let lockFd: number | undefined let heartbeat: NodeJS.Timeout | undefined @@ -184,7 +212,7 @@ export const withSettingsWriteLock = async (operation: () => T): Promise = try { fs.writeFileSync( lockFdLocal, - `${owner.pid}\n${owner.acquiredAt}\n${owner.hostname}\n${owner.token}\n`, + `${owner.pid}\n${owner.acquiredAt}\n${owner.hostname}\n${owner.token}\n${owner.pidStartedAt ?? ''}\n`, ) } catch (error) { // The lock exists before it has contents; leaving that empty file diff --git a/test/unit/utils/settings-config.spec.ts b/test/unit/utils/settings-config.spec.ts index 716f652d4..fc27f33e8 100644 --- a/test/unit/utils/settings-config.spec.ts +++ b/test/unit/utils/settings-config.spec.ts @@ -1094,17 +1094,41 @@ describe('settings write lock', () => { expect(fs.existsSync(lockPath)).to.equal(false) }) - it('reclaims a lock whose heartbeat stopped even when the pid was reused', async () => { + const linuxIt = fs.existsSync('/proc/self/stat') ? it : it.skip + + linuxIt('reclaims a lock whose pid was reused after a restart', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + // Same host, same live PID, but a different process start time: a restart + // reused the PID while the old lock file stayed on the shared volume. + fs.writeFileSync(lockPath, `${process.pid}\n${Date.now()}\n${os.hostname()}\nowner-token\n1\n`) + + expect(await withSettingsWriteLock(() => 'ran')).to.equal('ran') + expect(fs.existsSync(lockPath)).to.equal(false) + }) + + it('does not steal a slow same-host writer with an old heartbeat', async () => { ensureSettingsExists() const lockPath = getSettingsWriteLockPath() - // Same host and a live PID, but no heartbeat for over a minute: the owner died - // and a restart gave its PID to a new process. fs.writeFileSync(lockPath, `${process.pid}\n${Date.now()}\n${os.hostname()}\nowner-token\n`) const old = new Date(Date.now() - 61_000) fs.utimesSync(lockPath, old, old) - expect(await withSettingsWriteLock(() => 'ran')).to.equal('ran') - expect(fs.existsSync(lockPath)).to.equal(false) + let now = Date.now() + Sinon.stub(Date, 'now').callsFake(() => now) + setTimeout(() => { + now += 31_000 + }, 10) + + let error: Error | undefined + try { + await withSettingsWriteLock(() => 'ran') + } catch (err) { + error = err as Error + } + + expect(error?.message).to.equal('Timed out waiting for settings write lock') + expect(fs.existsSync(lockPath)).to.equal(true) }) it('does not remove a lock that was replaced while it was held', async () => {