diff --git a/.changeset/settings-write-lock-1039.md b/.changeset/settings-write-lock-1039.md new file mode 100644 index 00000000..d16d9217 --- /dev/null +++ b/.changeset/settings-write-lock-1039.md @@ -0,0 +1,11 @@ +--- +"nostream": patch +--- + +fix(admin): serialize every settings-file writer with the settings write lock + +The lock records its owner hostname and refreshes a heartbeat, so a relay in a +container no longer steals a live CLI lock just because the host PID does not +exist in its own PID namespace. On the same host the lock checks the owner's +process start time, so a reused PID is reclaimed without ever evicting a slow +writer, and async writers keep the lock until they settle. diff --git a/src/cli/commands/config.ts b/src/cli/commands/config.ts index c851ff7c..6391de71 100644 --- a/src/cli/commands/config.ts +++ b/src/cli/commands/config.ts @@ -11,6 +11,7 @@ import { toCategoryLabel, validatePathAgainstDefaults, validateSettings, + withSettingsWriteLock, } from '../../utils/settings-config' import { isSecretEnvKey, @@ -128,25 +129,38 @@ export const runConfigSet = async ( return 1 } - const settings = loadUserSettings() as unknown as Record - const next = setByPath(settings, path, parseTypedValue(rawValue, valueType)) + // Read and write under the settings-file lock so a CLI update cannot clobber + // a moderation ban written by a running relay in between. + const outcome = await withSettingsWriteLock< + | { ok: true } + | { ok: false; validationIssues: Array<{ path: string; message: string }> } + >(() => { + const settings = loadUserSettings() as unknown as Record + const next = setByPath(settings, path, parseTypedValue(rawValue, valueType)) + + if (options.validate !== false) { + const merged = loadMergedSettings() as unknown as Record + const mergedNext = setByPath(merged, path, getByPath(next, path)) + const validationIssues = validateSettings(mergedNext as any) + + if (validationIssues.length > 0) { + return { ok: false as const, validationIssues } + } + } - if (options.validate !== false) { - const merged = loadMergedSettings() as unknown as Record - const mergedNext = setByPath(merged, path, getByPath(next, path)) - const validationIssues = validateSettings(mergedNext as any) + saveSettings(next as any) - if (validationIssues.length > 0) { - logError('Config update rejected by validation:') - for (const issue of validationIssues) { - logError(`- ${issue.path}: ${issue.message}`) - } + return { ok: true as const } + }) - return 1 + if ('validationIssues' in outcome) { + logError('Config update rejected by validation:') + for (const issue of outcome.validationIssues) { + logError(`- ${issue.path}: ${issue.message}`) } - } - saveSettings(next as any) + return 1 + } logInfo(`Updated ${path}`) diff --git a/src/controllers/admin/patch-notifications-controller.ts b/src/controllers/admin/patch-notifications-controller.ts index 2e3545d4..d64e1dab 100644 --- a/src/controllers/admin/patch-notifications-controller.ts +++ b/src/controllers/admin/patch-notifications-controller.ts @@ -19,6 +19,7 @@ import { loadUserSettings, saveSettings, validateSettings, + withSettingsWriteLock, } from '../../utils/settings-config' import { validateSchema } from '../../utils/validation' import { recordAdminAuditFromRequest } from '../../utils/record-admin-audit' @@ -35,27 +36,44 @@ export class PatchAdminNotificationsController implements IController { return } - const merged = loadMergedSettings() - const current = getMergedAdminNotifications(merged) - const nextNotifications = mergeAdminNotificationsPatch(current, validation.value) + // Read and write under the same settings-file lock the moderation writer + // uses, so a notification save cannot clobber a ban written meanwhile. + const outcome = await withSettingsWriteLock< + | { ok: true; redacted: ReturnType } + | { ok: false; issues: Array<{ path: string; message: string }> } + >(() => { + const merged = loadMergedSettings() + const current = getMergedAdminNotifications(merged) + const nextNotifications = mergeAdminNotificationsPatch(current, validation.value) - const userSettings = loadUserSettings() as Settings - const nextUserSettings = mergeDeepRight(userSettings, { - admin: { - ...userSettings.admin, - notifications: nextNotifications, - }, - }) as Settings + const userSettings = loadUserSettings() as Settings + const nextUserSettings = mergeDeepRight(userSettings, { + admin: { + ...userSettings.admin, + notifications: nextNotifications, + }, + }) as Settings - const mergedNext = mergeDeepRight(loadDefaults(), nextUserSettings) as Settings + const mergedNext = mergeDeepRight(loadDefaults(), nextUserSettings) as Settings - const issues = validateSettings(mergedNext) - if (issues.length > 0) { - response.status(400).setHeader('content-type', 'application/json').send({ error: 'Validation failed', issues }) + const issues = validateSettings(mergedNext) + if (issues.length > 0) { + return { ok: false as const, issues } + } + + saveSettings(nextUserSettings) + + return { ok: true as const, redacted: getRedactedAdminNotifications() } + }) + + if ('issues' in outcome) { + response + .status(400) + .setHeader('content-type', 'application/json') + .send({ error: 'Validation failed', issues: outcome.issues }) return } - saveSettings(nextUserSettings) appendSettingsAuditLog({ action: 'settings.updated', changes: [{ path: 'admin.notifications', reload: 'hot-reload' }], @@ -74,7 +92,7 @@ export class PatchAdminNotificationsController implements IController { response.status(200).setHeader('content-type', 'application/json').send({ ok: true, - notifications: getRedactedAdminNotifications(), + notifications: outcome.redacted, }) } } diff --git a/src/controllers/admin/patch-settings-controller.ts b/src/controllers/admin/patch-settings-controller.ts index f67eb099..ac624c6e 100644 --- a/src/controllers/admin/patch-settings-controller.ts +++ b/src/controllers/admin/patch-settings-controller.ts @@ -12,6 +12,7 @@ import { setByPath, validatePathAgainstDefaults, validateSettings, + withSettingsWriteLock, } from '../../utils/settings-config' import { isWriteProtectedSettingsPath, @@ -50,27 +51,43 @@ export class PatchAdminSettingsController implements IController { return } - const userSettings = loadUserSettings() as unknown as Record - const nextUserSettings = changes.reduce( - (settings, change) => setByPath(settings, change.path, change.value), - userSettings, - ) - const merged = loadMergedSettings() as unknown as Record - const mergedNext = changes.reduce( - (settings, change) => setByPath(settings, change.path, getByPath(nextUserSettings, change.path)), - merged, - ) - const validationIssues = validateSettings(mergedNext as unknown as Settings) - - if (validationIssues.length > 0) { + // Every writer of settings.yaml takes this lock. Reading inside it is what + // stops an ordinary settings save from writing a document read before a + // moderation ban was applied, which would silently undo the ban. + const outcome = await withSettingsWriteLock< + | { ok: true; nextUserSettings: Record } + | { ok: false; issues: Array<{ path: string; message: string }> } + >(() => { + const userSettings = loadUserSettings() as unknown as Record + const nextUserSettings = changes.reduce( + (settings, change) => setByPath(settings, change.path, change.value), + userSettings, + ) + const merged = loadMergedSettings() as unknown as Record + const mergedNext = changes.reduce( + (settings, change) => setByPath(settings, change.path, getByPath(nextUserSettings, change.path)), + merged, + ) + const validationIssues = validateSettings(mergedNext as unknown as Settings) + + if (validationIssues.length > 0) { + return { ok: false as const, issues: validationIssues } + } + + saveSettings(nextUserSettings as unknown as Settings) + + return { ok: true as const, nextUserSettings } + }) + + if ('issues' in outcome) { response .status(400) .setHeader('content-type', 'application/json') - .send({ error: 'Validation failed', issues: validationIssues }) + .send({ error: 'Validation failed', issues: outcome.issues }) return } - saveSettings(nextUserSettings as unknown as Settings) + const nextUserSettings = outcome.nextUserSettings const updatedChanges = changes.map(({ path }) => ({ path, value: redactSettingsValue(path, getByPath(nextUserSettings, path)), diff --git a/src/controllers/admin/post-settings-restore-controller.ts b/src/controllers/admin/post-settings-restore-controller.ts index aded18e4..1bb3c65e 100644 --- a/src/controllers/admin/post-settings-restore-controller.ts +++ b/src/controllers/admin/post-settings-restore-controller.ts @@ -15,7 +15,7 @@ export class PostAdminSettingsRestoreController implements IController { } try { - restoreSettingsBackup(validation.value.filename) + await restoreSettingsBackup(validation.value.filename) appendSettingsAuditLog({ action: 'settings.restored', filename: validation.value.filename, diff --git a/src/utils/settings-config.ts b/src/utils/settings-config.ts index fdad3f41..338af45c 100644 --- a/src/utils/settings-config.ts +++ b/src/utils/settings-config.ts @@ -1,4 +1,6 @@ import fs from 'fs' +import os from 'os' +import { randomUUID } from 'node:crypto' import { setTimeout as delay } from 'node:timers/promises' import { join } from 'path' import yaml from 'js-yaml' @@ -32,40 +34,115 @@ export { const SETTINGS_WRITE_LOCK_STALE_MS = 60_000 const SETTINGS_WRITE_LOCK_RETRY_MS = 25 const SETTINGS_WRITE_LOCK_MAX_WAIT_MS = 30_000 +// A holder refreshes the lock mtime well inside the stale window. Without this a +// long write outlives the window and another process may steal a live lock. +const SETTINGS_WRITE_LOCK_HEARTBEAT_MS = 20_000 type SettingsWriteLockMetadata = { pid: number acquiredAt: number + /** Missing on locks written by older versions. */ + hostname?: string + /** Missing on locks written by older versions. */ + token?: string + /** Missing on locks written by older versions or platforms without /proc. */ + pidStartedAt?: string +} + +const readProcessStartTime = (pid: number): string | undefined => { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, 'utf-8') + // The comm field can contain spaces and parentheses, so split after its last ')'. + const end = stat.lastIndexOf(')') + if (end < 0) { + return undefined + } + // Field 22 is the start time; the remainder starts at field 3. + return stat + .slice(end + 1) + .trim() + .split(/\s+/)[19] + } catch { + return undefined + } } const readSettingsWriteLockMetadata = (lockPath: string): SettingsWriteLockMetadata | undefined => { try { const raw = fs.readFileSync(lockPath, 'utf-8') - const [pidLine, acquiredLine] = raw.split('\n') + const [pidLine, acquiredLine, hostnameLine, tokenLine, startedLine] = raw.split('\n') const pid = Number(pidLine) const acquiredAt = Number(acquiredLine) if (!Number.isFinite(pid) || !Number.isFinite(acquiredAt)) { return undefined } - return { pid, acquiredAt } + return { + pid, + acquiredAt, + hostname: hostnameLine || undefined, + token: tokenLine || undefined, + pidStartedAt: startedLine || undefined, + } } catch { return undefined } } -const isSettingsWriteLockStale = (metadata: SettingsWriteLockMetadata): boolean => { - if (Date.now() - metadata.acquiredAt > SETTINGS_WRITE_LOCK_STALE_MS) { - return true +const getSettingsWriteLockAgeMs = (lockPath: string): number => { + try { + return Date.now() - fs.statSync(lockPath).mtimeMs + } catch { + return Number.POSITIVE_INFINITY + } +} + +const isSettingsWriteLockStale = (lockPath: string, metadata: SettingsWriteLockMetadata): boolean => { + const age = getSettingsWriteLockAgeMs(lockPath) + + // A PID means nothing across container/host namespaces: the relay's PID 5 is + // not the host CLI's PID 5. Only the heartbeat age can prove abandonment there. + if (metadata.hostname !== os.hostname()) { + return age > SETTINGS_WRITE_LOCK_STALE_MS } try { process.kill(metadata.pid, 0) - return false } catch (error) { - // Only ESRCH proves the owner is gone. EPERM means the process exists under - // another user, and any other error is treated as alive so a failed check - // can never steal a live lock; the age rule above remains the fallback. - return (error as NodeJS.ErrnoException).code === 'ESRCH' + if ((error as NodeJS.ErrnoException).code === 'ESRCH') { + return true + } + // EPERM means the process exists under another user; fall through to identity. + } + + if (metadata.pidStartedAt) { + const currentStartedAt = readProcessStartTime(metadata.pid) + if (currentStartedAt) { + // A restarted process can reuse the old PID but never its start time. + return currentStartedAt !== metadata.pidStartedAt + } + } + + // Writers are synchronous, so the heartbeat cannot tick during a slow save. + // A live same-host PID is never evicted on age alone. + return false +} + +const removeSettingsWriteLockIfOwner = (lockPath: string, owner: SettingsWriteLockMetadata): void => { + try { + const current = readSettingsWriteLockMetadata(lockPath) + if ( + !current || + current.pid !== owner.pid || + current.acquiredAt !== owner.acquiredAt || + current.hostname !== owner.hostname || + current.token !== owner.token + ) { + return + } + + fs.rmSync(lockPath, { force: true }) + } catch { + // The lock is already gone or unreadable; nothing else to release. } } @@ -85,7 +162,7 @@ const attemptStealStaleSettingsWriteLock = ( !current || current.pid !== expected.pid || current.acquiredAt !== expected.acquiredAt || - !isSettingsWriteLockStale(current) + !isSettingsWriteLockStale(lockPath, current) ) { return false } @@ -119,13 +196,24 @@ export const withSettingsWriteLock = async (operation: () => T): Promise = ensureSettingsExists() const lockPath = getSettingsWriteLockPath() const deadline = Date.now() + SETTINGS_WRITE_LOCK_MAX_WAIT_MS + const owner: SettingsWriteLockMetadata = { + pid: process.pid, + acquiredAt: Date.now(), + hostname: os.hostname(), + token: randomUUID(), + pidStartedAt: readProcessStartTime(process.pid), + } let lockFd: number | undefined + let heartbeat: NodeJS.Timeout | undefined while (Date.now() < deadline) { try { const lockFdLocal = fs.openSync(lockPath, 'wx') try { - fs.writeFileSync(lockFdLocal, `${process.pid}\n${Date.now()}\n`) + fs.writeFileSync( + lockFdLocal, + `${owner.pid}\n${owner.acquiredAt}\n${owner.hostname}\n${owner.token}\n${owner.pidStartedAt ?? ''}\n`, + ) } catch (error) { // The lock exists before it has contents; leaving that empty file // behind would block every later writer until someone removes it. @@ -146,7 +234,7 @@ export const withSettingsWriteLock = async (operation: () => T): Promise = const metadata = readSettingsWriteLockMetadata(lockPath) if (metadata) { - if (isSettingsWriteLockStale(metadata)) { + if (isSettingsWriteLockStale(lockPath, metadata)) { attemptStealStaleSettingsWriteLock(lockPath, metadata) } } else { @@ -161,11 +249,29 @@ export const withSettingsWriteLock = async (operation: () => T): Promise = throw new Error('Timed out waiting for settings write lock') } + const heldFd = lockFd + heartbeat = setInterval(() => { + try { + const now = new Date() + // futimes works on the open file, so a stolen-and-replaced path cannot be + // kept alive by this holder's timer. + fs.futimesSync(heldFd, now, now) + } catch { + // The descriptor is gone; release will clear the timer. + } + }, SETTINGS_WRITE_LOCK_HEARTBEAT_MS) + heartbeat.unref() + try { - return operation() + // Await the result so an async writer keeps the lock until it settles; a plain + // `return operation()` would run the finally block as soon as the promise is created. + return await operation() } finally { - fs.closeSync(lockFd) - fs.rmSync(lockPath, { force: true }) + if (heartbeat) { + clearInterval(heartbeat) + } + fs.closeSync(heldFd) + removeSettingsWriteLockIfOwner(lockPath, owner) } } @@ -585,24 +691,26 @@ export const listSettingsBackups = (): SettingsBackupInfo[] => { .sort((left, right) => right.createdAt.localeCompare(left.createdAt)) } -export const restoreSettingsBackup = (filename: string): Settings => { +export const restoreSettingsBackup = async (filename: string): Promise => { if (!isSafeBackupFilename(filename)) { throw new Error('Invalid backup filename') } - const backupPath = join(getSettingsBackupDir(), filename) - if (!fs.existsSync(backupPath)) { - throw new Error('Backup not found') - } + return withSettingsWriteLock(() => { + const backupPath = join(getSettingsBackupDir(), filename) + if (!fs.existsSync(backupPath)) { + throw new Error('Backup not found') + } - const restored = yaml.load(fs.readFileSync(backupPath, 'utf-8')) as Settings - const issues = validateSettings(mergeDeepRight(loadDefaults(), restored) as Settings) - if (issues.length > 0) { - throw new Error(`Backup failed validation: ${issues.map((issue) => issue.path).join(', ')}`) - } + const restored = yaml.load(fs.readFileSync(backupPath, 'utf-8')) as Settings + const issues = validateSettings(mergeDeepRight(loadDefaults(), restored) as Settings) + if (issues.length > 0) { + throw new Error(`Backup failed validation: ${issues.map((issue) => issue.path).join(', ')}`) + } - saveSettings(restored) - return restored + saveSettings(restored) + return restored + }) } export const getTopLevelSettingCategories = (): string[] => { diff --git a/test/integration/features/settings/settings-write-lock.feature b/test/integration/features/settings/settings-write-lock.feature new file mode 100644 index 00000000..3d77a401 --- /dev/null +++ b/test/integration/features/settings/settings-write-lock.feature @@ -0,0 +1,23 @@ +@settings-write-lock +Feature: Settings write lock + + Every settings-file writer takes the same lock, and a lock held by a process + in another container or on the host is never stolen while its heartbeat is + fresh. + + Scenario: Two writers serialize and expose their lock owner + When two settings writers queue for the lock + Then the writers run one after the other + And the lock records the owner host and token + + Scenario: The notifications writer saves inside the lock + When the admin notifications controller saves settings + Then the notifications are saved and the lock is released + + Scenario: The settings writer saves inside the lock + When the admin settings controller saves a client subscription limit + Then the saved settings contain that limit and the lock is released + + Scenario: The restore writer saves inside the lock + When an admin settings backup is restored + Then the restored settings are saved and the lock is released diff --git a/test/integration/features/settings/settings-write-lock.feature.ts b/test/integration/features/settings/settings-write-lock.feature.ts new file mode 100644 index 00000000..fa024393 --- /dev/null +++ b/test/integration/features/settings/settings-write-lock.feature.ts @@ -0,0 +1,190 @@ +import { After, Before, Then, When, World } from '@cucumber/cucumber' +import { expect } from 'chai' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import Sinon from 'sinon' + +import { PatchAdminNotificationsController } from '../../../../src/controllers/admin/patch-notifications-controller' +import { PatchAdminSettingsController } from '../../../../src/controllers/admin/patch-settings-controller' +import { PostAdminSettingsRestoreController } from '../../../../src/controllers/admin/post-settings-restore-controller' +import { + ensureSettingsExists, + getSettingsBackupDir, + getSettingsFilePath, + getSettingsWriteLockPath, + loadUserSettings, + saveSettings, + withSettingsWriteLock, +} from '../../../../src/utils/settings-config' +import * as audit from '../../../../src/utils/record-admin-audit' + +const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)) + +const makeResponse = () => { + const response: any = { + status: Sinon.stub(), + setHeader: Sinon.stub(), + send: Sinon.stub(), + } + response.status.returns(response) + response.setHeader.returns(response) + response.send.returns(response) + + return response +} + +Before({ tags: '@settings-write-lock' }, function (this: World>) { + this.parameters.settingsConfigDir = process.env.NOSTR_CONFIG_DIR + this.parameters.settingsTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'nostream-settings-integration-')) + process.env.NOSTR_CONFIG_DIR = this.parameters.settingsTmpDir + this.parameters.settingsSandbox = Sinon.createSandbox() + this.parameters.settingsSandbox.stub(audit, 'recordAdminAuditFromRequest').resolves() + this.parameters.settingsOutbox = { enqueue: Sinon.stub().resolves() } + this.parameters.settingsResponses = {} + + ensureSettingsExists() +}) + +After({ tags: '@settings-write-lock' }, function (this: World>) { + this.parameters.settingsSandbox?.restore() + fs.rmSync(this.parameters.settingsTmpDir, { recursive: true, force: true }) + if (this.parameters.settingsConfigDir === undefined) { + delete process.env.NOSTR_CONFIG_DIR + } else { + process.env.NOSTR_CONFIG_DIR = this.parameters.settingsConfigDir + } + + this.parameters.settingsConfigDir = undefined + this.parameters.settingsTmpDir = undefined + this.parameters.settingsSandbox = undefined + this.parameters.settingsOutbox = undefined + this.parameters.settingsResponses = {} + this.parameters.settingsLockOrder = undefined + this.parameters.settingsLockMetadata = undefined +}) + +When('two settings writers queue for the lock', async function (this: World>) { + const lockPath = getSettingsWriteLockPath() + const order: string[] = [] + let acquired!: () => void + let release!: () => void + const firstAcquired = new Promise((resolve) => { + acquired = resolve + }) + const holdFirst = new Promise((resolve) => { + release = resolve + }) + + const first = withSettingsWriteLock(async () => { + order.push('first') + this.parameters.settingsLockMetadata = fs.readFileSync(lockPath, 'utf-8').trim().split('\n') + acquired() + await holdFirst + }) + + await firstAcquired + const second = withSettingsWriteLock(() => { + order.push('second') + }) + + await delay(50) + expect(order).to.deep.equal(['first']) + + release() + await Promise.all([first, second]) + this.parameters.settingsLockOrder = order +}) + +Then('the writers run one after the other', function (this: World>) { + expect(this.parameters.settingsLockOrder).to.deep.equal(['first', 'second']) +}) + +Then('the lock records the owner host and token', function (this: World>) { + const [pid, acquiredAt, hostname, token] = this.parameters.settingsLockMetadata as string[] + + expect(Number(pid)).to.equal(process.pid) + expect(Number(acquiredAt)).to.be.greaterThan(0) + expect(hostname).to.equal(os.hostname()) + expect(token).to.match(/^[0-9a-f-]{36}$/) + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) +}) + +When( + 'the admin notifications controller saves settings', + async function (this: World>) { + const controller = new PatchAdminNotificationsController(this.parameters.settingsOutbox) + const response = makeResponse() + this.parameters.settingsResponses.notifications = response + + await controller.handleRequest({ body: { enabled: true }, ip: '127.0.0.1' } as any, response) + }, +) + +Then( + 'the notifications are saved and the lock is released', + function (this: World>) { + const response = this.parameters.settingsResponses.notifications + const saved = loadUserSettings() as any + + Sinon.assert.calledWith(response.status, 200) + expect(saved.admin.notifications.enabled).to.equal(true) + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) + }, +) + +When( + 'the admin settings controller saves a client subscription limit', + async function (this: World>) { + const controller = new PatchAdminSettingsController(this.parameters.settingsOutbox) + const response = makeResponse() + this.parameters.settingsResponses.settings = response + + await controller.handleRequest( + { + body: { path: 'limits.client.subscription.maxSubscriptions', value: 12 }, + ip: '127.0.0.1', + } as any, + response, + ) + }, +) + +Then( + 'the saved settings contain that limit and the lock is released', + function (this: World>) { + const response = this.parameters.settingsResponses.settings + const saved = loadUserSettings() as any + + Sinon.assert.calledWith(response.status, 200) + expect(saved.limits.client.subscription.maxSubscriptions).to.equal(12) + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) + }, +) + +When('an admin settings backup is restored', async function (this: World>) { + const backupDir = getSettingsBackupDir() + fs.mkdirSync(backupDir, { recursive: true }) + const filename = 'settings.2026-11-02T12-00-00-000Z.yaml' + const current = loadUserSettings() + saveSettings({ ...current, info: { ...current.info, name: 'restored-from-integration' } } as any) + fs.copyFileSync(getSettingsFilePath(), path.join(backupDir, filename)) + + const controller = new PostAdminSettingsRestoreController() + const response = makeResponse() + this.parameters.settingsResponses.restore = response + + await controller.handleRequest({ body: { filename }, ip: '127.0.0.1' } as any, response) +}) + +Then( + 'the restored settings are saved and the lock is released', + function (this: World>) { + const response = this.parameters.settingsResponses.restore + const saved = loadUserSettings() as any + + Sinon.assert.calledWith(response.status, 200) + expect(saved.info.name).to.equal('restored-from-integration') + expect(fs.existsSync(getSettingsWriteLockPath())).to.equal(false) + }, +) diff --git a/test/unit/controllers/admin/patch-notifications-controller.spec.ts b/test/unit/controllers/admin/patch-notifications-controller.spec.ts index df34f311..ae6df86b 100644 --- a/test/unit/controllers/admin/patch-notifications-controller.spec.ts +++ b/test/unit/controllers/admin/patch-notifications-controller.spec.ts @@ -47,6 +47,7 @@ describe('PatchAdminNotificationsController', () => { send: sandbox.stub().returnsThis(), } + sandbox.stub(settingsConfig, 'withSettingsWriteLock').callsFake(async (operation: () => unknown) => operation()) sandbox.stub(settingsConfig, 'loadMergedSettings').returns(baseMergedSettings as any) sandbox.stub(settingsConfig, 'loadUserSettings').returns({ admin: { notifications: baseNotifications } } as any) sandbox.stub(settingsConfig, 'saveSettings') @@ -130,6 +131,30 @@ describe('PatchAdminNotificationsController', () => { expect(response.send.firstCall.args[0].ok).to.equal(true) }) + it('reads and saves inside the settings write lock', async () => { + const order: string[] = [] + ;(settingsConfig.withSettingsWriteLock as Sinon.SinonStub).callsFake(async (operation: () => unknown) => { + order.push('lock:enter') + try { + return operation() + } finally { + order.push('lock:exit') + } + }) + ;(settingsConfig.loadUserSettings as Sinon.SinonStub).callsFake(() => { + order.push('read') + return { admin: { notifications: baseNotifications } } + }) + ;(settingsConfig.saveSettings as Sinon.SinonStub).callsFake(() => { + order.push('save') + }) + + const controller = new PatchAdminNotificationsController(outboxRepository as any) + await controller.handleRequest({ body: { enabled: true }, ip: '127.0.0.1' } as any, response as any) + + expect(order).to.deep.equal(['lock:enter', 'read', 'save', 'lock:exit']) + }) + it('accepts the peering notification switches', async () => { const controller = new PatchAdminNotificationsController(outboxRepository as any) await controller.handleRequest( diff --git a/test/unit/controllers/admin/patch-settings-controller.spec.ts b/test/unit/controllers/admin/patch-settings-controller.spec.ts new file mode 100644 index 00000000..70dad7f3 --- /dev/null +++ b/test/unit/controllers/admin/patch-settings-controller.spec.ts @@ -0,0 +1,150 @@ +import chai from 'chai' +import Sinon from 'sinon' +import sinonChai from 'sinon-chai' + +import * as recordAdminAudit from '../../../../src/utils/record-admin-audit' +import * as settingsConfig from '../../../../src/utils/settings-config' + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { PatchAdminSettingsController } = require('../../../../src/controllers/admin/patch-settings-controller') + +chai.use(sinonChai) + +const { expect } = chai + +const baseSettings = { + payments: { enabled: false }, + info: { relay_url: 'wss://relay.example', name: 'test' }, +} + +describe('PatchAdminSettingsController settings write lock', () => { + let sandbox: Sinon.SinonSandbox + let outboxRepository: { enqueue: Sinon.SinonStub } + let response: { status: Sinon.SinonStub; setHeader: Sinon.SinonStub; send: Sinon.SinonStub } + let request: Record + + beforeEach(() => { + sandbox = Sinon.createSandbox() + outboxRepository = { enqueue: sandbox.stub().resolves() } + response = { + status: sandbox.stub().returnsThis(), + setHeader: sandbox.stub().returnsThis(), + send: sandbox.stub().returnsThis(), + } + request = { body: { path: 'payments.enabled', value: true }, ip: '127.0.0.1' } + + sandbox.stub(settingsConfig, 'withSettingsWriteLock').callsFake(async (operation: () => unknown) => operation()) + sandbox.stub(settingsConfig, 'loadUserSettings').returns(baseSettings as any) + sandbox.stub(settingsConfig, 'loadMergedSettings').returns(baseSettings as any) + sandbox.stub(settingsConfig, 'saveSettings') + sandbox.stub(settingsConfig, 'appendSettingsAuditLog') + sandbox.stub(settingsConfig, 'validatePathAgainstDefaults').returns([]) + sandbox.stub(settingsConfig, 'validateSettings').returns([]) + sandbox.stub(recordAdminAudit, 'recordAdminAuditFromRequest').resolves() + }) + + afterEach(() => { + sandbox.restore() + }) + + it('re-reads and saves inside the settings write lock', async () => { + const order: string[] = [] + ;(settingsConfig.withSettingsWriteLock as Sinon.SinonStub).callsFake(async (operation: () => unknown) => { + order.push('lock:enter') + try { + return operation() + } finally { + order.push('lock:exit') + } + }) + ;(settingsConfig.loadUserSettings as Sinon.SinonStub).callsFake(() => { + order.push('read') + return baseSettings + }) + ;(settingsConfig.loadMergedSettings as Sinon.SinonStub).callsFake(() => { + order.push('merged') + return baseSettings + }) + ;(settingsConfig.saveSettings as Sinon.SinonStub).callsFake(() => { + order.push('save') + }) + + const controller = new PatchAdminSettingsController(outboxRepository as any) + await controller.handleRequest(request as any, response as any) + + // The read has to happen after the lock is acquired. Otherwise a ban written + // by another worker between the read and the save would be overwritten. + expect(order).to.deep.equal(['lock:enter', 'read', 'merged', 'save', 'lock:exit']) + expect(response.status).to.have.been.calledOnceWithExactly(200) + }) + + it('keeps a moderation ban written before the lock is acquired', async () => { + let stored = { + payments: { enabled: false }, + limits: { event: { pubkey: { blacklist: [] as string[] } } }, + info: { relay_url: 'wss://relay.example', name: 'test' }, + } + ;(settingsConfig.withSettingsWriteLock as Sinon.SinonStub).callsFake(async (operation: () => unknown) => { + // Another worker bans a pubkey just before this writer gets the lock. + stored = { + ...stored, + limits: { event: { pubkey: { blacklist: ['banned-pubkey'] } } }, + } + return operation() + }) + ;(settingsConfig.loadUserSettings as Sinon.SinonStub).callsFake(() => stored) + ;(settingsConfig.loadMergedSettings as Sinon.SinonStub).callsFake(() => stored) + ;(settingsConfig.saveSettings as Sinon.SinonStub).callsFake((next: typeof stored) => { + stored = next + }) + + const controller = new PatchAdminSettingsController(outboxRepository as any) + await controller.handleRequest(request as any, response as any) + + // The ordinary save must not revert the ban; it also applies its own change. + expect(stored.limits.event.pubkey.blacklist).to.deep.equal(['banned-pubkey']) + expect(stored.payments.enabled).to.equal(true) + }) + + it('never reads settings outside the lock', async () => { + let locked = false + const readsWhileLocked: boolean[] = [] + ;(settingsConfig.withSettingsWriteLock as Sinon.SinonStub).callsFake(async (operation: () => unknown) => { + locked = true + try { + return operation() + } finally { + locked = false + } + }) + ;(settingsConfig.loadUserSettings as Sinon.SinonStub).callsFake(() => { + readsWhileLocked.push(locked) + return baseSettings + }) + ;(settingsConfig.loadMergedSettings as Sinon.SinonStub).callsFake(() => { + readsWhileLocked.push(locked) + return baseSettings + }) + + const controller = new PatchAdminSettingsController(outboxRepository as any) + await controller.handleRequest(request as any, response as any) + + expect(readsWhileLocked).to.deep.equal([true, true]) + }) + + it('does not save when merged validation fails', async () => { + ;(settingsConfig.validateSettings as Sinon.SinonStub).returns([{ path: 'payments', message: 'bad' }]) + + const controller = new PatchAdminSettingsController(outboxRepository as any) + await controller.handleRequest(request as any, response as any) + + expect(response.status).to.have.been.calledOnceWithExactly(400) + expect(response.send.firstCall.args[0]).to.deep.equal({ + error: 'Validation failed', + issues: [{ path: 'payments', message: 'bad' }], + }) + expect(settingsConfig.saveSettings).not.to.have.been.called + expect(settingsConfig.appendSettingsAuditLog).not.to.have.been.called + expect(outboxRepository.enqueue).not.to.have.been.called + }) +}) diff --git a/test/unit/utils/settings-config.spec.ts b/test/unit/utils/settings-config.spec.ts index fc6384e4..fc27f33e 100644 --- a/test/unit/utils/settings-config.spec.ts +++ b/test/unit/utils/settings-config.spec.ts @@ -3,10 +3,14 @@ import fs from 'fs' import os from 'os' import path from 'path' import Sinon from 'sinon' +import yaml from 'js-yaml' import { ensureSettingsExists, + getSettingsBackupDir, loadUserSettings, + restoreSettingsBackup, + saveSettings, toCategoryLabel, getByPath, getTopLevelSettingCategories, @@ -18,7 +22,7 @@ import { validateSettings, withSettingsWriteLock, } from '../../../src/utils/settings-config' -import { getSettingsWriteLockPath } from '../../../src/utils/settings-paths' +import { getSettingsFilePath, getSettingsWriteLockPath } from '../../../src/utils/settings-paths' import { toBech32 } from '../../../src/utils/transform' import { getGuidedSettingCategory, @@ -1047,18 +1051,132 @@ describe('settings write lock', () => { it('reclaims a lock whose owner pid is gone (ESRCH)', async () => { ensureSettingsExists() const lockPath = getSettingsWriteLockPath() - fs.writeFileSync(lockPath, `${process.pid}\n${Date.now()}\n`) + fs.writeFileSync(lockPath, `${process.pid}\n${Date.now()}\n${os.hostname()}\nowner-token\n`) Sinon.stub(process, 'kill').throws(Object.assign(new Error('no such process'), { code: 'ESRCH' })) expect(await withSettingsWriteLock(() => 'ran')).to.equal('ran') expect(fs.existsSync(lockPath)).to.equal(false) }) + it('does not steal a fresh lock from another host even when the pid is gone', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + const acquiredAt = Date.now() + // A Docker relay and a host CLI share .nostr but not their PID namespaces. + fs.writeFileSync(lockPath, `5\n${acquiredAt}\nrelay-container\nother-token\n`) + Sinon.stub(process, 'kill').throws(Object.assign(new Error('no such process'), { code: 'ESRCH' })) + + let now = acquiredAt + Sinon.stub(Date, 'now').callsFake(() => now) + setTimeout(() => { + now = acquiredAt + 31_000 + }, 10) + + let error: Error | undefined + try { + await withSettingsWriteLock(() => 'ran') + } catch (err) { + error = err as Error + } + + expect(error?.message).to.equal('Timed out waiting for settings write lock') + expect(fs.existsSync(lockPath)).to.equal(true) + }) + + it('reclaims a lock from another host once its heartbeat stops', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + fs.writeFileSync(lockPath, `5\n${Date.now()}\nrelay-container\nother-token\n`) + const old = new Date(Date.now() - 61_000) + fs.utimesSync(lockPath, old, old) + + expect(await withSettingsWriteLock(() => 'ran')).to.equal('ran') + expect(fs.existsSync(lockPath)).to.equal(false) + }) + + const linuxIt = fs.existsSync('/proc/self/stat') ? it : it.skip + + linuxIt('reclaims a lock whose pid was reused after a restart', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + // Same host, same live PID, but a different process start time: a restart + // reused the PID while the old lock file stayed on the shared volume. + fs.writeFileSync(lockPath, `${process.pid}\n${Date.now()}\n${os.hostname()}\nowner-token\n1\n`) + + expect(await withSettingsWriteLock(() => 'ran')).to.equal('ran') + expect(fs.existsSync(lockPath)).to.equal(false) + }) + + it('does not steal a slow same-host writer with an old heartbeat', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + fs.writeFileSync(lockPath, `${process.pid}\n${Date.now()}\n${os.hostname()}\nowner-token\n`) + const old = new Date(Date.now() - 61_000) + fs.utimesSync(lockPath, old, old) + + let now = Date.now() + Sinon.stub(Date, 'now').callsFake(() => now) + setTimeout(() => { + now += 31_000 + }, 10) + + let error: Error | undefined + try { + await withSettingsWriteLock(() => 'ran') + } catch (err) { + error = err as Error + } + + expect(error?.message).to.equal('Timed out waiting for settings write lock') + expect(fs.existsSync(lockPath)).to.equal(true) + }) + + it('does not remove a lock that was replaced while it was held', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + + await withSettingsWriteLock(() => { + fs.rmSync(lockPath) + fs.writeFileSync(lockPath, `999\n${Date.now()}\n${os.hostname()}\nreplacement-token\n`) + }) + + expect(fs.existsSync(lockPath)).to.equal(true) + expect(fs.readFileSync(lockPath, 'utf-8')).to.contain('replacement-token') + }) + + it('holds the lock until an async operation settles', async () => { + ensureSettingsExists() + const lockPath = getSettingsWriteLockPath() + let release!: () => void + const held = new Promise((resolve) => { + release = resolve + }) + + const first = withSettingsWriteLock(async () => { + await held + return 'first' + }) + + await new Promise((resolve) => setTimeout(resolve, 20)) + expect(fs.existsSync(lockPath)).to.equal(true) + + let secondRan = false + const second = withSettingsWriteLock(() => { + secondRan = true + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect(secondRan).to.equal(false) + + release() + expect(await Promise.all([first, second])).to.deep.equal(['first', undefined]) + expect(secondRan).to.equal(true) + }) + it('does not steal a live lock when the liveness check fails with EPERM', async () => { ensureSettingsExists() const lockPath = getSettingsWriteLockPath() const acquiredAt = Date.now() - fs.writeFileSync(lockPath, `${process.pid}\n${acquiredAt}\n`) + fs.writeFileSync(lockPath, `${process.pid}\n${acquiredAt}\n${os.hostname()}\nowner-token\n`) Sinon.stub(process, 'kill').throws(Object.assign(new Error('not permitted'), { code: 'EPERM' })) let now = acquiredAt @@ -1100,4 +1218,44 @@ describe('settings write lock', () => { expect(error?.message).to.equal('Timed out waiting for settings write lock') expect(fs.existsSync(lockPath)).to.equal(true) }) + + it('restores a backup while holding the settings write lock', async () => { + const backupDir = getSettingsBackupDir() + fs.mkdirSync(backupDir, { recursive: true }) + const filename = 'settings.2026-11-02T12-00-00-000Z.yaml' + const defaults = loadDefaults() + + fs.writeFileSync( + path.join(backupDir, filename), + yaml.dump({ ...defaults, info: { ...defaults.info, name: 'restored-from-backup' } }), + ) + + const lockPath = getSettingsWriteLockPath() + const settingsPath = getSettingsFilePath() + const renameSync = fs.renameSync.bind(fs) + let lockHeldDuringWrite = false + + // saveSettings() swaps the new document into place with renameSync(). Seeing + // the lock during that call proves restoreSettingsBackup() still holds it; + // the test would fail if the lock were removed from the restore path. + const renameStub = Sinon.stub(fs, 'renameSync').callsFake((from, to) => { + if (to === settingsPath) { + lockHeldDuringWrite = fs.existsSync(lockPath) + } + + return renameSync(from, to) + }) + + try { + const restored = await restoreSettingsBackup(filename) + + expect(restored.info?.name).to.equal('restored-from-backup') + expect(loadUserSettings().info?.name).to.equal('restored-from-backup') + } finally { + renameStub.restore() + } + + expect(lockHeldDuringWrite).to.equal(true) + expect(fs.existsSync(lockPath)).to.equal(false) + }) })