diff --git a/.changeset/ws-admission-buffer.md b/.changeset/ws-admission-buffer.md new file mode 100644 index 000000000..2465fac33 --- /dev/null +++ b/.changeset/ws-admission-buffer.md @@ -0,0 +1,14 @@ +--- +"nostream": patch +--- + +fix(relay): handle messages sent immediately after a WebSocket opens + +Attaches a temporary message listener before the awaited connection +rate-limit check, pauses the socket during admission, and buffers early +frames with bounded limits (64 messages / 1 MiB). Once admission succeeds the +adapter is created, buffered frames are replayed in arrival order, and the +socket resumes. Rate-limited connections are still terminated before any +message is processed, and buffer overflow closes with code 1009 instead of +silently dropping frames. Adds regression tests for early-message replay, +rate-limited rejection, and overflow handling. diff --git a/src/adapters/web-socket-server-adapter.ts b/src/adapters/web-socket-server-adapter.ts index 99d5d69fe..c85fd57d5 100644 --- a/src/adapters/web-socket-server-adapter.ts +++ b/src/adapters/web-socket-server-adapter.ts @@ -1,15 +1,15 @@ import { IncomingMessage, Server } from 'http' -import WebSocket, { CLOSED, CLOSING, OPEN, WebSocketServer } from 'ws' import { propEq } from 'ramda' +import WebSocket, { CLOSED, CLOSING, OPEN, RawData, WebSocketServer } from 'ws' import { IWebSocketAdapter, IWebSocketServerAdapter } from '../@types/adapters' +import { Factory } from '../@types/base' +import { Event } from '../@types/event' +import { Settings } from '../@types/settings' import { WebSocketAdapterEvent, WebSocketServerAdapterEvent } from '../constants/adapter' import { createLogger } from '../factories/logger-factory' -import { Event } from '../@types/event' -import { Factory } from '../@types/base' -import { getRemoteAddress } from '../utils/http' import { isRateLimited } from '../handlers/request-handlers/rate-limiter-middleware' -import { Settings } from '../@types/settings' +import { getRemoteAddress } from '../utils/http' import { getWsDrainTimeoutMs, isDraining } from '../utils/shutdown-state' import { WebServerAdapter } from './web-server-adapter' @@ -17,6 +17,24 @@ const logger = createLogger('web-socket-server-adapter') const WSS_CLIENT_HEALTH_PROBE_INTERVAL = 120000 +// A connection can send immediately after the handshake, before the async +// rate-limit check returns. Hold those frames briefly; bound the queue so a +// flood cannot turn a slow admission check into unbounded memory. +const MAX_EARLY_MESSAGE_COUNT = 64 +const MAX_EARLY_MESSAGE_BYTES = 1024 * 1024 + +const rawDataByteLength = (data: RawData): number => { + if (Buffer.isBuffer(data)) { + return data.byteLength + } + + if (Array.isArray(data)) { + return data.reduce((total, chunk) => total + chunk.byteLength, 0) + } + + return data.byteLength +} + export class WebSocketServerAdapter extends WebServerAdapter implements IWebSocketServerAdapter { private webSocketsAdapters: WeakMap @@ -134,13 +152,82 @@ export class WebSocketServerAdapter extends WebServerAdapter implements IWebSock logger('client %s connected: %o', remoteAddress, req.headers) - if (await isRateLimited(remoteAddress, currentSettings)) { - logger('client %s terminated: rate-limited', remoteAddress) + // Attach before the awaited admission check: ws emits to whichever + // listeners exist at the moment a frame arrives, and EventEmitter drops + // events with no listener. Pause adds backpressure while we decide. + const earlyMessages: { data: RawData; isBinary: boolean }[] = [] + let earlyBytes = 0 + let earlyOverflow = false + + const onEarlyMessage = (data: RawData, isBinary: boolean): void => { + if (earlyMessages.length >= MAX_EARLY_MESSAGE_COUNT) { + earlyOverflow = true + return + } + + const size = rawDataByteLength(data) + if (earlyBytes + size > MAX_EARLY_MESSAGE_BYTES) { + earlyOverflow = true + return + } + + earlyBytes += size + earlyMessages.push({ data, isBinary }) + } + + client.on('message', onEarlyMessage) + client.pause() + + let accepted = false + try { + if (await isRateLimited(remoteAddress, currentSettings)) { + logger('client %s terminated: rate-limited', remoteAddress) + client.terminate() + return + } + + if (earlyOverflow) { + logger.warn('client %s closed: message backlog exceeded during admission', remoteAddress) + client.close(1009, 'message backlog before admission') + // Let the close handshake complete even though the socket was paused. + client.resume() + return + } + + if (client.readyState !== OPEN) { + // Admission can finish after the peer started closing. Resume before + // leaving so the close handshake is not stuck behind the paused socket. + client.resume() + return + } + + accepted = true + } catch (error) { + logger.error('client %s admission failed: %o', remoteAddress, error) + client.terminate() + return + } finally { + if (!accepted) { + client.off('message', onEarlyMessage) + } + } + + try { + this.webSocketsAdapters.set(client, this.createWebSocketAdapter([client, req, this])) + } catch (error) { + logger.error('client %s adapter creation failed: %o', remoteAddress, error) + client.off('message', onEarlyMessage) client.terminate() return } - this.webSocketsAdapters.set(client, this.createWebSocketAdapter([client, req, this])) + // The adapter now owns the normal listener. Replay in arrival order before + // resuming, so no buffered frame races a newly arrived one. + client.off('message', onEarlyMessage) + for (const early of earlyMessages) { + client.emit('message', early.data, early.isBinary) + } + client.resume() } private onHeartbeat() { diff --git a/test/unit/adapters/web-socket-server-adapter-real.spec.ts b/test/unit/adapters/web-socket-server-adapter-real.spec.ts new file mode 100644 index 000000000..4af64cd91 --- /dev/null +++ b/test/unit/adapters/web-socket-server-adapter-real.spec.ts @@ -0,0 +1,114 @@ +import http from 'node:http' +import { AddressInfo } from 'node:net' + +import chai from 'chai' +import Sinon from 'sinon' +import WebSocket, { WebSocketServer } from 'ws' + +import { WebSocketServerAdapter } from '../../../src/adapters/web-socket-server-adapter' +import * as rateLimiterMiddleware from '../../../src/handlers/request-handlers/rate-limiter-middleware' + +const { expect } = chai + +const waitFor = async (predicate: () => boolean, timeoutMs = 3_000): Promise => { + const startedAt = Date.now() + while (!predicate()) { + if (Date.now() - startedAt > timeoutMs) { + throw new Error('timed out waiting for condition') + } + await new Promise((resolve) => setTimeout(resolve, 10)) + } +} + +describe('WebSocketServerAdapter real socket admission', () => { + let sandbox: Sinon.SinonSandbox + let server: http.Server + let webSocketServer: WebSocketServer + let adapter: WebSocketServerAdapter + let isRateLimitedStub: Sinon.SinonStub + let receivedFrames: Buffer[] | undefined + const clients: WebSocket[] = [] + + const settings = () => + ({ + network: { remoteIpHeader: '' }, + limits: { connection: { rateLimits: [] } }, + }) as any + + beforeEach(async () => { + sandbox = Sinon.createSandbox() + receivedFrames = undefined + isRateLimitedStub = sandbox.stub(rateLimiterMiddleware, 'isRateLimited').resolves(false) + + server = http.createServer() + webSocketServer = new WebSocketServer({ server }) + + adapter = new WebSocketServerAdapter( + server, + webSocketServer, + ([client]: [WebSocket, unknown, unknown]) => { + const received: Buffer[] = [] + receivedFrames = received + client.on('message', (data: Buffer) => received.push(data)) + + return { + emit: () => undefined, + drainAndClose: () => client.close(), + getClientId: () => 'real-client', + getClientAddress: () => '127.0.0.1', + } as any + }, + settings, + ) + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + }) + + afterEach(async () => { + for (const client of clients.splice(0)) { + if (client.readyState === WebSocket.OPEN) { + client.terminate() + } + } + + await new Promise((resolve) => adapter.close(() => resolve())) + sandbox.restore() + }) + + it('delivers frames sent immediately on open across a delayed admission check', async () => { + let releaseAdmission: ((limited: boolean) => void) | undefined + isRateLimitedStub.returns( + new Promise((resolve) => { + releaseAdmission = resolve + }), + ) + + const address = server.address() as AddressInfo + const client = new WebSocket(`ws://127.0.0.1:${address.port}`) + clients.push(client) + + await new Promise((resolve, reject) => { + client.once('open', () => resolve()) + client.once('error', reject) + }) + + const frames = [ + Buffer.from('["REQ","immediate-one",{"kinds":[1]}]'), + Buffer.from('["REQ","immediate-two",{"kinds":[1]}]'), + ] + + for (const frame of frames) { + client.send(frame) + } + + // Give the frames time to reach the relay while admission is still pending. + await new Promise((resolve) => setTimeout(resolve, 50)) + releaseAdmission?.(false) + + await waitFor(() => (receivedFrames?.length ?? 0) >= frames.length) + + expect((receivedFrames ?? []).map((frame) => frame.toString())).to.deep.equal( + frames.map((frame) => frame.toString()), + ) + }) +}) diff --git a/test/unit/adapters/web-socket-server-adapter.spec.ts b/test/unit/adapters/web-socket-server-adapter.spec.ts index 628128b92..91fa93ad9 100644 --- a/test/unit/adapters/web-socket-server-adapter.spec.ts +++ b/test/unit/adapters/web-socket-server-adapter.spec.ts @@ -1,17 +1,17 @@ -import * as rateLimiterMiddleware from '../../../src/handlers/request-handlers/rate-limiter-middleware' - import chai from 'chai' import chaiAsPromised from 'chai-as-promised' +import EventEmitter from 'events' import Sinon from 'sinon' import sinonChai from 'sinon-chai' +import * as rateLimiterMiddleware from '../../../src/handlers/request-handlers/rate-limiter-middleware' chai.use(sinonChai) chai.use(chaiAsPromised) const { expect } = chai -import { WebSocketAdapterEvent, WebSocketServerAdapterEvent } from '../../../src/constants/adapter' import { WebSocketServerAdapter } from '../../../src/adapters/web-socket-server-adapter' +import { WebSocketAdapterEvent, WebSocketServerAdapterEvent } from '../../../src/constants/adapter' import * as shutdownState from '../../../src/utils/shutdown-state' describe('WebSocketServerAdapter', () => { @@ -31,6 +31,16 @@ describe('WebSocketServerAdapter', () => { await sandbox.clock.runAllAsync() } + const createMockClient = (overrides: Record = {}) => + Object.assign(new EventEmitter(), { + readyState: 1, + pause: sandbox.stub(), + resume: sandbox.stub(), + terminate: sandbox.stub(), + close: sandbox.stub(), + ...overrides, + }) + beforeEach(() => { sandbox = Sinon.createSandbox() sandbox.useFakeTimers() @@ -133,8 +143,8 @@ describe('WebSocketServerAdapter', () => { it('drains connected WebSocket clients before closing the server', async () => { const drainAndCloseStub1 = sandbox.stub() const drainAndCloseStub2 = sandbox.stub() - const client1 = { readyState: 1, once: sandbox.stub().callsFake((_event: string, cb: () => void) => cb()) } - const client2 = { readyState: 1, once: sandbox.stub().callsFake((_event: string, cb: () => void) => cb()) } + const client1 = createMockClient() + const client2 = createMockClient() const mockAdapter1 = { drainAndClose: drainAndCloseStub1, @@ -230,10 +240,9 @@ describe('WebSocketServerAdapter', () => { describe('onBroadcast', () => { it('emits event to adapters of all OPEN clients', async () => { - const OPEN = 1 const emitStub = sandbox.stub() - const mockClient = { readyState: OPEN } + const mockClient = createMockClient() webSocketServer.clients = new Set([mockClient] as any) const mockAdapter = { @@ -276,9 +285,8 @@ describe('WebSocketServerAdapter', () => { describe('onHeartbeat', () => { it('emits heartbeat to connected adapters on the heartbeat interval', async () => { const emitStub = sandbox.stub() - const OPEN = 1 - const mockClient = { readyState: OPEN } + const mockClient = createMockClient() webSocketServer.clients = new Set([mockClient] as any) const mockWsAdapter = { @@ -316,7 +324,7 @@ describe('WebSocketServerAdapter', () => { .find((call: any) => call.args[0] === WebSocketServerAdapterEvent.Connection) const onConnection = connectionCall.args[1] - const mockClient = {} + const mockClient = createMockClient() const mockReq = { headers: {}, socket: { remoteAddress: '127.0.0.1' }, @@ -325,10 +333,120 @@ describe('WebSocketServerAdapter', () => { await onConnection(mockClient, mockReq) expect(createWebSocketAdapter).to.have.been.calledOnce + expect(mockClient.pause).to.have.been.calledOnce + expect(mockClient.resume).to.have.been.calledOnce + }) + + it('buffers and replays messages that arrive during the admission check', async () => { + let releaseRateLimit: ((limited: boolean) => void) | undefined + isRateLimitedStub.returns( + new Promise((resolve) => { + releaseRateLimit = resolve + }), + ) + + const received: Buffer[] = [] + createWebSocketAdapter.callsFake(([client]: [any]) => { + client.on('message', (data: Buffer) => received.push(data)) + return { getClientId: () => 'test-id', getClientAddress: () => '127.0.0.1' } + }) + + const connectionCall = webSocketServer.on + .getCalls() + .find((call: any) => call.args[0] === WebSocketServerAdapterEvent.Connection) + const onConnection = connectionCall.args[1] + const mockClient = createMockClient() + const mockReq = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } + + const connection = onConnection(mockClient, mockReq) + const first = Buffer.from('["REQ","r1",{"kinds":[1]}]') + const second = Buffer.from('["EVENT",{}]') + mockClient.emit('message', first, false) + mockClient.emit('message', second, false) + releaseRateLimit?.(false) + await connection + + expect(received).to.deep.equal([first, second]) + expect(mockClient.resume).to.have.been.calledOnce + expect(createWebSocketAdapter).to.have.been.calledOnce + }) + + it('terminates rate-limited connections without replaying buffered messages', async () => { + let releaseRateLimit: ((limited: boolean) => void) | undefined + isRateLimitedStub.returns( + new Promise((resolve) => { + releaseRateLimit = resolve + }), + ) + + const connectionCall = webSocketServer.on + .getCalls() + .find((call: any) => call.args[0] === WebSocketServerAdapterEvent.Connection) + const onConnection = connectionCall.args[1] + const mockClient = createMockClient() + const mockReq = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } + + const connection = onConnection(mockClient, mockReq) + mockClient.emit('message', Buffer.from('["REQ","r1",{}]'), false) + releaseRateLimit?.(true) + await connection + + expect(mockClient.terminate).to.have.been.calledOnce + expect(createWebSocketAdapter).not.to.have.been.called + expect(mockClient.resume).not.to.have.been.called + }) + + it('closes connections that overflow the early-message buffer', async () => { + let releaseRateLimit: ((limited: boolean) => void) | undefined + isRateLimitedStub.returns( + new Promise((resolve) => { + releaseRateLimit = resolve + }), + ) + + const connectionCall = webSocketServer.on + .getCalls() + .find((call: any) => call.args[0] === WebSocketServerAdapterEvent.Connection) + const onConnection = connectionCall.args[1] + const mockClient = createMockClient() + const mockReq = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } + + const connection = onConnection(mockClient, mockReq) + for (let index = 0; index < 65; index++) { + mockClient.emit('message', Buffer.from('[]'), false) + } + releaseRateLimit?.(false) + await connection + + expect(mockClient.close).to.have.been.calledWith(1009, 'message backlog before admission') + expect(mockClient.resume).to.have.been.calledOnce + expect(createWebSocketAdapter).not.to.have.been.called + }) + + it('resumes the socket when admission finishes after the client started closing', async () => { + let releaseRateLimit: ((limited: boolean) => void) | undefined + isRateLimitedStub.returns( + new Promise((resolve) => { + releaseRateLimit = resolve + }), + ) + + const connectionCall = webSocketServer.on + .getCalls() + .find((call: any) => call.args[0] === WebSocketServerAdapterEvent.Connection) + const onConnection = connectionCall.args[1] + const mockClient = createMockClient({ readyState: 2 }) + const mockReq = { headers: {}, socket: { remoteAddress: '127.0.0.1' } } + + const connection = onConnection(mockClient, mockReq) + releaseRateLimit?.(false) + await connection + + expect(mockClient.resume).to.have.been.calledOnce + expect(createWebSocketAdapter).not.to.have.been.called }) it('terminates rate-limited connections', async () => { - const terminateStub = sandbox.stub() isRateLimitedStub.resolves(true) const connectionCall = webSocketServer.on @@ -336,7 +454,7 @@ describe('WebSocketServerAdapter', () => { .find((call: any) => call.args[0] === WebSocketServerAdapterEvent.Connection) const onConnection = connectionCall.args[1] - const mockClient = { terminate: terminateStub } + const mockClient = createMockClient() const mockReq = { headers: {}, socket: { remoteAddress: '127.0.0.1' }, @@ -344,8 +462,9 @@ describe('WebSocketServerAdapter', () => { await onConnection(mockClient, mockReq) - expect(terminateStub).to.have.been.calledOnce + expect(mockClient.terminate).to.have.been.calledOnce expect(createWebSocketAdapter).not.to.have.been.called + expect(mockClient.resume).not.to.have.been.called }) it('rejects new connections while draining', async () => {