Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .changeset/ws-admission-buffer.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
---
"nostream": patch
---

fix(relay): handle messages sent immediately after a WebSocket opens

Attaches a temporary message listener before the awaited connection
rate-limit check, pauses the socket during admission, and buffers early
frames with bounded limits (64 messages / 1 MiB). Once admission succeeds the
adapter is created, buffered frames are replayed in arrival order, and the
socket resumes. Rate-limited connections are still terminated before any
message is processed, and buffer overflow closes with code 1009 instead of
silently dropping frames. Adds regression tests for early-message replay,
rate-limited rejection, and overflow handling.
103 changes: 95 additions & 8 deletions src/adapters/web-socket-server-adapter.ts
Original file line number Diff line number Diff line change
@@ -1,22 +1,40 @@
import { IncomingMessage, Server } from 'http'
import WebSocket, { CLOSED, CLOSING, OPEN, WebSocketServer } from 'ws'
import { propEq } from 'ramda'
import WebSocket, { CLOSED, CLOSING, OPEN, RawData, WebSocketServer } from 'ws'

import { IWebSocketAdapter, IWebSocketServerAdapter } from '../@types/adapters'
import { Factory } from '../@types/base'
import { Event } from '../@types/event'
import { Settings } from '../@types/settings'
import { WebSocketAdapterEvent, WebSocketServerAdapterEvent } from '../constants/adapter'
import { createLogger } from '../factories/logger-factory'
import { Event } from '../@types/event'
import { Factory } from '../@types/base'
import { getRemoteAddress } from '../utils/http'
import { isRateLimited } from '../handlers/request-handlers/rate-limiter-middleware'
import { Settings } from '../@types/settings'
import { getRemoteAddress } from '../utils/http'
import { getWsDrainTimeoutMs, isDraining } from '../utils/shutdown-state'
import { WebServerAdapter } from './web-server-adapter'

const logger = createLogger('web-socket-server-adapter')

const WSS_CLIENT_HEALTH_PROBE_INTERVAL = 120000

// A connection can send immediately after the handshake, before the async
// rate-limit check returns. Hold those frames briefly; bound the queue so a
// flood cannot turn a slow admission check into unbounded memory.
const MAX_EARLY_MESSAGE_COUNT = 64
const MAX_EARLY_MESSAGE_BYTES = 1024 * 1024

const rawDataByteLength = (data: RawData): number => {
if (Buffer.isBuffer(data)) {
return data.byteLength
}

if (Array.isArray(data)) {
return data.reduce((total, chunk) => total + chunk.byteLength, 0)
}

return data.byteLength
}

export class WebSocketServerAdapter extends WebServerAdapter implements IWebSocketServerAdapter {
private webSocketsAdapters: WeakMap<WebSocket, IWebSocketAdapter>

Expand Down Expand Up @@ -134,13 +152,82 @@ export class WebSocketServerAdapter extends WebServerAdapter implements IWebSock

logger('client %s connected: %o', remoteAddress, req.headers)

if (await isRateLimited(remoteAddress, currentSettings)) {
logger('client %s terminated: rate-limited', remoteAddress)
// Attach before the awaited admission check: ws emits to whichever
// listeners exist at the moment a frame arrives, and EventEmitter drops
// events with no listener. Pause adds backpressure while we decide.
const earlyMessages: { data: RawData; isBinary: boolean }[] = []
let earlyBytes = 0
let earlyOverflow = false

const onEarlyMessage = (data: RawData, isBinary: boolean): void => {
if (earlyMessages.length >= MAX_EARLY_MESSAGE_COUNT) {
earlyOverflow = true
return
}

const size = rawDataByteLength(data)
if (earlyBytes + size > MAX_EARLY_MESSAGE_BYTES) {
earlyOverflow = true
return
}

earlyBytes += size
earlyMessages.push({ data, isBinary })
}

client.on('message', onEarlyMessage)
client.pause()

let accepted = false
try {
if (await isRateLimited(remoteAddress, currentSettings)) {
logger('client %s terminated: rate-limited', remoteAddress)
client.terminate()
return
}

if (earlyOverflow) {
logger.warn('client %s closed: message backlog exceeded during admission', remoteAddress)
client.close(1009, 'message backlog before admission')
// Let the close handshake complete even though the socket was paused.
client.resume()
return
}

if (client.readyState !== OPEN) {
// Admission can finish after the peer started closing. Resume before
// leaving so the close handshake is not stuck behind the paused socket.
client.resume()
return
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

accepted = true
} catch (error) {
logger.error('client %s admission failed: %o', remoteAddress, error)
client.terminate()
return
} finally {
if (!accepted) {
client.off('message', onEarlyMessage)
}
}

try {
this.webSocketsAdapters.set(client, this.createWebSocketAdapter([client, req, this]))
} catch (error) {
logger.error('client %s adapter creation failed: %o', remoteAddress, error)
client.off('message', onEarlyMessage)
client.terminate()
return
}

this.webSocketsAdapters.set(client, this.createWebSocketAdapter([client, req, this]))
// The adapter now owns the normal listener. Replay in arrival order before
// resuming, so no buffered frame races a newly arrived one.
client.off('message', onEarlyMessage)
for (const early of earlyMessages) {
client.emit('message', early.data, early.isBinary)
}
client.resume()
}

private onHeartbeat() {
Expand Down
114 changes: 114 additions & 0 deletions test/unit/adapters/web-socket-server-adapter-real.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import http from 'node:http'
import { AddressInfo } from 'node:net'

import chai from 'chai'
import Sinon from 'sinon'
import WebSocket, { WebSocketServer } from 'ws'

import { WebSocketServerAdapter } from '../../../src/adapters/web-socket-server-adapter'
import * as rateLimiterMiddleware from '../../../src/handlers/request-handlers/rate-limiter-middleware'

const { expect } = chai

const waitFor = async (predicate: () => boolean, timeoutMs = 3_000): Promise<void> => {
const startedAt = Date.now()
while (!predicate()) {
if (Date.now() - startedAt > timeoutMs) {
throw new Error('timed out waiting for condition')
}
await new Promise((resolve) => setTimeout(resolve, 10))
}
}

describe('WebSocketServerAdapter real socket admission', () => {
let sandbox: Sinon.SinonSandbox
let server: http.Server
let webSocketServer: WebSocketServer
let adapter: WebSocketServerAdapter
let isRateLimitedStub: Sinon.SinonStub
let receivedFrames: Buffer[] | undefined
const clients: WebSocket[] = []

const settings = () =>
({
network: { remoteIpHeader: '' },
limits: { connection: { rateLimits: [] } },
}) as any

beforeEach(async () => {
sandbox = Sinon.createSandbox()
receivedFrames = undefined
isRateLimitedStub = sandbox.stub(rateLimiterMiddleware, 'isRateLimited').resolves(false)

server = http.createServer()
webSocketServer = new WebSocketServer({ server })

adapter = new WebSocketServerAdapter(
server,
webSocketServer,
([client]: [WebSocket, unknown, unknown]) => {
const received: Buffer[] = []
receivedFrames = received
client.on('message', (data: Buffer) => received.push(data))

return {
emit: () => undefined,
drainAndClose: () => client.close(),
getClientId: () => 'real-client',
getClientAddress: () => '127.0.0.1',
} as any
},
settings,
)

await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve))
})

afterEach(async () => {
for (const client of clients.splice(0)) {
if (client.readyState === WebSocket.OPEN) {
client.terminate()
}
}

await new Promise<void>((resolve) => adapter.close(() => resolve()))
sandbox.restore()
})

it('delivers frames sent immediately on open across a delayed admission check', async () => {
let releaseAdmission: ((limited: boolean) => void) | undefined
isRateLimitedStub.returns(
new Promise<boolean>((resolve) => {
releaseAdmission = resolve
}),
)

const address = server.address() as AddressInfo
const client = new WebSocket(`ws://127.0.0.1:${address.port}`)
clients.push(client)

await new Promise<void>((resolve, reject) => {
client.once('open', () => resolve())
client.once('error', reject)
})

const frames = [
Buffer.from('["REQ","immediate-one",{"kinds":[1]}]'),
Buffer.from('["REQ","immediate-two",{"kinds":[1]}]'),
]

for (const frame of frames) {
client.send(frame)
}

// Give the frames time to reach the relay while admission is still pending.
await new Promise((resolve) => setTimeout(resolve, 50))
releaseAdmission?.(false)

await waitFor(() => (receivedFrames?.length ?? 0) >= frames.length)

expect((receivedFrames ?? []).map((frame) => frame.toString())).to.deep.equal(
frames.map((frame) => frame.toString()),
)
})
})
Loading
Loading