Cap SSE subscribers per client IP instead of only globally (B-38)
GET /rounds/stream capped concurrent subscribers with one global counter (MAX_SUBSCRIBERS=500): anyone opening 500 connections degraded every other user to polling. The comment called it a defensive cap; it was actually the vector, since nothing stopped a single source from exhausting it alone. RoundEventBroadcaster now also tracks subscribers per client IP, capped much lower (MAX_SUBSCRIBERS_PER_IP=5). Past that cap, opening one more stream evicts that same IP's own oldest connection (woken via a new EVICTED sentinel so the SSE generator closes it promptly) rather than refusing the new one or letting one abusive IP crowd out unrelated clients under the old global-only cap. The global cap stays as a backstop against overall resource exhaustion regardless of source. Extracted client_ip() (X-Forwarded-For, since Caddy reverse-proxies every request) out of auth/routes.py into app/api/client_ip.py so the login/registration throttles (B-33) and this new per-IP cap share one definition instead of two that could drift apart. Not implemented: enforcing the connection cap at the Caddy layer itself, which the proposed fix also suggested - the standard Caddy image this project uses has no such directive without a third-party module, and building a custom image felt like a bigger, separate change than this fix warranted. Suite grows from 201 to 211 tests. BUGS.md moves B-38 to Previously fixed.
This commit is contained in:
1 parent
0b44fe632e
commit
702b37b319
7 files changed
+201
-45
No files matched your search
+54
-14
@@ -1,16 +1,31 @@
|
||||
import asyncio
|
||||
from collections import defaultdict
|
||||
|
||||
# Defensive cap on concurrent SSE subscribers. Expected load is on the order of
|
||||
# ~100 concurrent users; this is set well above that so it never engages under
|
||||
# normal use — it exists purely so a runaway/DoS-y number of open connections
|
||||
# degrades (new connections fall back to polling, see round_stream()) instead
|
||||
# of growing the in-memory subscriber set without bound. Revisit this number if
|
||||
# expected concurrency grows well past it.
|
||||
# Defensive backstop on concurrent SSE subscribers overall, regardless of source
|
||||
# — expected load is on the order of ~100 concurrent users, so this is set well
|
||||
# above that. The real defense against a single abusive source is the per-IP cap
|
||||
# below (B-38): a global-only cap was trivially exhausted by one client opening
|
||||
# MAX_SUBSCRIBERS connections, degrading every other user to polling — the
|
||||
# comment used to call it "defensive"; it was actually the vector.
|
||||
MAX_SUBSCRIBERS = 500
|
||||
|
||||
# How many concurrent streams a single client IP may hold. Deliberately small —
|
||||
# a real browser tab needs at most one, occasionally two briefly across a
|
||||
# reload — since this bounds one source's share of the global capacity, not a
|
||||
# legitimate per-user concurrency limit.
|
||||
MAX_SUBSCRIBERS_PER_IP = 5
|
||||
|
||||
# Put on a to-be-evicted subscriber's queue (B-38) to wake its generator
|
||||
# (app/api/routes/rounds.py:round_stream) promptly so it closes the connection
|
||||
# instead of lingering, silently uncounted, until the client's own network
|
||||
# timeout or the next keep-alive tick.
|
||||
EVICTED = object()
|
||||
|
||||
|
||||
class RoundEventCapacityError(Exception):
|
||||
"""Raised by subscribe() when MAX_SUBSCRIBERS is already reached."""
|
||||
"""Raised by subscribe() when MAX_SUBSCRIBERS — the global backstop — is
|
||||
already reached. The per-IP cap never raises this; it evicts instead (see
|
||||
subscribe())."""
|
||||
|
||||
|
||||
class RoundEventBroadcaster:
|
||||
@@ -26,22 +41,47 @@ class RoundEventBroadcaster:
|
||||
would need a shared channel (e.g. Redis pub/sub) instead.
|
||||
"""
|
||||
|
||||
def __init__(self, max_subscribers: int = MAX_SUBSCRIBERS):
|
||||
self._subscribers: set[asyncio.Queue] = set()
|
||||
def __init__(self, max_subscribers: int = MAX_SUBSCRIBERS, max_per_ip: int = MAX_SUBSCRIBERS_PER_IP):
|
||||
self._ip_by_queue: dict[asyncio.Queue, str] = {}
|
||||
self._queues_by_ip: dict[str, list[asyncio.Queue]] = defaultdict(list)
|
||||
self.max_subscribers = max_subscribers
|
||||
self.max_per_ip = max_per_ip
|
||||
|
||||
def subscribe(self) -> asyncio.Queue:
|
||||
if len(self._subscribers) >= self.max_subscribers:
|
||||
def subscribe(self, client_ip: str = "unknown") -> asyncio.Queue:
|
||||
if len(self._ip_by_queue) >= self.max_subscribers:
|
||||
raise RoundEventCapacityError(f"already at the {self.max_subscribers}-subscriber cap")
|
||||
|
||||
ip_queues = self._queues_by_ip[client_ip]
|
||||
if len(ip_queues) >= self.max_per_ip:
|
||||
# B-38: evict this IP's own oldest connection rather than refusing
|
||||
# the new one — bounds one source's footprint without turning a
|
||||
# legitimate reconnect storm (a flaky network retrying EventSource)
|
||||
# into an outright block, and without letting one abusive IP crowd
|
||||
# out unrelated clients the way the old global-only cap did.
|
||||
oldest = ip_queues.pop(0)
|
||||
self._ip_by_queue.pop(oldest, None)
|
||||
if not oldest.full():
|
||||
oldest.put_nowait(EVICTED)
|
||||
|
||||
queue: asyncio.Queue = asyncio.Queue(maxsize=1)
|
||||
self._subscribers.add(queue)
|
||||
self._ip_by_queue[queue] = client_ip
|
||||
ip_queues.append(queue)
|
||||
return queue
|
||||
|
||||
def unsubscribe(self, queue: asyncio.Queue) -> None:
|
||||
self._subscribers.discard(queue)
|
||||
client_ip = self._ip_by_queue.pop(queue, None)
|
||||
if client_ip is None:
|
||||
return
|
||||
ip_queues = self._queues_by_ip.get(client_ip)
|
||||
if ip_queues is None:
|
||||
return
|
||||
if queue in ip_queues:
|
||||
ip_queues.remove(queue)
|
||||
if not ip_queues:
|
||||
self._queues_by_ip.pop(client_ip, None)
|
||||
|
||||
def publish(self) -> None:
|
||||
for queue in self._subscribers:
|
||||
for queue in self._ip_by_queue:
|
||||
if queue.full():
|
||||
continue # a not-yet-delivered notification already covers this one
|
||||
queue.put_nowait(None)
|
||||
|
||||
Reference in new issue
Block a user