Cap SSE subscribers per client IP instead of only globally (B-38)

GET /rounds/stream capped concurrent subscribers with one global
counter (MAX_SUBSCRIBERS=500): anyone opening 500 connections degraded
every other user to polling. The comment called it a defensive cap;
it was actually the vector, since nothing stopped a single source from
exhausting it alone.

RoundEventBroadcaster now also tracks subscribers per client IP,
capped much lower (MAX_SUBSCRIBERS_PER_IP=5). Past that cap, opening
one more stream evicts that same IP's own oldest connection (woken via
a new EVICTED sentinel so the SSE generator closes it promptly) rather
than refusing the new one or letting one abusive IP crowd out unrelated
clients under the old global-only cap. The global cap stays as a
backstop against overall resource exhaustion regardless of source.

Extracted client_ip() (X-Forwarded-For, since Caddy reverse-proxies
every request) out of auth/routes.py into app/api/client_ip.py so the
login/registration throttles (B-33) and this new per-IP cap share one
definition instead of two that could drift apart.

Not implemented: enforcing the connection cap at the Caddy layer
itself, which the proposed fix also suggested - the standard Caddy
image this project uses has no such directive without a third-party
module, and building a custom image felt like a bigger, separate change
than this fix warranted.

Suite grows from 201 to 211 tests. BUGS.md moves B-38 to Previously
fixed.
This commit is contained in:
davide committed 2026-07-27 14:44:15 +02:00
1 parent 0b44fe632e
commit 702b37b319
7 files changed
+201 -45

No files matched your search

+17
View File
@@ -0,0 +1,17 @@
from fastapi import Request
def client_ip(request: Request) -> str:
"""The caller's real IP, from Caddy's X-Forwarded-For (see Caddyfile) —
request.client.host would otherwise be the reverse proxy's own address, not
the caller's. Falls back to request.client.host only if the header is
somehow missing (e.g. the app container hit directly, bypassing Caddy).
Shared by the login/registration throttles (B-33) and the SSE per-IP
subscriber cap (B-38) so the two can't drift into different notions of
"the client's IP".
"""
forwarded = request.headers.get("x-forwarded-for")
if forwarded:
return forwarded.split(",")[0].strip()
return request.client.host if request.client else "unknown"
+11 -3
View File
@@ -8,12 +8,13 @@ from pydantic import BaseModel
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.client_ip import client_ip
from app.api.timeutil import isoformat_utc
from app.auth.dependencies import get_optional_user
from app.db.models import RoundParticipant, User
from app.db.session import get_session
from app.rounds.config import get_round_config
from app.rounds.events import RoundEventCapacityError, broadcaster
from app.rounds.events import EVICTED, RoundEventCapacityError, broadcaster
from app.rounds.service import get_active_round
router = APIRouter(prefix="/rounds", tags=["rounds"])
@@ -46,9 +47,14 @@ async def round_stream(request: Request) -> Response:
That's also what happens past MAX_SUBSCRIBERS (app/rounds/events.py): this
returns 503 rather than opening a stream, and the browser's EventSource
just retries later while the frontend keeps working off polling meanwhile.
Concurrent streams are additionally capped per client IP (B-38): past
MAX_SUBSCRIBERS_PER_IP, opening one more evicts that IP's own oldest
connection rather than refusing the new one or letting a single source
exhaust the global cap and degrade every other user.
"""
try:
queue = broadcaster.subscribe()
queue = broadcaster.subscribe(client_ip(request))
except RoundEventCapacityError:
return JSONResponse(status_code=503, content={"detail": "too many concurrent update streams"})
@@ -59,7 +65,9 @@ async def round_stream(request: Request) -> Response:
if await request.is_disconnected():
break
try:
await asyncio.wait_for(queue.get(), timeout=_SSE_DISCONNECT_CHECK_SECONDS)
item = await asyncio.wait_for(queue.get(), timeout=_SSE_DISCONNECT_CHECK_SECONDS)
if item is EVICTED:
break # this IP opened another stream past its per-IP cap
yield "event: update\ndata: {}\n\n".format(json.dumps({}))
ticks_since_keepalive = 0
except asyncio.TimeoutError:
+1 -10
View File
@@ -4,6 +4,7 @@ from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
from app.api.client_ip import client_ip as _client_ip
from app.api.errors import http_error
from app.auth.rate_limit import AuthRateLimiters
from app.auth.security import MIN_PASSWORD_LENGTH, create_access_token, hash_password, verify_password
@@ -34,16 +35,6 @@ def _rate_limiters(request: Request) -> AuthRateLimiters:
return request.app.state.auth_rate_limiters
def _client_ip(request: Request) -> str:
# Caddy (see Caddyfile) reverse-proxies every request, so request.client.host
# is the proxy's address, not the caller's — fall back to it only if the
# header is somehow missing (e.g. hitting the app container directly).
forwarded = request.headers.get("x-forwarded-for")
if forwarded:
return forwarded.split(",")[0].strip()
return request.client.host if request.client else "unknown"
def _rate_limited_error(retry_after: float):
return http_error(
status.HTTP_429_TOO_MANY_REQUESTS,
+54 -14
View File
@@ -1,16 +1,31 @@
import asyncio
from collections import defaultdict
# Defensive cap on concurrent SSE subscribers. Expected load is on the order of
# ~100 concurrent users; this is set well above that so it never engages under
# normal use — it exists purely so a runaway/DoS-y number of open connections
# degrades (new connections fall back to polling, see round_stream()) instead
# of growing the in-memory subscriber set without bound. Revisit this number if
# expected concurrency grows well past it.
# Defensive backstop on concurrent SSE subscribers overall, regardless of source
# — expected load is on the order of ~100 concurrent users, so this is set well
# above that. The real defense against a single abusive source is the per-IP cap
# below (B-38): a global-only cap was trivially exhausted by one client opening
# MAX_SUBSCRIBERS connections, degrading every other user to polling — the
# comment used to call it "defensive"; it was actually the vector.
MAX_SUBSCRIBERS = 500
# How many concurrent streams a single client IP may hold. Deliberately small —
# a real browser tab needs at most one, occasionally two briefly across a
# reload — since this bounds one source's share of the global capacity, not a
# legitimate per-user concurrency limit.
MAX_SUBSCRIBERS_PER_IP = 5
# Put on a to-be-evicted subscriber's queue (B-38) to wake its generator
# (app/api/routes/rounds.py:round_stream) promptly so it closes the connection
# instead of lingering, silently uncounted, until the client's own network
# timeout or the next keep-alive tick.
EVICTED = object()
class RoundEventCapacityError(Exception):
"""Raised by subscribe() when MAX_SUBSCRIBERS is already reached."""
"""Raised by subscribe() when MAX_SUBSCRIBERS — the global backstop — is
already reached. The per-IP cap never raises this; it evicts instead (see
subscribe())."""
class RoundEventBroadcaster:
@@ -26,22 +41,47 @@ class RoundEventBroadcaster:
would need a shared channel (e.g. Redis pub/sub) instead.
"""
def __init__(self, max_subscribers: int = MAX_SUBSCRIBERS):
self._subscribers: set[asyncio.Queue] = set()
def __init__(self, max_subscribers: int = MAX_SUBSCRIBERS, max_per_ip: int = MAX_SUBSCRIBERS_PER_IP):
self._ip_by_queue: dict[asyncio.Queue, str] = {}
self._queues_by_ip: dict[str, list[asyncio.Queue]] = defaultdict(list)
self.max_subscribers = max_subscribers
self.max_per_ip = max_per_ip
def subscribe(self) -> asyncio.Queue:
if len(self._subscribers) >= self.max_subscribers:
def subscribe(self, client_ip: str = "unknown") -> asyncio.Queue:
if len(self._ip_by_queue) >= self.max_subscribers:
raise RoundEventCapacityError(f"already at the {self.max_subscribers}-subscriber cap")
ip_queues = self._queues_by_ip[client_ip]
if len(ip_queues) >= self.max_per_ip:
# B-38: evict this IP's own oldest connection rather than refusing
# the new one — bounds one source's footprint without turning a
# legitimate reconnect storm (a flaky network retrying EventSource)
# into an outright block, and without letting one abusive IP crowd
# out unrelated clients the way the old global-only cap did.
oldest = ip_queues.pop(0)
self._ip_by_queue.pop(oldest, None)
if not oldest.full():
oldest.put_nowait(EVICTED)
queue: asyncio.Queue = asyncio.Queue(maxsize=1)
self._subscribers.add(queue)
self._ip_by_queue[queue] = client_ip
ip_queues.append(queue)
return queue
def unsubscribe(self, queue: asyncio.Queue) -> None:
self._subscribers.discard(queue)
client_ip = self._ip_by_queue.pop(queue, None)
if client_ip is None:
return
ip_queues = self._queues_by_ip.get(client_ip)
if ip_queues is None:
return
if queue in ip_queues:
ip_queues.remove(queue)
if not ip_queues:
self._queues_by_ip.pop(client_ip, None)
def publish(self) -> None:
for queue in self._subscribers:
for queue in self._ip_by_queue:
if queue.full():
continue # a not-yet-delivered notification already covers this one
queue.put_nowait(None)