Cap SSE subscribers per client IP instead of only globally (B-38)
GET /rounds/stream capped concurrent subscribers with one global counter (MAX_SUBSCRIBERS=500): anyone opening 500 connections degraded every other user to polling. The comment called it a defensive cap; it was actually the vector, since nothing stopped a single source from exhausting it alone. RoundEventBroadcaster now also tracks subscribers per client IP, capped much lower (MAX_SUBSCRIBERS_PER_IP=5). Past that cap, opening one more stream evicts that same IP's own oldest connection (woken via a new EVICTED sentinel so the SSE generator closes it promptly) rather than refusing the new one or letting one abusive IP crowd out unrelated clients under the old global-only cap. The global cap stays as a backstop against overall resource exhaustion regardless of source. Extracted client_ip() (X-Forwarded-For, since Caddy reverse-proxies every request) out of auth/routes.py into app/api/client_ip.py so the login/registration throttles (B-33) and this new per-IP cap share one definition instead of two that could drift apart. Not implemented: enforcing the connection cap at the Caddy layer itself, which the proposed fix also suggested - the standard Caddy image this project uses has no such directive without a third-party module, and building a custom image felt like a bigger, separate change than this fix warranted. Suite grows from 201 to 211 tests. BUGS.md moves B-38 to Previously fixed.
This commit is contained in:
1 parent
0b44fe632e
commit
702b37b319
7 files changed
+201
-45
No files matched your search
@@ -0,0 +1,17 @@
|
||||
from fastapi import Request
|
||||
|
||||
|
||||
def client_ip(request: Request) -> str:
|
||||
"""The caller's real IP, from Caddy's X-Forwarded-For (see Caddyfile) —
|
||||
request.client.host would otherwise be the reverse proxy's own address, not
|
||||
the caller's. Falls back to request.client.host only if the header is
|
||||
somehow missing (e.g. the app container hit directly, bypassing Caddy).
|
||||
|
||||
Shared by the login/registration throttles (B-33) and the SSE per-IP
|
||||
subscriber cap (B-38) so the two can't drift into different notions of
|
||||
"the client's IP".
|
||||
"""
|
||||
forwarded = request.headers.get("x-forwarded-for")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()
|
||||
return request.client.host if request.client else "unknown"
|
||||
@@ -8,12 +8,13 @@ from pydantic import BaseModel
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.client_ip import client_ip
|
||||
from app.api.timeutil import isoformat_utc
|
||||
from app.auth.dependencies import get_optional_user
|
||||
from app.db.models import RoundParticipant, User
|
||||
from app.db.session import get_session
|
||||
from app.rounds.config import get_round_config
|
||||
from app.rounds.events import RoundEventCapacityError, broadcaster
|
||||
from app.rounds.events import EVICTED, RoundEventCapacityError, broadcaster
|
||||
from app.rounds.service import get_active_round
|
||||
|
||||
router = APIRouter(prefix="/rounds", tags=["rounds"])
|
||||
@@ -46,9 +47,14 @@ async def round_stream(request: Request) -> Response:
|
||||
That's also what happens past MAX_SUBSCRIBERS (app/rounds/events.py): this
|
||||
returns 503 rather than opening a stream, and the browser's EventSource
|
||||
just retries later while the frontend keeps working off polling meanwhile.
|
||||
|
||||
Concurrent streams are additionally capped per client IP (B-38): past
|
||||
MAX_SUBSCRIBERS_PER_IP, opening one more evicts that IP's own oldest
|
||||
connection rather than refusing the new one or letting a single source
|
||||
exhaust the global cap and degrade every other user.
|
||||
"""
|
||||
try:
|
||||
queue = broadcaster.subscribe()
|
||||
queue = broadcaster.subscribe(client_ip(request))
|
||||
except RoundEventCapacityError:
|
||||
return JSONResponse(status_code=503, content={"detail": "too many concurrent update streams"})
|
||||
|
||||
@@ -59,7 +65,9 @@ async def round_stream(request: Request) -> Response:
|
||||
if await request.is_disconnected():
|
||||
break
|
||||
try:
|
||||
await asyncio.wait_for(queue.get(), timeout=_SSE_DISCONNECT_CHECK_SECONDS)
|
||||
item = await asyncio.wait_for(queue.get(), timeout=_SSE_DISCONNECT_CHECK_SECONDS)
|
||||
if item is EVICTED:
|
||||
break # this IP opened another stream past its per-IP cap
|
||||
yield "event: update\ndata: {}\n\n".format(json.dumps({}))
|
||||
ticks_since_keepalive = 0
|
||||
except asyncio.TimeoutError:
|
||||
|
||||
+1
-10
@@ -4,6 +4,7 @@ from sqlalchemy import func, select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.api.client_ip import client_ip as _client_ip
|
||||
from app.api.errors import http_error
|
||||
from app.auth.rate_limit import AuthRateLimiters
|
||||
from app.auth.security import MIN_PASSWORD_LENGTH, create_access_token, hash_password, verify_password
|
||||
@@ -34,16 +35,6 @@ def _rate_limiters(request: Request) -> AuthRateLimiters:
|
||||
return request.app.state.auth_rate_limiters
|
||||
|
||||
|
||||
def _client_ip(request: Request) -> str:
|
||||
# Caddy (see Caddyfile) reverse-proxies every request, so request.client.host
|
||||
# is the proxy's address, not the caller's — fall back to it only if the
|
||||
# header is somehow missing (e.g. hitting the app container directly).
|
||||
forwarded = request.headers.get("x-forwarded-for")
|
||||
if forwarded:
|
||||
return forwarded.split(",")[0].strip()
|
||||
return request.client.host if request.client else "unknown"
|
||||
|
||||
|
||||
def _rate_limited_error(retry_after: float):
|
||||
return http_error(
|
||||
status.HTTP_429_TOO_MANY_REQUESTS,
|
||||
|
||||
+54
-14
@@ -1,16 +1,31 @@
|
||||
import asyncio
|
||||
from collections import defaultdict
|
||||
|
||||
# Defensive cap on concurrent SSE subscribers. Expected load is on the order of
|
||||
# ~100 concurrent users; this is set well above that so it never engages under
|
||||
# normal use — it exists purely so a runaway/DoS-y number of open connections
|
||||
# degrades (new connections fall back to polling, see round_stream()) instead
|
||||
# of growing the in-memory subscriber set without bound. Revisit this number if
|
||||
# expected concurrency grows well past it.
|
||||
# Defensive backstop on concurrent SSE subscribers overall, regardless of source
|
||||
# — expected load is on the order of ~100 concurrent users, so this is set well
|
||||
# above that. The real defense against a single abusive source is the per-IP cap
|
||||
# below (B-38): a global-only cap was trivially exhausted by one client opening
|
||||
# MAX_SUBSCRIBERS connections, degrading every other user to polling — the
|
||||
# comment used to call it "defensive"; it was actually the vector.
|
||||
MAX_SUBSCRIBERS = 500
|
||||
|
||||
# How many concurrent streams a single client IP may hold. Deliberately small —
|
||||
# a real browser tab needs at most one, occasionally two briefly across a
|
||||
# reload — since this bounds one source's share of the global capacity, not a
|
||||
# legitimate per-user concurrency limit.
|
||||
MAX_SUBSCRIBERS_PER_IP = 5
|
||||
|
||||
# Put on a to-be-evicted subscriber's queue (B-38) to wake its generator
|
||||
# (app/api/routes/rounds.py:round_stream) promptly so it closes the connection
|
||||
# instead of lingering, silently uncounted, until the client's own network
|
||||
# timeout or the next keep-alive tick.
|
||||
EVICTED = object()
|
||||
|
||||
|
||||
class RoundEventCapacityError(Exception):
|
||||
"""Raised by subscribe() when MAX_SUBSCRIBERS is already reached."""
|
||||
"""Raised by subscribe() when MAX_SUBSCRIBERS — the global backstop — is
|
||||
already reached. The per-IP cap never raises this; it evicts instead (see
|
||||
subscribe())."""
|
||||
|
||||
|
||||
class RoundEventBroadcaster:
|
||||
@@ -26,22 +41,47 @@ class RoundEventBroadcaster:
|
||||
would need a shared channel (e.g. Redis pub/sub) instead.
|
||||
"""
|
||||
|
||||
def __init__(self, max_subscribers: int = MAX_SUBSCRIBERS):
|
||||
self._subscribers: set[asyncio.Queue] = set()
|
||||
def __init__(self, max_subscribers: int = MAX_SUBSCRIBERS, max_per_ip: int = MAX_SUBSCRIBERS_PER_IP):
|
||||
self._ip_by_queue: dict[asyncio.Queue, str] = {}
|
||||
self._queues_by_ip: dict[str, list[asyncio.Queue]] = defaultdict(list)
|
||||
self.max_subscribers = max_subscribers
|
||||
self.max_per_ip = max_per_ip
|
||||
|
||||
def subscribe(self) -> asyncio.Queue:
|
||||
if len(self._subscribers) >= self.max_subscribers:
|
||||
def subscribe(self, client_ip: str = "unknown") -> asyncio.Queue:
|
||||
if len(self._ip_by_queue) >= self.max_subscribers:
|
||||
raise RoundEventCapacityError(f"already at the {self.max_subscribers}-subscriber cap")
|
||||
|
||||
ip_queues = self._queues_by_ip[client_ip]
|
||||
if len(ip_queues) >= self.max_per_ip:
|
||||
# B-38: evict this IP's own oldest connection rather than refusing
|
||||
# the new one — bounds one source's footprint without turning a
|
||||
# legitimate reconnect storm (a flaky network retrying EventSource)
|
||||
# into an outright block, and without letting one abusive IP crowd
|
||||
# out unrelated clients the way the old global-only cap did.
|
||||
oldest = ip_queues.pop(0)
|
||||
self._ip_by_queue.pop(oldest, None)
|
||||
if not oldest.full():
|
||||
oldest.put_nowait(EVICTED)
|
||||
|
||||
queue: asyncio.Queue = asyncio.Queue(maxsize=1)
|
||||
self._subscribers.add(queue)
|
||||
self._ip_by_queue[queue] = client_ip
|
||||
ip_queues.append(queue)
|
||||
return queue
|
||||
|
||||
def unsubscribe(self, queue: asyncio.Queue) -> None:
|
||||
self._subscribers.discard(queue)
|
||||
client_ip = self._ip_by_queue.pop(queue, None)
|
||||
if client_ip is None:
|
||||
return
|
||||
ip_queues = self._queues_by_ip.get(client_ip)
|
||||
if ip_queues is None:
|
||||
return
|
||||
if queue in ip_queues:
|
||||
ip_queues.remove(queue)
|
||||
if not ip_queues:
|
||||
self._queues_by_ip.pop(client_ip, None)
|
||||
|
||||
def publish(self) -> None:
|
||||
for queue in self._subscribers:
|
||||
for queue in self._ip_by_queue:
|
||||
if queue.full():
|
||||
continue # a not-yet-delivered notification already covers this one
|
||||
queue.put_nowait(None)
|
||||
|
||||
Reference in new issue
Block a user