From bb1ccfe59046cbf3437794658cf07072f49cca03 Mon Sep 17 00:00:00 2001 From: Deimos Date: Fri, 24 Jan 2020 07:38:28 -0700 Subject: [PATCH] Set a maximum length for Redis event streams Nothing should get anywhere near this limit for a very long time, but it's better to have it set up in case anything ever gets out of control. --- tildes/scripts/postgresql_redis_bridge.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tildes/scripts/postgresql_redis_bridge.py b/tildes/scripts/postgresql_redis_bridge.py index fe2f823..a93da4e 100644 --- a/tildes/scripts/postgresql_redis_bridge.py +++ b/tildes/scripts/postgresql_redis_bridge.py @@ -20,6 +20,11 @@ from tildes.lib.event_stream import REDIS_KEY_PREFIX NOTIFY_CHANNEL = "postgresql_events" +# Stream entries seem to generally require about 20-50 bytes each, depending on which +# data fields they include. A max length of a million should mean that any individual +# stream shouldn't be able to take up more memory than 50 MB or so. +STREAM_MAX_LENGTH = 1_000_000 + def postgresql_redis_bridge(config_path: str) -> None: """Listen for NOTIFY events and add them to Redis streams.""" @@ -61,7 +66,12 @@ def postgresql_redis_bridge(config_path: str) -> None: except json.decoder.JSONDecodeError: continue - pipe.xadd(f"{REDIS_KEY_PREFIX}{stream_name}", fields) + pipe.xadd( + f"{REDIS_KEY_PREFIX}{stream_name}", + fields, + maxlen=STREAM_MAX_LENGTH, + approximate=True, + ) pipe.execute()