Set a maximum length for Redis event streams

Nothing should get anywhere near this limit for a very long time, but
it's better to have it set up in case anything ever gets out of control.
This commit is contained in:
Deimos
2020-01-24 07:38:28 -07:00
parent 9eec00cc6a
commit bb1ccfe590

View File

@@ -20,6 +20,11 @@ from tildes.lib.event_stream import REDIS_KEY_PREFIX
NOTIFY_CHANNEL = "postgresql_events"
# Stream entries seem to generally require about 20-50 bytes each, depending on which
# data fields they include. A max length of a million should mean that any individual
# stream shouldn't be able to take up more memory than 50 MB or so.
STREAM_MAX_LENGTH = 1_000_000
def postgresql_redis_bridge(config_path: str) -> None:
"""Listen for NOTIFY events and add them to Redis streams."""
@@ -61,7 +66,12 @@ def postgresql_redis_bridge(config_path: str) -> None:
except json.decoder.JSONDecodeError:
continue
pipe.xadd(f"{REDIS_KEY_PREFIX}{stream_name}", fields)
pipe.xadd(
f"{REDIS_KEY_PREFIX}{stream_name}",
fields,
maxlen=STREAM_MAX_LENGTH,
approximate=True,
)
pipe.execute()