Skip to content

Commit

Permalink
chore: make deletion batch size configurable via setting
Browse files Browse the repository at this point in the history
  • Loading branch information
Fabian Nobis committed Feb 3, 2025
1 parent 5451456 commit 73abc3a
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 3 deletions.
12 changes: 12 additions & 0 deletions docs/v3/develop/settings-ref.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -1580,6 +1580,18 @@ The maximum number of seconds between flushes of the event persister.
**Supported environment variables**:
`PREFECT_SERVER_SERVICES_EVENT_PERSISTER_FLUSH_INTERVAL`, `PREFECT_API_SERVICES_EVENT_PERSISTER_FLUSH_INTERVAL`

### `batch_size_delete`
The number of expired events and event resources the event persister will attempt to delete in one batch.

**Type**: `integer`

**Default**: `10000`

**TOML dotted key path**: `server.services.event_persister.batch_size_delete`

**Supported environment variables**:
`PREFECT_SERVER_SERVICES_EVENT_PERSISTER_BATCH_SIZE_DELETE`

---
## ServerServicesFlowRunNotificationsSettings
Settings for controlling the flow run notifications service
Expand Down
10 changes: 10 additions & 0 deletions schemas/settings.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -1416,6 +1416,16 @@
],
"title": "Flush Interval",
"type": "number"
},
"batch_size_delete": {
"default": 10000,
"description": "The number of expired events and event resources the event persister will attempt to delete in one batch.",
"exclusiveMinimum": 0,
"supported_environment_variables": [
"PREFECT_SERVER_SERVICES_EVENT_PERSISTER_BATCH_SIZE_DELETE"
],
"title": "Batch Size Delete",
"type": "integer"
}
},
"title": "ServerServicesEventPersisterSettings",
Expand Down
12 changes: 9 additions & 3 deletions src/prefect/server/events/services/event_persister.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
PREFECT_API_SERVICES_EVENT_PERSISTER_BATCH_SIZE,
PREFECT_API_SERVICES_EVENT_PERSISTER_FLUSH_INTERVAL,
PREFECT_EVENTS_RETENTION_PERIOD,
PREFECT_SERVER_SERVICES_EVENT_PERSISTER_BATCH_SIZE_DELETE,
)
from prefect.settings.context import get_current_settings
from prefect.settings.models.server.services import ServicesBaseSetting
Expand Down Expand Up @@ -167,18 +168,23 @@ async def flush() -> None:

async def trim() -> None:
older_than = DateTime.now("UTC") - PREFECT_EVENTS_RETENTION_PERIOD.value()

delete_batch_size = (
PREFECT_SERVER_SERVICES_EVENT_PERSISTER_BATCH_SIZE_DELETE.value()
)
try:
async with db.session_context() as session:
resource_count = await batch_delete(
session,
db.EventResource,
db.EventResource.updated < older_than,
batch_size=10_000,
batch_size=delete_batch_size,
)

event_count = await batch_delete(
session, db.Event, db.Event.occurred < older_than, batch_size=10_000
session,
db.Event,
db.Event.occurred < older_than,
batch_size=delete_batch_size,
)

if resource_count or event_count:
Expand Down
10 changes: 10 additions & 0 deletions src/prefect/settings/models/server/services.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,16 @@ class ServerServicesEventPersisterSettings(ServicesBaseSetting):
),
)

batch_size_delete: int = Field(
default=10_000,
gt=0,
description="The number of expired events and event resources the event persister will attempt to delete in one batch.",
validation_alias=AliasChoices(
AliasPath("batch_size_delete"),
"prefect_server_services_event_persister_batch_size_delete",
),
)


class ServerServicesEventLoggerSettings(ServicesBaseSetting):
"""
Expand Down
1 change: 1 addition & 0 deletions tests/test_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -362,6 +362,7 @@
"PREFECT_SERVER_SERVICES_CANCELLATION_CLEANUP_LOOP_SECONDS": {"test_value": 10.0},
"PREFECT_SERVER_SERVICES_EVENT_LOGGER_ENABLED": {"test_value": True},
"PREFECT_SERVER_SERVICES_EVENT_PERSISTER_BATCH_SIZE": {"test_value": 10},
"PREFECT_SERVER_SERVICES_EVENT_PERSISTER_BATCH_SIZE_DELETE": {"test_value": 20},
"PREFECT_SERVER_SERVICES_EVENT_PERSISTER_ENABLED": {"test_value": True},
"PREFECT_SERVER_SERVICES_EVENT_PERSISTER_FLUSH_INTERVAL": {"test_value": 10.0},
"PREFECT_SERVER_SERVICES_FLOW_RUN_NOTIFICATIONS_ENABLED": {"test_value": True},
Expand Down

0 comments on commit 73abc3a

Please sign in to comment.