Add schedule for task purge

It is guarded by a setting and off by default.

fixes #5378

.github/workflows/scripts/install.sh

@@ -78,7 +78,7 @@ VARSYAML
 
 cat >> vars/main.yaml << VARSYAML
 pulp_env: {}
-pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "content_path_prefix": "/somewhere/else/", "orphan_protection_time": 0}
+pulp_settings: {"allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "content_path_prefix": "/somewhere/else/", "orphan_protection_time": 0, "task_protection_time": 10, "tmpfile_protection_time": 10, "upload_protection_time": 10}
 pulp_scheme: https
 pulp_default_container: ghcr.io/pulp/pulp-ci-centos9:latest
 VARSYAML

CHANGES/5378.feature

@@ -0,0 +1,2 @@
+Added a scheduled version of task purge.
+You need to adjust the "TASK_PROTECTION_TIME" setting in order to use it.

pulpcore/app/settings.py

@@ -256,8 +256,11 @@
 
 # how long to protect ephemeral items in minutes
 ORPHAN_PROTECTION_TIME = 24 * 60
-# if set to 0, upload and tmpfile cleanup task is disabled
+
+# Custom cleaup intervals
+# for the following, if set to 0, the corresponding cleanup task is disabled
 UPLOAD_PROTECTION_TIME = 0
+TASK_PROTECTION_TIME = 0
 TMPFILE_PROTECTION_TIME = 0
 
 REMOTE_USER_ENVIRON_NAME = "REMOTE_USER"

pulpcore/app/tasks/purge.py

@@ -1,15 +1,17 @@
 from gettext import gettext as _
 from logging import getLogger
 
+from django.conf import settings
 from django.db.models.deletion import ProtectedError
+from django.utils import timezone
 
 from pulpcore.app.models import (
     ProgressReport,
     Task,
 )
 from pulpcore.app.role_util import get_objects_for_user
 from pulpcore.app.util import get_domain, get_current_authenticated_user
-from pulpcore.constants import TASK_STATES
+from pulpcore.constants import TASK_STATES, TASK_FINAL_STATES
 
 log = getLogger(__name__)
 
@@ -55,7 +57,7 @@ def _details_reporting(current_reports, current_details, totals_pb):
     return current_reports
 
 
-def purge(finished_before, states):
+def purge(finished_before=None, states=None):
     """
     This task purges from the database records of tasks which finished prior to the specified time.
 
@@ -70,10 +72,15 @@ def purge(finished_before, states):
     by deleting a Task.
 
     Args:
-        finished_before (DateTime): Earliest finished-time to **NOT** purge.
-        states (List[str]): List of task-states we want to purge.
+        finished_before (Optional[DateTime]): Earliest finished-time to **NOT** purge.
+        states (Optional[List[str]]): List of task-states we want to purge.
 
     """
+    if finished_before is None:
+        assert settings.TASK_PROTECTION_TIME > 0
+        finished_before = timezone.now() - timezone.timedelta(minutes=settings.TASK_PROTECTION_TIME)
+    if states is None:
+        states = TASK_FINAL_STATES
     current_user = get_current_authenticated_user()
     domain = get_domain()
     # Tasks, prior to the specified date, in the specified state, owned by the current-user, in the

pulpcore/app/util.py

@@ -379,6 +379,7 @@ def configure_cleanup():
             "pulpcore.app.tasks.orphan.tmpfile_cleanup",
             settings.TMPFILE_PROTECTION_TIME,
         ),
+        ("tasks", "pulpcore.app.tasks.purge.purge", settings.TASK_PROTECTION_TIME),
     ]:
         if protection_time > 0:
             dispatch_interval = timedelta(minutes=protection_time)

staging_docs/admin/learn/settings.md

@@ -379,11 +379,11 @@ up before the task finishes. Default is 1440 minutes (24 hours).
 
 
 
-### UPLOAD_PROTECTION_TIME and TMPFILE_PROTECTION_TIME
+### UPLOAD_PROTECTION_TIME, TMPFILE_PROTECTION_TIME and TASK_PROTECTION_TIME
 
-Pulp uses `uploads` and `pulp temporary files` to pass data from the api to worker tasks.
-These options allow to specify a timeinterval in minutes used for cleaning up stale entries. If
-set to 0, automatic cleanup is disabled, which is the default.
+Pulp uses `tasks`, `uploads` and `pulp temporary files` to pass data from the api to worker tasks.
+These options allow to specify a timeinterval in minutes used for cleaning up stale entries.
+If set to 0, automatic cleanup is disabled, which is the default.
 
 
 

template_config.yml

@@ -65,6 +65,9 @@ pulp_settings:
   - /tmp
   content_path_prefix: /somewhere/else/
   orphan_protection_time: 0
+  upload_protection_time: 10
+  tmpfile_protection_time: 10
+  task_protection_time: 10
 pulp_settings_azure:
   domain_enabled: true
 pulp_settings_gcp: null