diff --git a/README.md b/README.md index ded972a..9af9fe3 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,45 @@ This repository is a collection of useful scripts and tools for TAs and professors to manage students workloads. -## get_url.py +## Cronjobs + +### ope-notebook-culler + +This cronjob runs once every 24 hours at 7am, shutting down all notebooks from the rhods-notebooks namespace. To add resources to the rhods-notebooks namespace: + +NOTE: PVCs persist this shutdown process. + +1. Ensure you are logged in to your OpenShift account via the CLI and you have access to rhods-notebooks namespace. +2. Switch to rhods-notebooks namespace: +``` + oc project rhods-notebooks +``` + +3. From cronjobs/notebook-culler/ directory run: +``` + oc apply -k . +``` + + This will deploy all the necessary resources for the cronjob to run on the specified schedule. + +Alternatively, to run the script immediately: + +1. Ensure you followed the steps above +2. Verify the cronjob nb-culler exists +``` + oc get cronjob nb-culler +``` + +3. Run: +``` + kubectl create -n rhods-notebooks job --from=cronjob/nb-culler nb-culler +``` + + This will trigger the cronjob to spawn a job manually. + +## Scripts + +### get_url.py This script is used to retrieve the URL for a particular notebook associated with one student. To execute this script: diff --git a/cronjobs/notebok-culler/cronjob.yaml b/cronjobs/notebok-culler/cronjob.yaml new file mode 100644 index 0000000..62d21d1 --- /dev/null +++ b/cronjobs/notebok-culler/cronjob.yaml @@ -0,0 +1,68 @@ +kind: CronJob +apiVersion: batch/v1 +metadata: + name: nb-culler + labels: + component.opendatahub.io/name: nb-culler + opendatahub.io/component: 'true' + opendatahub.io/modified: 'false' +spec: + schedule: '0 7 * * *' + startingDeadlineSeconds: 200 + concurrencyPolicy: Replace + suspend: false + jobTemplate: + metadata: + labels: + component.opendatahub.io/name: nb-culler + opendatahub.io/component: 'true' + spec: + template: + metadata: + labels: + component.opendatahub.io/name: nb-culler + opendatahub.io/component: 'true' + parent: nb-culler + spec: + restartPolicy: Never + serviceAccountName: ope-notebook-culler + schedulerName: default-scheduler + terminationGracePeriodSeconds: 30 + securityContext: {} + containers: + - name: oc-cli + image: >- + registry.redhat.io/openshift4/ose-cli@sha256:25fef269ac6e7491cb8340119a9b473acbeb53bc6970ad029fdaae59c3d0ca61 + command: ["/bin/bash", "-c", "--"] + args: + - | + #threshold to stop running notebooks. Currently set to 24 hours + cutoff_time=86400 + current_time=$(date +%s) + notebooks=$(oc get notebooks -n rhods-notebooks -o jsonpath='{range .items[?(@.status.containerState.running)]}{.metadata.name}{" "}{.metadata.namespace}{" "}{.status.containerState.running.startedAt}{"\n"}{end}') + if [ -z "$notebooks" ]; then + echo "No running notebooks found" + exit 0 + fi + # Loop through each notebook + while read -r nb ns ts; do + timestamp=$(date -d $ts +%s) + difference=$((current_time - timestamp)) + if [ $difference -gt $cutoff_time ]; then + echo "$nb is more than 24 hours old, stopping the notebook" + oc patch notebook $nb -n $ns --type merge -p '{"metadata":{"annotations":{"kubeflow-resource-stopped":"'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'"}}}' + fi + done <<< "$notebooks" + resources: + limits: + memory: 800Mi + requests: + memory: 400Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + imagePullPolicy: IfNotPresent + serviceAccount: nb-culler + dnsPolicy: ClusterFirst + successfulJobsHistoryLimit: 7 + failedJobsHistoryLimit: 7 + \ No newline at end of file diff --git a/cronjobs/notebok-culler/kustomization.yaml b/cronjobs/notebok-culler/kustomization.yaml new file mode 100644 index 0000000..a534896 --- /dev/null +++ b/cronjobs/notebok-culler/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - cronjob.yaml + - rolebinding.yaml + - serviceaccount.yaml +namespace: rhods-notebooks diff --git a/cronjobs/notebok-culler/rolebinding.yaml b/cronjobs/notebok-culler/rolebinding.yaml new file mode 100644 index 0000000..b5372b8 --- /dev/null +++ b/cronjobs/notebok-culler/rolebinding.yaml @@ -0,0 +1,11 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nb-culler +subjects: + - kind: ServiceAccount + name: nb-culler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: edit diff --git a/cronjobs/notebok-culler/serviceaccount.yaml b/cronjobs/notebok-culler/serviceaccount.yaml new file mode 100644 index 0000000..7937c8a --- /dev/null +++ b/cronjobs/notebok-culler/serviceaccount.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: nb-culler