Merge pull request #6072 from pvisweswar/SAAS-14342/autorestart_of_ec…

…2_servers

adding script to auto restart task

environments/staging/public.yml

@@ -231,3 +231,9 @@ allow_aws_ssm_proxy: true
 release_bucket: dimagi-commcare-staging-release
 
 formplayer_java_version: "{{ java_17_bin_path }}/java"
+
+# formplayer & web_worker
+
+maintenance_start_time: "06:00"
+
+maintenance_end_time: "08:00"

src/commcare_cloud/ansible/roles/devops_scripts/tasks/main.yml

@@ -8,6 +8,29 @@
     mode: "u=rwx,g=rwx,o=r"
   when: "'postgresql' in group_names or 'citusdb' in group_names or 'pg_standby' in group_names"
 
+- name: Copy autoreboot.sh script to /usr/local/sbin
+  template:
+    src: autoreboot.sh.j2
+    dest: /usr/local/sbin/autoreboot.sh
+    owner: ansible
+    group: ansible
+    mode: "u=rwx,g=rwx,o=r"
+  when: "'formplayer' in group_names or 'webworkers' in group_names"
+
+- name: Creates a cron to check the autoreboot details
+  become: yes
+  cron:
+    name: Run autoreboot.sh at minute 0 and 30 on Tuesday, Wednesday, and Thursday
+    minute: "0,30"
+    hour: "*"
+    day: "*"
+    month: "*"
+    weekday: "2-4" # Tuesday (2) through Thursday (4)
+    user: root
+    job: "/usr/local/sbin/autoreboot.sh"
+    cron_file: autoreboot_sh
+  when: maintenance_start_time is defined and ("'formplayer' in group_names or 'webworkers' in group_names")
+
 - name: Copy intruder.sh script to /usr/local/sbin
   copy:
     src: intruder.sh

src/commcare_cloud/ansible/roles/devops_scripts/templates/autoreboot.sh.j2

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+current_availability_zone=$(ec2metadata --availability-zone | awk -F'-' '{print $3}' | cut -c2-)
+maintenance_start_time={{ maintenance_start_time }}
+maintenance_end_time={{ maintenance_end_time }}
+current_time=$(date +%R)
+uptime_days=$(uptime | awk '/days?/ {print $3; next}; {print 0}')
+
+if [ $uptime_days -ge 90 ] && [[ $current_time > $maintenance_start_time ]] && [[ $current_time < $maintenance_end_time ]]; then
+  if [ "$current_availability_zone" == "a" ]; then
+    logger '[Maintenance] Rebooting server as uptime is greater than or equal to ninety days in az:A'
+    echo "Starting reboot"
+    shutdown -r +1
+  elif [ "$current_availability_zone" == "b" ]; then
+    logger '[Maintenance] Rebooting server as uptime is greater than or equal to ninety days in az:B'
+    echo "Starting reboot"
+    shutdown -r +30
+  elif [ "$current_availability_zone" == "c" ]; then
+    logger '[Maintenance] Rebooting server as uptime is greater than or equal to ninety days in az:C'
+    echo "Starting reboot"
+    shutdown -r +60
+  fi
+fi