Add files via upload

Splunk_Status_Monitor.sh

@@ -0,0 +1,110 @@
+#!/bin/bash
+
+# Log file path
+LOG_FILE="/var/log/Splunk_Status.log"
+
+# Splunk installation path
+SPLUNK_PATH="/opt/splunk/bin"
+
+# Function to generate process_id
+generate_process_id() {
+    openssl rand -hex 3 | tr 'a-f' 'A-F'
+}
+
+# Function to log messages with timestamp and process_id
+log_message() {
+    local message="$1"
+    local process_id="$2"
+    echo "timestamp=\"$(date '+%Y-%m-%d %H:%M:%S %Z')\" process_id=\"$process_id\" message=\"$message\"" >> $LOG_FILE
+}
+
+# Function to check the service status
+check_service() {
+    # Check service status on port 443 (expecting "303 See Other" and "Server: Splunkd")
+    curl_443=$(curl -k --silent --head https://127.0.0.1)
+    echo "$curl_443" | grep -q "303 See Other" && echo "$curl_443" | grep -q "Server: Splunkd"
+    local status_443=$?
+
+    # Check service status on port 8089 (expecting "200 OK" and "Server: Splunkd")
+    curl_8089=$(curl -k --silent --head https://127.0.0.1:8089)
+    echo "$curl_8089" | grep -q "200 OK" && echo "$curl_8089" | grep -q "Server: Splunkd"
+    local status_8089=$?
+
+    # Check if splunkd is running
+    $SPLUNK_PATH/splunk status | grep "splunkd is not running." > /dev/null
+    local splunk_status=$?
+
+    # Return 0 if all checks pass, otherwise return 1
+    if [ $status_443 -eq 0 ] && [ $status_8089 -eq 0 ] && [ $splunk_status -ne 0 ]; then
+        return 0
+    else
+        # Log which check failed
+        if [ $status_443 -ne 0 ]; then
+            log_message "Service down: port 443 check failed." "$1"
+        elif [ $status_8089 -ne 0 ]; then
+            log_message "Service down: port 8089 check failed." "$1"
+        elif [ $splunk_status -eq 0 ]; then
+            log_message "Service down: splunkd is not running." "$1"
+        fi
+        return 1
+    fi
+}
+
+# Function to start Splunk service
+start_splunk() {
+    log_message "Starting Splunk service..." "$1"
+    $SPLUNK_PATH/splunk start
+}
+
+# Function to restart Splunk service
+restart_splunk() {
+    log_message "Restarting Splunk service..." "$1"
+    $SPLUNK_PATH/splunk restart
+}
+
+# Main loop
+while true; do
+    process_id=$(generate_process_id)
+    check_service "$process_id"
+    if [ $? -ne 0 ]; then
+        log_message "Service is down. Waiting for 3 minutes before rechecking." "$process_id"
+        sleep 180  # Wait 3 minutes before rechecking
+
+        # Recheck service status after waiting
+        check_service "$process_id"
+        if [ $? -ne 0 ]; then
+            log_message "Service is still down after waiting. Attempting to start Splunk." "$process_id"
+
+            start_splunk "$process_id"
+            sleep 180  # Wait 3 minutes for the service to have time to start up
+
+            check_service "$process_id"
+            if [ $? -ne 0 ]; then
+                log_message "Service is still down after first start attempt. Attempting to start Splunk again." "$process_id"
+
+                start_splunk "$process_id"  # Attempt to start Splunk again
+                sleep 180  # Wait 3 minutes for the service to have time to start up again
+
+                check_service "$process_id"
+                if [ $? -ne 0 ]; then
+                    log_message "Service is still down after second start attempt. Attempting to restart Splunk." "$process_id"
+
+                    restart_splunk "$process_id"  # Attempt to restart Splunk
+                    sleep 180  # Wait 3 minutes for the service to have time to restart
+
+                    check_service "$process_id"
+                    if [ $? -ne 0 ]; then
+                        log_message "Service is still down after restart." "$process_id"
+                    else
+                        log_message "Service is up after restart." "$process_id"
+                    fi
+                else
+                    log_message "Service is up after second start attempt." "$process_id"
+                fi
+            else
+                log_message "Service is up after first start attempt." "$process_id"
+            fi
+        fi
+    fi
+    sleep 30  # Wait 30 seconds before the next check
+done

Splunk_Status_Monitor_Service.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+# Variables
+SERVICE_FILE="/etc/systemd/system/Splunk_Status_Monitor.service"
+TIMER_FILE="/etc/systemd/system/Splunk_Status_Monitor.timer"
+SCRIPT_PATH="/root/scripts/Splunk_Status_Monitor.sh"
+
+# Check if the script exists
+if [ ! -f "$SCRIPT_PATH" ]; then
+    echo "Script $SCRIPT_PATH not found. Please check the correct path."
+    exit 1
+fi
+
+# Give execution permissions to the script
+chmod 750 $SCRIPT_PATH
+
+# Create the service file
+echo "Creating service file..."
+cat <<EOL | sudo tee $SERVICE_FILE > /dev/null
+[Unit]
+Description=Splunk Status Monitor Service
+After=network.target
+
+[Service]
+Type=simple
+ExecStart=$SCRIPT_PATH
+Restart=on-failure
+RestartSec=30s
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target
+EOL
+
+# Create the timer file
+echo "Creating timer file..."
+cat <<EOL | sudo tee $TIMER_FILE > /dev/null
+[Unit]
+Description=Run Splunk Status Monitor 5 minutes after boot
+
+[Timer]
+OnBootSec=5min
+Unit=Splunk_Status_Monitor.service
+
+[Install]
+WantedBy=timers.target
+EOL
+
+# Enable and start the service and timer
+echo "Enabling and starting the service and timer..."
+sudo systemctl daemon-reload
+sudo systemctl enable Splunk_Status_Monitor.service
+sudo systemctl enable Splunk_Status_Monitor.timer
+sudo systemctl start Splunk_Status_Monitor.timer
+
+echo "Service and timer have been successfully created and enabled."