#!/bin/bash

# Note: This script is only used on systems with Systemd >=254,
# due to relying on Systemd features `RestartSteps` and `RestartMaxDelaySec`.
#
# Purpose:
#   Selectively reset the automatically-incrementing Systemd service restart delay, and 
# ensure the orchid service restarts quickly, if it had sufficient uptime.
#
# Background:
#   Systemd's `RestartSteps` and `RestartMaxDelaySec` options increase the restart delay of a service
# after each abnormal exit, which is close to our desired behavior. What is missing, however,
# is a way to reset that delay.
#
# Natively, service restart time is based only on those two options, along with the `NRestarts` property, which
# increases by 1 for every abnormal exit. This property is only automatically reset with a system restart,
# meaning that once orchid has crashed a few times, its restart time will remain elevated
# even if its functionality later recovers.
#
# Desired behavior:
#   - Typically, if orchid exits abnormally, we want it to immediately restart.
#   - If orchid is repeatedly crashing, we want it to be restarted with longer delays, in order to
# give the system time to recover (from, say, repeated OOM kills) or to allow for manual intervention.
#   - If the server then runs successfully for a while, we want to revert back to the initial state,
# with a quick restart if it unexpectedly goes down.
#
#                        _______________________________________________
#                        |                                              |  <--------
#                        V                                              |          |
# start -> crash -> quick restart -> up for a while -> reset delay -> crash        |
#                        |                  ^                                      |
#                        V                  |                                      |
#          -------> quickly crash           |                                      |
#          |             |                  |  <----------------------             |
#          |   loop      |                  |                        |             |
#          |             V                  |   this script handles this part and this part
#          |______ ++delay (capped) ________|
#
# Solution:
#   This script is invoked when orchid exits abnormally, and if the service has been running long enough,
# this resets the service's `NRestarts` property, so that Systemd treats it as though it's the first crash.

# If orchid was running for this many seconds, do not wait for systemd restart.
# This number of seconds is "long enough" or "a while" in the above diagram.
readonly MIN_UPTIME_FOR_QUICK_RESTART=60

readonly SERVICE='orchid.service'

# Timestamp of now, in Epoch seconds.
readonly NOW=$(date '+%s')

# input: Service property name
# returns: The value of the property for orchid,
#           without the `PropertyName=` portion
query_property() {
    systemctl show $SERVICE --property="$1" | cut -d '=' -f 2
}

# Timestamp of the last time orchid started, in Epoch seconds.
readonly last_start_time=$(query_property 'ExecMainStartTimestamp' | xargs -0 date '+%s' -d)

# Uptime of last orchid process in seconds.
readonly last_uptime=$(( $NOW - $last_start_time ))

echo "orchid went down at $(date -d "@$NOW"), and was last started at $(date -d "@$last_start_time")."
echo "Last uptime was approximately $last_uptime seconds."

readonly nrestarts="$(query_property 'NRestarts')"

echo "NRestarts=$nrestarts"

# If orchid had been running for long enough before terminating, reset the Systemd delay sequence count.
if [[ $last_uptime -ge $MIN_UPTIME_FOR_QUICK_RESTART ]]; then
    echo "Resetting orchid restart count due to > 1 minute uptime."
    # Reset `NRestarts` to 0, returning the Systemd restart delay sequence back to the beginning, so
    # the next automatic restart will be immediate.
    systemctl reset-failed $SERVICE
fi

# If orchid has restarted at most a couple times since the last restart count reset,
# the built-in Systemd restart delay is still very quick, and we don't want to independently perform a restart
# that might interfere with Systemd's, regardless of the uptime.
#
# Otherwise, the redundant/conflicting restart attempts can result in a longer effective restart delay.
if [[ "$nrestarts" -le 2 ]]; then
    echo "Relying on automatic Systemd restart, due to low NRestarts."
    exit
fi

# If orchid had been running for long enough (and restart count is more than 2 -- see above), 
# then in addition to having reset the delay count, we want to restart quickly right now.
if [[ $last_uptime -ge $MIN_UPTIME_FOR_QUICK_RESTART ]]; then
    echo "Restarting orchid immediately due to > 1 minute uptime."
    sleep 0.1

    # Even though we reset `NRestarts` earlier, this doesn't interrupt the Systemd delay already in progress.
    # Rather than wait up to the full `RestartMaxDelaySec`, we restart the service ourselves.
    systemctl restart $SERVICE
fi

# Otherwise, the orchid service has not run long enough without an abnormal restart, so
# we let Systemd restart it automatically, with a delay that increases each restart up until the max delay. 
