#!/bin/bash
NTFY_URL="${NTFY_URL:-invalid}"
RECOVERY_WAIT=15 # seconds to wait before declaring recovery
declare -A last_notified # debounce: last alert time per container
declare -A unhealthy # track containers we've alerted on
notify() {
local title="$1"
local msg="$2"
local priority="${3:-default}"
local tags="${4:-docker}"
curl -s \
-H "Title: $title" \
-H "Priority: $priority" \
-H "Tags: $tags" \
-d "$msg" \
"$NTFY_URL"
}
is_running() {
local name="$1"
# exits 0 if container exists and is running, 1 otherwise
docker inspect --format '{{.State.Running}}' "$name" 2>/dev/null | grep -q "true"
}
docker events \
--filter 'event=die' \
--filter 'event=oom' \
--filter 'event=start' \
--format '{{.Action}}|{{.Actor.Attributes.name}}|{{.Actor.Attributes.exitCode}}' \
| while IFS='|' read -r event_status container_name exit_code; do
now=$(date +%s)
case "$event_status" in
die)
# Skip clean/intentional stops
[ "$exit_code" = "0" ] && continue
# Debounce: skip if we already alerted within 60s
last=${last_notified[$container_name]:-0}
(( now - last < 60 )) && continue
last_notified[$container_name]=$now
unhealthy[$container_name]=1
notify "💀 Container Died" \
"$container_name exited unexpectedly (exit code $exit_code)" \
"high" "docker,skull"
;;
oom)
last=${last_notified[$container_name]:-0}
(( now - last < 60 )) && continue
last_notified[$container_name]=$now
unhealthy[$container_name]=1
notify "🧠 OOM Kill" \
"$container_name was killed — hit its memory limit!" \
"urgent" "docker,warning"
;;
start)
# Only care about containers we previously flagged as unhealthy
[ -z "${unhealthy[$container_name]}" ] && continue
# Wait a bit, then confirm it actually stayed up
sleep "$RECOVERY_WAIT"
if is_running "$container_name"; then
unset unhealthy[$container_name]
unset last_notified[$container_name]
notify "✅ Container Recovered" \
"$container_name is back up and running" \
"default" "docker,white_check_mark"
fi
# If it's not running after the wait, it crashed again —
# the die handler will catch that and fire a new alert
;;
esac
done