#!/bin/bash
#
# pve-uplink-watchdog.sh
#
# Watches this node's production (external) network path. If it fails while the
# internal cluster network stays healthy, puts the node into HA maintenance mode
# so HA-managed guests drain to a peer. Reverses automatically on recovery.
#
# REQUIRES: migration network pinned to the internal CIDR in
# /etc/pve/datacenter.cfg, e.g.
# migration: type=secure,network=10.10.10.0/24
# Without that, the drain fails for the same reason manual migration does.
#
# Install: /usr/local/sbin/pve-uplink-watchdog.sh (chmod 750, root:root)
#
set -uo pipefail
### ------------------------- configuration -------------------------------- ###
# External targets that prove production reachability. Use the gateway plus at
# least one peer node external address. Any single success means we are UP.
EXTERNAL_TARGETS=("10.0.0.254" "10.0.0.2" "10.0.0.3")
# Peer addresses on the INTERNAL network. Used to sanity check whether this is a
# local fault or a site-wide outage.
INTERNAL_PEERS=("10.10.10.2" "10.10.10.3")
# Address on the internal network each peer should be able to reach externally.
# Usually the production gateway.
PEER_EXTERNAL_PROBE="10.0.0.254"
# Consecutive failed cycles before draining. At 10s interval, 6 = ~60s.
FAIL_THRESHOLD=6
# Consecutive successful cycles before undoing maintenance. Deliberately higher
# than FAIL_THRESHOLD so a flapping link does not ping-pong guests.
RECOVER_THRESHOLD=18
INTERVAL=10
PING_COUNT=2
PING_TIMEOUT=2
SSH_TIMEOUT=5
STATE_DIR="/run/pve-uplink-watchdog"
STATE_FILE="${STATE_DIR}/state"
# Set to 1 for a dry run. Logs the decision, changes nothing.
DRY_RUN=0
### ------------------------------------------------------------------------ ###
NODE="$(hostname -s)"
mkdir -p "$STATE_DIR"
log() { logger -t pve-uplink-watchdog -p "daemon.${1}" -- "$2"; }
# --- health probes ---------------------------------------------------------
external_up() {
local t
for t in "${EXTERNAL_TARGETS[@]}"; do
if ping -c "$PING_COUNT" -W "$PING_TIMEOUT" -q "$t" &>/dev/null; then
return 0
fi
done
return 1
}
internal_up() {
local p
for p in "${INTERNAL_PEERS[@]}"; do
if ping -c "$PING_COUNT" -W "$PING_TIMEOUT" -q "$p" &>/dev/null; then
return 0
fi
done
return 1
}
# Guard against a site-wide outage. Ask a peer, over the internal link, whether
# IT can still reach the production gateway. If no peer can, the fault is
# upstream and draining would only stampede guests onto equally blind nodes.
peer_confirms_local_fault() {
local p
for p in "${INTERNAL_PEERS[@]}"; do
if ssh -o BatchMode=yes \
-o StrictHostKeyChecking=accept-new \
-o ConnectTimeout="$SSH_TIMEOUT" \
"root@${p}" \
"ping -c 2 -W 2 -q ${PEER_EXTERNAL_PROBE}" &>/dev/null; then
log info "peer ${p} still reaches ${PEER_EXTERNAL_PROBE}; fault is local to ${NODE}"
return 0
fi
done
log warning "no peer can reach ${PEER_EXTERNAL_PROBE}; treating as site-wide outage, not draining"
return 1
}
quorate() {
pvecm status 2>/dev/null | grep -qi "Quorate:.*Yes"
}
# --- actions ---------------------------------------------------------------
enter_maintenance() {
if [[ "$DRY_RUN" -eq 1 ]]; then
log notice "DRY RUN: would enable node-maintenance on ${NODE}"
return 0
fi
if ha-manager crm-command node-maintenance enable "$NODE"; then
log notice "node-maintenance ENABLED on ${NODE}; HA guests draining"
echo "maintenance" > "$STATE_FILE"
return 0
fi
log err "failed to enable node-maintenance on ${NODE}"
return 1
}
exit_maintenance() {
if [[ "$DRY_RUN" -eq 1 ]]; then
log notice "DRY RUN: would disable node-maintenance on ${NODE}"
return 0
fi
if ha-manager crm-command node-maintenance disable "$NODE"; then
log notice "node-maintenance DISABLED on ${NODE}; uplink recovered"
echo "normal" > "$STATE_FILE"
return 0
fi
log err "failed to disable node-maintenance on ${NODE}"
return 1
}
# --- main loop -------------------------------------------------------------
[[ -f "$STATE_FILE" ]] || echo "normal" > "$STATE_FILE"
fail_count=0
ok_count=0
log info "started on ${NODE} (dry_run=${DRY_RUN}, threshold=${FAIL_THRESHOLD})"
while true; do
state="$(cat "$STATE_FILE" 2>/dev/null || echo normal)"
if external_up; then
fail_count=0
ok_count=$((ok_count + 1))
if [[ "$state" == "maintenance" && "$ok_count" -ge "$RECOVER_THRESHOLD" ]]; then
exit_maintenance && ok_count=0
fi
else
ok_count=0
fail_count=$((fail_count + 1))
log warning "external path down (${fail_count}/${FAIL_THRESHOLD})"
if [[ "$state" != "maintenance" && "$fail_count" -ge "$FAIL_THRESHOLD" ]]; then
if ! internal_up; then
log err "internal network also down; HA/fencing owns this, standing down"
fail_count=0
elif ! quorate; then
log err "node not quorate; /etc/pve read-only, cannot drain"
fail_count=0
elif peer_confirms_local_fault; then
enter_maintenance
fail_count=0
else
fail_count=0
fi
fi
fi
sleep "$INTERVAL"
done
