Homelabv4/scripts/ha-failover-watcher.sh

#!/bin/bash
# Multi-site active failover pilot (see
# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - the VPS's half
# of the failover-watcher. Runs as a systemd service on the VPS
# (172.93.53.139), NOT in k3s - it needs to keep running even if the VPS's
# own k3s/CNPG cluster is unhealthy, and it's the one thing in this whole
# pilot that's genuinely new/bespoke rather than reusing an existing
# operator.
#
# Deploy (manual, matches every other VPS systemd unit this session - not
# git-applied automatically):
#   scp this file to the VPS as /usr/local/bin/ha-failover-watcher.sh
#   chmod +x it, then install scripts/ha-failover-watcher.service
#   (see that file) and `systemctl enable --now ha-failover-watcher`.
#
# Logic (deliberately simple - see the plan doc's "what CNPG genuinely
# does NOT provide" paragraph for why this exists at all):
#   1. Every 10s, read /ha-failover/home-heartbeat from etcd via THIS
#      node's own local etcd member (127.0.0.1:2379) using a linearizable
#      (default, quorum-backed) read - home's heartbeat-writer
#      (infrastructure/ha-failover/manifests/heartbeat-writer.yaml)
#      refreshes this key every 10s while home is healthy.
#   2. If the read itself fails/times out, this VPS can't reach a
#      majority of the 3-member etcd cluster (needs 2 of 3) - meaning
#      EITHER home is genuinely down AND the witness is also unreachable
#      from here, OR this VPS itself is the one that's partitioned.
#      Either way, we cannot safely tell which, so we do NOT promote -
#      this is the split-brain-prevention property etcd's own Raft
#      consensus gives us for free, no custom quorum-counting needed.
#   3. If the read succeeds and the heartbeat is fresher than
#      STALE_THRESHOLD seconds, home is confirmed up - no-op.
#   4. If the read succeeds (so we DO have majority/quorum) and the
#      heartbeat is older than STALE_THRESHOLD - or missing entirely -
#      home is confirmed down by majority agreement. Promote, once:
#        a. Skip if already primary (checked via the Cluster CR itself,
#           idempotent - safe to run this loop forever).
#        b. kubectl patch the local pg-authentik Cluster:
#           spec.replica.{self,primary,source} = vps. No promotionToken -
#           confirmed live via dry-run that CNPG's admission webhook does
#           NOT require one (it's optional, used for graceful/planned
#           switchover to cross-check LSNs - not available for a genuine
#           unplanned outage since home isn't reachable to generate one).
#           This means promotion accepts whatever the VPS replica had
#           already streamed - typically a couple seconds of async lag,
#           an accepted tradeoff of async cross-WAN replication (there is
#           no realistic sync-replication option over a home/VPS WAN
#           link without crippling write latency).
#        c. Flip the pg-authentik.ha.huskypup.net Cloudflare A record to
#           this VPS's public IP - both sites listen on the SAME external
#           port 61432 (home via UniFi WAN forward, VPS via
#           pg-authentik-forward.service's local socat forward)
#           specifically so a single floating hostname:port works for
#           both sites without the app tier needing per-site config.
#        d. Record the promotion in etcd (/ha-failover/promoted-at,
#           /ha-failover/promoted-by) - both for the idempotency check
#           above surviving a script restart, and as an audit trail for
#           whoever does the (deliberately manual - see the plan doc)
#           failback later.
#
# Failback is NOT automated by this script on purpose - see
# scripts/ha-failback-authentik.sh, run by a human once home is
# confirmed healthy again.
set -u

ETCD="http://127.0.0.1:2379"
STALE_THRESHOLD=45  # ~4-5 missed 10s heartbeats before acting - avoids flapping on one blip
CHECK_INTERVAL=10
CLOUDFLARE_TOKEN_FILE="/etc/ha-failover/cloudflare-token"
CF_ZONE_NAME="huskypup.net"
CF_RECORD_NAME="pg-authentik.ha.huskypup.net"
VPS_PUBLIC_IP="172.93.53.139"
LOG_TAG="ha-failover-watcher"

log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) ${LOG_TAG}: $*"; }

b64() { printf '%s' "$1" | base64 | tr -d '\n'; }

etcd_get() {
  # $1 = key. Prints the decoded value, or nothing + returns 1 if the
  # read failed (unreachable/no quorum) or the key doesn't exist.
  local key_b64 resp val_b64
  key_b64="$(b64 "$1")"
  resp="$(curl -sf --max-time 5 -X POST "${ETCD}/v3/kv/range" \
    -d "{\"key\":\"${key_b64}\"}")" || return 1
  val_b64="$(echo "$resp" | jq -r '.kvs[0].value // empty')"
  [ -n "$val_b64" ] || return 1
  echo "$val_b64" | base64 -d
}

etcd_put() {
  local key_b64 val_b64
  key_b64="$(b64 "$1")"
  val_b64="$(b64 "$2")"
  curl -sf --max-time 5 -X POST "${ETCD}/v3/kv/put" \
    -d "{\"key\":\"${key_b64}\",\"value\":\"${val_b64}\"}" >/dev/null
}

flip_dns_to_vps() {
  local token zone_id record_id
  token="$(cat "$CLOUDFLARE_TOKEN_FILE")"
  zone_id="$(curl -sf -H "Authorization: Bearer ${token}" \
    "https://api.cloudflare.com/client/v4/zones?name=${CF_ZONE_NAME}" | jq -r '.result[0].id')"
  if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then
    log "ERROR: could not resolve Cloudflare zone id for ${CF_ZONE_NAME}"
    return 1
  fi
  record_id="$(curl -sf -H "Authorization: Bearer ${token}" \
    "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${CF_RECORD_NAME}&type=A" \
    | jq -r '.result[0].id // empty')"
  if [ -z "$record_id" ]; then
    log "ERROR: no existing A record for ${CF_RECORD_NAME} - refusing to create one blind, fix manually"
    return 1
  fi
  curl -sf -X PATCH -H "Authorization: Bearer ${token}" -H "Content-Type: application/json" \
    -d "{\"type\":\"A\",\"name\":\"${CF_RECORD_NAME}\",\"content\":\"${VPS_PUBLIC_IP}\",\"ttl\":60,\"proxied\":false}" \
    "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" >/dev/null
}

promote() {
  local current_primary
  current_primary="$(kubectl -n authentik get cluster pg-authentik -o jsonpath='{.spec.replica.primary}' 2>/dev/null)"
  if [ "$current_primary" = "vps" ]; then
    return 0  # already promoted, nothing to do
  fi
  log "PROMOTING: home confirmed down by etcd majority (heartbeat stale/missing). Flipping pg-authentik to vps."
  if ! kubectl -n authentik patch cluster pg-authentik --type merge \
      -p '{"spec":{"replica":{"self":"vps","primary":"vps","source":"vps"}}}'; then
    log "ERROR: kubectl patch failed - Cluster CR NOT promoted, will retry next loop"
    return 1
  fi
  if ! flip_dns_to_vps; then
    log "ERROR: Cluster CR promoted but Cloudflare DNS flip failed - fix pg-authentik.ha.huskypup.net manually, it still points at home"
  fi
  etcd_put "/ha-failover/promoted-at" "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  etcd_put "/ha-failover/promoted-by" "vps"
  log "Promotion complete."
}

log "starting, stale threshold=${STALE_THRESHOLD}s check interval=${CHECK_INTERVAL}s"
while true; do
  hb="$(etcd_get /ha-failover/home-heartbeat)"
  if [ $? -ne 0 ]; then
    log "cannot reach etcd quorum (or no heartbeat key yet) - not acting, will retry"
    sleep "$CHECK_INTERVAL"
    continue
  fi
  now="$(date +%s)"
  age=$((now - hb))
  if [ "$age" -gt "$STALE_THRESHOLD" ]; then
    log "home heartbeat is ${age}s stale (threshold ${STALE_THRESHOLD}s)"
    promote
  fi
  sleep "$CHECK_INTERVAL"
done