#!/bin/bash
# Multi-site active failover pilot (see
# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - the deliberate
# MANUAL counterpart to scripts/ha-failover-watcher.sh's automatic
# promotion. Run this by hand, on the VPS, once you've confirmed home is
# genuinely healthy again and want to move primary back - never automated,
# by design (auto-flipping back immediately on reconnect risks flapping,
# and a human should confirm home's data/state before handing writes back
# to it).
#
# Order matters:
#   1. Confirm home's pg-authentik Cluster is healthy and NOT still
#      thinking it's primary (it shouldn't be, since it was down/
#      unreachable when the VPS promoted - but check
#      status.currentPrimary on home before proceeding).
#   2. Re-point home's Cluster to replicate FROM the vps (it needs to
#      catch up on everything written to the VPS while it was down)
#      before flipping primary back - home has to actually BE a caught-up
#      replica of vps first, or this loses the writes the VPS accepted
#      during the outage.
#   3. Only once home shows it's streaming and caught up, flip primary
#      back to home and update DNS.
#
# This script only does step 3 (the fast, symmetric part - same shape as
# the watcher's own promote() but in reverse). Steps 1-2 are a judgment
# call requiring you to actually look at both clusters' status first -
# not scripted here on purpose.
set -euo pipefail

echo "This will flip pg-authentik's primary back to home and repoint DNS."
echo "Before continuing, you MUST have already confirmed:"
echo "  - home's pg-authentik Cluster is healthy and replicating FROM vps"
echo "    (kubectl -n authentik get cluster pg-authentik -o jsonpath='{.status}')"
echo "  - home is caught up (no meaningful replication lag from vps)"
read -r -p "Confirmed both of the above? [y/N] " ans
if [ "${ans:-N}" != "y" ] && [ "${ans:-N}" != "Y" ]; then
  echo "Aborted."
  exit 1
fi

echo "Flipping home's Cluster CR to primary..."
kubectl -n authentik patch cluster pg-authentik --type merge \
  -p '{"spec":{"replica":{"self":"home","primary":"home","source":"home"}}}'

echo "Flipping VPS's Cluster CR back to a replica of home..."
ssh root@172.93.53.139 "kubectl -n authentik patch cluster pg-authentik --type merge \
  -p '{\"spec\":{\"replica\":{\"self\":\"vps\",\"primary\":\"home\",\"source\":\"home\"}}}'"

echo "Flipping pg-authentik.ha.huskypup.net back to home's IP..."
HOME_PUBLIC_IP="$(dig +short home.kube.huskypup.net @1.1.1.1 | tail -1)"
if [ -z "$HOME_PUBLIC_IP" ]; then
  echo "ERROR: could not resolve home.kube.huskypup.net - fix the DNS record manually"
  exit 1
fi
TOKEN="$(kubectl -n cert-manager get secret cloudflare-token-secret -o jsonpath='{.data.cloudflare-token}' | base64 -d)"
ZONE_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://api.cloudflare.com/client/v4/zones?name=huskypup.net" | jq -r '.result[0].id')"
RECORD_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=pg-authentik.ha.huskypup.net&type=A" | jq -r '.result[0].id')"
curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
  -d "{\"type\":\"A\",\"name\":\"pg-authentik.ha.huskypup.net\",\"content\":\"${HOME_PUBLIC_IP}\",\"ttl\":60,\"proxied\":false}" \
  "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" >/dev/null

echo "Clearing the etcd promotion record..."
curl -sf --max-time 5 -X POST http://172.28.101.41:32379/v3/kv/deleterange \
  -d "{\"key\":\"$(printf '%s' '/ha-failover/promoted-at' | base64)\"}" >/dev/null || true
curl -sf --max-time 5 -X POST http://172.28.101.41:32379/v3/kv/deleterange \
  -d "{\"key\":\"$(printf '%s' '/ha-failover/promoted-by' | base64)\"}" >/dev/null || true

echo "Failback complete. home is primary again, pg-authentik.ha.huskypup.net -> ${HOME_PUBLIC_IP}"