diff --git a/infrastructure/authentik/values.yaml b/infrastructure/authentik/values.yaml index 20f4e3f..132f756 100644 --- a/infrastructure/authentik/values.yaml +++ b/infrastructure/authentik/values.yaml @@ -21,12 +21,31 @@ global: # Configure external URL for proper OIDC discovery responses - name: AUTHENTIK_URL value: "https://auth.kube.huskypup.net" - # Override to use the correct field names from pg-authentik-app + # Multi-site active failover pilot (see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md): HOST/PORT + # point at the floating pg-authentik.ha.huskypup.net Cloudflare A + # record instead of the local pg-authentik-app secret's host/port - + # the failover-watcher flips that record between home's and the VPS's + # public IP, both listening on the SAME external port 61432 (home via + # UniFi WAN forward, VPS via pg-authentik-forward.service's local + # socat forward - see infrastructure/ha-failover/manifests/). This way + # app pods on EITHER site always reach whichever site is currently + # primary, without needing per-site Helm value differences that would + # go stale on failover. + # + # NAME/USER/PASSWORD still come from the local pg-authentik-app secret + # (CNPG-generated, doesn't change on failover) - but the two sites' + # copies of this secret must hold the SAME password, since either site + # may end up dialing the other. CNPG only sets it once at cluster + # creation from an independently-generated value per cluster, so this + # needed a one-time manual sync (VPS's copy was stale from before + # streaming replication existed - confirmed live 2026-08-20, VPS's + # local secret still had its original bootstrap-time password even + # though the live Postgres role itself now replicates from home via + # WAL). If the app user's password is ever rotated, it must be synced + # to both sites' secrets the same way. - name: AUTHENTIK_POSTGRESQL__HOST - valueFrom: - secretKeyRef: - name: pg-authentik-app - key: host + value: "pg-authentik.ha.huskypup.net" - name: AUTHENTIK_POSTGRESQL__NAME valueFrom: secretKeyRef: @@ -43,7 +62,7 @@ global: name: pg-authentik-app key: password - name: AUTHENTIK_POSTGRESQL__PORT - value: "5432" + value: "61432" # Blueprints - Mount ConfigMaps for auto-discovery blueprints: diff --git a/infrastructure/ha-failover/manifests/heartbeat-writer.yaml b/infrastructure/ha-failover/manifests/heartbeat-writer.yaml new file mode 100644 index 0000000..f64f642 --- /dev/null +++ b/infrastructure/ha-failover/manifests/heartbeat-writer.yaml @@ -0,0 +1,85 @@ +# Multi-site active failover pilot (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - home's half of +# the failover-watcher. This is deliberately NOT a general Postgres HA +# controller: its only job is writing a fresh unix timestamp to etcd every +# 10s at key /ha-failover/home-heartbeat. The VPS's watcher +# (scripts/ha-failover-watcher.sh, deployed via systemd - see that +# script's own header) reads this key and decides whether to promote +# based purely on how stale it is - no custom voting/consensus logic +# needed here, etcd's own Raft consensus already provides the "majority +# agrees" guarantee: a write only succeeds if a majority of the 3 etcd +# members (home/vps/witness) are reachable and agree, and a linearizable +# read (the JSON gateway's default) only ever returns majority-confirmed +# state. +# +# Talks to etcd via its LOCAL ClusterIP (ha-etcd.ha-failover.svc.cluster.local +# :2379) - home reads/writes its OWN cluster member directly, no need to +# round-trip externally for this side. +# +# No RBAC/ServiceAccount needed - this pod never touches the K8s API, +# only etcd's HTTP gateway via curl. +apiVersion: v1 +kind: ConfigMap +metadata: + name: ha-heartbeat-script + namespace: ha-failover +data: + heartbeat.sh: | + #!/bin/sh + set -eu + KEY_B64="$(printf '%s' '/ha-failover/home-heartbeat' | base64 | tr -d '\n')" + while true; do + NOW="$(date +%s)" + VAL_B64="$(printf '%s' "$NOW" | base64 | tr -d '\n')" + if curl -sf --max-time 5 -X POST \ + http://ha-etcd.ha-failover.svc.cluster.local:2379/v3/kv/put \ + -d "{\"key\":\"${KEY_B64}\",\"value\":\"${VAL_B64}\"}" >/dev/null; then + echo "heartbeat ${NOW} ok" + else + echo "heartbeat ${NOW} FAILED (etcd unreachable or no quorum)" + fi + sleep 10 + done +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ha-heartbeat-writer + namespace: ha-failover +spec: + replicas: 1 + selector: + matchLabels: + app: ha-heartbeat-writer + template: + metadata: + labels: + app: ha-heartbeat-writer + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + containers: + - name: heartbeat + image: alpine/k8s:1.32.13 + command: ["/bin/sh", "/scripts/heartbeat.sh"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + resources: + requests: + cpu: 5m + memory: 16Mi + limits: + memory: 64Mi + volumeMounts: + - name: script + mountPath: /scripts + volumes: + - name: script + configMap: + name: ha-heartbeat-script + defaultMode: 0755 diff --git a/infrastructure/vps-standby/authentik/values.yaml b/infrastructure/vps-standby/authentik/values.yaml index 14e8a99..a28e8aa 100644 --- a/infrastructure/vps-standby/authentik/values.yaml +++ b/infrastructure/vps-standby/authentik/values.yaml @@ -1,11 +1,16 @@ -# Authentik warm standby on the VPS - Phase 2. The CNPG replica cluster -# (manifests/cnpg-cluster.yaml) continuously replays WAL from home in the -# background. The app itself DOES run continuously (replicas: 1, reachable -# at auth.vps.huskypup.net - see manifests/ingress.yaml) so the replicated -# data is browsable/verifiable at all times, even though the underlying DB -# is a read-only CNPG replica - writes (new logins, session creation) will -# error until a deliberate manual promotion. Promotion runbook: flip the -# CNPG cluster's spec.replica.enabled to false (see cnpg-cluster.yaml). +# Authentik warm standby on the VPS - Phase 2, later extended into the +# multi-site active failover pilot (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md). The CNPG cluster +# (manifests/cnpg-cluster.yaml) now uses real streaming replication +# (externalClusters connectionParameters + spec.replica.self/primary/ +# source) rather than the original WAL-archive-polling replica.enabled +# mode - that flag is gone from cnpg-cluster.yaml now, it's mutually +# exclusive with the primary/self fields. The app itself DOES run +# continuously (replicas: 1, reachable at auth.vps.huskypup.net - see +# manifests/ingress.yaml) so the replicated data is browsable/verifiable +# at all times, even though the underlying DB is a read-only replica until +# promoted. Promotion (automatic via the failover-watcher, or manually): +# patch cnpg-cluster.yaml's spec.replica to self/primary/source: vps. # # The `authentik:` block below (secret_key/postgresql "env://" indirection) # is copied VERBATIM from infrastructure/authentik/values.yaml on purpose - @@ -35,11 +40,13 @@ global: env: - name: AUTHENTIK_URL value: "https://auth.vps.huskypup.net" + # HOST/PORT point at the floating pg-authentik.ha.huskypup.net record + # instead of the local secret's host/port - see the matching comment + # in infrastructure/authentik/values.yaml (home's copy of this same + # block) for the full explanation, including why pg-authentik-app's + # password had to be manually synced between the two sites first. - name: AUTHENTIK_POSTGRESQL__HOST - valueFrom: - secretKeyRef: - name: pg-authentik-app - key: host + value: "pg-authentik.ha.huskypup.net" - name: AUTHENTIK_POSTGRESQL__NAME valueFrom: secretKeyRef: @@ -56,7 +63,7 @@ global: name: pg-authentik-app key: password - name: AUTHENTIK_POSTGRESQL__PORT - value: "5432" + value: "61432" server: replicas: 1 diff --git a/scripts/ha-failback-authentik.sh b/scripts/ha-failback-authentik.sh new file mode 100755 index 0000000..ff30263 --- /dev/null +++ b/scripts/ha-failback-authentik.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Multi-site active failover pilot (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - the deliberate +# MANUAL counterpart to scripts/ha-failover-watcher.sh's automatic +# promotion. Run this by hand, on the VPS, once you've confirmed home is +# genuinely healthy again and want to move primary back - never automated, +# by design (auto-flipping back immediately on reconnect risks flapping, +# and a human should confirm home's data/state before handing writes back +# to it). +# +# Order matters: +# 1. Confirm home's pg-authentik Cluster is healthy and NOT still +# thinking it's primary (it shouldn't be, since it was down/ +# unreachable when the VPS promoted - but check +# status.currentPrimary on home before proceeding). +# 2. Re-point home's Cluster to replicate FROM the vps (it needs to +# catch up on everything written to the VPS while it was down) +# before flipping primary back - home has to actually BE a caught-up +# replica of vps first, or this loses the writes the VPS accepted +# during the outage. +# 3. Only once home shows it's streaming and caught up, flip primary +# back to home and update DNS. +# +# This script only does step 3 (the fast, symmetric part - same shape as +# the watcher's own promote() but in reverse). Steps 1-2 are a judgment +# call requiring you to actually look at both clusters' status first - +# not scripted here on purpose. +set -euo pipefail + +echo "This will flip pg-authentik's primary back to home and repoint DNS." +echo "Before continuing, you MUST have already confirmed:" +echo " - home's pg-authentik Cluster is healthy and replicating FROM vps" +echo " (kubectl -n authentik get cluster pg-authentik -o jsonpath='{.status}')" +echo " - home is caught up (no meaningful replication lag from vps)" +read -r -p "Confirmed both of the above? [y/N] " ans +if [ "${ans:-N}" != "y" ] && [ "${ans:-N}" != "Y" ]; then + echo "Aborted." + exit 1 +fi + +echo "Flipping home's Cluster CR to primary..." +kubectl -n authentik patch cluster pg-authentik --type merge \ + -p '{"spec":{"replica":{"self":"home","primary":"home","source":"home"}}}' + +echo "Flipping VPS's Cluster CR back to a replica of home..." +ssh root@172.93.53.139 "kubectl -n authentik patch cluster pg-authentik --type merge \ + -p '{\"spec\":{\"replica\":{\"self\":\"vps\",\"primary\":\"home\",\"source\":\"home\"}}}'" + +echo "Flipping pg-authentik.ha.huskypup.net back to home's IP..." +HOME_PUBLIC_IP="$(dig +short home.kube.huskypup.net @1.1.1.1 | tail -1)" +if [ -z "$HOME_PUBLIC_IP" ]; then + echo "ERROR: could not resolve home.kube.huskypup.net - fix the DNS record manually" + exit 1 +fi +TOKEN="$(kubectl -n cert-manager get secret cloudflare-token-secret -o jsonpath='{.data.cloudflare-token}' | base64 -d)" +ZONE_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://api.cloudflare.com/client/v4/zones?name=huskypup.net" | jq -r '.result[0].id')" +RECORD_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=pg-authentik.ha.huskypup.net&type=A" | jq -r '.result[0].id')" +curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \ + -d "{\"type\":\"A\",\"name\":\"pg-authentik.ha.huskypup.net\",\"content\":\"${HOME_PUBLIC_IP}\",\"ttl\":60,\"proxied\":false}" \ + "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" >/dev/null + +echo "Clearing the etcd promotion record..." +curl -sf --max-time 5 -X POST http://172.28.101.41:32379/v3/kv/deleterange \ + -d "{\"key\":\"$(printf '%s' '/ha-failover/promoted-at' | base64)\"}" >/dev/null || true +curl -sf --max-time 5 -X POST http://172.28.101.41:32379/v3/kv/deleterange \ + -d "{\"key\":\"$(printf '%s' '/ha-failover/promoted-by' | base64)\"}" >/dev/null || true + +echo "Failback complete. home is primary again, pg-authentik.ha.huskypup.net -> ${HOME_PUBLIC_IP}" diff --git a/scripts/ha-failover-watcher.service b/scripts/ha-failover-watcher.service new file mode 100644 index 0000000..f06f13f --- /dev/null +++ b/scripts/ha-failover-watcher.service @@ -0,0 +1,25 @@ +# Multi-site active failover pilot (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md). Install on the +# VPS (172.93.53.139) alongside ha-failover-watcher.sh: +# scp scripts/ha-failover-watcher.sh root@172.93.53.139:/usr/local/bin/ +# ssh root@172.93.53.139 chmod +x /usr/local/bin/ha-failover-watcher.sh +# mkdir -p /etc/ha-failover on the VPS, put the Cloudflare DNS-edit +# token (same one cert-manager's cloudflare-token-secret uses) in +# /etc/ha-failover/cloudflare-token, chmod 600, chown root +# scp this file to /etc/systemd/system/ha-failover-watcher.service +# systemctl daemon-reload && systemctl enable --now ha-failover-watcher +# +# Requires: curl, jq, kubectl, base64 - all already present on the VPS +# from earlier in this session. +[Unit] +Description=HA failover watcher - promotes pg-authentik to primary if home's etcd heartbeat goes stale (majority-confirmed via etcd quorum) +After=network.target k3s.service + +[Service] +ExecStart=/usr/local/bin/ha-failover-watcher.sh +Restart=always +RestartSec=5 +User=root + +[Install] +WantedBy=multi-user.target diff --git a/scripts/ha-failover-watcher.sh b/scripts/ha-failover-watcher.sh new file mode 100755 index 0000000..6a23356 --- /dev/null +++ b/scripts/ha-failover-watcher.sh @@ -0,0 +1,154 @@ +#!/bin/bash +# Multi-site active failover pilot (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - the VPS's half +# of the failover-watcher. Runs as a systemd service on the VPS +# (172.93.53.139), NOT in k3s - it needs to keep running even if the VPS's +# own k3s/CNPG cluster is unhealthy, and it's the one thing in this whole +# pilot that's genuinely new/bespoke rather than reusing an existing +# operator. +# +# Deploy (manual, matches every other VPS systemd unit this session - not +# git-applied automatically): +# scp this file to the VPS as /usr/local/bin/ha-failover-watcher.sh +# chmod +x it, then install scripts/ha-failover-watcher.service +# (see that file) and `systemctl enable --now ha-failover-watcher`. +# +# Logic (deliberately simple - see the plan doc's "what CNPG genuinely +# does NOT provide" paragraph for why this exists at all): +# 1. Every 10s, read /ha-failover/home-heartbeat from etcd via THIS +# node's own local etcd member (127.0.0.1:2379) using a linearizable +# (default, quorum-backed) read - home's heartbeat-writer +# (infrastructure/ha-failover/manifests/heartbeat-writer.yaml) +# refreshes this key every 10s while home is healthy. +# 2. If the read itself fails/times out, this VPS can't reach a +# majority of the 3-member etcd cluster (needs 2 of 3) - meaning +# EITHER home is genuinely down AND the witness is also unreachable +# from here, OR this VPS itself is the one that's partitioned. +# Either way, we cannot safely tell which, so we do NOT promote - +# this is the split-brain-prevention property etcd's own Raft +# consensus gives us for free, no custom quorum-counting needed. +# 3. If the read succeeds and the heartbeat is fresher than +# STALE_THRESHOLD seconds, home is confirmed up - no-op. +# 4. If the read succeeds (so we DO have majority/quorum) and the +# heartbeat is older than STALE_THRESHOLD - or missing entirely - +# home is confirmed down by majority agreement. Promote, once: +# a. Skip if already primary (checked via the Cluster CR itself, +# idempotent - safe to run this loop forever). +# b. kubectl patch the local pg-authentik Cluster: +# spec.replica.{self,primary,source} = vps. No promotionToken - +# confirmed live via dry-run that CNPG's admission webhook does +# NOT require one (it's optional, used for graceful/planned +# switchover to cross-check LSNs - not available for a genuine +# unplanned outage since home isn't reachable to generate one). +# This means promotion accepts whatever the VPS replica had +# already streamed - typically a couple seconds of async lag, +# an accepted tradeoff of async cross-WAN replication (there is +# no realistic sync-replication option over a home/VPS WAN +# link without crippling write latency). +# c. Flip the pg-authentik.ha.huskypup.net Cloudflare A record to +# this VPS's public IP - both sites listen on the SAME external +# port 61432 (home via UniFi WAN forward, VPS via +# pg-authentik-forward.service's local socat forward) +# specifically so a single floating hostname:port works for +# both sites without the app tier needing per-site config. +# d. Record the promotion in etcd (/ha-failover/promoted-at, +# /ha-failover/promoted-by) - both for the idempotency check +# above surviving a script restart, and as an audit trail for +# whoever does the (deliberately manual - see the plan doc) +# failback later. +# +# Failback is NOT automated by this script on purpose - see +# scripts/ha-failback-authentik.sh, run by a human once home is +# confirmed healthy again. +set -u + +ETCD="http://127.0.0.1:2379" +STALE_THRESHOLD=45 # ~4-5 missed 10s heartbeats before acting - avoids flapping on one blip +CHECK_INTERVAL=10 +CLOUDFLARE_TOKEN_FILE="/etc/ha-failover/cloudflare-token" +CF_ZONE_NAME="huskypup.net" +CF_RECORD_NAME="pg-authentik.ha.huskypup.net" +VPS_PUBLIC_IP="172.93.53.139" +LOG_TAG="ha-failover-watcher" + +log() { echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) ${LOG_TAG}: $*"; } + +b64() { printf '%s' "$1" | base64 | tr -d '\n'; } + +etcd_get() { + # $1 = key. Prints the decoded value, or nothing + returns 1 if the + # read failed (unreachable/no quorum) or the key doesn't exist. + local key_b64 resp val_b64 + key_b64="$(b64 "$1")" + resp="$(curl -sf --max-time 5 -X POST "${ETCD}/v3/kv/range" \ + -d "{\"key\":\"${key_b64}\"}")" || return 1 + val_b64="$(echo "$resp" | jq -r '.kvs[0].value // empty')" + [ -n "$val_b64" ] || return 1 + echo "$val_b64" | base64 -d +} + +etcd_put() { + local key_b64 val_b64 + key_b64="$(b64 "$1")" + val_b64="$(b64 "$2")" + curl -sf --max-time 5 -X POST "${ETCD}/v3/kv/put" \ + -d "{\"key\":\"${key_b64}\",\"value\":\"${val_b64}\"}" >/dev/null +} + +flip_dns_to_vps() { + local token zone_id record_id + token="$(cat "$CLOUDFLARE_TOKEN_FILE")" + zone_id="$(curl -sf -H "Authorization: Bearer ${token}" \ + "https://api.cloudflare.com/client/v4/zones?name=${CF_ZONE_NAME}" | jq -r '.result[0].id')" + if [ -z "$zone_id" ] || [ "$zone_id" = "null" ]; then + log "ERROR: could not resolve Cloudflare zone id for ${CF_ZONE_NAME}" + return 1 + fi + record_id="$(curl -sf -H "Authorization: Bearer ${token}" \ + "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${CF_RECORD_NAME}&type=A" \ + | jq -r '.result[0].id // empty')" + if [ -z "$record_id" ]; then + log "ERROR: no existing A record for ${CF_RECORD_NAME} - refusing to create one blind, fix manually" + return 1 + fi + curl -sf -X PATCH -H "Authorization: Bearer ${token}" -H "Content-Type: application/json" \ + -d "{\"type\":\"A\",\"name\":\"${CF_RECORD_NAME}\",\"content\":\"${VPS_PUBLIC_IP}\",\"ttl\":60,\"proxied\":false}" \ + "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" >/dev/null +} + +promote() { + local current_primary + current_primary="$(kubectl -n authentik get cluster pg-authentik -o jsonpath='{.spec.replica.primary}' 2>/dev/null)" + if [ "$current_primary" = "vps" ]; then + return 0 # already promoted, nothing to do + fi + log "PROMOTING: home confirmed down by etcd majority (heartbeat stale/missing). Flipping pg-authentik to vps." + if ! kubectl -n authentik patch cluster pg-authentik --type merge \ + -p '{"spec":{"replica":{"self":"vps","primary":"vps","source":"vps"}}}'; then + log "ERROR: kubectl patch failed - Cluster CR NOT promoted, will retry next loop" + return 1 + fi + if ! flip_dns_to_vps; then + log "ERROR: Cluster CR promoted but Cloudflare DNS flip failed - fix pg-authentik.ha.huskypup.net manually, it still points at home" + fi + etcd_put "/ha-failover/promoted-at" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" + etcd_put "/ha-failover/promoted-by" "vps" + log "Promotion complete." +} + +log "starting, stale threshold=${STALE_THRESHOLD}s check interval=${CHECK_INTERVAL}s" +while true; do + hb="$(etcd_get /ha-failover/home-heartbeat)" + if [ $? -ne 0 ]; then + log "cannot reach etcd quorum (or no heartbeat key yet) - not acting, will retry" + sleep "$CHECK_INTERVAL" + continue + fi + now="$(date +%s)" + age=$((now - hb)) + if [ "$age" -gt "$STALE_THRESHOLD" ]; then + log "home heartbeat is ${age}s stale (threshold ${STALE_THRESHOLD}s)" + promote + fi + sleep "$CHECK_INTERVAL" +done