mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-20 23:16:49 +00:00
Scooby HuskyandClaude Sonnet 5
f7a9a03816
Builds the one genuinely new component from the plan - everything else
reuses CNPG/etcd as-is.
- infrastructure/ha-failover/manifests/heartbeat-writer.yaml: home writes
a fresh timestamp to etcd (/ha-failover/home-heartbeat) every 10s via
its local etcd member. No custom quorum/voting logic - etcd's own Raft
consensus (majority write/linearizable read) already gives the 'a
majority agrees' guarantee.
- scripts/ha-failover-watcher.sh + .service: runs on the VPS as a
systemd service (not k3s - must survive the VPS's own cluster being
unhealthy). Reads the heartbeat via its own local etcd member; if the
read itself fails, we can't tell if home is down or if this VPS is the
one partitioned, so it does NOT act (etcd's consensus requirement
provides the split-brain safety here, not custom code). If the read
succeeds and the heartbeat is stale (>45s, ~4-5 missed beats), home is
confirmed down by majority - promotes pg-authentik's Cluster CR
(spec.replica self/primary/source -> vps, no promotionToken - confirmed
live via dry-run that CNPG's webhook doesn't require one, it's only for
planned/graceful switchover LSN cross-checks that aren't available
during a genuine unplanned outage) and flips the
pg-authentik.ha.huskypup.net Cloudflare A record to the VPS's IP.
- scripts/ha-failback-authentik.sh: the deliberately manual reverse -
human confirms home is healthy and caught up before running this.
- Floating hostname plumbing: both sites' Authentik values now read
AUTHENTIK_POSTGRESQL__HOST/PORT as pg-authentik.ha.huskypup.net:61432
instead of the local pg-authentik-app secret's host/port, so app pods
on either site always reach whichever site is actually primary. Both
sites listen on the same external port (VPS gets a new
pg-authentik-forward.service socat forward, mirroring the existing
minio-forward.service pattern, so its NodePort 32433 is externally
reachable on 61432 same as home's UniFi-forwarded port).
- Found and worked around two real bugs surfaced while wiring this up
(fixed manually via scripts handed to the user - both blocked by the
Claude Code auto-mode classifier as credential-transmission /
cluster-DNS-edit actions):
- VPS's pg-authentik-app secret had a STALE password from before
streaming replication existed - the live Postgres role password now
replicates from home via WAL, but VPS's local K8s secret never got
updated to match. Needs a one-time sync (and again on any future
rotation).
- This UniFi does not support NAT hairpin/loopback for its own WAN
port-forwards - home's own pods resolving the floating hostname need
a local CoreDNS rewrite straight to pg-authentik-rw, confirmed live by
a DNS-resolves-but-TCP-connect-fails test.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
126 lines
4.3 KiB
YAML
126 lines
4.3 KiB
YAML
# Authentik warm standby on the VPS - Phase 2, later extended into the
|
|
# multi-site active failover pilot (see
|
|
# /home/scooby/.claude/plans/jiggly-snacking-iverson.md). The CNPG cluster
|
|
# (manifests/cnpg-cluster.yaml) now uses real streaming replication
|
|
# (externalClusters connectionParameters + spec.replica.self/primary/
|
|
# source) rather than the original WAL-archive-polling replica.enabled
|
|
# mode - that flag is gone from cnpg-cluster.yaml now, it's mutually
|
|
# exclusive with the primary/self fields. The app itself DOES run
|
|
# continuously (replicas: 1, reachable at auth.vps.huskypup.net - see
|
|
# manifests/ingress.yaml) so the replicated data is browsable/verifiable
|
|
# at all times, even though the underlying DB is a read-only replica until
|
|
# promoted. Promotion (automatic via the failover-watcher, or manually):
|
|
# patch cnpg-cluster.yaml's spec.replica to self/primary/source: vps.
|
|
#
|
|
# The `authentik:` block below (secret_key/postgresql "env://" indirection)
|
|
# is copied VERBATIM from infrastructure/authentik/values.yaml on purpose -
|
|
# the authentik chart deterministically renders its own generated Secret
|
|
# from these literal values.yaml strings (not randomly), so keeping this
|
|
# block byte-identical between home and VPS means both clusters
|
|
# independently arrive at the same AUTHENTIK_SECRET_KEY without needing to
|
|
# manually copy it - required for promotion to work, since that key
|
|
# decrypts things (stored provider secrets, cert private keys) that live
|
|
# encrypted in the DB being replicated.
|
|
authentik:
|
|
secret_key: env://AUTHENTIK_SECRET_KEY
|
|
postgresql:
|
|
host: env://AUTHENTIK_POSTGRESQL__HOST
|
|
port: 5432
|
|
name: env://AUTHENTIK_POSTGRESQL__NAME
|
|
user: env://AUTHENTIK_POSTGRESQL__USER
|
|
password: env://AUTHENTIK_POSTGRESQL__PASSWORD
|
|
|
|
global:
|
|
envFrom:
|
|
- secretRef:
|
|
name: authentik
|
|
- secretRef:
|
|
name: pg-authentik-app
|
|
prefix: AUTHENTIK_POSTGRESQL__
|
|
env:
|
|
- name: AUTHENTIK_URL
|
|
value: "https://auth.vps.huskypup.net"
|
|
# HOST/PORT point at the floating pg-authentik.ha.huskypup.net record
|
|
# instead of the local secret's host/port - see the matching comment
|
|
# in infrastructure/authentik/values.yaml (home's copy of this same
|
|
# block) for the full explanation, including why pg-authentik-app's
|
|
# password had to be manually synced between the two sites first.
|
|
- name: AUTHENTIK_POSTGRESQL__HOST
|
|
value: "pg-authentik.ha.huskypup.net"
|
|
- name: AUTHENTIK_POSTGRESQL__NAME
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: pg-authentik-app
|
|
key: dbname
|
|
- name: AUTHENTIK_POSTGRESQL__USER
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: pg-authentik-app
|
|
key: username
|
|
- name: AUTHENTIK_POSTGRESQL__PASSWORD
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: pg-authentik-app
|
|
key: password
|
|
- name: AUTHENTIK_POSTGRESQL__PORT
|
|
value: "61432"
|
|
|
|
server:
|
|
replicas: 1
|
|
# Default failureThreshold: 3 (40s runway) is too strict running against
|
|
# a read-only CNPG replica - the worker constantly retries a scheduled-
|
|
# task query that needs a write lock (harmless, expected, logged as
|
|
# 'cannot execute SELECT FOR UPDATE in a read-only transaction'), and
|
|
# any transient DB hiccup during that trips the server's health check
|
|
# and gets it killed (confirmed live: CrashLoopBackOff, 'connection
|
|
# refused' during a brief blip). Widened so brief hiccups don't kill it.
|
|
livenessProbe:
|
|
failureThreshold: 10
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
readinessProbe:
|
|
failureThreshold: 10
|
|
initialDelaySeconds: 10
|
|
periodSeconds: 10
|
|
timeoutSeconds: 5
|
|
containerSecurityContext:
|
|
runAsNonRoot: true
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 512Mi
|
|
limits:
|
|
memory: 1Gi
|
|
metrics:
|
|
enabled: false
|
|
|
|
worker:
|
|
replicas: 1
|
|
containerSecurityContext:
|
|
runAsNonRoot: true
|
|
allowPrivilegeEscalation: false
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
resources:
|
|
requests:
|
|
cpu: 50m
|
|
memory: 512Mi
|
|
limits:
|
|
memory: 1Gi
|
|
metrics:
|
|
enabled: false
|
|
|
|
# Disable bundled Postgres - using the CNPG replica cluster instead
|
|
postgresql:
|
|
enabled: false
|