From b77f0865d08ad77f842bfa963c8bd7e84c81b2a3 Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Wed, 19 Aug 2026 00:08:23 -0500 Subject: [PATCH] Wire CNPG streaming replication + distributed topology for pg-authentik pilot Adds connectionParameters (real streaming, not just barmanObjectStore WAL-archive polling) to both sides' existing externalClusters entry, plus replica.self/primary fields declaring the distributed topology (both currently agree home is primary). Auth reuses CNPG's own auto-generated streaming_replica client certs, cross-copied between clusters manually (kubectl, not git - same pattern as every other cross-cluster secret tonight) since they're the exact credential each side's pg_hba.conf already trusts. Also adds the VPS-side NodePort exposing its own pg-authentik primary back to home (home->VPS already works via the existing netbird-egress route, no UniFi port-forward needed for that direction - only the reverse, VPS/witness reaching into home, needed the WAN workaround). --- .../authentik/manifests/cnpg-cluster.yaml | 50 +++++++++++++++++++ .../authentik/manifests/cnpg-cluster.yaml | 38 ++++++++++++++ .../manifests/ha-postgres-nodeport.yaml | 27 ++++++++++ 3 files changed, 115 insertions(+) create mode 100644 infrastructure/vps-standby/authentik/manifests/ha-postgres-nodeport.yaml diff --git a/infrastructure/authentik/manifests/cnpg-cluster.yaml b/infrastructure/authentik/manifests/cnpg-cluster.yaml index c11abf8..35c8220 100644 --- a/infrastructure/authentik/manifests/cnpg-cluster.yaml +++ b/infrastructure/authentik/manifests/cnpg-cluster.yaml @@ -76,3 +76,53 @@ spec: monitoring: enablePodMonitor: true + # Multi-site active failover pilot (see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - streaming + # replication to/from the VPS's pg-authentik cluster, alongside (not + # replacing) the barmanObjectStore backup above. + # + # Auth: reuses CNPG's own generated streaming_replica client cert + # (CN=streaming_replica, issued by the VPS's own pg-authentik CA) - + # copied here manually as pg-authentik-vps-{ca,replication} (kubectl, + # not git - same pattern as every other cross-cluster secret tonight). + # This is the exact credential CNPG's own pg_hba.conf already trusts + # for streaming replication, nothing new to provision. + # + # sslmode: verify-ca (not verify-full) - the VPS's server cert was + # issued for its in-cluster DNS names, not for the NodePort IP address + # (100.108.113.41) home actually dials; verify-ca still validates the + # cert chain/CA trust and encrypts the connection, just skips hostname + # matching. + # + # Reachable via 100.108.113.41 (VPS's Netbird IP) - home reaching OUT + # to the VPS already works today via the netbird-egress DaemonSet's + # route (same path used for MinIO/CrowdSec backups all session) - no + # UniFi port-forward needed for this direction, only the reverse + # (VPS/witness reaching INTO home) needed that. + externalClusters: + - name: vps + connectionParameters: + host: 100.108.113.41 + port: "32433" + dbname: app + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-authentik-vps-replication + key: tls.crt + sslKey: + name: pg-authentik-vps-replication + key: tls.key + sslRootCert: + name: pg-authentik-vps-ca + key: ca.crt + + # Distributed topology: both sides agree home is primary today. Failover + # flips the VPS's replica.primary to "vps" (+ promotion token) - see the + # plan doc's failover-watcher section. Failback is the same in reverse, + # deliberately manual. + replica: + self: home + primary: home + source: vps # where home would replicate FROM if it were ever a replica (not active now - primary: home) + diff --git a/infrastructure/vps-standby/authentik/manifests/cnpg-cluster.yaml b/infrastructure/vps-standby/authentik/manifests/cnpg-cluster.yaml index 4f24d9a..5d7aad8 100644 --- a/infrastructure/vps-standby/authentik/manifests/cnpg-cluster.yaml +++ b/infrastructure/vps-standby/authentik/manifests/cnpg-cluster.yaml @@ -70,6 +70,26 @@ spec: recovery: source: home-backup + # Multi-site active failover pilot (see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - the + # bootstrap.recovery above (barmanObjectStore) stays exactly as-is for + # the initial seed; connectionParameters below adds real streaming + # replication for ongoing sync, replacing the old WAL-archive-polling + # behavior (the source of the crash-looping/probe-tuning fights from + # earlier tonight - streaming doesn't need to restart postgres to check + # for new WAL). + # + # Auth: reuses CNPG's own generated streaming_replica client cert + # (CN=streaming_replica, issued by home's own pg-authentik CA) - copied + # here manually as pg-authentik-home-{ca,replication} (kubectl, not + # git). Same credential CNPG's own pg_hba.conf already trusts, nothing + # new to provision. + # + # Reachable via home.kube.huskypup.net:61432 - a UniFi WAN port-forward + # to talos-cp-01's real LAN IP (172.28.101.41), source-IP-restricted to + # just the VPS and witness public IPs. NOT the Netbird mesh directly - + # UniFi has no route to that CIDR, confirmed live (see + # infrastructure/ha-failover/manifests/etcd.yaml for the full story). externalClusters: - name: home-backup barmanObjectStore: @@ -91,9 +111,27 @@ spec: secretAccessKey: name: vps-minio-secret key: secretkey + connectionParameters: + host: home.kube.huskypup.net + port: "61432" + dbname: app + user: streaming_replica + sslmode: verify-ca + sslCert: + name: pg-authentik-home-replication + key: tls.crt + sslKey: + name: pg-authentik-home-replication + key: tls.key + sslRootCert: + name: pg-authentik-home-ca + key: ca.crt + # Distributed topology: both sides agree home is primary today. replica: enabled: true + self: vps + primary: home source: home-backup monitoring: diff --git a/infrastructure/vps-standby/authentik/manifests/ha-postgres-nodeport.yaml b/infrastructure/vps-standby/authentik/manifests/ha-postgres-nodeport.yaml new file mode 100644 index 0000000..b181776 --- /dev/null +++ b/infrastructure/vps-standby/authentik/manifests/ha-postgres-nodeport.yaml @@ -0,0 +1,27 @@ +--- +# Exposes the VPS's pg-authentik primary (currently a read-only replica - +# see cnpg-cluster.yaml's replica.enabled) to home over the Netbird mesh, +# for the multi-site active-failover pilot. Same selector as CNPG's own +# generated pg-authentik-rw ClusterIP Service, just NodePort instead. +# +# Unlike home's side of this (infrastructure/authentik/manifests/ +# ha-postgres-nodeport.yaml), no UniFi port-forward or Kyverno +# PolicyException needed here - the VPS's k3s has no NodePort +# restriction, and home reaching OUT to the VPS already works today via +# the netbird-egress DaemonSet's route (confirmed live all session, same +# path used for MinIO/CrowdSec) - it's only the reverse direction +# (external peers reaching INTO home) that needed the UniFi workaround. +apiVersion: v1 +kind: Service +metadata: + name: ha-authentik-postgres + namespace: authentik +spec: + type: NodePort + selector: + cnpg.io/cluster: pg-authentik + cnpg.io/instanceRole: primary + ports: + - port: 5432 + targetPort: 5432 + nodePort: 32433