diff --git a/infrastructure/authentik/manifests/ha-postgres-peerauth.yaml b/infrastructure/authentik/manifests/ha-postgres-peerauth.yaml new file mode 100644 index 0000000..f270b1e --- /dev/null +++ b/infrastructure/authentik/manifests/ha-postgres-peerauth.yaml @@ -0,0 +1,40 @@ +# Multi-site active failover pilot (see +# /home/scooby/.claude/plans/jiggly-snacking-iverson.md) - CNPG streaming +# replication from the VPS was silently failing: `pg_stat_wal_receiver` on +# the VPS replica showed 0 rows, and its logs showed a repeating +# "could not connect to the primary server: ... server closed the +# connection unexpectedly" every few minutes (confirmed live 2026-08-19). +# +# Root cause: the `authentik` namespace is enrolled in Istio's ambient mesh +# (`istio.io/dataplane-mode: ambient`) and the mesh-wide default +# PeerAuthentication (istio-system/default) is STRICT - ztunnel requires a +# valid mesh (HBONE/SPIFHE) identity for ALL traffic to pods in this +# namespace, including traffic arriving via the ha-authentik-postgres +# NodePort from the VPS/witness (which have no mesh identity at all - +# they're not in this cluster). ztunnel accepts the raw TCP connection then +# resets it once it can't complete an mTLS handshake it never receives - +# exactly matching the "server closed the connection unexpectedly" +# symptom. Confirmed via `openssl s_client -starttls postgres`: TCP +# connects, the postgres SSLRequest byte is sent, 0 bytes come back. +# +# Same root cause and same fix as the existing precedent for this exact +# problem (infrastructure/istio/manifests/mesh/peer-authentication-webhooks.yaml +# - CrowdSec's hostNetwork bouncer / the API server's webhook calls): allow +# PERMISSIVE (mTLS or plaintext) inbound. Scoped here to just the CNPG +# primary pod's port 5432 via portLevelMtls, rather than the whole +# namespace like that precedent does - Authentik's own in-mesh east-west +# traffic (server/worker -> everything else) should stay STRICT. +apiVersion: security.istio.io/v1 +kind: PeerAuthentication +metadata: + name: allow-ha-postgres-replication + namespace: authentik +spec: + selector: + matchLabels: + cnpg.io/cluster: pg-authentik + mtls: + mode: STRICT + portLevelMtls: + "5432": + mode: PERMISSIVE diff --git a/infrastructure/istio/manifests/mesh/authorization-policies.yaml b/infrastructure/istio/manifests/mesh/authorization-policies.yaml index 6a5352b..c025e06 100644 --- a/infrastructure/istio/manifests/mesh/authorization-policies.yaml +++ b/infrastructure/istio/manifests/mesh/authorization-policies.yaml @@ -117,6 +117,20 @@ spec: - source: namespaces: - prometheus + # Multi-site active failover pilot (see + # /home/scooby/.claude/plans/jiggly-snacking-iverson.md): CNPG + # streaming replication from the VPS/witness, arriving via the + # ha-authentik-postgres NodePort - no mesh identity at all (they're + # not in this cluster), so no `source.namespaces` rule above can ever + # match them. Scoped by destination port instead of source, matching + # the port-scoped PERMISSIVE PeerAuthentication in + # ha-postgres-peerauth.yaml (same root cause, same fix, one layer up - + # mTLS being allowed through isn't enough by itself, this ALLOW policy + # independently denies anything not matching one of the rules above). + - to: + - operation: + ports: + - "5432" --- # --- Grafana: Allow ingress + Prometheus datasource queries + scraping ---