From 185e9c292e6965d9a91e386b308f7646d3df1af1 Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Thu, 20 Aug 2026 18:47:50 -0500 Subject: [PATCH] Fix real cause of VPS authentik's DB timeout loop + repo secrets cleanup Root cause of the 'PostgreSQL connection failed, retrying... (connection timeout expired)' loop that survived every earlier fix (streaming replication, the password sync, the port fix, the CoreDNS override): authentik's Helm values had TWO sources for AUTHENTIK_POSTGRESQL__HOST/ PORT - an explicit uppercase env: override (pointed at the floating pg-authentik.ha.huskypup.net:61432) AND a chart-generated envFrom secretRef (pg-authentik-app, prefix: AUTHENTIK_POSTGRESQL__) whose keys are lowercase (host, port, ...), producing a SEPARATE AUTHENTIK_POSTGRESQL__host/port pair pointing at the old local pg-authentik-rw:5432. Kubernetes treats these as two unrelated env vars (case-sensitive), but authentik's own generic AUTHENTIK_*-prefixed env-var scanner apparently doesn't, and was resolving to the lowercase (stale, local) values regardless of the explicit override - confirmed live by dumping the pod's actual env: both HOST and host were present with different values. Removed the now-fully-redundant envFrom entry (every key it provided is already explicitly set via valueFrom). Also: user correctly pointed out plaintext secrets don't belong in the repo. Audited for the same class of issue as the already-fixed Cloudflare token (infrastructure/cert-manager/manifests/secret-cf-token.yaml): - infrastructure/external-dns/manifests/secret-external-dns-unifi.yaml had a live UniFi API key in plaintext - moved to Vault+ExternalSecret, same pattern as the Cloudflare token fix. The key itself is still the original (now Vault-stored) value - rotating it requires the UniFi web UI (no self-service API), noted in the file as a separate pending step. - infrastructure/vps-eso/manifests/clustersecretstore.yaml had the AppRole's roleId inline (added this session) - moved to roleRef, sourced from the same Secret as secretId, consistent with 'nothing sensitive in git' regardless of how sensitive one field is alone. NOT touched, flagged separately for the user: infrastructure/authentik/ *-blueprint.yaml (vault, gitlab, n8n, nextcloud, rancher, grafana, argocd, guacamole) all have live-looking high-entropy client_secret values hardcoded in plaintext - same class of issue but much larger blast radius (8 apps' SSO), needs its own coordinated rotation, not bundled into this commit. Co-Authored-By: Claude Sonnet 5 --- infrastructure/authentik/values.yaml | 21 +++++++++--- .../manifests/secret-external-dns-unifi.yaml | 33 ++++++++++++++++--- .../vps-eso/manifests/clustersecretstore.yaml | 12 ++++--- .../vps-standby/authentik/values.yaml | 9 +++-- 4 files changed, 60 insertions(+), 15 deletions(-) diff --git a/infrastructure/authentik/values.yaml b/infrastructure/authentik/values.yaml index 132f756..dbdd6ca 100644 --- a/infrastructure/authentik/values.yaml +++ b/infrastructure/authentik/values.yaml @@ -13,10 +13,23 @@ global: # First secret: Authentik-specific settings (AUTHENTIK_SECRET_KEY) - secretRef: name: authentik - # Second secret: Database credentials from CNPG - - secretRef: - name: pg-authentik-app - prefix: AUTHENTIK_POSTGRESQL__ + # The second secretRef (pg-authentik-app, prefix: AUTHENTIK_POSTGRESQL__) + # that used to live here is GONE - found live 2026-08-20 that it was + # actively breaking things, not just redundant. That prefix mechanism + # copies the raw secret's lowercase keys (host, port, dbname, username, + # password) into lowercase-suffixed env vars (AUTHENTIK_POSTGRESQL__host, + # etc.) - env var names are case-sensitive to Kubernetes, but authentik's + # own generic AUTHENTIK_*-prefixed env-var scanner is NOT, and it was + # picking up the lowercase AUTHENTIK_POSTGRESQL__host=pg-authentik-rw + # instead of the explicit uppercase AUTHENTIK_POSTGRESQL__HOST override + # below - silently connecting to the local CNPG service on port 5432 + # the whole time regardless of what HOST/PORT were explicitly set to, + # surfacing as an unexplained "connection timeout expired" loop (the + # local hostname resolves fine in-cluster but on the VPS the request + # apparently doesn't complete - never fully root-caused, just confirmed + # dropping this duplicate source fixes it). Every key that secretRef + # used to provide (dbname/username/password) is already explicitly set + # via valueFrom below, so nothing is lost by removing it. env: # Configure external URL for proper OIDC discovery responses - name: AUTHENTIK_URL diff --git a/infrastructure/external-dns/manifests/secret-external-dns-unifi.yaml b/infrastructure/external-dns/manifests/secret-external-dns-unifi.yaml index 57bfba7..4bbfbb5 100644 --- a/infrastructure/external-dns/manifests/secret-external-dns-unifi.yaml +++ b/infrastructure/external-dns/manifests/secret-external-dns-unifi.yaml @@ -1,8 +1,33 @@ --- -apiVersion: v1 -kind: Secret +# Was a plain Secret with a live UniFi API key committed in plaintext - +# found and fixed 2026-08-20 (same class of issue as the Cloudflare token +# in infrastructure/cert-manager/manifests/secret-cf-token.yaml, which was +# already rotated to this pattern - this one was missed at the time). +# Moved to Vault+ExternalSecret, matching that precedent. +# +# The value in Vault right now (secret/unifi-api-key) is still the +# ORIGINAL key that was exposed in git history - rotating it (Settings -> +# Admins -> API Keys on the UniFi controller at https://172.28.110.1, no +# self-service API for this) is a separate, still-pending step. Once +# rotated: +# vault kv put secret/unifi-api-key api-key= +# ESO picks it up automatically (refreshInterval below), no restart +# needed on external-dns's side. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret metadata: name: external-dns-unifi-secret namespace: external-dns -stringData: - api-key: 3Qha5hupHsCkkFyQb8z-T2kh-BasIGSH \ No newline at end of file +spec: + refreshInterval: 1h + secretStoreRef: + kind: ClusterSecretStore + name: vault-backend + target: + name: external-dns-unifi-secret + creationPolicy: Owner + data: + - secretKey: api-key + remoteRef: + key: unifi-api-key + property: api-key diff --git a/infrastructure/vps-eso/manifests/clustersecretstore.yaml b/infrastructure/vps-eso/manifests/clustersecretstore.yaml index ba2ee9a..232001a 100644 --- a/infrastructure/vps-eso/manifests/clustersecretstore.yaml +++ b/infrastructure/vps-eso/manifests/clustersecretstore.yaml @@ -57,13 +57,14 @@ # vault write -f auth/approle/role/vps-eso/secret-id # -> secret_id # # vault-approle-creds is a plain Secret created manually on the VPS -# (kubectl, not git - same reasoning as every other VPS secret): +# (kubectl, not git - same reasoning as every other VPS secret, and +# consistent regardless of "how sensitive" a given field is on its own - +# role_id and secret_id both live in the same Secret, neither in git): # kubectl -n external-secrets create secret generic vault-approle-creds \ # --from-literal=role_id= \ # --from-literal=secret_id= # Rotate the secret_id periodically by writing a new one and patching this -# Secret - role_id is stable and not sensitive on its own (useless without -# a valid secret_id). +# Secret. apiVersion: external-secrets.io/v1 kind: ClusterSecretStore metadata: @@ -77,7 +78,10 @@ spec: auth: appRole: path: approle - roleId: ecaf2eda-d922-f7cf-1143-690bbbb4d8ea # not sensitive alone, see note above + roleRef: + name: vault-approle-creds + namespace: external-secrets + key: role_id secretRef: name: vault-approle-creds namespace: external-secrets diff --git a/infrastructure/vps-standby/authentik/values.yaml b/infrastructure/vps-standby/authentik/values.yaml index a28e8aa..e4ced4b 100644 --- a/infrastructure/vps-standby/authentik/values.yaml +++ b/infrastructure/vps-standby/authentik/values.yaml @@ -34,9 +34,12 @@ global: envFrom: - secretRef: name: authentik - - secretRef: - name: pg-authentik-app - prefix: AUTHENTIK_POSTGRESQL__ + # The pg-authentik-app secretRef that used to live here is GONE - see + # the matching comment in infrastructure/authentik/values.yaml (home's + # copy of this same block) for why: its lowercase-suffixed env vars + # (AUTHENTIK_POSTGRESQL__host etc.) were silently winning over the + # explicit uppercase HOST/PORT overrides below, connecting straight to + # the local CNPG service the whole time regardless of what was set. env: - name: AUTHENTIK_URL value: "https://auth.vps.huskypup.net"