From 7b3669e9a6015fdff9304f6e52abf9844f551364 Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Mon, 17 Aug 2026 16:41:11 -0500 Subject: [PATCH] Fix CNPG operator -> instance status communication + add CNPG health check Two related fixes, found while working through why authentik/n8n/gitlab ArgoCD Applications showed Unknown/stuck health despite pods being fine: 1. ArgoCD has no built-in health check for CNPG's Cluster CRD, so it always reported "Unknown" - and since app-level health rolls up to the worst resource status, every app with a CNPG Cluster showed Unknown/Progressing regardless of actual state. Added a Lua health check (resource.customizations.health.postgresql.cnpg.io_Cluster) reading .status.conditions[Ready] / .status.phase. 2. Once that started reporting real status instead of masking it, pg-authentik showed a genuine problem: CNPG's operator couldn't reach its Postgres instances' status API (port 8000) - "Cannot extract Pod status ... context deadline exceeded" - because: a) authentik-ingress's CiliumNetworkPolicy never allowed cnpg-system as a source (crowdsec-ingress already had this exception, authentik-ingress was just missing it - inconsistency, not deliberate). b) Even after fixing (a), still blocked - pg-authentik's pods are ambient-mesh-enrolled, so the connection actually goes through ztunnel's HBONE tunnel (port 15008) first, same underlying issue as the argocd-redis fix from earlier today. Rather than keep finding and patching this per-namespace as it recurs, added a cluster-wide CiliumClusterwideNetworkPolicy allowing HBONE broadly - ztunnel's own mTLS/SPIFFE identity verification is the real security boundary for mesh traffic; Cilium blocking the tunnel port itself was only breaking legitimate traffic, not adding meaningful protection on top. Verified live: CNPG operator immediately went from failing status extraction to successfully reconciling (recreating a pod to reattach its PVC) once both fixes were in place. --- apps/argocd/values.yaml | 38 +++++++++++++++++++ .../manifests/network-policies/baseline.yaml | 8 ++++ .../manifests/network-policies/hbone.yaml | 34 +++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 infrastructure/cilium/manifests/network-policies/hbone.yaml diff --git a/apps/argocd/values.yaml b/apps/argocd/values.yaml index bb3c5d0..579ec5b 100644 --- a/apps/argocd/values.yaml +++ b/apps/argocd/values.yaml @@ -36,6 +36,44 @@ configs: jqPathExpressions: - .spec.volumeClaimTemplates[].apiVersion - .spec.volumeClaimTemplates[].kind + # ArgoCD has no built-in health check for CNPG's Cluster CRD, so it + # defaults to "Unknown" - and since app-level health rolls up to the + # worst status of any resource, every app with a CNPG Cluster (authentik, + # n8n, gitlab, nextcloud, crowdsec) shows Unknown/Progressing overall even + # when everything is actually fine. Confirmed 2026-08-17: no amount of + # refreshing/resyncing individual apps fixed it, because it isn't a cache + # issue - there was just nothing telling ArgoCD how to read CNPG's status. + resource.customizations.health.postgresql.cnpg.io_Cluster: | + health_status = {} + if obj.status ~= nil and obj.status.conditions ~= nil then + for i, condition in ipairs(obj.status.conditions) do + if condition.type == "Ready" then + if condition.status == "True" then + health_status.status = "Healthy" + health_status.message = "Cluster is Ready" + return health_status + else + health_status.status = "Progressing" + health_status.message = condition.message or "Cluster is not Ready" + return health_status + end + end + end + end + if obj.status ~= nil and obj.status.phase ~= nil then + if obj.status.phase == "Cluster in healthy state" then + health_status.status = "Healthy" + elseif string.find(obj.status.phase, "Failed") then + health_status.status = "Degraded" + else + health_status.status = "Progressing" + end + health_status.message = obj.status.phase + return health_status + end + health_status.status = "Unknown" + health_status.message = "No status information available yet" + return health_status oidc.config: | name: Authentik issuer: https://auth.kube.huskypup.net/application/o/argocd/ diff --git a/infrastructure/cilium/manifests/network-policies/baseline.yaml b/infrastructure/cilium/manifests/network-policies/baseline.yaml index 145fc39..19cc3f8 100644 --- a/infrastructure/cilium/manifests/network-policies/baseline.yaml +++ b/infrastructure/cilium/manifests/network-policies/baseline.yaml @@ -69,6 +69,14 @@ spec: - fromEndpoints: - matchLabels: k8s:io.kubernetes.pod.namespace: authentik + # CNPG operator - status port (8000) queries on pg-authentik instances. + # crowdsec-ingress below already has this; was missing here, which broke + # CNPG's own health/status reporting for pg-authentik ("Cannot extract Pod + # status ... context deadline exceeded" in the operator logs) even though + # the database itself was working fine. Confirmed 2026-08-17. + - fromEndpoints: + - matchLabels: + k8s:io.kubernetes.pod.namespace: cnpg-system # Apps doing OIDC token exchange - fromEndpoints: - matchLabels: diff --git a/infrastructure/cilium/manifests/network-policies/hbone.yaml b/infrastructure/cilium/manifests/network-policies/hbone.yaml new file mode 100644 index 0000000..75b8f1a --- /dev/null +++ b/infrastructure/cilium/manifests/network-policies/hbone.yaml @@ -0,0 +1,34 @@ +# Allow ztunnel's HBONE tunnel port (15008) cluster-wide. +# +# Istio ambient mode routes ALL pod-to-pod traffic through ztunnel's mTLS +# HBONE tunnel on port 15008 first, regardless of which "real" port the +# traffic is ultimately for. Per-namespace CiliumNetworkPolicies that only +# allow the application port (e.g. 6379 for Redis, 8000 for CNPG's instance +# status API) block the HBONE tunnel itself even when the intended port is +# allowed - ztunnel's own encrypted connection never gets through. +# +# Hit this same bug twice in one session (2026-08-17): argocd-redis (fixed +# with a namespace-scoped NetworkPolicy) and cnpg-system -> pg-authentik's +# status port (fixed the same way first, still broken - needed this too). +# Rather than keep discovering and patching this per-namespace, allow HBONE +# broadly: ztunnel already enforces real security here via mTLS + SPIFFE +# identity verification (and any CiliumNetworkPolicy/AuthorizationPolicy +# ingress rules on the target still apply to the traffic's real intent) - +# Cilium blocking the tunnel port itself only breaks legitimate mesh traffic, +# it doesn't add a meaningful security boundary on top of what ztunnel +# already does. +apiVersion: cilium.io/v2 +kind: CiliumClusterwideNetworkPolicy +metadata: + name: allow-hbone +spec: + description: "Allows Istio ambient's ztunnel HBONE tunnel port cluster-wide - see comments" + enableDefaultDeny: + egress: false + ingress: false + endpointSelector: {} + ingress: + - toPorts: + - ports: + - port: "15008" + protocol: TCP