Files
Scooby Husky 7b3669e9a6 Fix CNPG operator -> instance status communication + add CNPG health check
Two related fixes, found while working through why authentik/n8n/gitlab
ArgoCD Applications showed Unknown/stuck health despite pods being fine:

1. ArgoCD has no built-in health check for CNPG's Cluster CRD, so it
   always reported "Unknown" - and since app-level health rolls up to
   the worst resource status, every app with a CNPG Cluster showed
   Unknown/Progressing regardless of actual state. Added a Lua health
   check (resource.customizations.health.postgresql.cnpg.io_Cluster)
   reading .status.conditions[Ready] / .status.phase.

2. Once that started reporting real status instead of masking it,
   pg-authentik showed a genuine problem: CNPG's operator couldn't
   reach its Postgres instances' status API (port 8000) - "Cannot
   extract Pod status ... context deadline exceeded" - because:
   a) authentik-ingress's CiliumNetworkPolicy never allowed cnpg-system
      as a source (crowdsec-ingress already had this exception,
      authentik-ingress was just missing it - inconsistency, not
      deliberate).
   b) Even after fixing (a), still blocked - pg-authentik's pods are
      ambient-mesh-enrolled, so the connection actually goes through
      ztunnel's HBONE tunnel (port 15008) first, same underlying issue
      as the argocd-redis fix from earlier today. Rather than keep
      finding and patching this per-namespace as it recurs, added a
      cluster-wide CiliumClusterwideNetworkPolicy allowing HBONE
      broadly - ztunnel's own mTLS/SPIFFE identity verification is the
      real security boundary for mesh traffic; Cilium blocking the
      tunnel port itself was only breaking legitimate traffic, not
      adding meaningful protection on top.

Verified live: CNPG operator immediately went from failing status
extraction to successfully reconciling (recreating a pod to reattach
its PVC) once both fixes were in place.
2026-08-17 16:41:11 -05:00

167 lines
5.1 KiB
YAML

# values/argocd.values.yaml
global:
domain: argocd.kube.huskypup.net
configs:
cm:
url: https://argocd.kube.huskypup.net
resource.customizations.ignoreDifferences.all: |
managedFieldsManagers:
- external-secrets
- istio-system
jqPathExpressions:
- .metadata.finalizers
resource.customizations.ignoreDifferences.external-secrets.io_ExternalSecret: |
jqPathExpressions:
- .metadata.annotations."force-sync"
- .metadata.annotations."reconcile.external-secrets.io/force-sync"
- .metadata.finalizers
- .spec.data[].remoteRef.conversionStrategy
- .spec.data[].remoteRef.decodingStrategy
- .spec.data[].remoteRef.metadataPolicy
resource.customizations.ignoreDifferences.gateway.networking.k8s.io_HTTPRoute: |
jqPathExpressions:
- .metadata.annotations
- .spec.parentRefs[].port
resource.customizations.ignoreDifferences.monitoring.coreos.com_ServiceMonitor: |
jqPathExpressions:
- .spec.endpoints[].relabelings[].action
- .spec.endpoints[].metricRelabelings[].action
resource.customizations.ignoreDifferences.monitoring.coreos.com_PodMonitor: |
jqPathExpressions:
- .spec.podMetricsEndpoints[].relabelings[].action
- .spec.podMetricsEndpoints[].metricRelabelings[].action
resource.customizations.ignoreDifferences.apps_StatefulSet: |
jqPathExpressions:
- .spec.volumeClaimTemplates[].apiVersion
- .spec.volumeClaimTemplates[].kind
# ArgoCD has no built-in health check for CNPG's Cluster CRD, so it
# defaults to "Unknown" - and since app-level health rolls up to the
# worst status of any resource, every app with a CNPG Cluster (authentik,
# n8n, gitlab, nextcloud, crowdsec) shows Unknown/Progressing overall even
# when everything is actually fine. Confirmed 2026-08-17: no amount of
# refreshing/resyncing individual apps fixed it, because it isn't a cache
# issue - there was just nothing telling ArgoCD how to read CNPG's status.
resource.customizations.health.postgresql.cnpg.io_Cluster: |
health_status = {}
if obj.status ~= nil and obj.status.conditions ~= nil then
for i, condition in ipairs(obj.status.conditions) do
if condition.type == "Ready" then
if condition.status == "True" then
health_status.status = "Healthy"
health_status.message = "Cluster is Ready"
return health_status
else
health_status.status = "Progressing"
health_status.message = condition.message or "Cluster is not Ready"
return health_status
end
end
end
end
if obj.status ~= nil and obj.status.phase ~= nil then
if obj.status.phase == "Cluster in healthy state" then
health_status.status = "Healthy"
elseif string.find(obj.status.phase, "Failed") then
health_status.status = "Degraded"
else
health_status.status = "Progressing"
end
health_status.message = obj.status.phase
return health_status
end
health_status.status = "Unknown"
health_status.message = "No status information available yet"
return health_status
oidc.config: |
name: Authentik
issuer: https://auth.kube.huskypup.net/application/o/argocd/
clientID: $oidc.authentik.clientId
clientSecret: $oidc.authentik.clientSecret
requestedScopes:
- openid
- profile
- email
- groups
rbac:
policy.default: role:readonly
scopes: '[groups, preferred_username]'
policy.csv: |
g, Authentik Admins, role:admin
g, ArgoCD Admins, role:admin
g, Scooby, role:admin
server:
replicas: 1
extraArgs:
- --insecure # TLS is terminated at Istio gateway
hostAliases:
- ip: "172.28.101.244"
hostnames:
- auth.kube.huskypup.net
- gitlab.kube.huskypup.net
ingress:
enabled: false
env:
- name: oidc.authentik.clientId
valueFrom:
secretKeyRef:
name: argocd-oauth-secret
key: client-id
- name: oidc.authentik.clientSecret
valueFrom:
secretKeyRef:
name: argocd-oauth-secret
key: client-secret
dex:
enabled: false # using Authentik instead of built-in Dex
# Enable Prometheus metrics
metrics:
enabled: true
serviceMonitor:
enabled: true
additionalLabels:
release: prometheus
controller:
metrics:
enabled: true
serviceMonitor:
enabled: true
additionalLabels:
release: prometheus
repoServer:
replicas: 2
hostAliases:
- ip: "172.28.101.244"
hostnames:
- gitlab.kube.huskypup.net
- auth.kube.huskypup.net
metrics:
enabled: true
serviceMonitor:
enabled: true
additionalLabels:
release: prometheus
applicationSet:
hostAliases:
- ip: "172.28.101.244"
hostnames:
- gitlab.kube.huskypup.net
- auth.kube.huskypup.net
metrics:
enabled: true
serviceMonitor:
enabled: true
additionalLabels:
release: prometheus