From 983671ef5042b7a7a955503da50938984902ea1d Mon Sep 17 00:00:00 2001
From: Scooby Husky <garrettstone499@gmail.com>
Date: Mon, 17 Aug 2026 15:08:33 -0500
Subject: [PATCH] Fix ArgoCD Redis connectivity broken by ambient mesh HBONE
 NetworkPolicy gap

Root cause (confirmed via ztunnel logs): the argo-cd chart's default
NetworkPolicies only allow each component's application port (e.g. 6379
for redis), but Istio ambient mode routes ALL pod-to-pod traffic through
ztunnel's HBONE tunnel on port 15008 first - so the tunnel itself was
being blocked even though the "real" port was allowed. Every inter-pod
connection in the argocd namespace hung for exactly 10s then reset;
ztunnel's own log named it directly ("connection timed out, maybe a
NetworkPolicy is blocking HBONE port 15008"). This broke argocd-server's
Redis-backed session/cluster-info caching cluster-wide and was silently
preventing the root Application from picking up new child Applications.

Fix: additive NetworkPolicy allowing ingress on 15008 for all argocd
pods (NetworkPolicies union across multiple policies selecting a pod,
so this doesn't touch/replace the chart's rendered ones - safe across
Helm upgrades).

Also: dropped argocd from the namespace-enrollment job's waypoint list.
argocd's only AuthorizationPolicy (allow-argocd-access) is a plain
source-namespace/IP match with no L7 rules - its own status shows
"attached to ztunnel", not waypoint - so forcing L7 waypoint processing
onto the namespace was unnecessary overhead, not a security requirement.
ztunnel's mTLS still fully covers it. (This was a red herring for the
HBONE bug itself, not the fix, but a valid simplification found along
the way.)
---
 .../argocd/manifests/hbone-networkpolicy.yaml | 32 +++++++++++++++++++
 .../apps/namespace-enrollment-job.yaml        | 15 +++++++--
 2 files changed, 45 insertions(+), 2 deletions(-)
 create mode 100644 apps/argocd/manifests/hbone-networkpolicy.yaml

diff --git a/apps/argocd/manifests/hbone-networkpolicy.yaml b/apps/argocd/manifests/hbone-networkpolicy.yaml
new file mode 100644
index 0000000..67e5650
--- /dev/null
+++ b/apps/argocd/manifests/hbone-networkpolicy.yaml
@@ -0,0 +1,32 @@
+---
+# Istio ambient mesh routes ALL pod-to-pod traffic through ztunnel's HBONE
+# tunnel on port 15008, not the application's own port - the app-port-only
+# NetworkPolicies the argo-cd chart renders by default (e.g. argocd-redis
+# only allowing port 6379) block the actual HBONE connection even though the
+# "real" port is allowed, because ztunnel's encrypted tunnel itself never
+# gets past the policy. Symptom: every inter-pod connection in the argocd
+# namespace hangs for exactly 10s then resets - ztunnel logs the reason
+# directly: "connection timed out, maybe a NetworkPolicy is blocking HBONE
+# port 15008". Confirmed 2026-08-17: this was breaking argocd-server's
+# Redis-backed session/cluster-info caching cluster-wide.
+#
+# NetworkPolicies are additive (union of all policies selecting a pod), so
+# this supplements the chart's rendered policies rather than replacing them -
+# safe across Helm upgrades.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: argocd-allow-hbone
+  namespace: argocd
+spec:
+  podSelector:
+    matchLabels:
+      app.kubernetes.io/instance: argocd
+  policyTypes:
+    - Ingress
+  ingress:
+    - from:
+        - namespaceSelector: {}
+      ports:
+        - protocol: TCP
+          port: 15008
diff --git a/infrastructure/istio/manifests/apps/namespace-enrollment-job.yaml b/infrastructure/istio/manifests/apps/namespace-enrollment-job.yaml
index 7297553..7d07a9e 100644
--- a/infrastructure/istio/manifests/apps/namespace-enrollment-job.yaml
+++ b/infrastructure/istio/manifests/apps/namespace-enrollment-job.yaml
@@ -32,12 +32,23 @@ spec:
                 kubectl label namespace "$ns" istio-injection- 2>/dev/null || true
               done
 
-              # Attach waypoint proxies for L7 policy enforcement
+              # Attach waypoint proxies for L7 policy enforcement.
+              # argocd excluded: its only AuthorizationPolicy (allow-argocd-access)
+              # is a plain source-namespace/IP match with no L7 rules - it's
+              # enforced directly by ztunnel (see its status: "attached to
+              # ztunnel", not waypoint). Forcing waypoint L7 processing onto the
+              # namespace anyway broke argocd-redis: waypoint doesn't handle
+              # Redis's long-lived RESP protocol well, causing ~10s hang-then-
+              # close on every connection (i/o timeout errors in argocd-server,
+              # cluster info/session caching failing, new Applications from git
+              # never getting picked up). ztunnel's plain mTLS still fully
+              # covers argocd's actual security requirement here.
               echo "Attaching waypoint proxies to app namespaces..."
-              WAYPOINT_APP_NAMESPACES=(argocd gitlab n8n nextcloud teslamate home-assistant frigate guacamole)
+              WAYPOINT_APP_NAMESPACES=(gitlab n8n nextcloud teslamate home-assistant frigate guacamole)
               for ns in "${WAYPOINT_APP_NAMESPACES[@]}"; do
                 kubectl label namespace "$ns" istio.io/use-waypoint=waypoint --overwrite 2>/dev/null || true
               done
+              kubectl label namespace argocd istio.io/use-waypoint- 2>/dev/null || true
 
               # Annotate services for NetBird operator auto-discovery (netbird.io/expose triggers the operator)
               echo "Annotating services for NetBird operator exposure..."