mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-21 11:36:50 +00:00
Scooby HuskyandClaude Sonnet 5
3e7643e67e
CNPG's new externalClusters connectionParameters were configured correctly but streaming replication was silently failing - pg_stat_wal_receiver on the VPS replica showed 0 rows, logs repeated 'could not connect to the primary server: ... server closed the connection unexpectedly' every few minutes. Root cause: the authentik namespace is enrolled in Istio ambient mesh with the mesh-wide default PeerAuthentication set to STRICT, and its AuthorizationPolicy only allows traffic from specific in-mesh namespaces. Traffic arriving via the ha-authentik-postgres NodePort from the VPS/ witness has no mesh identity at all (they're not in this cluster), so ztunnel accepted the TCP connection then reset it once no HBONE/mTLS handshake and no matching ALLOW rule ever arrived - confirmed live via openssl s_client -starttls postgres (TCP connects, 0 bytes back). Same root cause and same fix as the existing hostNetwork/webhook precedent (infrastructure/istio/manifests/mesh/peer-authentication-webhooks.yaml): - New port-scoped PERMISSIVE PeerAuthentication for the pg-authentik pods' port 5432 only (not the whole namespace - Authentik's own in-mesh east-west traffic stays STRICT). - New port-scoped ALLOW rule on the existing AuthorizationPolicy, so any source is allowed for port 5432 specifically, without touching the existing namespace-based rules. Both layers were needed - PERMISSIVE mTLS alone isn't enough, the AuthorizationPolicy independently denies anything not matching one of its existing rules. Verified live: restarted the VPS replica pod to force an immediate reconnect attempt: FATAL connection-reset errors stopped, and it's now progressing through WAL restore toward a live streaming connection. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
387 lines
9.9 KiB
YAML
387 lines
9.9 KiB
YAML
# ==========================================================================
|
|
# Zero Trust Authorization Policies - Deny by Default, Allow Explicitly
|
|
# ==========================================================================
|
|
#
|
|
# Policy hierarchy:
|
|
# 1. Mesh-wide DENY (default - everything blocked)
|
|
# 2. Ingress Gateway ALLOW (external traffic entry point)
|
|
# 3. Service-to-service ALLOW (explicit inter-service communication)
|
|
# 4. Monitoring ALLOW (Prometheus scraping, Kiali queries)
|
|
#
|
|
# In ambient mode, L7 policies are enforced by waypoint proxies in each
|
|
# namespace. L4 policies (source namespace/principal) are enforced by ztunnel.
|
|
# Each namespace with ALLOW/CUSTOM policies must have a waypoint Gateway.
|
|
# ==========================================================================
|
|
|
|
# --- Ingress Gateway: Allow all external traffic through the edge gateway ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-ingress-gateway
|
|
namespace: istio-system
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
istio: ingressgateway
|
|
action: ALLOW
|
|
rules:
|
|
- {}
|
|
|
|
---
|
|
# --- Allow Prometheus to scrape Istio control plane + gateway metrics ---
|
|
# Selector scopes this to istio-system workloads only.
|
|
# Without a selector, policies in the root namespace (istio-system) apply
|
|
# mesh-wide in ambient mode, creating implicit deny for all ambient workloads.
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-prometheus-scraping
|
|
namespace: istio-system
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app.kubernetes.io/part-of: istio
|
|
action: ALLOW
|
|
rules:
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
to:
|
|
- operation:
|
|
ports:
|
|
- "15014" # istiod control plane metrics
|
|
- "15020" # sidecar/gateway merged metrics
|
|
- "15090" # Envoy admin metrics
|
|
|
|
---
|
|
# --- Allow Kiali to query istiod ---
|
|
# Selector scopes this to istiod only (ambient root namespace caveat above).
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-kiali
|
|
namespace: istio-system
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: istiod
|
|
action: ALLOW
|
|
rules:
|
|
- from:
|
|
- source:
|
|
principals:
|
|
- cluster.local/ns/istio-system/sa/kiali-service-account
|
|
|
|
---
|
|
# --- Authentik: Allow traffic from ingress + apps doing OIDC ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-authentik-access
|
|
namespace: authentik
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace (server ↔ worker ↔ postgres)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- authentik
|
|
# CNPG operator managing database instances
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- cnpg-system
|
|
# Ingress gateway for browser flows
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- istio-system
|
|
# Apps doing OIDC token exchange
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- argocd
|
|
- gitlab
|
|
- grafana
|
|
- nextcloud
|
|
- home-assistant
|
|
- guacamole
|
|
- netbird
|
|
- cattle-system
|
|
- frigate
|
|
- teslamate
|
|
# Prometheus scraping (L4-only; L7 path checks deferred to waypoint)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
# Multi-site active failover pilot (see
|
|
# /home/scooby/.claude/plans/jiggly-snacking-iverson.md): CNPG
|
|
# streaming replication from the VPS/witness, arriving via the
|
|
# ha-authentik-postgres NodePort - no mesh identity at all (they're
|
|
# not in this cluster), so no `source.namespaces` rule above can ever
|
|
# match them. Scoped by destination port instead of source, matching
|
|
# the port-scoped PERMISSIVE PeerAuthentication in
|
|
# ha-postgres-peerauth.yaml (same root cause, same fix, one layer up -
|
|
# mTLS being allowed through isn't enough by itself, this ALLOW policy
|
|
# independently denies anything not matching one of the rules above).
|
|
- to:
|
|
- operation:
|
|
ports:
|
|
- "5432"
|
|
|
|
---
|
|
# --- Grafana: Allow ingress + Prometheus datasource queries + scraping ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-grafana-access
|
|
namespace: grafana
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- grafana
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- istio-system
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
# NetBird VPN cluster routers (non-mesh, use ipBlocks)
|
|
- from:
|
|
- source:
|
|
ipBlocks:
|
|
- "10.244.0.0/16"
|
|
|
|
---
|
|
# --- Prometheus: Allow ingress + self-scraping + Grafana ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-prometheus-access
|
|
namespace: prometheus
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace (Prometheus ↔ alertmanager ↔ node-exporter)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- istio-system
|
|
- grafana
|
|
# NetBird VPN cluster routers (non-mesh, use ipBlocks)
|
|
- from:
|
|
- source:
|
|
ipBlocks:
|
|
- "10.244.0.0/16"
|
|
|
|
---
|
|
# --- MQTT: Allow Home Assistant + Frigate + ESPHome + Prometheus ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-mqtt-access
|
|
namespace: mqtt
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- mqtt
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- istio-system
|
|
- home-assistant
|
|
- frigate
|
|
- teslamate
|
|
# Prometheus scraping (L4-only; L7 path checks deferred to waypoint)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
|
|
---
|
|
# --- External DNS: Allow internal access + Prometheus ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-external-dns
|
|
namespace: external-dns
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- external-dns
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- istio-system
|
|
# Prometheus scraping (L4-only; L7 path checks deferred to waypoint)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
|
|
---
|
|
# --- Unpoller: Allow Prometheus scraping ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-unpoller-access
|
|
namespace: unpoller
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- unpoller
|
|
# Prometheus scraping (L4-only; L7 path checks deferred to waypoint)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
|
|
---
|
|
# --- Netbird: Allow ingress + Prometheus ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-netbird-access
|
|
namespace: netbird
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- netbird
|
|
# CNPG operator managing database instances
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- cnpg-system
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- istio-system
|
|
# Netbird operator querying management API
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- netbird
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
# kube-apiserver webhook calls to netbird-operator webhook service (node IPs)
|
|
- from:
|
|
- source:
|
|
ipBlocks:
|
|
- "172.28.101.0/24"
|
|
|
|
---
|
|
# --- Netbird Operator: Allow intra-namespace + Prometheus ---
|
|
# Operator runs in the "netbird" namespace, not "netbird-operator" (which
|
|
# doesn't exist) - fixed 2026-08-17, was blocking istio-mesh-config from
|
|
# ever going Synced (kubectl apply failed: "namespaces netbird-operator
|
|
# not found").
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-netbird-operator-access
|
|
namespace: netbird
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- netbird
|
|
# Prometheus scraping (L4-only)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
|
|
---
|
|
# --- CrowdSec: Allow intra-namespace + CNPG + Prometheus ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-crowdsec-access
|
|
namespace: crowdsec
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace (LAPI <-> agent)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- crowdsec
|
|
# CNPG operator managing database instances
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- cnpg-system
|
|
# Prometheus scraping (L4-only; L7 path checks deferred to waypoint)
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
# Firewall bouncer (hostNetwork DaemonSet) connects from node IPs
|
|
- from:
|
|
- source:
|
|
ipBlocks:
|
|
- "172.28.101.0/24"
|
|
|
|
---
|
|
# --- Scylla Manager: Allow ingress + Prometheus + intra-namespace ---
|
|
apiVersion: security.istio.io/v1
|
|
kind: AuthorizationPolicy
|
|
metadata:
|
|
name: allow-scylla-manager-access
|
|
namespace: scylla-manager
|
|
spec:
|
|
action: ALLOW
|
|
rules:
|
|
# Intra-namespace
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- scylla-manager
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- istio-system
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- prometheus
|
|
# Scylla operator managing clusters
|
|
- from:
|
|
- source:
|
|
namespaces:
|
|
- scylla-operator
|