diff --git a/apps/n8n/manifests/cnpg-cluster.yaml b/apps/n8n/manifests/cnpg-cluster.yaml index 7ce7cb8..c45cff2 100644 --- a/apps/n8n/manifests/cnpg-cluster.yaml +++ b/apps/n8n/manifests/cnpg-cluster.yaml @@ -47,29 +47,31 @@ spec: database: n8n owner: n8n - # Backup to VPS MinIO - DISABLED 2026-08-17. No node in the cluster has - # any route into the Netbird mesh for pod-originated (egress) traffic; - # the per-namespace netbird "router" pods are inbound-only infrastructure. - # continuousArchiving kept failing to connect to vps-minio.netbird.internal, - # which held Ready=False permanently (real condition, not cosmetic - this - # is what drove n8n's CNPG health flapping in ArgoCD). Re-enable once real - # pod-egress routing to the VPS exists (tracked as a separate task) - no - # other change needed, this block is otherwise complete/correct. - # backup: - # barmanObjectStore: - # destinationPath: s3://cnpg-backups/pg-n8n - # endpointURL: http://vps-minio.netbird.internal:30900 - # s3Credentials: - # accessKeyId: - # name: vps-minio-secret - # key: accesskey - # secretAccessKey: - # name: vps-minio-secret - # key: secretkey - # wal: - # compression: gzip - # maxParallel: 2 - # retentionPolicy: "30d" + # Backup to VPS MinIO - RE-ENABLED 2026-08-18. Real pod-egress routing to + # the VPS now exists (netbird-egress DaemonSet, hostNetwork + per-node + # route into 100.108.0.0/16 - see infrastructure/netbird/manifests/ + # egress-daemonset.yaml). Endpoint changed from the NodePort + # (vps-minio.netbird.internal:30900 - never resolvable anyway, Netbird has + # no DNS configured, and separately blocked by Netbird's own ACL model, + # which only permits *established* forwarded connections through a peer, + # never new ones) to a locally-terminated socat forward on the VPS host + # itself (100.108.113.41:9000 - see minio-forward.service on the VPS), + # mirroring the pattern that already worked for the CrowdSec CAPI proxy. + backup: + barmanObjectStore: + destinationPath: s3://cnpg-backups/pg-n8n + endpointURL: http://100.108.113.41:9000 + s3Credentials: + accessKeyId: + name: vps-minio-secret + key: accesskey + secretAccessKey: + name: vps-minio-secret + key: secretkey + wal: + compression: gzip + maxParallel: 2 + retentionPolicy: "30d" monitoring: enablePodMonitor: true diff --git a/apps/nextcloud/manifests/cnpg-cluster.yaml b/apps/nextcloud/manifests/cnpg-cluster.yaml index 0cef3c0..516bb08 100644 --- a/apps/nextcloud/manifests/cnpg-cluster.yaml +++ b/apps/nextcloud/manifests/cnpg-cluster.yaml @@ -44,30 +44,33 @@ spec: database: nextcloud owner: nextcloud - # Backup to VPS MinIO - DISABLED 2026-08-17. No node in the cluster has - # any route into the Netbird mesh for pod-originated (egress) traffic; - # the per-namespace netbird "router" pods are inbound-only infrastructure. - # continuousArchiving kept failing to connect to vps-minio.netbird.internal, - # which held Ready=False permanently (real condition, not cosmetic). - # Re-enable once real pod-egress routing to the VPS exists (tracked as a - # separate task) - no other change needed, this block is otherwise - # complete/correct. This covers the DB only - file PVC content is - # separate, see nextcloud-pvc-sync-cronjob.yaml (same underlying gap). - # backup: - # barmanObjectStore: - # destinationPath: s3://cnpg-backups/pg-nextcloud - # endpointURL: http://vps-minio.netbird.internal:30900 - # s3Credentials: - # accessKeyId: - # name: vps-minio-secret - # key: accesskey - # secretAccessKey: - # name: vps-minio-secret - # key: secretkey - # wal: - # compression: gzip - # maxParallel: 2 - # retentionPolicy: "30d" + # Backup to VPS MinIO - RE-ENABLED 2026-08-18. Real pod-egress routing to + # the VPS now exists (netbird-egress DaemonSet, hostNetwork + per-node + # route into 100.108.0.0/16 - see infrastructure/netbird/manifests/ + # egress-daemonset.yaml). Endpoint changed from the NodePort + # (vps-minio.netbird.internal:30900 - never resolvable anyway, Netbird has + # no DNS configured, and separately blocked by Netbird's own ACL model, + # which only permits *established* forwarded connections through a peer, + # never new ones) to a locally-terminated socat forward on the VPS host + # itself (100.108.113.41:9000 - see minio-forward.service on the VPS), + # mirroring the pattern that already worked for the CrowdSec CAPI proxy. + # This covers the DB only - file PVC content is separate, see + # nextcloud-pvc-sync-cronjob.yaml (same fix applies there too). + backup: + barmanObjectStore: + destinationPath: s3://cnpg-backups/pg-nextcloud + endpointURL: http://100.108.113.41:9000 + s3Credentials: + accessKeyId: + name: vps-minio-secret + key: accesskey + secretAccessKey: + name: vps-minio-secret + key: secretkey + wal: + compression: gzip + maxParallel: 2 + retentionPolicy: "30d" monitoring: enablePodMonitor: true diff --git a/apps/nextcloud/manifests/pvc-sync-cronjob.yaml b/apps/nextcloud/manifests/pvc-sync-cronjob.yaml index 3a5c5a3..e93d302 100644 --- a/apps/nextcloud/manifests/pvc-sync-cronjob.yaml +++ b/apps/nextcloud/manifests/pvc-sync-cronjob.yaml @@ -100,10 +100,13 @@ spec: echo "==> Done." env: - # VPS's Netbird address - replace once bootstrapped, matches + # Locally-terminated socat forward on the VPS host to + # MinIO's ClusterIP, not the NodePort - see # infrastructure/vault/manifests/raft-snapshot-cronjob.yaml + # for why (Netbird has no DNS, and blocks new forwarded + # connections through a peer by default). - name: VPS_MINIO_ENDPOINT - value: "vps-minio.netbird.internal:30900" + value: "100.108.113.41:9000" - name: MINIO_ACCESS_KEY valueFrom: secretKeyRef: diff --git a/infrastructure/authentik/manifests/cnpg-cluster.yaml b/infrastructure/authentik/manifests/cnpg-cluster.yaml index 23850bf..c11abf8 100644 --- a/infrastructure/authentik/manifests/cnpg-cluster.yaml +++ b/infrastructure/authentik/manifests/cnpg-cluster.yaml @@ -47,29 +47,31 @@ spec: database: app owner: app - # Backup to VPS MinIO - DISABLED 2026-08-17. No node in the cluster has - # any route into the Netbird mesh for pod-originated (egress) traffic; - # the per-namespace netbird "router" pods are inbound-only infrastructure. - # continuousArchiving kept failing to connect to vps-minio.netbird.internal, - # which held Ready=False permanently (real condition, not cosmetic). - # Re-enable once real pod-egress routing to the VPS exists (tracked as a - # separate task) - no other change needed, this block is otherwise - # complete/correct. - # backup: - # barmanObjectStore: - # destinationPath: s3://cnpg-backups/pg-authentik - # endpointURL: http://vps-minio.netbird.internal:30900 - # s3Credentials: - # accessKeyId: - # name: vps-minio-secret - # key: accesskey - # secretAccessKey: - # name: vps-minio-secret - # key: secretkey - # wal: - # compression: gzip - # maxParallel: 2 - # retentionPolicy: "30d" + # Backup to VPS MinIO - RE-ENABLED 2026-08-18. Real pod-egress routing to + # the VPS now exists (netbird-egress DaemonSet, hostNetwork + per-node + # route into 100.108.0.0/16 - see infrastructure/netbird/manifests/ + # egress-daemonset.yaml). Endpoint changed from the NodePort + # (vps-minio.netbird.internal:30900 - never resolvable anyway, Netbird has + # no DNS configured, and separately blocked by Netbird's own ACL model, + # which only permits *established* forwarded connections through a peer, + # never new ones) to a locally-terminated socat forward on the VPS host + # itself (100.108.113.41:9000 - see minio-forward.service on the VPS), + # mirroring the pattern that already worked for the CrowdSec CAPI proxy. + backup: + barmanObjectStore: + destinationPath: s3://cnpg-backups/pg-authentik + endpointURL: http://100.108.113.41:9000 + s3Credentials: + accessKeyId: + name: vps-minio-secret + key: accesskey + secretAccessKey: + name: vps-minio-secret + key: secretkey + wal: + compression: gzip + maxParallel: 2 + retentionPolicy: "30d" monitoring: enablePodMonitor: true diff --git a/infrastructure/crowdsec/values.yaml b/infrastructure/crowdsec/values.yaml index 0cb80c4..402c16e 100644 --- a/infrastructure/crowdsec/values.yaml +++ b/infrastructure/crowdsec/values.yaml @@ -34,29 +34,20 @@ lapi: name: crowdsec-bouncer-key key: api-key optional: true - # CAPI/console-enroll disabled for now (2026-08-17). Root causes found: - # 1) Home's public IP gets a 403 from AWS WAF in front of api.crowdsec.net - # (even on a bare unauthenticated request) - an IP-level block - # unrelated to credentials; ENROLL_KEY above is valid and correctly - # wired, but can't get past this from home. - # 2) Tried routing around it via a proxy on the VPS over Netbird - - # confirmed no node in this cluster has any route into the Netbird - # mesh CIDR for pod-originated (egress) traffic at all. The - # per-namespace netbird "router" pods (gitlab, vault, argocd, etc.) - # are inbound-only infrastructure (let external Netbird peers reach - # K8s services) - nothing provides the reverse path. Building that - # properly needs either a Talos-native static route (fragile - would - # point at a pod IP that changes on every restart) or a deliberate - # exception to the disallow-host-namespaces STIG policy that - # correctly blocks the hostNetwork-based approach. Out of scope for - # a same-night fix - tracked as a separate future task. - # LAPI treats a failed CAPI/enroll attempt as fatal at startup (crashes - # the whole server, taking the agent + bouncer down with it via their - # dependency on LAPI), so disable it entirely rather than crash-loop. - # ENROLL_KEY above stays set - flip this back to false (or remove it) - # once real pod-egress routing to the VPS exists, no other change needed. - - name: DISABLE_ONLINE_API - value: "true" + # CAPI/console-enroll RE-ENABLED 2026-08-18. Home's public IP still gets + # a 403 from AWS WAF in front of api.crowdsec.net (an IP-level block, + # unrelated to credentials), so this routes through a tinyproxy on the + # VPS over Netbird instead - confirmed working end-to-end now that real + # pod-egress routing to the VPS exists (netbird-egress DaemonSet, see + # infrastructure/netbird/manifests/egress-daemonset.yaml). This was + # tried once before and failed because that egress route didn't exist + # yet - not a proxy or credentials problem. + - name: HTTPS_PROXY + value: "http://100.108.113.41:8888" + - name: HTTP_PROXY + value: "http://100.108.113.41:8888" + - name: NO_PROXY + value: "localhost,127.0.0.1,.svc,.svc.cluster.local,10.244.0.0/16,10.96.0.0/16" persistentVolume: data: enabled: false diff --git a/infrastructure/kyverno/policies/netbird-egress-exception.yaml b/infrastructure/kyverno/policies/netbird-egress-exception.yaml new file mode 100644 index 0000000..f95a847 --- /dev/null +++ b/infrastructure/kyverno/policies/netbird-egress-exception.yaml @@ -0,0 +1,38 @@ +# Scoped exception to the disallow-host-namespaces STIG policy (V-242386), +# narrowly for the netbird-egress DaemonSet only - not a namespace-wide +# exclusion like crowdsec's (see disallow-host-namespaces ClusterPolicy). +# +# Why hostNetwork is genuinely needed here: no node in this cluster has any +# route into the Netbird mesh CIDR (100.108.0.0/16) for pod-originated +# egress traffic - confirmed 2026-08-17 while debugging CrowdSec/CNPG +# backups reaching the VPS. The existing per-namespace "router" pods +# (NBRoutingPeer-based) are inbound-only and run fine in ordinary pod +# network namespaces. An EGRESS gateway is architecturally different: its +# wt0 interface must live in each node's real network namespace so the +# node's own kernel routing table can send matched pod traffic out through +# it directly - a pod-netns router would need SNAT/forwarding indirection +# on top, and per-pod route injection isn't possible without hostPID +# (also STIG-blocked) or a Cilium egress-gateway feature that doesn't +# actually solve this (it selects source IP for already-routable traffic, +# it doesn't create the missing route). +apiVersion: kyverno.io/v2 +kind: PolicyException +metadata: + name: netbird-egress-exemption + namespace: kyverno +spec: + exceptions: + - policyName: disallow-host-namespaces + ruleNames: + - host-namespaces + - autogen-host-namespaces + match: + any: + - resources: + kinds: + - DaemonSet + - Pod + namespaces: + - netbird + names: + - netbird-egress* diff --git a/infrastructure/netbird/manifests/egress-daemonset.yaml b/infrastructure/netbird/manifests/egress-daemonset.yaml new file mode 100644 index 0000000..81bddd8 --- /dev/null +++ b/infrastructure/netbird/manifests/egress-daemonset.yaml @@ -0,0 +1,96 @@ +# Pod-egress routing into the Netbird mesh. +# +# The existing per-namespace `router` Deployments (NBRoutingPeer CRs - +# gitlab, vault, argocd, nextcloud, etc.) are INBOUND-only infrastructure: +# they let external Netbird peers reach into those namespaces' services. +# Nothing programs the reverse - no node ever gets a route sending pod +# traffic OUT through any of those router pods' wt0 interfaces. Confirmed +# 2026-08-17 while debugging CrowdSec's CAPI enrollment: `ip route get +# 100.108.113.41` on a node running a pod that needed to reach the VPS's +# Netbird IP just showed the plain LAN default gateway - none of the +# "connected" router pods were ever actually in the path, and a tcpdump on +# their wt0 during live attempts showed zero packets. +# +# This DaemonSet is the missing outbound half: one netbird client per node, +# running with hostNetwork so its wt0 interface lives directly in the node's +# real network namespace (avoiding all the SNAT/forwarding complexity a +# pod-netns subnet router would need), plus a sidecar that adds a host route +# sending 100.108.0.0/16 out via that interface. Once packets leave a node +# via wt0 with a real Netbird-mesh source identity, return routing already +# works via the same route-advertisement mechanism the inbound routers use. +# +# hostNetwork requires infrastructure/kyverno/policies/netbird-egress-exception.yaml +# - a scoped PolicyException to the disallow-host-namespaces STIG policy, +# not a broad exclusion. See that file for the full justification. +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: netbird-egress + namespace: netbird + labels: + app.kubernetes.io/name: netbird-egress +spec: + selector: + matchLabels: + app.kubernetes.io/name: netbird-egress + template: + metadata: + labels: + app.kubernetes.io/name: netbird-egress + annotations: + # Same reasoning as netbird-cluster-router: ztunnel iptables rules + # interfere with WireGuard packet forwarding. + ambient.istio.io/redirection: disabled + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: netbird + image: netbirdio/netbird:0.66.0 + imagePullPolicy: IfNotPresent + env: + - name: NB_SETUP_KEY + valueFrom: + secretKeyRef: + name: netbird-router-setup-key + key: setup-key + - name: NB_MANAGEMENT_URL + value: "https://netbird.kube.huskypup.net" + securityContext: + capabilities: + add: + - NET_ADMIN + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + memory: 128Mi + # Adds the host route sending mesh-bound traffic out via wt0 once + # the netbird container brings the interface up. Re-asserts on a + # loop since wt0 can be recreated on reconnect (route otherwise + # silently disappears with it). + - name: route-manager + image: alpine:3.20 + command: + - sh + - -c + - | + apk add --no-cache iproute2 >/dev/null 2>&1 + while true; do + if ip link show wt0 >/dev/null 2>&1; then + ip route replace 100.108.0.0/16 dev wt0 2>/dev/null + fi + sleep 10 + done + securityContext: + capabilities: + add: + - NET_ADMIN + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 64Mi + terminationGracePeriodSeconds: 30 diff --git a/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml b/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml index 1636d19..3a8da9a 100644 --- a/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml +++ b/infrastructure/vault/manifests/raft-snapshot-cronjob.yaml @@ -45,7 +45,14 @@ data: # VPS's Netbird address - replace with the actual peer IP/hostname once # the VPS is bootstrapped and joined to the mesh (scripts/vps-bootstrap.sh). - VPS_MINIO_ENDPOINT="${VPS_MINIO_ENDPOINT:-vps-minio.netbird.internal:30900}" + # 100.108.113.41:9000 is a locally-terminated socat forward on the VPS + # host (minio-forward.service) to MinIO's ClusterIP, not the NodePort. + # vps-minio.netbird.internal was never resolvable (Netbird has no DNS + # configured) and the NodePort itself is unreachable from other peers - + # Netbird's own ACL model only permits established forwarded + # connections through a peer, never new ones. See egress-daemonset.yaml + # for the pod-egress route this now travels over. + VPS_MINIO_ENDPOINT="${VPS_MINIO_ENDPOINT:-100.108.113.41:9000}" BUCKET="vault-raft-snapshots" SNAP_NAME="vault-raft-$(date -u +%Y%m%dT%H%M%SZ).snap" @@ -102,13 +109,9 @@ metadata: namespace: vault spec: schedule: "0 */6 * * *" # every 6 hours - # Suspended 2026-08-17: the upload step targets vps-minio.netbird.internal, - # which no pod in this cluster can currently reach - no node has an - # egress route into the Netbird mesh (see commit history around this - # date for the full investigation). The leader-detection fix above is - # real and independent of this, so it's kept fixed rather than reverted. - # Unsuspend once real pod-egress routing to the VPS exists. - suspend: true + # Re-enabled 2026-08-18: real pod-egress routing to the VPS now exists + # (netbird-egress DaemonSet) and the endpoint above was fixed to use the + # locally-terminated socat forward instead of the unreachable NodePort. concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 3