From 2c8d29c51f85e6a802fcfb1b13d1666b8b6a64b0 Mon Sep 17 00:00:00 2001 From: Scooby Husky Date: Mon, 17 Aug 2026 19:45:04 -0500 Subject: [PATCH] Disable VPS MinIO backup on n8n/nextcloud/authentik CNPG clusters Discovered while debugging pg-n8n's recurring 'Instance Status Extraction Error': the root cause is the same missing pod-egress route to the Netbird mesh found tonight while fixing CrowdSec - continuousArchiving kept failing to connect to vps-minio.netbird.internal, and CNPG correctly holds Ready=False while archiving is broken (a real condition, not cosmetic). That's what was driving these three apps' ArgoCD health flapping. No node in the cluster has any route into 100.108.0.0/16 for pod-originated traffic - the per-namespace netbird 'router' pods (gitlab, vault, argocd, etc.) are inbound-only infrastructure, not egress gateways. gitlab's own CNPG backup is unaffected (points at a local in-cluster MinIO, not the VPS). Commented out rather than deleted - re-enable once real pod-egress routing exists, tracked as a separate task. No data loss: this is WAL archiving/backup, not the live database. Co-Authored-By: Claude Sonnet 5 --- apps/n8n/manifests/cnpg-cluster.yaml | 42 ++++++++++-------- apps/nextcloud/manifests/cnpg-cluster.yaml | 43 +++++++++++-------- .../authentik/manifests/cnpg-cluster.yaml | 41 ++++++++++-------- 3 files changed, 70 insertions(+), 56 deletions(-) diff --git a/apps/n8n/manifests/cnpg-cluster.yaml b/apps/n8n/manifests/cnpg-cluster.yaml index b64df8a..7ce7cb8 100644 --- a/apps/n8n/manifests/cnpg-cluster.yaml +++ b/apps/n8n/manifests/cnpg-cluster.yaml @@ -47,25 +47,29 @@ spec: database: n8n owner: n8n - # Backup to the VPS MinIO backup receiver (Phase 1b). Previously had no - # backup block at all. VPS_MINIO_ENDPOINT placeholder matches - # infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace with - # the VPS's actual Netbird address once bootstrapped. - backup: - barmanObjectStore: - destinationPath: s3://cnpg-backups/pg-n8n - endpointURL: http://vps-minio.netbird.internal:30900 - s3Credentials: - accessKeyId: - name: vps-minio-secret - key: accesskey - secretAccessKey: - name: vps-minio-secret - key: secretkey - wal: - compression: gzip - maxParallel: 2 - retentionPolicy: "30d" + # Backup to VPS MinIO - DISABLED 2026-08-17. No node in the cluster has + # any route into the Netbird mesh for pod-originated (egress) traffic; + # the per-namespace netbird "router" pods are inbound-only infrastructure. + # continuousArchiving kept failing to connect to vps-minio.netbird.internal, + # which held Ready=False permanently (real condition, not cosmetic - this + # is what drove n8n's CNPG health flapping in ArgoCD). Re-enable once real + # pod-egress routing to the VPS exists (tracked as a separate task) - no + # other change needed, this block is otherwise complete/correct. + # backup: + # barmanObjectStore: + # destinationPath: s3://cnpg-backups/pg-n8n + # endpointURL: http://vps-minio.netbird.internal:30900 + # s3Credentials: + # accessKeyId: + # name: vps-minio-secret + # key: accesskey + # secretAccessKey: + # name: vps-minio-secret + # key: secretkey + # wal: + # compression: gzip + # maxParallel: 2 + # retentionPolicy: "30d" monitoring: enablePodMonitor: true diff --git a/apps/nextcloud/manifests/cnpg-cluster.yaml b/apps/nextcloud/manifests/cnpg-cluster.yaml index ece9be6..0cef3c0 100644 --- a/apps/nextcloud/manifests/cnpg-cluster.yaml +++ b/apps/nextcloud/manifests/cnpg-cluster.yaml @@ -44,25 +44,30 @@ spec: database: nextcloud owner: nextcloud - # Backup to the VPS MinIO backup receiver. VPS_MINIO_ENDPOINT placeholder - # matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace - # with the VPS's actual Netbird address once bootstrapped. This covers the - # DB only - file PVC content is separate, see nextcloud-pvc-sync-cronjob.yaml. - backup: - barmanObjectStore: - destinationPath: s3://cnpg-backups/pg-nextcloud - endpointURL: http://vps-minio.netbird.internal:30900 - s3Credentials: - accessKeyId: - name: vps-minio-secret - key: accesskey - secretAccessKey: - name: vps-minio-secret - key: secretkey - wal: - compression: gzip - maxParallel: 2 - retentionPolicy: "30d" + # Backup to VPS MinIO - DISABLED 2026-08-17. No node in the cluster has + # any route into the Netbird mesh for pod-originated (egress) traffic; + # the per-namespace netbird "router" pods are inbound-only infrastructure. + # continuousArchiving kept failing to connect to vps-minio.netbird.internal, + # which held Ready=False permanently (real condition, not cosmetic). + # Re-enable once real pod-egress routing to the VPS exists (tracked as a + # separate task) - no other change needed, this block is otherwise + # complete/correct. This covers the DB only - file PVC content is + # separate, see nextcloud-pvc-sync-cronjob.yaml (same underlying gap). + # backup: + # barmanObjectStore: + # destinationPath: s3://cnpg-backups/pg-nextcloud + # endpointURL: http://vps-minio.netbird.internal:30900 + # s3Credentials: + # accessKeyId: + # name: vps-minio-secret + # key: accesskey + # secretAccessKey: + # name: vps-minio-secret + # key: secretkey + # wal: + # compression: gzip + # maxParallel: 2 + # retentionPolicy: "30d" monitoring: enablePodMonitor: true diff --git a/infrastructure/authentik/manifests/cnpg-cluster.yaml b/infrastructure/authentik/manifests/cnpg-cluster.yaml index 55f5982..23850bf 100644 --- a/infrastructure/authentik/manifests/cnpg-cluster.yaml +++ b/infrastructure/authentik/manifests/cnpg-cluster.yaml @@ -47,24 +47,29 @@ spec: database: app owner: app - # Backup to the VPS MinIO backup receiver (Phase 1b). VPS_MINIO_ENDPOINT - # placeholder matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - - # replace with the VPS's actual Netbird address once bootstrapped. - backup: - barmanObjectStore: - destinationPath: s3://cnpg-backups/pg-authentik - endpointURL: http://vps-minio.netbird.internal:30900 - s3Credentials: - accessKeyId: - name: vps-minio-secret - key: accesskey - secretAccessKey: - name: vps-minio-secret - key: secretkey - wal: - compression: gzip - maxParallel: 2 - retentionPolicy: "30d" + # Backup to VPS MinIO - DISABLED 2026-08-17. No node in the cluster has + # any route into the Netbird mesh for pod-originated (egress) traffic; + # the per-namespace netbird "router" pods are inbound-only infrastructure. + # continuousArchiving kept failing to connect to vps-minio.netbird.internal, + # which held Ready=False permanently (real condition, not cosmetic). + # Re-enable once real pod-egress routing to the VPS exists (tracked as a + # separate task) - no other change needed, this block is otherwise + # complete/correct. + # backup: + # barmanObjectStore: + # destinationPath: s3://cnpg-backups/pg-authentik + # endpointURL: http://vps-minio.netbird.internal:30900 + # s3Credentials: + # accessKeyId: + # name: vps-minio-secret + # key: accesskey + # secretAccessKey: + # name: vps-minio-secret + # key: secretkey + # wal: + # compression: gzip + # maxParallel: 2 + # retentionPolicy: "30d" monitoring: enablePodMonitor: true