Add VPS warm-standby/backup site (Phase 0-1b)

Foundation for a DR/backup path using an always-on VPS as a second
ArgoCD-managed cluster, plus DB/backup standardization work that fell
out of it:

- vps-standby ArgoCD cluster destination + AppProject, MinIO backup
  receiver, VPS bootstrap script (k3s, Netbird, cert-manager)
- Dual-site DNS failover watcher + home-IP DDNS CronJob, Cloudflare
  token moved out of git into Vault+ExternalSecret
- Nextcloud migrated from ad-hoc MariaDB to CNPG + redis-operator
  (matches n8n/Authentik/GitLab's backup-native pattern)
- Authentik's CNPG manifests moved into the actual ArgoCD-synced
  manifests/ path (were present but never wired into the sync path)
- Vault raft-snapshot CronJob, CNPG barmanObjectStore backups
  (Authentik/n8n/Nextcloud), Nextcloud file-PVC restic sync - all
  targeting the new VPS MinIO receiver

See VPS Warm-Standby plan doc for full design rationale.
This commit is contained in:
Scooby Husky
2026-08-17 14:59:26 -05:00
parent 5163403e24
commit 7990f1fa47
25 changed files with 1161 additions and 139 deletions
@@ -0,0 +1,68 @@
apiVersion: postgresql.cnpg.io/v1
kind: Cluster
metadata:
name: pg-nextcloud
namespace: nextcloud
spec:
imageName: ghcr.io/cloudnative-pg/postgresql:16
instances: 2
resources:
requests:
memory: "512Mi"
cpu: "50m"
limits:
memory: "2Gi"
cpu: "500m"
affinity:
topologyKey: kubernetes.io/hostname
storage:
size: 20Gi
storageClass: rook-ceph-block
primaryUpdateStrategy: unsupervised
postgresql:
parameters:
max_connections: "200"
shared_buffers: "512MB"
effective_cache_size: "1536MB"
maintenance_work_mem: "128MB"
checkpoint_completion_target: "0.9"
wal_buffers: "16MB"
default_statistics_target: "100"
random_page_cost: "1.1"
effective_io_concurrency: "200"
work_mem: "2621kB"
min_wal_size: "1GB"
max_wal_size: "4GB"
bootstrap:
initdb:
database: nextcloud
owner: nextcloud
# Backup to the VPS MinIO backup receiver. VPS_MINIO_ENDPOINT placeholder
# matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace
# with the VPS's actual Netbird address once bootstrapped. This covers the
# DB only - file PVC content is separate, see nextcloud-pvc-sync-cronjob.yaml.
backup:
barmanObjectStore:
destinationPath: s3://cnpg-backups/pg-nextcloud
endpointURL: http://vps-minio.netbird.internal:30900
s3Credentials:
accessKeyId:
name: vps-minio-secret
key: accesskey
secretAccessKey:
name: vps-minio-secret
key: secretkey
wal:
compression: gzip
maxParallel: 2
retentionPolicy: "30d"
monitoring:
enablePodMonitor: true
@@ -0,0 +1,42 @@
---
# Mirrors apps/n8n/manifests/cnpg-secrets.yaml: CNPG creates pg-nextcloud-app
# with a generated password on cluster init; ESO overrides it with a
# generator-managed password so it's rotated/tracked the same way as every
# other app secret in this repo.
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
metadata:
name: nextcloud-cnpg-secret
namespace: nextcloud
spec:
length: 42
digits: 5
symbols: 5
symbolCharacters: "-_$@"
noUpper: false
allowRepeat: true
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: nextcloud-cnpg-secret
namespace: nextcloud
spec:
refreshInterval: "24h"
target:
# Merges into the pg-nextcloud-app secret CNPG creates automatically
name: pg-nextcloud-app
creationPolicy: Merge
template:
metadata:
labels:
cnpg.io/reload: "true"
data:
password: "{{ .password }}"
dataFrom:
- sourceRef:
generatorRef:
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
name: nextcloud-cnpg-secret
@@ -1,92 +0,0 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: mariadb-nextcloud
namespace: nextcloud
labels:
app: mariadb-nextcloud
spec:
replicas: 1
selector:
matchLabels:
app: mariadb-nextcloud
template:
metadata:
labels:
app: mariadb-nextcloud
spec:
securityContext:
runAsUser: 999
runAsGroup: 999
fsGroup: 999
containers:
- name: mariadb
image: mariadb:11.4
securityContext:
runAsUser: 999
runAsGroup: 999
runAsNonRoot: true
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
args:
- --character-set-server=utf8mb4
- --collation-server=utf8mb4_unicode_ci
- --init-connect=SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci
env:
- name: MARIADB_ROOT_PASSWORD
valueFrom:
secretKeyRef:
name: mariadb-nextcloud
key: root-password
- name: MARIADB_PASSWORD
valueFrom:
secretKeyRef:
name: mariadb-nextcloud
key: password
- name: MARIADB_USER
value: nextcloud
- name: MARIADB_DATABASE
value: nextcloud
ports:
- containerPort: 3306
volumeMounts:
- name: mariadb-data
mountPath: /var/lib/mysql
- name: tmp
mountPath: /tmp
- name: run-mysqld
mountPath: /run/mysqld
volumes:
- name: tmp
emptyDir: {}
- name: run-mysqld
emptyDir: {}
- name: mariadb-data
persistentVolumeClaim:
claimName: mariadb-nextcloud-pvc
---
apiVersion: v1
kind: Service
metadata:
name: mariadb-nextcloud
namespace: nextcloud
spec:
selector:
app: mariadb-nextcloud
ports:
- port: 3306
targetPort: 3306
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: mariadb-nextcloud-pvc
namespace: nextcloud
spec:
accessModes:
- ReadWriteOnce
storageClassName: rook-ceph-block
resources:
requests:
storage: 10Gi
@@ -1,35 +0,0 @@
---
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
metadata:
name: nextcloud-mariadb-password
namespace: nextcloud
spec:
length: 32
digits: 5
symbols: 3
symbolCharacters: "-_$"
noUpper: false
allowRepeat: true
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: nextcloud-mariadb-secret
namespace: nextcloud
spec:
refreshInterval: "24h"
target:
name: mariadb-nextcloud
creationPolicy: Owner
template:
data:
password: "{{ .password }}"
root-password: "{{ .password }}"
user: nextcloud
dataFrom:
- sourceRef:
generatorRef:
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
name: nextcloud-mariadb-password
@@ -0,0 +1,124 @@
---
# Syncs Nextcloud's file PVC content (nextcloud-nextcloud, 200Gi) to the VPS
# MinIO backup receiver via restic. The one piece of the Nextcloud backup story
# no operator covers - actual file blobs, not database rows (DB is handled by
# CNPG's barmanObjectStore in cnpg-cluster.yaml).
#
# Caveat: this backs up the live-mounted volume with no application-level
# quiesce (no Nextcloud maintenance-mode pause around the snapshot). Acceptable
# for a no-real-data test environment; for anything with real user data, pair
# this with `occ maintenance:mode --on` before / `--off` after.
#
# Uses podAffinity to land on the same node as a running Nextcloud pod, since
# the PVC is RWO (Ceph RBD) - RWO allows multiple pods to mount it concurrently
# only when co-located on the same node. Verify the label selector below
# matches the actual Nextcloud chart's pod labels before relying on this.
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
metadata:
name: nextcloud-restic-password
namespace: nextcloud
spec:
length: 48
digits: 5
symbols: 5
symbolCharacters: "-_$@"
noUpper: false
allowRepeat: true
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: nextcloud-restic-password
namespace: nextcloud
spec:
refreshInterval: "0" # generate once - changing this after the repo is initialized would lock you out of existing backups
target:
name: nextcloud-restic-password
creationPolicy: Owner
data: []
dataFrom:
- sourceRef:
generatorRef:
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
name: nextcloud-restic-password
---
apiVersion: batch/v1
kind: CronJob
metadata:
name: nextcloud-pvc-sync
namespace: nextcloud
spec:
schedule: "0 2 * * *" # daily at 02:00 - bulky, don't run it as often as the DB backups
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 3
failedJobsHistoryLimit: 3
jobTemplate:
spec:
backoffLimit: 2
template:
spec:
restartPolicy: Never
affinity:
podAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app.kubernetes.io/name: nextcloud
topologyKey: kubernetes.io/hostname
containers:
- name: restic-backup
image: restic/restic:0.16.4
command:
- /bin/sh
- -c
- |
set -eu
export RESTIC_REPOSITORY="s3:http://${VPS_MINIO_ENDPOINT}/nextcloud-files/restic-repo"
export RESTIC_PASSWORD_FILE=/restic-secret/password
export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
restic snapshots >/dev/null 2>&1 || restic init
echo "==> Backing up /data..."
restic backup /data --tag nextcloud --host nextcloud-k8s
echo "==> Pruning: keep 7 daily, 4 weekly, 6 monthly..."
restic forget --tag nextcloud --host nextcloud-k8s \
--keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
echo "==> Done."
env:
# VPS's Netbird address - replace once bootstrapped, matches
# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
- name: VPS_MINIO_ENDPOINT
value: "vps-minio.netbird.internal:30900"
- name: MINIO_ACCESS_KEY
valueFrom:
secretKeyRef:
name: vps-minio-secret
key: accesskey
- name: MINIO_SECRET_KEY
valueFrom:
secretKeyRef:
name: vps-minio-secret
key: secretkey
volumeMounts:
- name: nextcloud-data
mountPath: /data
readOnly: true
- name: restic-secret
mountPath: /restic-secret
readOnly: true
volumes:
- name: nextcloud-data
persistentVolumeClaim:
claimName: nextcloud-nextcloud
- name: restic-secret
secret:
secretName: nextcloud-restic-password
items:
- key: password
path: password
+72
View File
@@ -0,0 +1,72 @@
---
# Standalone Redis via the ot-container-kit redis-operator that's already
# deployed as infra (argocd-apps/infrastructure/redis-operator.yaml) but
# currently unused - GitLab's Redis is a hand-rolled StatefulSet instead
# (apps/gitlab/manifests/redis-cluster.yaml), not this operator. This is the
# first real consumer of it in the repo.
#
# NOTE: verify this CR against `kubectl explain redis.spec` (or the chart's
# CRD source) for the redis-operator 0.15.0 actually deployed before first
# apply - the ot-container-kit CRD schema has shifted across versions and
# this wasn't checked against a live cluster.
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
metadata:
name: nextcloud-redis-password
namespace: nextcloud
spec:
length: 32
digits: 5
symbols: 0
noUpper: false
allowRepeat: true
---
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: nextcloud-redis-password
namespace: nextcloud
spec:
refreshInterval: "0" # generate once, don't rotate (Redis CR reads this at pod start only)
target:
name: redis-nextcloud-secret
creationPolicy: Owner
template:
data:
password: "{{ .password }}"
dataFrom:
- sourceRef:
generatorRef:
apiVersion: generators.external-secrets.io/v1alpha1
kind: Password
name: nextcloud-redis-password
---
apiVersion: redis.redis.opstreelabs.in/v1beta2
kind: Redis
metadata:
name: redis-nextcloud
namespace: nextcloud
spec:
kubernetesConfig:
image: quay.io/opstree/redis:v7.0.12
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 25m
memory: 128Mi
limits:
cpu: 250m
memory: 256Mi
redisExporter:
enabled: false
redisSecret:
name: redis-nextcloud-secret
key: password
storage:
volumeClaimTemplate:
spec:
accessModes: ["ReadWriteOnce"]
storageClassName: rook-ceph-block
resources:
requests:
storage: 2Gi
@@ -0,0 +1,27 @@
---
# VPS MinIO credentials for CNPG's barmanObjectStore backup target, and reused
# by the Phase 1b PVC-content sync CronJob. Same Vault source as
# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
apiVersion: external-secrets.io/v1
kind: ExternalSecret
metadata:
name: vps-minio-credentials
namespace: nextcloud
spec:
refreshInterval: 1h
secretStoreRef:
kind: ClusterSecretStore
name: vault-backend
target:
name: vps-minio-secret
creationPolicy: Owner
data:
- secretKey: accesskey
remoteRef:
key: vps-minio-credentials
property: access-key
- secretKey: secretkey
remoteRef:
key: vps-minio-credentials
property: secret-key