mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-20 23:16:49 +00:00
Add VPS warm-standby/backup site (Phase 0-1b)
Foundation for a DR/backup path using an always-on VPS as a second ArgoCD-managed cluster, plus DB/backup standardization work that fell out of it: - vps-standby ArgoCD cluster destination + AppProject, MinIO backup receiver, VPS bootstrap script (k3s, Netbird, cert-manager) - Dual-site DNS failover watcher + home-IP DDNS CronJob, Cloudflare token moved out of git into Vault+ExternalSecret - Nextcloud migrated from ad-hoc MariaDB to CNPG + redis-operator (matches n8n/Authentik/GitLab's backup-native pattern) - Authentik's CNPG manifests moved into the actual ArgoCD-synced manifests/ path (were present but never wired into the sync path) - Vault raft-snapshot CronJob, CNPG barmanObjectStore backups (Authentik/n8n/Nextcloud), Nextcloud file-PVC restic sync - all targeting the new VPS MinIO receiver See VPS Warm-Standby plan doc for full design rationale.
This commit is contained in:
@@ -0,0 +1,68 @@
|
||||
apiVersion: postgresql.cnpg.io/v1
|
||||
kind: Cluster
|
||||
metadata:
|
||||
name: pg-nextcloud
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
imageName: ghcr.io/cloudnative-pg/postgresql:16
|
||||
instances: 2
|
||||
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "50m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "500m"
|
||||
|
||||
affinity:
|
||||
topologyKey: kubernetes.io/hostname
|
||||
|
||||
storage:
|
||||
size: 20Gi
|
||||
storageClass: rook-ceph-block
|
||||
|
||||
primaryUpdateStrategy: unsupervised
|
||||
|
||||
postgresql:
|
||||
parameters:
|
||||
max_connections: "200"
|
||||
shared_buffers: "512MB"
|
||||
effective_cache_size: "1536MB"
|
||||
maintenance_work_mem: "128MB"
|
||||
checkpoint_completion_target: "0.9"
|
||||
wal_buffers: "16MB"
|
||||
default_statistics_target: "100"
|
||||
random_page_cost: "1.1"
|
||||
effective_io_concurrency: "200"
|
||||
work_mem: "2621kB"
|
||||
min_wal_size: "1GB"
|
||||
max_wal_size: "4GB"
|
||||
|
||||
bootstrap:
|
||||
initdb:
|
||||
database: nextcloud
|
||||
owner: nextcloud
|
||||
|
||||
# Backup to the VPS MinIO backup receiver. VPS_MINIO_ENDPOINT placeholder
|
||||
# matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace
|
||||
# with the VPS's actual Netbird address once bootstrapped. This covers the
|
||||
# DB only - file PVC content is separate, see nextcloud-pvc-sync-cronjob.yaml.
|
||||
backup:
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg-backups/pg-nextcloud
|
||||
endpointURL: http://vps-minio.netbird.internal:30900
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: vps-minio-secret
|
||||
key: accesskey
|
||||
secretAccessKey:
|
||||
name: vps-minio-secret
|
||||
key: secretkey
|
||||
wal:
|
||||
compression: gzip
|
||||
maxParallel: 2
|
||||
retentionPolicy: "30d"
|
||||
|
||||
monitoring:
|
||||
enablePodMonitor: true
|
||||
@@ -0,0 +1,42 @@
|
||||
---
|
||||
# Mirrors apps/n8n/manifests/cnpg-secrets.yaml: CNPG creates pg-nextcloud-app
|
||||
# with a generated password on cluster init; ESO overrides it with a
|
||||
# generator-managed password so it's rotated/tracked the same way as every
|
||||
# other app secret in this repo.
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: nextcloud-cnpg-secret
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
length: 42
|
||||
digits: 5
|
||||
symbols: 5
|
||||
symbolCharacters: "-_$@"
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nextcloud-cnpg-secret
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: "24h"
|
||||
target:
|
||||
# Merges into the pg-nextcloud-app secret CNPG creates automatically
|
||||
name: pg-nextcloud-app
|
||||
creationPolicy: Merge
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
cnpg.io/reload: "true"
|
||||
data:
|
||||
password: "{{ .password }}"
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: nextcloud-cnpg-secret
|
||||
@@ -1,92 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: mariadb-nextcloud
|
||||
namespace: nextcloud
|
||||
labels:
|
||||
app: mariadb-nextcloud
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: mariadb-nextcloud
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: mariadb-nextcloud
|
||||
spec:
|
||||
securityContext:
|
||||
runAsUser: 999
|
||||
runAsGroup: 999
|
||||
fsGroup: 999
|
||||
containers:
|
||||
- name: mariadb
|
||||
image: mariadb:11.4
|
||||
securityContext:
|
||||
runAsUser: 999
|
||||
runAsGroup: 999
|
||||
runAsNonRoot: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
args:
|
||||
- --character-set-server=utf8mb4
|
||||
- --collation-server=utf8mb4_unicode_ci
|
||||
- --init-connect=SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci
|
||||
env:
|
||||
- name: MARIADB_ROOT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mariadb-nextcloud
|
||||
key: root-password
|
||||
- name: MARIADB_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: mariadb-nextcloud
|
||||
key: password
|
||||
- name: MARIADB_USER
|
||||
value: nextcloud
|
||||
- name: MARIADB_DATABASE
|
||||
value: nextcloud
|
||||
ports:
|
||||
- containerPort: 3306
|
||||
volumeMounts:
|
||||
- name: mariadb-data
|
||||
mountPath: /var/lib/mysql
|
||||
- name: tmp
|
||||
mountPath: /tmp
|
||||
- name: run-mysqld
|
||||
mountPath: /run/mysqld
|
||||
volumes:
|
||||
- name: tmp
|
||||
emptyDir: {}
|
||||
- name: run-mysqld
|
||||
emptyDir: {}
|
||||
- name: mariadb-data
|
||||
persistentVolumeClaim:
|
||||
claimName: mariadb-nextcloud-pvc
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: mariadb-nextcloud
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
selector:
|
||||
app: mariadb-nextcloud
|
||||
ports:
|
||||
- port: 3306
|
||||
targetPort: 3306
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: mariadb-nextcloud-pvc
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: rook-ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
@@ -1,35 +0,0 @@
|
||||
---
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: nextcloud-mariadb-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
length: 32
|
||||
digits: 5
|
||||
symbols: 3
|
||||
symbolCharacters: "-_$"
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nextcloud-mariadb-secret
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: "24h"
|
||||
target:
|
||||
name: mariadb-nextcloud
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
data:
|
||||
password: "{{ .password }}"
|
||||
root-password: "{{ .password }}"
|
||||
user: nextcloud
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: nextcloud-mariadb-password
|
||||
@@ -0,0 +1,124 @@
|
||||
---
|
||||
# Syncs Nextcloud's file PVC content (nextcloud-nextcloud, 200Gi) to the VPS
|
||||
# MinIO backup receiver via restic. The one piece of the Nextcloud backup story
|
||||
# no operator covers - actual file blobs, not database rows (DB is handled by
|
||||
# CNPG's barmanObjectStore in cnpg-cluster.yaml).
|
||||
#
|
||||
# Caveat: this backs up the live-mounted volume with no application-level
|
||||
# quiesce (no Nextcloud maintenance-mode pause around the snapshot). Acceptable
|
||||
# for a no-real-data test environment; for anything with real user data, pair
|
||||
# this with `occ maintenance:mode --on` before / `--off` after.
|
||||
#
|
||||
# Uses podAffinity to land on the same node as a running Nextcloud pod, since
|
||||
# the PVC is RWO (Ceph RBD) - RWO allows multiple pods to mount it concurrently
|
||||
# only when co-located on the same node. Verify the label selector below
|
||||
# matches the actual Nextcloud chart's pod labels before relying on this.
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: nextcloud-restic-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
length: 48
|
||||
digits: 5
|
||||
symbols: 5
|
||||
symbolCharacters: "-_$@"
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nextcloud-restic-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: "0" # generate once - changing this after the repo is initialized would lock you out of existing backups
|
||||
target:
|
||||
name: nextcloud-restic-password
|
||||
creationPolicy: Owner
|
||||
data: []
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: nextcloud-restic-password
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: nextcloud-pvc-sync
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
schedule: "0 2 * * *" # daily at 02:00 - bulky, don't run it as often as the DB backups
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
affinity:
|
||||
podAffinity:
|
||||
requiredDuringSchedulingIgnoredDuringExecution:
|
||||
- labelSelector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: nextcloud
|
||||
topologyKey: kubernetes.io/hostname
|
||||
containers:
|
||||
- name: restic-backup
|
||||
image: restic/restic:0.16.4
|
||||
command:
|
||||
- /bin/sh
|
||||
- -c
|
||||
- |
|
||||
set -eu
|
||||
export RESTIC_REPOSITORY="s3:http://${VPS_MINIO_ENDPOINT}/nextcloud-files/restic-repo"
|
||||
export RESTIC_PASSWORD_FILE=/restic-secret/password
|
||||
export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
|
||||
export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
|
||||
|
||||
restic snapshots >/dev/null 2>&1 || restic init
|
||||
|
||||
echo "==> Backing up /data..."
|
||||
restic backup /data --tag nextcloud --host nextcloud-k8s
|
||||
|
||||
echo "==> Pruning: keep 7 daily, 4 weekly, 6 monthly..."
|
||||
restic forget --tag nextcloud --host nextcloud-k8s \
|
||||
--keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
|
||||
|
||||
echo "==> Done."
|
||||
env:
|
||||
# VPS's Netbird address - replace once bootstrapped, matches
|
||||
# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
|
||||
- name: VPS_MINIO_ENDPOINT
|
||||
value: "vps-minio.netbird.internal:30900"
|
||||
- name: MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-secret
|
||||
key: accesskey
|
||||
- name: MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-secret
|
||||
key: secretkey
|
||||
volumeMounts:
|
||||
- name: nextcloud-data
|
||||
mountPath: /data
|
||||
readOnly: true
|
||||
- name: restic-secret
|
||||
mountPath: /restic-secret
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: nextcloud-data
|
||||
persistentVolumeClaim:
|
||||
claimName: nextcloud-nextcloud
|
||||
- name: restic-secret
|
||||
secret:
|
||||
secretName: nextcloud-restic-password
|
||||
items:
|
||||
- key: password
|
||||
path: password
|
||||
@@ -0,0 +1,72 @@
|
||||
---
|
||||
# Standalone Redis via the ot-container-kit redis-operator that's already
|
||||
# deployed as infra (argocd-apps/infrastructure/redis-operator.yaml) but
|
||||
# currently unused - GitLab's Redis is a hand-rolled StatefulSet instead
|
||||
# (apps/gitlab/manifests/redis-cluster.yaml), not this operator. This is the
|
||||
# first real consumer of it in the repo.
|
||||
#
|
||||
# NOTE: verify this CR against `kubectl explain redis.spec` (or the chart's
|
||||
# CRD source) for the redis-operator 0.15.0 actually deployed before first
|
||||
# apply - the ot-container-kit CRD schema has shifted across versions and
|
||||
# this wasn't checked against a live cluster.
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
metadata:
|
||||
name: nextcloud-redis-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
length: 32
|
||||
digits: 5
|
||||
symbols: 0
|
||||
noUpper: false
|
||||
allowRepeat: true
|
||||
---
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: nextcloud-redis-password
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: "0" # generate once, don't rotate (Redis CR reads this at pod start only)
|
||||
target:
|
||||
name: redis-nextcloud-secret
|
||||
creationPolicy: Owner
|
||||
template:
|
||||
data:
|
||||
password: "{{ .password }}"
|
||||
dataFrom:
|
||||
- sourceRef:
|
||||
generatorRef:
|
||||
apiVersion: generators.external-secrets.io/v1alpha1
|
||||
kind: Password
|
||||
name: nextcloud-redis-password
|
||||
---
|
||||
apiVersion: redis.redis.opstreelabs.in/v1beta2
|
||||
kind: Redis
|
||||
metadata:
|
||||
name: redis-nextcloud
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
kubernetesConfig:
|
||||
image: quay.io/opstree/redis:v7.0.12
|
||||
imagePullPolicy: IfNotPresent
|
||||
resources:
|
||||
requests:
|
||||
cpu: 25m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
redisExporter:
|
||||
enabled: false
|
||||
redisSecret:
|
||||
name: redis-nextcloud-secret
|
||||
key: password
|
||||
storage:
|
||||
volumeClaimTemplate:
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
storageClassName: rook-ceph-block
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
@@ -0,0 +1,27 @@
|
||||
---
|
||||
# VPS MinIO credentials for CNPG's barmanObjectStore backup target, and reused
|
||||
# by the Phase 1b PVC-content sync CronJob. Same Vault source as
|
||||
# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
|
||||
# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vps-minio-credentials
|
||||
namespace: nextcloud
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: vps-minio-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: accesskey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: access-key
|
||||
- secretKey: secretkey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: secret-key
|
||||
Reference in New Issue
Block a user