mirror of
https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
synced 2026-08-20 23:16:49 +00:00
Add VPS warm-standby/backup site (Phase 0-1b)
Foundation for a DR/backup path using an always-on VPS as a second ArgoCD-managed cluster, plus DB/backup standardization work that fell out of it: - vps-standby ArgoCD cluster destination + AppProject, MinIO backup receiver, VPS bootstrap script (k3s, Netbird, cert-manager) - Dual-site DNS failover watcher + home-IP DDNS CronJob, Cloudflare token moved out of git into Vault+ExternalSecret - Nextcloud migrated from ad-hoc MariaDB to CNPG + redis-operator (matches n8n/Authentik/GitLab's backup-native pattern) - Authentik's CNPG manifests moved into the actual ArgoCD-synced manifests/ path (were present but never wired into the sync path) - Vault raft-snapshot CronJob, CNPG barmanObjectStore backups (Authentik/n8n/Nextcloud), Nextcloud file-PVC restic sync - all targeting the new VPS MinIO receiver See VPS Warm-Standby plan doc for full design rationale.
This commit is contained in:
Scooby Husky
+19
-1
@@ -47,7 +47,25 @@ spec:
|
||||
database: app
|
||||
owner: app
|
||||
|
||||
# Backup to the VPS MinIO backup receiver (Phase 1b). VPS_MINIO_ENDPOINT
|
||||
# placeholder matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml -
|
||||
# replace with the VPS's actual Netbird address once bootstrapped.
|
||||
backup:
|
||||
barmanObjectStore:
|
||||
destinationPath: s3://cnpg-backups/pg-authentik
|
||||
endpointURL: http://vps-minio.netbird.internal:30900
|
||||
s3Credentials:
|
||||
accessKeyId:
|
||||
name: vps-minio-secret
|
||||
key: accesskey
|
||||
secretAccessKey:
|
||||
name: vps-minio-secret
|
||||
key: secretkey
|
||||
wal:
|
||||
compression: gzip
|
||||
maxParallel: 2
|
||||
retentionPolicy: "30d"
|
||||
|
||||
monitoring:
|
||||
enablePodMonitor: true
|
||||
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
---
|
||||
# VPS MinIO credentials for CNPG's barmanObjectStore backup target.
|
||||
# Same Vault source as infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
|
||||
# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vps-minio-credentials
|
||||
namespace: authentik
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: vps-minio-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: accesskey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: access-key
|
||||
- secretKey: secretkey
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: secret-key
|
||||
@@ -0,0 +1,102 @@
|
||||
---
|
||||
# Keeps home.kube.huskypup.net pointed at this cluster's current public IP in
|
||||
# Cloudflare. This is the health-check target the VPS's DNS failover watcher
|
||||
# (scripts/vps-dns-failover.sh) uses to decide whether home is reachable -
|
||||
# it only needs to run while home is up, which is exactly when it can run.
|
||||
#
|
||||
# Reuses the cloudflare-token-secret already wired via ExternalSecret for
|
||||
# cert-manager's DNS-01 solver (see secret-cf-token.yaml) - same zone, same
|
||||
# token, no new secret plumbing.
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: home-ip-ddns-script
|
||||
namespace: cert-manager
|
||||
data:
|
||||
update.sh: |
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ZONE_NAME="kube.huskypup.net"
|
||||
RECORD_NAME="home.kube.huskypup.net"
|
||||
TOKEN="$(cat /etc/cf/cloudflare-token)"
|
||||
|
||||
CURRENT_IP="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)"
|
||||
if [ -z "$CURRENT_IP" ]; then
|
||||
echo "ERROR: could not determine public IP"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ZONE_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" \
|
||||
| jq -r '.result[0].id')"
|
||||
|
||||
RECORD_JSON="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${RECORD_NAME}&type=A")"
|
||||
RECORD_ID="$(echo "$RECORD_JSON" | jq -r '.result[0].id // empty')"
|
||||
EXISTING_IP="$(echo "$RECORD_JSON" | jq -r '.result[0].content // empty')"
|
||||
|
||||
if [ "$EXISTING_IP" = "$CURRENT_IP" ]; then
|
||||
echo "home.kube.huskypup.net already up to date (${CURRENT_IP})"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
BODY="{\"type\":\"A\",\"name\":\"${RECORD_NAME}\",\"content\":\"${CURRENT_IP}\",\"ttl\":120,\"proxied\":false}"
|
||||
|
||||
if [ -n "$RECORD_ID" ]; then
|
||||
echo "Updating ${RECORD_NAME}: ${EXISTING_IP} -> ${CURRENT_IP}"
|
||||
curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "$BODY" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" >/dev/null
|
||||
else
|
||||
echo "Creating ${RECORD_NAME} -> ${CURRENT_IP}"
|
||||
curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
|
||||
-d "$BODY" \
|
||||
"https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" >/dev/null
|
||||
fi
|
||||
echo "done"
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: home-ip-ddns
|
||||
namespace: cert-manager
|
||||
spec:
|
||||
schedule: "*/10 * * * *" # every 10 minutes; cheap, and only matters while home is up
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
containers:
|
||||
- name: ddns-update
|
||||
image: alpine/k8s:1.32.13 # already has curl + jq (see MEMORY.md kubectl image note)
|
||||
command: ["/bin/sh", "/scripts/update.sh"]
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: ["ALL"]
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
- name: cf-token
|
||||
mountPath: /etc/cf
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: home-ip-ddns-script
|
||||
defaultMode: 0755
|
||||
- name: cf-token
|
||||
secret:
|
||||
secretName: cloudflare-token-secret
|
||||
items:
|
||||
- key: cloudflare-token
|
||||
path: cloudflare-token
|
||||
@@ -1,9 +1,22 @@
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Secret
|
||||
# Was previously a plain Secret with the Cloudflare token committed in git.
|
||||
# Rotated to ExternalSecret+Vault (see scripts/rotate-cloudflare-token.sh) -
|
||||
# never commit a live token here again.
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: cloudflare-token-secret
|
||||
namespace: cert-manager
|
||||
type: Opaque
|
||||
stringData:
|
||||
cloudflare-token: Mg9Yx1ku3-rIn7VR7Wf_PZ1uir7AqUsx3IZVpuRX # be sure you are generating an API token and not a global API key https://cert-manager.io/docs/configuration/acme/dns01/cloudflare/#api-tokens
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: cloudflare-token-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: cloudflare-token
|
||||
remoteRef:
|
||||
key: cloudflare-dns-token
|
||||
property: token
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
---
|
||||
# Periodic Vault raft snapshot, shipped to the VPS MinIO backup receiver.
|
||||
# This is the DR path for Vault's data independent of the unseal-key custody
|
||||
# story - a Ceph/cluster-loss disaster is recovered by standing up a fresh
|
||||
# Vault and `vault operator raft snapshot restore`ing the latest one of these,
|
||||
# not by anything to do with the unseal key itself.
|
||||
#
|
||||
# Requires a one-time manual step after VPS bootstrap: store the VPS MinIO
|
||||
# root credentials (see infrastructure/vps-standby/minio/values.yaml) into
|
||||
# this cluster's Vault so ESO can hand them to the CronJob:
|
||||
# vault kv put secret/vps-minio-credentials \
|
||||
# access-key=<minio-root-user> secret-key=<minio-root-password>
|
||||
apiVersion: external-secrets.io/v1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: vps-minio-credentials
|
||||
namespace: vault
|
||||
spec:
|
||||
refreshInterval: 1h
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: vault-backend
|
||||
target:
|
||||
name: vps-minio-credentials
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: access-key
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: access-key
|
||||
- secretKey: secret-key
|
||||
remoteRef:
|
||||
key: vps-minio-credentials
|
||||
property: secret-key
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: vault-raft-snapshot-script
|
||||
namespace: vault
|
||||
data:
|
||||
snapshot.sh: |
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# VPS's Netbird address - replace with the actual peer IP/hostname once
|
||||
# the VPS is bootstrapped and joined to the mesh (scripts/vps-bootstrap.sh).
|
||||
VPS_MINIO_ENDPOINT="${VPS_MINIO_ENDPOINT:-vps-minio.netbird.internal:30900}"
|
||||
BUCKET="vault-raft-snapshots"
|
||||
SNAP_NAME="vault-raft-$(date -u +%Y%m%dT%H%M%SZ).snap"
|
||||
|
||||
ROOT_TOKEN="$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)"
|
||||
|
||||
echo "==> Taking raft snapshot from vault-0..."
|
||||
kubectl -n vault exec vault-0 -- env VAULT_TOKEN="$ROOT_TOKEN" \
|
||||
vault operator raft snapshot save "/tmp/${SNAP_NAME}"
|
||||
|
||||
echo "==> Copying snapshot out of vault-0..."
|
||||
kubectl -n vault cp "vault-0:/tmp/${SNAP_NAME}" "/tmp/${SNAP_NAME}"
|
||||
kubectl -n vault exec vault-0 -- rm -f "/tmp/${SNAP_NAME}"
|
||||
|
||||
echo "==> Installing mc (MinIO client)..."
|
||||
curl -sf https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
|
||||
chmod +x /usr/local/bin/mc
|
||||
mc alias set vps-minio "http://${VPS_MINIO_ENDPOINT}" \
|
||||
"${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null
|
||||
|
||||
echo "==> Uploading ${SNAP_NAME} to vps-minio/${BUCKET}..."
|
||||
mc cp "/tmp/${SNAP_NAME}" "vps-minio/${BUCKET}/${SNAP_NAME}"
|
||||
rm -f "/tmp/${SNAP_NAME}"
|
||||
|
||||
echo "==> Pruning snapshots older than 30 days..."
|
||||
mc find "vps-minio/${BUCKET}" --older-than 30d --exec "mc rm {}" || true
|
||||
|
||||
echo "==> Done: ${SNAP_NAME}"
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: vault-raft-snapshot
|
||||
namespace: vault
|
||||
spec:
|
||||
schedule: "0 */6 * * *" # every 6 hours
|
||||
concurrencyPolicy: Forbid
|
||||
successfulJobsHistoryLimit: 3
|
||||
failedJobsHistoryLimit: 3
|
||||
jobTemplate:
|
||||
spec:
|
||||
backoffLimit: 2
|
||||
template:
|
||||
spec:
|
||||
serviceAccountName: argocd-hook-sa # already has kubectl exec rights in this namespace (see vault-init-job.yaml)
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: raft-snapshot
|
||||
image: alpine/k8s:1.32.13
|
||||
command: ["/bin/bash", "/scripts/snapshot.sh"]
|
||||
env:
|
||||
- name: MINIO_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-credentials
|
||||
key: access-key
|
||||
- name: MINIO_SECRET_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vps-minio-credentials
|
||||
key: secret-key
|
||||
volumeMounts:
|
||||
- name: scripts
|
||||
mountPath: /scripts
|
||||
volumes:
|
||||
- name: scripts
|
||||
configMap:
|
||||
name: vault-raft-snapshot-script
|
||||
defaultMode: 0755
|
||||
@@ -0,0 +1,55 @@
|
||||
# MinIO on the VPS - shared backup-receiver bucket for the vps-standby site.
|
||||
# Standalone mode: single node, single VPS disk, no erasure coding needed here -
|
||||
# this is a backup *copy*, not itself something requiring HA.
|
||||
#
|
||||
# rootUser/rootPassword are intentionally left unset: the chart auto-generates
|
||||
# a random root password and stores it in a Secret (minio/minio chart default
|
||||
# behavior) rather than committing credentials to git. Retrieve after first
|
||||
# deploy with:
|
||||
# kubectl --context vps-standby -n minio get secret minio -o jsonpath='{.data.rootPassword}' | base64 -d
|
||||
|
||||
mode: standalone
|
||||
|
||||
persistence:
|
||||
enabled: true
|
||||
storageClass: local-path
|
||||
size: 150Gi # budget out of the VPS's 360GB disk; leaves room for Phase 2 standby PVCs
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: 50m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
|
||||
# Buckets used by the backup/standby plumbing (Phase 1b / Phase 2). Created on
|
||||
# first deploy; safe to append to as later phases land.
|
||||
buckets:
|
||||
- name: vault-raft-snapshots
|
||||
policy: none
|
||||
purge: false
|
||||
- name: cnpg-backups
|
||||
policy: none
|
||||
purge: false
|
||||
- name: nextcloud-files
|
||||
policy: none
|
||||
purge: false
|
||||
|
||||
# No ingress here - MinIO is reached over Netbird (while home is up) or from
|
||||
# workloads inside the vps-standby cluster itself. It never needs to be public.
|
||||
ingress:
|
||||
enabled: false
|
||||
|
||||
# NodePort so home-cluster CronJobs (vault raft snapshots, CNPG barman backups,
|
||||
# Nextcloud PVC sync) can reach this over the Netbird tunnel at
|
||||
# <VPS_NETBIRD_IP>:30900 - single-node cluster, so NodePort is simplest here.
|
||||
# Verify the exact key path against `helm show values minio/minio` for the
|
||||
# deployed chart version (5.4.0) before applying - not confirmed live.
|
||||
service:
|
||||
type: NodePort
|
||||
nodePort: 30900
|
||||
|
||||
metrics:
|
||||
serviceMonitor:
|
||||
enabled: false # no Prometheus on the VPS cluster (out of scope for this build)
|
||||
Reference in New Issue
Block a user