Add VPS warm-standby/backup site (Phase 0-1b)

Foundation for a DR/backup path using an always-on VPS as a second
ArgoCD-managed cluster, plus DB/backup standardization work that fell
out of it:

- vps-standby ArgoCD cluster destination + AppProject, MinIO backup
  receiver, VPS bootstrap script (k3s, Netbird, cert-manager)
- Dual-site DNS failover watcher + home-IP DDNS CronJob, Cloudflare
  token moved out of git into Vault+ExternalSecret
- Nextcloud migrated from ad-hoc MariaDB to CNPG + redis-operator
  (matches n8n/Authentik/GitLab's backup-native pattern)
- Authentik's CNPG manifests moved into the actual ArgoCD-synced
  manifests/ path (were present but never wired into the sync path)
- Vault raft-snapshot CronJob, CNPG barmanObjectStore backups
  (Authentik/n8n/Nextcloud), Nextcloud file-PVC restic sync - all
  targeting the new VPS MinIO receiver

See VPS Warm-Standby plan doc for full design rationale.

apps/n8n/manifests/cnpg-cluster.yaml

@@ -47,5 +47,25 @@ spec:
       database: n8n
       owner: n8n
 
+  # Backup to the VPS MinIO backup receiver (Phase 1b). Previously had no
+  # backup block at all. VPS_MINIO_ENDPOINT placeholder matches
+  # infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace with
+  # the VPS's actual Netbird address once bootstrapped.
+  backup:
+    barmanObjectStore:
+      destinationPath: s3://cnpg-backups/pg-n8n
+      endpointURL: http://vps-minio.netbird.internal:30900
+      s3Credentials:
+        accessKeyId:
+          name: vps-minio-secret
+          key: accesskey
+        secretAccessKey:
+          name: vps-minio-secret
+          key: secretkey
+      wal:
+        compression: gzip
+        maxParallel: 2
+    retentionPolicy: "30d"
+
   monitoring:
     enablePodMonitor: true

apps/n8n/manifests/vps-minio-secret.yaml

@@ -0,0 +1,26 @@
+---
+# VPS MinIO credentials for CNPG's barmanObjectStore backup target.
+# Same Vault source as infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
+# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: vps-minio-credentials
+  namespace: n8n
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault-backend
+  target:
+    name: vps-minio-secret
+    creationPolicy: Owner
+  data:
+    - secretKey: accesskey
+      remoteRef:
+        key: vps-minio-credentials
+        property: access-key
+    - secretKey: secretkey
+      remoteRef:
+        key: vps-minio-credentials
+        property: secret-key

apps/nextcloud/manifests/cnpg-cluster.yaml

@@ -0,0 +1,68 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: pg-nextcloud
+  namespace: nextcloud
+spec:
+  imageName: ghcr.io/cloudnative-pg/postgresql:16
+  instances: 2
+
+  resources:
+    requests:
+      memory: "512Mi"
+      cpu: "50m"
+    limits:
+      memory: "2Gi"
+      cpu: "500m"
+
+  affinity:
+    topologyKey: kubernetes.io/hostname
+
+  storage:
+    size: 20Gi
+    storageClass: rook-ceph-block
+
+  primaryUpdateStrategy: unsupervised
+
+  postgresql:
+    parameters:
+      max_connections: "200"
+      shared_buffers: "512MB"
+      effective_cache_size: "1536MB"
+      maintenance_work_mem: "128MB"
+      checkpoint_completion_target: "0.9"
+      wal_buffers: "16MB"
+      default_statistics_target: "100"
+      random_page_cost: "1.1"
+      effective_io_concurrency: "200"
+      work_mem: "2621kB"
+      min_wal_size: "1GB"
+      max_wal_size: "4GB"
+
+  bootstrap:
+    initdb:
+      database: nextcloud
+      owner: nextcloud
+
+  # Backup to the VPS MinIO backup receiver. VPS_MINIO_ENDPOINT placeholder
+  # matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml - replace
+  # with the VPS's actual Netbird address once bootstrapped. This covers the
+  # DB only - file PVC content is separate, see nextcloud-pvc-sync-cronjob.yaml.
+  backup:
+    barmanObjectStore:
+      destinationPath: s3://cnpg-backups/pg-nextcloud
+      endpointURL: http://vps-minio.netbird.internal:30900
+      s3Credentials:
+        accessKeyId:
+          name: vps-minio-secret
+          key: accesskey
+        secretAccessKey:
+          name: vps-minio-secret
+          key: secretkey
+      wal:
+        compression: gzip
+        maxParallel: 2
+    retentionPolicy: "30d"
+
+  monitoring:
+    enablePodMonitor: true

apps/nextcloud/manifests/cnpg-secrets.yaml

@@ -0,0 +1,42 @@
+---
+# Mirrors apps/n8n/manifests/cnpg-secrets.yaml: CNPG creates pg-nextcloud-app
+# with a generated password on cluster init; ESO overrides it with a
+# generator-managed password so it's rotated/tracked the same way as every
+# other app secret in this repo.
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: nextcloud-cnpg-secret
+  namespace: nextcloud
+spec:
+  length: 42
+  digits: 5
+  symbols: 5
+  symbolCharacters: "-_$@"
+  noUpper: false
+  allowRepeat: true
+
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: nextcloud-cnpg-secret
+  namespace: nextcloud
+spec:
+  refreshInterval: "24h"
+  target:
+    # Merges into the pg-nextcloud-app secret CNPG creates automatically
+    name: pg-nextcloud-app
+    creationPolicy: Merge
+    template:
+      metadata:
+        labels:
+          cnpg.io/reload: "true"
+      data:
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: nextcloud-cnpg-secret

apps/nextcloud/manifests/mariadb-deployment.yaml

@@ -1,92 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: mariadb-nextcloud
-  namespace: nextcloud
-  labels:
-    app: mariadb-nextcloud
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: mariadb-nextcloud
-  template:
-    metadata:
-      labels:
-        app: mariadb-nextcloud
-    spec:
-      securityContext:
-        runAsUser: 999
-        runAsGroup: 999
-        fsGroup: 999
-      containers:
-      - name: mariadb
-        image: mariadb:11.4
-        securityContext:
-          runAsUser: 999
-          runAsGroup: 999
-          runAsNonRoot: true
-          allowPrivilegeEscalation: false
-          capabilities:
-            drop: ["ALL"]
-        args:
-          - --character-set-server=utf8mb4
-          - --collation-server=utf8mb4_unicode_ci
-          - --init-connect=SET NAMES utf8mb4 COLLATE utf8mb4_unicode_ci
-        env:
-        - name: MARIADB_ROOT_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: mariadb-nextcloud
-              key: root-password
-        - name: MARIADB_PASSWORD
-          valueFrom:
-            secretKeyRef:
-              name: mariadb-nextcloud
-              key: password
-        - name: MARIADB_USER
-          value: nextcloud
-        - name: MARIADB_DATABASE
-          value: nextcloud
-        ports:
-        - containerPort: 3306
-        volumeMounts:
-        - name: mariadb-data
-          mountPath: /var/lib/mysql
-        - name: tmp
-          mountPath: /tmp
-        - name: run-mysqld
-          mountPath: /run/mysqld
-      volumes:
-      - name: tmp
-        emptyDir: {}
-      - name: run-mysqld
-        emptyDir: {}
-      - name: mariadb-data
-        persistentVolumeClaim:
-          claimName: mariadb-nextcloud-pvc
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: mariadb-nextcloud
-  namespace: nextcloud
-spec:
-  selector:
-    app: mariadb-nextcloud
-  ports:
-  - port: 3306
-    targetPort: 3306
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: mariadb-nextcloud-pvc
-  namespace: nextcloud
-spec:
-  accessModes:
-    - ReadWriteOnce
-  storageClassName: rook-ceph-block
-  resources:
-    requests:
-      storage: 10Gi

apps/nextcloud/manifests/mariadb-secrets.yaml

@@ -1,35 +0,0 @@
----
-apiVersion: generators.external-secrets.io/v1alpha1
-kind: Password
-metadata:
-  name: nextcloud-mariadb-password
-  namespace: nextcloud
-spec:
-  length: 32
-  digits: 5
-  symbols: 3
-  symbolCharacters: "-_$"
-  noUpper: false
-  allowRepeat: true
----
-apiVersion: external-secrets.io/v1
-kind: ExternalSecret
-metadata:
-  name: nextcloud-mariadb-secret
-  namespace: nextcloud
-spec:
-  refreshInterval: "24h"
-  target:
-    name: mariadb-nextcloud
-    creationPolicy: Owner
-    template:
-      data:
-        password: "{{ .password }}"
-        root-password: "{{ .password }}"
-        user: nextcloud
-  dataFrom:
-    - sourceRef:
-        generatorRef:
-          apiVersion: generators.external-secrets.io/v1alpha1
-          kind: Password
-          name: nextcloud-mariadb-password

apps/nextcloud/manifests/pvc-sync-cronjob.yaml

@@ -0,0 +1,124 @@
+---
+# Syncs Nextcloud's file PVC content (nextcloud-nextcloud, 200Gi) to the VPS
+# MinIO backup receiver via restic. The one piece of the Nextcloud backup story
+# no operator covers - actual file blobs, not database rows (DB is handled by
+# CNPG's barmanObjectStore in cnpg-cluster.yaml).
+#
+# Caveat: this backs up the live-mounted volume with no application-level
+# quiesce (no Nextcloud maintenance-mode pause around the snapshot). Acceptable
+# for a no-real-data test environment; for anything with real user data, pair
+# this with `occ maintenance:mode --on` before / `--off` after.
+#
+# Uses podAffinity to land on the same node as a running Nextcloud pod, since
+# the PVC is RWO (Ceph RBD) - RWO allows multiple pods to mount it concurrently
+# only when co-located on the same node. Verify the label selector below
+# matches the actual Nextcloud chart's pod labels before relying on this.
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: nextcloud-restic-password
+  namespace: nextcloud
+spec:
+  length: 48
+  digits: 5
+  symbols: 5
+  symbolCharacters: "-_$@"
+  noUpper: false
+  allowRepeat: true
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: nextcloud-restic-password
+  namespace: nextcloud
+spec:
+  refreshInterval: "0"  # generate once - changing this after the repo is initialized would lock you out of existing backups
+  target:
+    name: nextcloud-restic-password
+    creationPolicy: Owner
+  data: []
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: nextcloud-restic-password
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: nextcloud-pvc-sync
+  namespace: nextcloud
+spec:
+  schedule: "0 2 * * *"  # daily at 02:00 - bulky, don't run it as often as the DB backups
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: Never
+          affinity:
+            podAffinity:
+              requiredDuringSchedulingIgnoredDuringExecution:
+                - labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: nextcloud
+                  topologyKey: kubernetes.io/hostname
+          containers:
+            - name: restic-backup
+              image: restic/restic:0.16.4
+              command:
+                - /bin/sh
+                - -c
+                - |
+                  set -eu
+                  export RESTIC_REPOSITORY="s3:http://${VPS_MINIO_ENDPOINT}/nextcloud-files/restic-repo"
+                  export RESTIC_PASSWORD_FILE=/restic-secret/password
+                  export AWS_ACCESS_KEY_ID="${MINIO_ACCESS_KEY}"
+                  export AWS_SECRET_ACCESS_KEY="${MINIO_SECRET_KEY}"
+
+                  restic snapshots >/dev/null 2>&1 || restic init
+
+                  echo "==> Backing up /data..."
+                  restic backup /data --tag nextcloud --host nextcloud-k8s
+
+                  echo "==> Pruning: keep 7 daily, 4 weekly, 6 monthly..."
+                  restic forget --tag nextcloud --host nextcloud-k8s \
+                    --keep-daily 7 --keep-weekly 4 --keep-monthly 6 --prune
+
+                  echo "==> Done."
+              env:
+                # VPS's Netbird address - replace once bootstrapped, matches
+                # infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
+                - name: VPS_MINIO_ENDPOINT
+                  value: "vps-minio.netbird.internal:30900"
+                - name: MINIO_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: vps-minio-secret
+                      key: accesskey
+                - name: MINIO_SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: vps-minio-secret
+                      key: secretkey
+              volumeMounts:
+                - name: nextcloud-data
+                  mountPath: /data
+                  readOnly: true
+                - name: restic-secret
+                  mountPath: /restic-secret
+                  readOnly: true
+          volumes:
+            - name: nextcloud-data
+              persistentVolumeClaim:
+                claimName: nextcloud-nextcloud
+            - name: restic-secret
+              secret:
+                secretName: nextcloud-restic-password
+                items:
+                  - key: password
+                    path: password

apps/nextcloud/manifests/redis-cr.yaml

@@ -0,0 +1,72 @@
+---
+# Standalone Redis via the ot-container-kit redis-operator that's already
+# deployed as infra (argocd-apps/infrastructure/redis-operator.yaml) but
+# currently unused - GitLab's Redis is a hand-rolled StatefulSet instead
+# (apps/gitlab/manifests/redis-cluster.yaml), not this operator. This is the
+# first real consumer of it in the repo.
+#
+# NOTE: verify this CR against `kubectl explain redis.spec` (or the chart's
+# CRD source) for the redis-operator 0.15.0 actually deployed before first
+# apply - the ot-container-kit CRD schema has shifted across versions and
+# this wasn't checked against a live cluster.
+apiVersion: generators.external-secrets.io/v1alpha1
+kind: Password
+metadata:
+  name: nextcloud-redis-password
+  namespace: nextcloud
+spec:
+  length: 32
+  digits: 5
+  symbols: 0
+  noUpper: false
+  allowRepeat: true
+---
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: nextcloud-redis-password
+  namespace: nextcloud
+spec:
+  refreshInterval: "0"  # generate once, don't rotate (Redis CR reads this at pod start only)
+  target:
+    name: redis-nextcloud-secret
+    creationPolicy: Owner
+    template:
+      data:
+        password: "{{ .password }}"
+  dataFrom:
+    - sourceRef:
+        generatorRef:
+          apiVersion: generators.external-secrets.io/v1alpha1
+          kind: Password
+          name: nextcloud-redis-password
+---
+apiVersion: redis.redis.opstreelabs.in/v1beta2
+kind: Redis
+metadata:
+  name: redis-nextcloud
+  namespace: nextcloud
+spec:
+  kubernetesConfig:
+    image: quay.io/opstree/redis:v7.0.12
+    imagePullPolicy: IfNotPresent
+    resources:
+      requests:
+        cpu: 25m
+        memory: 128Mi
+      limits:
+        cpu: 250m
+        memory: 256Mi
+  redisExporter:
+    enabled: false
+  redisSecret:
+    name: redis-nextcloud-secret
+    key: password
+  storage:
+    volumeClaimTemplate:
+      spec:
+        accessModes: ["ReadWriteOnce"]
+        storageClassName: rook-ceph-block
+        resources:
+          requests:
+            storage: 2Gi

apps/nextcloud/manifests/vps-minio-secret.yaml

@@ -0,0 +1,27 @@
+---
+# VPS MinIO credentials for CNPG's barmanObjectStore backup target, and reused
+# by the Phase 1b PVC-content sync CronJob. Same Vault source as
+# infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
+# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: vps-minio-credentials
+  namespace: nextcloud
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault-backend
+  target:
+    name: vps-minio-secret
+    creationPolicy: Owner
+  data:
+    - secretKey: accesskey
+      remoteRef:
+        key: vps-minio-credentials
+        property: access-key
+    - secretKey: secretkey
+      remoteRef:
+        key: vps-minio-credentials
+        property: secret-key

apps/nextcloud/values.yaml

@@ -73,16 +73,27 @@ nextcloud:
 
 externalDatabase:
    enabled: true
-   type: mysql
+   # Moved from MariaDB to CNPG-backed Postgres (apps/nextcloud/manifests/cnpg-cluster.yaml)
-   host: mariadb-nextcloud
+   # for backup/DR consistency with the rest of the stack (n8n, Authentik, GitLab
-   port: 3306
+   # all use CNPG's native barmanObjectStore backup - MariaDB needed a bespoke
+   # mysqldump job instead). pg-nextcloud-rw is CNPG's generated read-write
+   # Service name for the "pg-nextcloud" Cluster.
+   #
+   # IMPORTANT: switching type here does NOT migrate existing data. Nextcloud
+   # requires an explicit `occ db:convert-type pgsql ...` run before cutover if
+   # there's real data in the old MariaDB instance. usernameKey/passwordKey below
+   # match CNPG's generated app-secret keys (confirmed against the existing,
+   # already-working infrastructure/authentik/values.yaml pg-authentik-app usage).
+   type: postgresql
+   host: pg-nextcloud-rw
+   port: 5432
    user: nextcloud
    database: nextcloud
    existingSecret:
       enabled: true
-      secretName: mariadb-nextcloud
+      secretName: pg-nextcloud-app
       passwordKey: password
-      usernameKey: user
+      usernameKey: username  # matches CNPG's generated app-secret key, confirmed against infrastructure/authentik/values.yaml's existing pg-authentik-app usage
 
 persistence:
    enabled: true
@@ -99,7 +110,16 @@ resources:
      memory: 2Gi
 
 redis:
-   enabled: false
+   # Was disabled; now points at the standalone Redis CR (redis-operator) in
+   # apps/nextcloud/manifests/redis-cr.yaml instead of the chart's bundled
+   # subchart - fixes Nextcloud's file-locking/caching as a side benefit of
+   # this migration. Verify these key names against `helm show values
+   # nextcloud/nextcloud` for the deployed chart version before applying.
+   enabled: true
+   host: redis-nextcloud
+   port: 6379
+   existingSecret: redis-nextcloud-secret
+   existingSecretPasswordKey: password
 
 metrics:
    enabled: false

argocd-apps/projects.yaml

@@ -57,3 +57,26 @@ spec:
   namespaceResourceWhitelist:
     - group: '*'
       kind: '*'
+---
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: vps-standby
+  namespace: argocd
+spec:
+  # VPS warm-standby / backup-site services. Deployed to the "vps-standby" cluster
+  # registered via `argocd cluster add ... --name vps-standby` (see
+  # scripts/vps-bootstrap.sh). Scoped to that cluster only — nothing in this
+  # project should ever target the home cluster.
+  description: VPS warm-standby / backup-site services
+  sourceRepos:
+    - '*'
+  destinations:
+    - namespace: '*'
+      name: vps-standby
+  clusterResourceWhitelist:
+    - group: '*'
+      kind: '*'
+  namespaceResourceWhitelist:
+    - group: '*'
+      kind: '*'

argocd-apps/vps-standby/minio.yaml

@@ -0,0 +1,30 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: vps-minio
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"  # first thing on the vps-standby cluster - everything else backs up to it
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: vps-standby
+  sources:
+    - repoURL: https://charts.min.io/
+      chart: minio
+      targetRevision: 5.4.0
+      helm:
+        valueFiles:
+          - $values/infrastructure/vps-standby/minio/values.yaml
+    - repoURL: https://gitlab.kube.huskypup.net/Scooby/Homelabv4.git
+      targetRevision: main
+      ref: values
+  destination:
+    name: vps-standby
+    namespace: minio
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

infrastructure/authentik/manifests/cnpg-cluster.yaml

@@ -47,7 +47,25 @@ spec:
       database: app
       owner: app
 
+  # Backup to the VPS MinIO backup receiver (Phase 1b). VPS_MINIO_ENDPOINT
+  # placeholder matches infrastructure/vault/manifests/raft-snapshot-cronjob.yaml -
+  # replace with the VPS's actual Netbird address once bootstrapped.
+  backup:
+    barmanObjectStore:
+      destinationPath: s3://cnpg-backups/pg-authentik
+      endpointURL: http://vps-minio.netbird.internal:30900
+      s3Credentials:
+        accessKeyId:
+          name: vps-minio-secret
+          key: accesskey
+        secretAccessKey:
+          name: vps-minio-secret
+          key: secretkey
+      wal:
+        compression: gzip
+        maxParallel: 2
+    retentionPolicy: "30d"
+
   monitoring:
     enablePodMonitor: true
 
-

infrastructure/authentik/manifests/vps-minio-secret.yaml

@@ -0,0 +1,26 @@
+---
+# VPS MinIO credentials for CNPG's barmanObjectStore backup target.
+# Same Vault source as infrastructure/vault/manifests/raft-snapshot-cronjob.yaml
+# (secret/vps-minio-credentials) - populated once, manually, after VPS bootstrap.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: vps-minio-credentials
+  namespace: authentik
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault-backend
+  target:
+    name: vps-minio-secret
+    creationPolicy: Owner
+  data:
+    - secretKey: accesskey
+      remoteRef:
+        key: vps-minio-credentials
+        property: access-key
+    - secretKey: secretkey
+      remoteRef:
+        key: vps-minio-credentials
+        property: secret-key

infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml

@@ -0,0 +1,102 @@
+---
+# Keeps home.kube.huskypup.net pointed at this cluster's current public IP in
+# Cloudflare. This is the health-check target the VPS's DNS failover watcher
+# (scripts/vps-dns-failover.sh) uses to decide whether home is reachable -
+# it only needs to run while home is up, which is exactly when it can run.
+#
+# Reuses the cloudflare-token-secret already wired via ExternalSecret for
+# cert-manager's DNS-01 solver (see secret-cf-token.yaml) - same zone, same
+# token, no new secret plumbing.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: home-ip-ddns-script
+  namespace: cert-manager
+data:
+  update.sh: |
+    #!/bin/sh
+    set -eu
+
+    ZONE_NAME="kube.huskypup.net"
+    RECORD_NAME="home.kube.huskypup.net"
+    TOKEN="$(cat /etc/cf/cloudflare-token)"
+
+    CURRENT_IP="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)"
+    if [ -z "$CURRENT_IP" ]; then
+      echo "ERROR: could not determine public IP"
+      exit 1
+    fi
+
+    ZONE_ID="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+      "https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" \
+      | jq -r '.result[0].id')"
+
+    RECORD_JSON="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+      "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records?name=${RECORD_NAME}&type=A")"
+    RECORD_ID="$(echo "$RECORD_JSON" | jq -r '.result[0].id // empty')"
+    EXISTING_IP="$(echo "$RECORD_JSON" | jq -r '.result[0].content // empty')"
+
+    if [ "$EXISTING_IP" = "$CURRENT_IP" ]; then
+      echo "home.kube.huskypup.net already up to date (${CURRENT_IP})"
+      exit 0
+    fi
+
+    BODY="{\"type\":\"A\",\"name\":\"${RECORD_NAME}\",\"content\":\"${CURRENT_IP}\",\"ttl\":120,\"proxied\":false}"
+
+    if [ -n "$RECORD_ID" ]; then
+      echo "Updating ${RECORD_NAME}: ${EXISTING_IP} -> ${CURRENT_IP}"
+      curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
+        -d "$BODY" \
+        "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records/${RECORD_ID}" >/dev/null
+    else
+      echo "Creating ${RECORD_NAME} -> ${CURRENT_IP}"
+      curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
+        -d "$BODY" \
+        "https://api.cloudflare.com/client/v4/zones/${ZONE_ID}/dns_records" >/dev/null
+    fi
+    echo "done"
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: home-ip-ddns
+  namespace: cert-manager
+spec:
+  schedule: "*/10 * * * *"  # every 10 minutes; cheap, and only matters while home is up
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          restartPolicy: Never
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65534
+          containers:
+            - name: ddns-update
+              image: alpine/k8s:1.32.13  # already has curl + jq (see MEMORY.md kubectl image note)
+              command: ["/bin/sh", "/scripts/update.sh"]
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop: ["ALL"]
+              volumeMounts:
+                - name: script
+                  mountPath: /scripts
+                - name: cf-token
+                  mountPath: /etc/cf
+                  readOnly: true
+          volumes:
+            - name: script
+              configMap:
+                name: home-ip-ddns-script
+                defaultMode: 0755
+            - name: cf-token
+              secret:
+                secretName: cloudflare-token-secret
+                items:
+                  - key: cloudflare-token
+                    path: cloudflare-token

infrastructure/cert-manager/manifests/secret-cf-token.yaml

@@ -1,9 +1,22 @@
 ---
-apiVersion: v1
+# Was previously a plain Secret with the Cloudflare token committed in git.
-kind: Secret
+# Rotated to ExternalSecret+Vault (see scripts/rotate-cloudflare-token.sh) -
+# never commit a live token here again.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
 metadata:
   name: cloudflare-token-secret
   namespace: cert-manager
-type: Opaque
+spec:
-stringData:
+  refreshInterval: 1h
-  cloudflare-token: Mg9Yx1ku3-rIn7VR7Wf_PZ1uir7AqUsx3IZVpuRX # be sure you are generating an API token and not a global API key https://cert-manager.io/docs/configuration/acme/dns01/cloudflare/#api-tokens
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault-backend
+  target:
+    name: cloudflare-token-secret
+    creationPolicy: Owner
+  data:
+    - secretKey: cloudflare-token
+      remoteRef:
+        key: cloudflare-dns-token
+        property: token

infrastructure/vault/manifests/raft-snapshot-cronjob.yaml

@@ -0,0 +1,116 @@
+---
+# Periodic Vault raft snapshot, shipped to the VPS MinIO backup receiver.
+# This is the DR path for Vault's data independent of the unseal-key custody
+# story - a Ceph/cluster-loss disaster is recovered by standing up a fresh
+# Vault and `vault operator raft snapshot restore`ing the latest one of these,
+# not by anything to do with the unseal key itself.
+#
+# Requires a one-time manual step after VPS bootstrap: store the VPS MinIO
+# root credentials (see infrastructure/vps-standby/minio/values.yaml) into
+# this cluster's Vault so ESO can hand them to the CronJob:
+#   vault kv put secret/vps-minio-credentials \
+#     access-key=<minio-root-user> secret-key=<minio-root-password>
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: vps-minio-credentials
+  namespace: vault
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: vault-backend
+  target:
+    name: vps-minio-credentials
+    creationPolicy: Owner
+  data:
+    - secretKey: access-key
+      remoteRef:
+        key: vps-minio-credentials
+        property: access-key
+    - secretKey: secret-key
+      remoteRef:
+        key: vps-minio-credentials
+        property: secret-key
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: vault-raft-snapshot-script
+  namespace: vault
+data:
+  snapshot.sh: |
+    #!/bin/bash
+    set -euo pipefail
+
+    # VPS's Netbird address - replace with the actual peer IP/hostname once
+    # the VPS is bootstrapped and joined to the mesh (scripts/vps-bootstrap.sh).
+    VPS_MINIO_ENDPOINT="${VPS_MINIO_ENDPOINT:-vps-minio.netbird.internal:30900}"
+    BUCKET="vault-raft-snapshots"
+    SNAP_NAME="vault-raft-$(date -u +%Y%m%dT%H%M%SZ).snap"
+
+    ROOT_TOKEN="$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)"
+
+    echo "==> Taking raft snapshot from vault-0..."
+    kubectl -n vault exec vault-0 -- env VAULT_TOKEN="$ROOT_TOKEN" \
+      vault operator raft snapshot save "/tmp/${SNAP_NAME}"
+
+    echo "==> Copying snapshot out of vault-0..."
+    kubectl -n vault cp "vault-0:/tmp/${SNAP_NAME}" "/tmp/${SNAP_NAME}"
+    kubectl -n vault exec vault-0 -- rm -f "/tmp/${SNAP_NAME}"
+
+    echo "==> Installing mc (MinIO client)..."
+    curl -sf https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
+    chmod +x /usr/local/bin/mc
+    mc alias set vps-minio "http://${VPS_MINIO_ENDPOINT}" \
+      "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}" >/dev/null
+
+    echo "==> Uploading ${SNAP_NAME} to vps-minio/${BUCKET}..."
+    mc cp "/tmp/${SNAP_NAME}" "vps-minio/${BUCKET}/${SNAP_NAME}"
+    rm -f "/tmp/${SNAP_NAME}"
+
+    echo "==> Pruning snapshots older than 30 days..."
+    mc find "vps-minio/${BUCKET}" --older-than 30d --exec "mc rm {}" || true
+
+    echo "==> Done: ${SNAP_NAME}"
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: vault-raft-snapshot
+  namespace: vault
+spec:
+  schedule: "0 */6 * * *"  # every 6 hours
+  concurrencyPolicy: Forbid
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          serviceAccountName: argocd-hook-sa  # already has kubectl exec rights in this namespace (see vault-init-job.yaml)
+          restartPolicy: Never
+          containers:
+            - name: raft-snapshot
+              image: alpine/k8s:1.32.13
+              command: ["/bin/bash", "/scripts/snapshot.sh"]
+              env:
+                - name: MINIO_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: vps-minio-credentials
+                      key: access-key
+                - name: MINIO_SECRET_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: vps-minio-credentials
+                      key: secret-key
+              volumeMounts:
+                - name: scripts
+                  mountPath: /scripts
+          volumes:
+            - name: scripts
+              configMap:
+                name: vault-raft-snapshot-script
+                defaultMode: 0755

infrastructure/vps-standby/minio/values.yaml

@@ -0,0 +1,55 @@
+# MinIO on the VPS - shared backup-receiver bucket for the vps-standby site.
+# Standalone mode: single node, single VPS disk, no erasure coding needed here -
+# this is a backup *copy*, not itself something requiring HA.
+#
+# rootUser/rootPassword are intentionally left unset: the chart auto-generates
+# a random root password and stores it in a Secret (minio/minio chart default
+# behavior) rather than committing credentials to git. Retrieve after first
+# deploy with:
+#   kubectl --context vps-standby -n minio get secret minio -o jsonpath='{.data.rootPassword}' | base64 -d
+
+mode: standalone
+
+persistence:
+  enabled: true
+  storageClass: local-path
+  size: 150Gi  # budget out of the VPS's 360GB disk; leaves room for Phase 2 standby PVCs
+
+resources:
+  requests:
+    cpu: 50m
+    memory: 256Mi
+  limits:
+    cpu: 500m
+    memory: 1Gi
+
+# Buckets used by the backup/standby plumbing (Phase 1b / Phase 2). Created on
+# first deploy; safe to append to as later phases land.
+buckets:
+  - name: vault-raft-snapshots
+    policy: none
+    purge: false
+  - name: cnpg-backups
+    policy: none
+    purge: false
+  - name: nextcloud-files
+    policy: none
+    purge: false
+
+# No ingress here - MinIO is reached over Netbird (while home is up) or from
+# workloads inside the vps-standby cluster itself. It never needs to be public.
+ingress:
+  enabled: false
+
+# NodePort so home-cluster CronJobs (vault raft snapshots, CNPG barman backups,
+# Nextcloud PVC sync) can reach this over the Netbird tunnel at
+# <VPS_NETBIRD_IP>:30900 - single-node cluster, so NodePort is simplest here.
+# Verify the exact key path against `helm show values minio/minio` for the
+# deployed chart version (5.4.0) before applying - not confirmed live.
+service:
+  type: NodePort
+  nodePort: 30900
+
+metrics:
+  serviceMonitor:
+    enabled: false  # no Prometheus on the VPS cluster (out of scope for this build)

scripts/rotate-cloudflare-token.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+# rotate-cloudflare-token.sh - Rotate the Cloudflare DNS-01 token out of git and into Vault
+#
+# infrastructure/cert-manager/manifests/secret-cf-token.yaml used to contain a live
+# Cloudflare API token committed in plaintext. It's now an ExternalSecret pulling
+# from Vault at secret/cloudflare-dns-token#token - this script populates that path.
+#
+# This does NOT create or revoke the Cloudflare token itself - that's a manual step
+# in the Cloudflare dashboard, deliberately not automated here since it's a live,
+# outward-facing credential change:
+#
+#   1. Cloudflare dashboard -> My Profile -> API Tokens -> Create Token
+#      Scope: Zone:DNS:Edit, restricted to the kube.huskypup.net zone only
+#   2. Run this script with the new token
+#   3. Confirm cert-manager can still issue certs (kubectl get certificaterequests -A)
+#   4. THEN go back to Cloudflare and revoke the old token
+#      (the one that was committed in git - assume it's compromised)
+#
+# Usage:
+#   ./scripts/rotate-cloudflare-token.sh <new-cloudflare-token>
+#
+# Prerequisites:
+#   - Vault initialized and unsealed
+#   - kubectl configured for the home cluster
+
+set -euo pipefail
+
+NEW_TOKEN="${1:?Usage: $0 <new-cloudflare-token>}"
+
+echo "=== Cloudflare DNS-01 Token Rotation ==="
+
+echo "Storing new token in Vault at secret/cloudflare-dns-token..."
+ROOT_TOKEN=$(kubectl -n vault get secret vault-init-keys -o jsonpath='{.data.VAULT_ROOT_TOKEN}' | base64 -d)
+kubectl exec -n vault vault-0 -- env "VAULT_TOKEN=${ROOT_TOKEN}" \
+  vault kv put secret/cloudflare-dns-token token="${NEW_TOKEN}"
+
+echo "Forcing ExternalSecret refresh..."
+kubectl -n cert-manager annotate externalsecret cloudflare-token-secret \
+  force-sync="$(date +%s)" --overwrite
+
+echo ""
+echo "=== Done ==="
+echo "Next:"
+echo "  1. Verify: kubectl -n cert-manager get secret cloudflare-token-secret -o jsonpath='{.data.cloudflare-token}' | base64 -d"
+echo "  2. Verify a cert still renews cleanly (or delete one Certificate to force a test issuance)"
+echo "  3. Revoke the OLD token in the Cloudflare dashboard once confirmed working"

scripts/vps-bootstrap.sh

@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+# vps-bootstrap.sh - Phase 0: turn a bare VPS into the vps-standby ArgoCD destination
+#
+# Run this ON THE VPS itself (as root, or via sudo), not against the home cluster.
+# Installs k3s (single node), joins the existing self-hosted Netbird mesh, and
+# installs cert-manager with the same Cloudflare DNS-01 ClusterIssuer pattern used
+# at home — so TLS issuance works identically regardless of which site is "live"
+# (DNS-01 only needs DNS control, not public HTTP reachability).
+#
+# This script does NOT register the cluster with ArgoCD — that's a one-time manual
+# step run from your workstation/home cluster once this script prints the kubeconfig
+# (ArgoCD can't reach the VPS until it exists, and shouldn't hold cluster-admin creds
+# for a box it doesn't manage yet).
+#
+# Usage:
+#   sudo ./scripts/vps-bootstrap.sh <netbird-setup-key> <cloudflare-dns-edit-token>
+#
+# Prerequisites:
+#   - A Netbird setup key (Netbird dashboard → Settings → Setup Keys → create
+#     a reusable, non-ephemeral key)
+#   - A Cloudflare API token scoped to Zone:DNS:Edit for kube.huskypup.net only
+#     (create a NEW token for this — do not reuse the one from
+#     infrastructure/cert-manager/manifests/secret-cf-token.yaml, that one is
+#     being rotated/retired; see Phase 0.5)
+#   - Ubuntu/Debian VPS with a public IP, run as root
+
+set -euo pipefail
+
+NETBIRD_SETUP_KEY="${1:?Usage: $0 <netbird-setup-key> <cloudflare-dns-edit-token>}"
+CLOUDFLARE_TOKEN="${2:?Usage: $0 <netbird-setup-key> <cloudflare-dns-edit-token>}"
+
+NETBIRD_MGMT_URL="https://netbird.kube.huskypup.net"
+LETSENCRYPT_EMAIL="garrettstone499@gmail.com"
+DNS_ZONE="kube.huskypup.net"
+CERT_MANAGER_VERSION="v1.13.2"  # matches infrastructure/cert-manager chart version at home
+
+if [ "$(id -u)" -ne 0 ]; then
+  echo "ERROR: run as root (sudo $0 ...)" >&2
+  exit 1
+fi
+
+echo "================================================="
+echo "VPS Standby Bootstrap - Phase 0"
+echo "================================================="
+echo ""
+
+# --- k3s -----------------------------------------------------------------
+if command -v k3s >/dev/null 2>&1; then
+  echo "✅ k3s already installed, skipping install"
+else
+  echo "Installing k3s (single node)..."
+  # Keep the built-in Traefik ingress controller — this is a lean standby box,
+  # not a mirror of home's Istio/Envoy-Gateway mesh. servicelb is fine too
+  # since this is a single node with a real public IP.
+  curl -sfL https://get.k3s.io | sh -s - \
+    --write-kubeconfig-mode 644 \
+    --disable metrics-server
+  echo "✅ k3s installed"
+fi
+
+echo "Waiting for k3s node to be Ready..."
+for i in $(seq 1 30); do
+  if k3s kubectl get nodes 2>/dev/null | grep -q " Ready"; then
+    echo "✅ node is Ready"
+    break
+  fi
+  sleep 5
+done
+k3s kubectl get nodes
+
+export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+
+# --- Netbird ---------------------------------------------------------------
+if command -v netbird >/dev/null 2>&1 && netbird status 2>/dev/null | grep -q "Management: Connected"; then
+  echo "✅ Netbird already connected, skipping"
+else
+  echo "Installing Netbird client..."
+  curl -fsSL https://pkgs.netbird.io/install.sh | sh
+  echo "Joining Netbird mesh (${NETBIRD_MGMT_URL})..."
+  netbird up --management-url "${NETBIRD_MGMT_URL}" --setup-key "${NETBIRD_SETUP_KEY}"
+  echo "✅ Netbird joined"
+fi
+echo "Note: this Netbird session is used opportunistically for backup/sync traffic"
+echo "while home is up. It is NOT the path used to reach this VPS when home is down —"
+echo "that's direct SSH on this box's public IP. See plan doc, decision #2."
+
+# --- cert-manager ------------------------------------------------------------
+echo ""
+echo "Installing cert-manager ${CERT_MANAGER_VERSION}..."
+if ! command -v helm >/dev/null 2>&1; then
+  curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+fi
+
+helm repo add jetstack https://charts.jetstack.io 2>/dev/null || true
+helm repo update jetstack
+
+if helm -n cert-manager status cert-manager >/dev/null 2>&1; then
+  echo "✅ cert-manager already installed"
+else
+  helm install cert-manager jetstack/cert-manager \
+    --namespace cert-manager \
+    --create-namespace \
+    --version "${CERT_MANAGER_VERSION}" \
+    --set installCRDs=true \
+    --wait --timeout 300s
+  echo "✅ cert-manager installed"
+fi
+
+echo "Waiting for cert-manager webhook to be ready..."
+k3s kubectl -n cert-manager rollout status deployment/cert-manager-webhook --timeout=120s
+
+# --- Cloudflare DNS-01 ClusterIssuer (same pattern as home) -----------------
+echo ""
+echo "Applying Cloudflare token Secret + ClusterIssuer..."
+cat <<EOF | k3s kubectl apply -f -
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloudflare-token-secret
+  namespace: cert-manager
+type: Opaque
+stringData:
+  cloudflare-token: "${CLOUDFLARE_TOKEN}"
+---
+apiVersion: cert-manager.io/v1
+kind: ClusterIssuer
+metadata:
+  name: letsencrypt-production
+spec:
+  acme:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: ${LETSENCRYPT_EMAIL}
+    privateKeySecretRef:
+      name: letsencrypt-production
+    solvers:
+      - dns01:
+          cloudflare:
+            email: ${LETSENCRYPT_EMAIL}
+            apiTokenSecretRef:
+              name: cloudflare-token-secret
+              key: cloudflare-token
+        selector:
+          dnsZones:
+            - "${DNS_ZONE}"
+EOF
+echo "✅ ClusterIssuer letsencrypt-production ready"
+
+# --- Output for ArgoCD registration -----------------------------------------
+echo ""
+echo "================================================="
+echo "✅ VPS foundation bootstrap complete"
+echo "================================================="
+echo ""
+echo "Next step (run from your workstation, NOT this VPS):"
+echo ""
+echo "  1. Copy this VPS's kubeconfig to your workstation, e.g.:"
+echo "       scp root@<vps-ip>:/etc/rancher/k3s/k3s.yaml ~/vps-standby-kubeconfig.yaml"
+echo "     Then edit the 'server:' line inside it to use this VPS's public IP"
+echo "     instead of 127.0.0.1."
+echo ""
+echo "  2. Register it with ArgoCD:"
+echo "       KUBECONFIG=~/vps-standby-kubeconfig.yaml argocd cluster add default --name vps-standby"
+echo ""
+echo "  3. Confirm registration:"
+echo "       argocd cluster list"
+echo ""
+echo "This box is otherwise reachable via:"
+echo "  - Netbird (while home's self-hosted mesh is up)"
+echo "  - Direct SSH on its public IP (always, break-glass path)"

scripts/vps-dns-failover/vps-dns-failover.service

@@ -0,0 +1,10 @@
+[Unit]
+Description=VPS dual-site DNS failover check (Homelabv4 vps-standby)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/vps-dns-failover.sh
+# Deliberately no dependency on k3s/docker being up - this must keep working
+# even if the VPS's own cluster is unhealthy.

scripts/vps-dns-failover/vps-dns-failover.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+# vps-dns-failover.sh - Phase 0.5 dual-site DNS failover watcher
+#
+# Runs ON THE VPS as a systemd timer (see vps-dns-failover.timer/.service in this
+# directory) - deliberately plain systemd, not a k8s CronJob, so it works even if
+# the VPS's own k3s is unhealthy. It must never depend on anything inside the home
+# cluster (that's the thing it's watching) or the VPS's own k3s (that's a separate
+# failure domain from this box's basic OS-level networking).
+#
+# Health-checks home.kube.huskypup.net (kept current by an in-cluster CronJob while
+# home is up - see infrastructure/cert-manager/manifests/home-ip-ddns-cronjob.yaml)
+# and flips Cloudflare A records for the standby-service hostnames between home's
+# public IP and this VPS's own public IP, with a consecutive-check threshold so a
+# single blip doesn't cause a flap.
+#
+# State (current active site + streak counters) persists in $STATE_DIR between
+# runs since each systemd timer firing is a fresh process.
+#
+# Install:
+#   sudo mkdir -p /etc/vps-dns-failover
+#   sudo sh -c 'echo "<dns-edit-scoped-cloudflare-token>" > /etc/vps-dns-failover/cloudflare-token'
+#   sudo chmod 600 /etc/vps-dns-failover/cloudflare-token
+#   sudo cp vps-dns-failover.sh /usr/local/bin/vps-dns-failover.sh
+#   sudo chmod +x /usr/local/bin/vps-dns-failover.sh
+#   sudo cp vps-dns-failover.service vps-dns-failover.timer /etc/systemd/system/
+#   sudo systemctl daemon-reload
+#   sudo systemctl enable --now vps-dns-failover.timer
+
+set -euo pipefail
+
+TOKEN_FILE="/etc/vps-dns-failover/cloudflare-token"
+STATE_DIR="/var/lib/vps-dns-failover"
+ZONE_NAME="kube.huskypup.net"
+HOME_CHECK_HOST="home.kube.huskypup.net"
+HOME_CHECK_PORT=443
+FAILURE_THRESHOLD=3   # consecutive failed checks before flipping to the VPS
+SUCCESS_THRESHOLD=3   # consecutive successful checks before flipping back to home
+STANDBY_HOSTNAMES=(
+  vault.kube.huskypup.net
+  auth.kube.huskypup.net
+  gitea.kube.huskypup.net
+  n8n.kube.huskypup.net
+  nextcloud.kube.huskypup.net
+)
+
+mkdir -p "$STATE_DIR"
+TOKEN="$(cat "$TOKEN_FILE")"
+STATE_FILE="${STATE_DIR}/state"   # format: "<active-site> <fail-streak> <success-streak>"
+
+if [ -f "$STATE_FILE" ]; then
+  read -r ACTIVE FAIL_STREAK SUCCESS_STREAK < "$STATE_FILE"
+else
+  ACTIVE="home"
+  FAIL_STREAK=0
+  SUCCESS_STREAK=0
+fi
+
+# --- health check ------------------------------------------------------------
+HOME_IP="$(dig +short A "$HOME_CHECK_HOST" @1.1.1.1 | tail -n1)"
+if [ -n "$HOME_IP" ] && timeout 5 bash -c "cat < /dev/null > /dev/tcp/${HOME_IP}/${HOME_CHECK_PORT}" 2>/dev/null; then
+  HEALTHY=1
+else
+  HEALTHY=0
+fi
+
+if [ "$HEALTHY" = 1 ]; then
+  FAIL_STREAK=0
+  SUCCESS_STREAK=$((SUCCESS_STREAK + 1))
+else
+  SUCCESS_STREAK=0
+  FAIL_STREAK=$((FAIL_STREAK + 1))
+fi
+
+echo "$(date -u +%FT%TZ) active=${ACTIVE} healthy=${HEALTHY} fail_streak=${FAIL_STREAK} success_streak=${SUCCESS_STREAK} home_ip=${HOME_IP:-none}"
+
+# --- Cloudflare helpers --------------------------------------------------------
+cf_zone_id() {
+  curl -sf -H "Authorization: Bearer ${TOKEN}" \
+    "https://api.cloudflare.com/client/v4/zones?name=${ZONE_NAME}" | jq -r '.result[0].id'
+}
+
+cf_set_record() {
+  local zone_id="$1" hostname="$2" target_ip="$3"
+  local record_json record_id
+  record_json="$(curl -sf -H "Authorization: Bearer ${TOKEN}" \
+    "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records?name=${hostname}&type=A")"
+  record_id="$(echo "$record_json" | jq -r '.result[0].id // empty')"
+  local body="{\"type\":\"A\",\"name\":\"${hostname}\",\"content\":\"${target_ip}\",\"ttl\":60,\"proxied\":false}"
+  if [ -n "$record_id" ]; then
+    curl -sf -X PATCH -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
+      -d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records/${record_id}" >/dev/null
+  else
+    curl -sf -X POST -H "Authorization: Bearer ${TOKEN}" -H "Content-Type: application/json" \
+      -d "$body" "https://api.cloudflare.com/client/v4/zones/${zone_id}/dns_records" >/dev/null
+  fi
+  echo "  ${hostname} -> ${target_ip}"
+}
+
+flip_to() {
+  local target="$1"
+  local target_ip
+  if [ "$target" = "vps" ]; then
+    target_ip="$(curl -sf https://cloudflare.com/cdn-cgi/trace | grep -o '^ip=.*' | cut -d= -f2)"
+  else
+    target_ip="$HOME_IP"
+  fi
+  if [ -z "$target_ip" ]; then
+    echo "ERROR: could not determine target IP for '${target}', not flipping"
+    return 1
+  fi
+  echo "Flipping standby hostnames to ${target} (${target_ip})..."
+  local zone_id
+  zone_id="$(cf_zone_id)"
+  for h in "${STANDBY_HOSTNAMES[@]}"; do
+    cf_set_record "$zone_id" "$h" "$target_ip"
+  done
+}
+
+# --- decide ------------------------------------------------------------------
+if [ "$ACTIVE" = "home" ] && [ "$FAIL_STREAK" -ge "$FAILURE_THRESHOLD" ]; then
+  flip_to "vps"
+  ACTIVE="vps"
+  FAIL_STREAK=0
+  SUCCESS_STREAK=0
+elif [ "$ACTIVE" = "vps" ] && [ "$SUCCESS_STREAK" -ge "$SUCCESS_THRESHOLD" ]; then
+  flip_to "home"
+  ACTIVE="home"
+  FAIL_STREAK=0
+  SUCCESS_STREAK=0
+fi
+
+echo "${ACTIVE} ${FAIL_STREAK} ${SUCCESS_STREAK}" > "$STATE_FILE"

scripts/vps-dns-failover/vps-dns-failover.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run vps-dns-failover check every 2 minutes
+
+[Timer]
+OnBootSec=1min
+OnUnitActiveSec=2min
+AccuracySec=10s
+
+[Install]
+WantedBy=timers.target